diff --git a/api/openapi.yaml b/api/openapi.yaml
index 3d49b67b7..f4e240e0c 100644
--- a/api/openapi.yaml
+++ b/api/openapi.yaml
@@ -1984,7 +1984,7 @@ components:
additionalProperties: true
properties:
code:
- description: "Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status."
+ description: "Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, sending_paused, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental), sending_paused (403, outbound sending is paused for the account by the platform abuse controls; queued mail is held, new sends are refused until an operator resumes; experimental). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status."
type: string
x-e2a-error-contracts:
address_in_trash:
@@ -2259,6 +2259,11 @@ components:
retryable: false
statuses:
- 409
+ sending_paused:
+ family: auth
+ retryable: false
+ statuses:
+ - 403
starter_template_not_found:
family: not_found
retryable: false
@@ -2317,6 +2322,7 @@ components:
- 400
x-experimental-values:
- blocked_by_policy
+ - sending_paused
details:
additionalProperties: true
description: Optional structured context, polymorphic by code. Treat it as an open object keyed off code; unknown codes and fields must be preserved.
@@ -2891,6 +2897,8 @@ components:
- submission.provider_rejected
- submission.local_retries_exhausted
- submission.cancelled
+ - submission.policy_budget_expired
+ - submission.sending_setup_expired
- delivery.recipient_server_accepted
- delivery.temporary_delay
- delivery.permanent_bounce
diff --git a/cmd/e2a/main.go b/cmd/e2a/main.go
index aa8ec21a7..7e13c1726 100644
--- a/cmd/e2a/main.go
+++ b/cmd/e2a/main.go
@@ -38,7 +38,6 @@ import (
"github.com/tokencanopy/e2a/internal/limits"
"github.com/tokencanopy/e2a/internal/oauth"
"github.com/tokencanopy/e2a/internal/outbound"
- "github.com/tokencanopy/e2a/internal/outboundsend"
"github.com/tokencanopy/e2a/internal/relay"
"github.com/tokencanopy/e2a/internal/senderidentity"
"github.com/tokencanopy/e2a/internal/sendingpolicy"
@@ -343,29 +342,28 @@ func main() {
// Outbound delivery is queue-first and at-least-once for GA. The accept-tx
// enqueues an outbound_send job in the same transaction as the message row;
- // there is no submit-inline fallback.
+ // there is no submit-inline fallback. Every provider call passes through
+ // the sending-protection gate and the authorized submitter — see
+ // newOutboundSending, whose wiring test pins that composition.
rampStore := sendramp.NewStore(pool)
- outboundRamp := agent.NewOutboundRampGate(
- rampStore,
- sendramp.NewSchedule(cfg.SendingRamp.StartDaily, cfg.SendingRamp.TargetDaily, cfg.SendingRamp.RampDays),
- cfg.SendingRamp.Enabled,
- )
- if cfg.SendingRamp.Enabled {
- log.Printf("Outbound sending ramp enabled: %d→%d recipients over %d qualified days", cfg.SendingRamp.StartDaily, cfg.SendingRamp.TargetDaily, cfg.SendingRamp.RampDays)
- }
outboundSendStore := agent.NewOutboundSendStore(store, webhookOutbox, usageTracker)
store.SetScheduledSendFinalizer(outboundSendStore)
- outboundJobs := outboundsend.NewJobs(
- outboundSendStore,
- agent.NewOutboundDeliverer(sender),
- pool,
- outboundRamp,
- ).WithMetrics(metrics).
+ outboundSending := newOutboundSending(outboundSendingDeps{
+ pool: pool,
+ store: outboundSendStore,
+ relay: smtpRelay,
+ secrets: spSecrets,
+ source: spSource,
+ policy: spPolicy,
+ sesConfigSet: cfg.DeliveryFeedback.SESConfigurationSet,
+ metrics: metrics,
// Fire-time per-agent rate limit (60 submissions/min/agent sliding
// window, durable in Postgres): the cross-replica counterpart of the
// acceptance-time in-memory limiter, enforced immediately before
// provider submission so scheduled-send bursts can't exceed it.
- WithRateGate(sendrate.NewStore(pool, time.Minute, 60))
+ rate: sendrate.NewStore(pool, time.Minute, 60),
+ })
+ outboundJobs := outboundSending.jobs
registrars = append(registrars, outboundJobs)
registrars = append(registrars, sendramp.NewMaintenanceJobs(rampStore))
// Queue depth/age gauges: a 30s maintenance periodic sampling river_job
diff --git a/cmd/e2a/outbound_wiring.go b/cmd/e2a/outbound_wiring.go
new file mode 100644
index 000000000..462cdea66
--- /dev/null
+++ b/cmd/e2a/outbound_wiring.go
@@ -0,0 +1,51 @@
+package main
+
+import (
+ "github.com/jackc/pgx/v5/pgxpool"
+
+ "github.com/tokencanopy/e2a/internal/agent"
+ "github.com/tokencanopy/e2a/internal/outbound"
+ "github.com/tokencanopy/e2a/internal/outboundsend"
+ "github.com/tokencanopy/e2a/internal/sendingpolicy"
+)
+
+// outboundSendingDeps is everything the outbound composition root needs. It
+// is a struct rather than positional arguments so the wiring test can build
+// the production composition from synthetic inputs and inspect the result.
+type outboundSendingDeps struct {
+ pool *pgxpool.Pool
+ store outboundsend.Store
+ relay *outbound.SMTPRelay
+ secrets sendingpolicy.Secrets
+ source sendingpolicy.PolicySource
+ policy sendingpolicy.RuntimePolicy
+ sesConfigSet string
+ metrics outboundsend.Metrics
+ rate outboundsend.RateGate
+}
+
+// outboundSending is the composed outbound send path.
+type outboundSending struct {
+ gate sendingpolicy.Gate
+ submitter *outbound.ProviderSubmitter
+ jobs *outboundsend.Jobs
+}
+
+// newOutboundSending is the ONE composition root for provider-bound customer
+// mail. The gate is the deployment's policy authority; the submitter is the
+// only object that opens a socket to the provider and it refuses to do so
+// without a token from that gate; the jobs bundle prepares an operation at
+// enqueue and authorizes every worker execution through the same gate. No
+// raw sender and no direct ramp store reach the worker from here.
+func newOutboundSending(d outboundSendingDeps) outboundSending {
+ gate := sendingpolicy.NewGate(d.pool, d.secrets, d.source, d.policy)
+ submitter := outbound.NewProviderSubmitter(d.relay, gate)
+ // Delivery feedback: tag outbound with the SES configuration set so SES
+ // publishes delivery/bounce/complaint events. Empty = off.
+ submitter.SetSESConfigurationSet(d.sesConfigSet)
+ jobs := outboundsend.NewJobs(d.store, agent.NewOutboundDeliverer(submitter), d.pool).
+ WithGate(gate).
+ WithMetrics(d.metrics).
+ WithRateGate(d.rate)
+ return outboundSending{gate: gate, submitter: submitter, jobs: jobs}
+}
diff --git a/cmd/e2a/sending_policy_wiring_test.go b/cmd/e2a/sending_policy_wiring_test.go
new file mode 100644
index 000000000..a7eebc1a4
--- /dev/null
+++ b/cmd/e2a/sending_policy_wiring_test.go
@@ -0,0 +1,78 @@
+package main
+
+import (
+ "context"
+ "fmt"
+ "strings"
+ "testing"
+
+ "github.com/riverqueue/river"
+
+ "github.com/tokencanopy/e2a/internal/config"
+ "github.com/tokencanopy/e2a/internal/outbound"
+ "github.com/tokencanopy/e2a/internal/sendingpolicy"
+ "github.com/tokencanopy/e2a/internal/testutil/testdb"
+)
+
+// TestSendingPolicyWiring builds the production outbound composition from
+// synthetic inputs and proves the registered send path holds the concrete
+// Gate and the authorized submitter. It exists so that a refactor that
+// reintroduced a raw sender or a direct ramp gate in the worker's path could
+// not pass CI: the only deliverer the composition root may produce is the one
+// over outbound.ProviderSubmitter, and the only admission authority is the
+// sendingpolicy module.
+func TestSendingPolicyWiring(t *testing.T) {
+ pool := testdb.TestDB(t)
+ relay := outbound.NewSMTPRelay(&config.OutboundSMTPConfig{Host: "relay.invalid", Port: 587, FromDomain: "test.e2a.dev"})
+
+ composed := newOutboundSending(outboundSendingDeps{
+ pool: pool,
+ store: nil, // the store is not exercised by construction
+ relay: relay,
+ secrets: sendingpolicy.Secrets{},
+ source: sendingpolicy.PolicySourceConfig,
+ policy: sendingpolicy.DisabledPolicy(),
+ sesConfigSet: "e2a-delivery-test",
+ })
+
+ if _, ok := composed.gate.(*sendingpolicy.Module); !ok {
+ t.Fatalf("gate is %T, want the concrete *sendingpolicy.Module", composed.gate)
+ }
+ if composed.submitter == nil {
+ t.Fatal("no authorized submitter composed")
+ }
+ if got := composed.submitter.SESConfigurationSet(); got != "e2a-delivery-test" {
+ t.Fatalf("submitter configuration set = %q, want the deployment's — delivery feedback must stay on", got)
+ }
+ if composed.jobs.Gate() != composed.gate {
+ t.Fatal("the jobs bundle does not hold the composed gate")
+ }
+ // The worker RegisterJobs registers is what runs in production; it, not
+ // the bundle, must carry the gate and the legacy resolver. Without the
+ // resolver every job in flight at cutover would fail closed.
+ // Register exactly as main does and inspect what River received — the
+ // constructor alone would not catch a RegisterJobs that bypassed it.
+ composed.jobs.RegisterJobs(river.NewWorkers())
+ worker := composed.jobs.RegisteredSendWorker()
+ if worker == nil {
+ t.Fatal("RegisterJobs registered no send worker")
+ }
+ if worker.Gate() != composed.gate {
+ t.Fatal("the registered send worker does not hold the composed gate")
+ }
+ if !worker.HasOperationResolver() {
+ t.Fatal("the registered send worker has no legacy operation resolver")
+ }
+ if composed.jobs.TerminalReconcileWorker() == nil {
+ t.Fatal("no terminal reconciler composed")
+ }
+ if got := fmt.Sprintf("%T", composed.jobs.Deliverer()); !strings.HasSuffix(got, "agent.outboundDeliverer") {
+ t.Fatalf("worker deliverer is %s, want the ProviderSubmitter-backed agent.outboundDeliverer", got)
+ }
+
+ // The composed gate is live: a config-source module answers policy reads
+ // against the real database, which is what the worker will do.
+ if _, err := composed.gate.LookupOperation(context.Background(), "op_wiring_probe"); err == nil {
+ t.Fatal("a never-prepared operation resolved")
+ }
+}
diff --git a/docs/api.md b/docs/api.md
index c4aad7739..db8b9b960 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -85,7 +85,8 @@ stable field are beta, `x-experimental-values` on that field):
the screening + review-hold event types (`email.flagged`, `email.blocked`,
`email.review_requested`, `email.review_approved`, `email.review_rejected` —
marked via `x-experimental-values` on the stable `type` field). The stable
-`error.code` vocabulary likewise marks only `blocked_by_policy` experimental.
+`error.code` vocabulary likewise marks only `blocked_by_policy` and
+`sending_paused` experimental.
See [events.md](events.md).
The exact operation-level list is repeated with methods and paths in
@@ -313,6 +314,7 @@ retryable ones (the per-row retry notes in the table below are authoritative).
| `unauthorized` | 401 | Missing or invalid credentials (REST and the WebSocket handshake). |
| `forbidden` | 403 | Authenticated but not allowed (key scope, cross-tenant access). |
| `blocked_by_policy` | 403 | **Experimental.** The outbound message was blocked by the agent's outbound policy gate. |
+| `sending_paused` | 403 | **Experimental.** Outbound sending is paused for the account by the platform abuse controls. Nothing was queued; queued mail is held until an operator resumes. |
| **Validation** | | |
| `invalid_request` | 400 / 422 | The canonical input-validation code — malformed (400) or semantically invalid (422). `error.details` carries the per-field list. |
| `invalid_cursor` | 400 | Bad pagination cursor — drop it and re-fetch from the start. |
@@ -465,7 +467,8 @@ every `/v1` operation not listed here is covered by the GA freeze.
`x-experimental-values` listing exactly those values — the field itself
stays stable, the listed values (and their payloads) may still change, and
every unlisted value is stable. The stable `ErrorBody.code` discriminator
- similarly marks only `blocked_by_policy` experimental. Anything not marked
+ similarly marks only `blocked_by_policy` and `sending_paused` experimental.
+ Anything not marked
beta or experimental is stable surface. One deliberate schema-level use of
the beta marker under a **stable** operation: the account export's interior
record schemas (`GET /v1/account/export`) are beta-marked because they are
@@ -859,6 +862,8 @@ retryability; clients must not reinterpret those fields independently:
| `submission.provider_rejected` | `submission` | `failed` | false |
| `submission.local_retries_exhausted` | `submission` | `failed` | true |
| `submission.cancelled` | `submission` | `failed` | false |
+| `submission.policy_budget_expired` | `submission` | `failed` | true |
+| `submission.sending_setup_expired` | `submission` | `failed` | true |
| `delivery.recipient_server_accepted` | `delivery` | `delivered` | false |
| `delivery.temporary_delay` | `delivery` | `deferred` | true |
| `delivery.permanent_bounce` | `delivery` | `bounced` | false |
diff --git a/docs/design/async-message-pipeline.md b/docs/design/async-message-pipeline.md
index 1a27614d0..7aff7c3de 100644
--- a/docs/design/async-message-pipeline.md
+++ b/docs/design/async-message-pipeline.md
@@ -256,3 +256,39 @@ Still open:
6. **Residual-window reconciler** (header-tagged SNS feedback vs a `sending` row): ~~alert-only v1, auto-heal later~~ **shipped as auto-heal (2026-07-16)**: header-tagged evidence is recorded on the row, the re-driven worker/terminal reconciler settles evidence-bearing `accepted`/`sending` rows as sent, and the §3.1 correction rule heals an already-written local `failed` when correlated delivery feedback arrives.
7. **Inbound (I2): raw-blob retention** — `river_job.args` holds full raw messages for pending inbound jobs; cap size / age-out policy for a backlog.
8. **`email.accepted` event — emit or not?** Currently **not** emitted: the caller learns `accepted` synchronously (the 200 body + `delivery_status='accepted'` on the row), and contract §4's *push* vocabulary is deliberately terminal-only (`sent`/`failed`/`deferred`). Optional addition: a one-line `PublishTx` of `email.accepted` in the accept-tx would populate the `webhook_events` log (visible in `GET /v1/events`) and deliver only to anyone who *explicitly* subscribes — harmless, but it widens the event vocabulary. Decide: accept-time event-log entry for observability vs. keep the push vocabulary terminal-only. (Leaning: skip at GA — the sync 200 already carries `accepted`; revisit if subscribers ask for an accept-time signal.)
+
+## Addendum (2026-09-05): the sending-protection gate owns admission
+
+Slice B6 of the sending abuse prevention plan (`e2a-ops` docs/superpowers) moved
+every provider-bound decision behind `internal/sendingpolicy`'s `Gate`. The
+worker-owned `RampGate` and `agent.NewOutboundRampGate` are gone; the
+custom-domain ramp is composed inside the gate (B4) and the SMTP seam is the
+token-requiring `outbound.ProviderSubmitter` (B5). The worker order is now
+fixed:
+
+1. `Reserve` the durable attempt (idempotent per ordinal; a confirmed ordinal
+ is followed by a fresh one, allocated by the gate, never by the worker);
+2. an early hold snoozes without provider I/O;
+3. the per-agent rate gate `DeferAttempt`s and snoozes; a final suppression
+ match `CancelAttempt`s and fails;
+4. `ConsumeAttempt` is the last serialized decision; a hold here is handled
+ like an early one;
+5. the authorized submitter redeems the token immediately before the socket
+ opens and settles the provider's answer (`SettleProvider`); a lost 250 is
+ `ErrProviderAcceptanceUnknown` — retried as a new ordinal, never settled.
+
+The accept transaction prepares the operation (`PrepareExternalTx`) between the
+message insert and the River insert; a paused account is refused at the door
+(`ErrSendingPaused` → HTTP 403 `sending_paused`). Jobs enqueued by a pre-floor
+slot carry no reference and are resolved at fire time through the same path
+(`Jobs.ResolveLegacyOperation`).
+
+Finite holds persist `messages.local_hold_class` / `local_hold_anchor`
+(migration 116); the deadline is always derived — 72 hours for
+`rate_ramp_or_provider` and `tenant_setup`, seven days for `policy_budget` —
+and expiry emits `submission.local_retries_exhausted`,
+`submission.sending_setup_expired`, or `submission.policy_budget_expired`
+respectively. An account pause has no clock and starts no hold, but a deadline
+already running keeps running. Terminal reconciliation is settlement-only: an
+evidence-settled row also settles the attempt that dialed
+(`Gate.SettleOperation`).
diff --git a/docs/events.md b/docs/events.md
index 4ec7e0105..40ba564d4 100644
--- a/docs/events.md
+++ b/docs/events.md
@@ -106,7 +106,7 @@ The event-to-reason mapping is:
|---|---|
| `email.received` | `acceptance.inbound_smtp` (or `acceptance.local_loopback`); DMARC `pass` → `authentication.dmarc_pass`, DMARC `fail` → `authentication.dmarc_fail`, DMARC `none` → `authentication.dmarc_none`, DMARC `temperror` → `authentication.dmarc_temporary_error`, and DMARC `permerror` → `authentication.dmarc_permanent_error`; plus `queue.inbound_processing` when async intake was durably queued. |
| `email.sent` | `submission.upstream_accepted` or `submission.local_loopback_accepted`. |
-| `email.failed` | `submission.provider_rejected`, `submission.local_retries_exhausted`, or `submission.cancelled`, matching the terminal cause. Temporary attempts use `submission.temporary_failure` in the ledger but do not emit a terminal `email.failed` event. |
+| `email.failed` | `submission.provider_rejected`, `submission.local_retries_exhausted`, `submission.cancelled`, `submission.policy_budget_expired`, or `submission.sending_setup_expired`, matching the terminal cause. Temporary attempts use `submission.temporary_failure` in the ledger but do not emit a terminal `email.failed` event. |
| `email.delivered` | `delivery.recipient_server_accepted` for `delivered_to`. |
| `email.bounced` | `delivery.permanent_bounce`, `delivery.transient_bounce`, or `delivery.undetermined_bounce` for `delivered_to`. |
| `email.complained` | `complaint.recipient_reported` for `delivered_to`. |
diff --git a/internal/agent/api.go b/internal/agent/api.go
index 42bf91119..fe16be741 100644
--- a/internal/agent/api.go
+++ b/internal/agent/api.go
@@ -34,6 +34,7 @@ import (
"github.com/tokencanopy/e2a/internal/logredact"
"github.com/tokencanopy/e2a/internal/oauth"
"github.com/tokencanopy/e2a/internal/outbound"
+ "github.com/tokencanopy/e2a/internal/outboundsend"
"github.com/tokencanopy/e2a/internal/piguard"
"github.com/tokencanopy/e2a/internal/ratelimit"
"github.com/tokencanopy/e2a/internal/telemetry"
@@ -1607,6 +1608,12 @@ func (a *API) DeliverOutbound(ctx context.Context, user *identity.User, agent *i
accepted = msg
return nil
}); txErr != nil {
+ if errors.Is(txErr, outboundsend.ErrSendingPaused) {
+ // The account is paused for sending abuse: refuse at the door
+ // rather than queue mail that can never leave. Nothing was
+ // committed — the message row rolled back with the job.
+ return nil, &OutboundError{Status: http.StatusForbidden, Code: "sending_paused", Msg: "sending is paused for this account"}
+ }
log.Printf("[api] async accept tx failed: agent=%s to_count=%d to_domains=%v error=%v", agent.Domain, len(req.To), logredact.AddressDomains(req.To), txErr)
return nil, &OutboundError{Status: http.StatusInternalServerError, Code: "internal_error", Msg: "failed to accept message for send"}
}
@@ -1728,6 +1735,9 @@ func (a *API) acceptPlatformSend(ctx context.Context, agent *identity.AgentIdent
accepted = msg
return nil
}); txErr != nil {
+ if errors.Is(txErr, outboundsend.ErrSendingPaused) {
+ return nil, &OutboundError{Status: http.StatusForbidden, Code: "sending_paused", Msg: "sending is paused for this account"}
+ }
log.Printf("[api] platform accept tx failed: agent=%s to_count=%d to_domains=%v error=%v", agent.Domain, len(req.To), logredact.AddressDomains(req.To), txErr)
return nil, &OutboundError{Status: http.StatusInternalServerError, Code: "internal_error", Msg: "failed to accept message for send"}
}
diff --git a/internal/agent/hitl_api.go b/internal/agent/hitl_api.go
index fe36bf636..38a7c7cd8 100644
--- a/internal/agent/hitl_api.go
+++ b/internal/agent/hitl_api.go
@@ -15,6 +15,7 @@ import (
"github.com/tokencanopy/e2a/internal/limits"
"github.com/tokencanopy/e2a/internal/logredact"
"github.com/tokencanopy/e2a/internal/outbound"
+ "github.com/tokencanopy/e2a/internal/outboundsend"
)
// approveRequest is the JSON body accepted by the approve endpoint. Every
@@ -280,6 +281,10 @@ func approveAsyncError(agentID, messageID string, err error) *OutboundError {
return &OutboundError{Status: http.StatusConflict, Code: "message_not_pending", Msg: "message is not pending approval"}
case errors.Is(err, identity.ErrMessageNotFound):
return &OutboundError{Status: http.StatusNotFound, Code: "not_found", Msg: "message not found"}
+ case errors.Is(err, outboundsend.ErrSendingPaused):
+ // The draft stays pending_review (the approval transaction rolled
+ // back); the reviewer learns why rather than seeing a 500.
+ return &OutboundError{Status: http.StatusForbidden, Code: "sending_paused", Msg: "sending is paused for this account; the draft remains pending"}
default:
var ve *outbound.ValidationError
if errors.As(err, &ve) {
diff --git a/internal/agent/outbound_async.go b/internal/agent/outbound_async.go
index 4408be993..55b46411d 100644
--- a/internal/agent/outbound_async.go
+++ b/internal/agent/outbound_async.go
@@ -2,6 +2,7 @@ package agent
import (
"context"
+ "errors"
"fmt"
"hash/fnv"
"log"
@@ -17,7 +18,7 @@ import (
"github.com/tokencanopy/e2a/internal/messagelifecycle"
"github.com/tokencanopy/e2a/internal/outbound"
"github.com/tokencanopy/e2a/internal/outboundsend"
- "github.com/tokencanopy/e2a/internal/sendramp"
+ "github.com/tokencanopy/e2a/internal/sendingpolicy"
"github.com/tokencanopy/e2a/internal/usage"
"github.com/tokencanopy/e2a/internal/webhookpub"
)
@@ -106,71 +107,6 @@ func NewOutboundSendStore(store *identity.Store, outbox webhookpub.Outbox, usage
return &outboundSendStore{store: store, outbox: outbox, usage: usageTracker}
}
-type outboundRampGate struct {
- store *sendramp.Store
- schedule sendramp.Schedule
- enabled bool
- now func() time.Time
-}
-
-// NewOutboundRampGate adapts the durable sendramp store to the worker-owned
-// gate contract. The schedule is snapshotted by Store on the first eligible
-// send; config changes therefore affect only domains that have not armed yet.
-func NewOutboundRampGate(store *sendramp.Store, schedule sendramp.Schedule, enabled bool, clocks ...func() time.Time) outboundsend.RampGate {
- now := time.Now
- if len(clocks) > 0 && clocks[0] != nil {
- now = clocks[0]
- }
- return &outboundRampGate{store: store, schedule: schedule, enabled: enabled, now: now}
-}
-
-func (g *outboundRampGate) Reserve(ctx context.Context, req outboundsend.RampRequest) (outboundsend.RampDecision, error) {
- if !g.enabled {
- // Disabled is pass-through: reserve nothing, count nothing, stamp
- // nothing. The domain stays 'inactive'.
- //
- // An earlier revision stamped the domain 'exempt' here, reasoning that
- // a sender allowed to send unthrottled must not be re-throttled if the
- // ramp is later enabled. That turned every eligible send into a silent,
- // unmarked grandfathering decision, and 'exempt' has since grown
- // meaning beyond "skip the ramp": an exempt domain reads as an
- // established sender, so it also stops consuming the shared probation
- // pool that bounds Sybil abuse. Widening that set from the send path,
- // once per send, is not a decision this gate gets to make.
- //
- // Grandfathering belongs to the audited one-shot that already exists
- // for it: sendingpolicy's ActivationRequest.GrandfatherCurrentSendingDomains,
- // which writes a replay marker, locks the domains table against
- // concurrent sender transitions, and can never widen its set twice.
- return outboundsend.RampDecision{Allowed: true}, nil
- }
- d, err := g.store.Reserve(ctx, sendramp.ReserveRequest{
- MessageID: req.MessageID,
- UserID: req.UserID,
- Domain: req.Domain,
- Units: req.Units,
- Day: g.now().UTC(),
- Schedule: g.schedule,
- })
- return outboundsend.RampDecision{Allowed: d.Allowed, RetryAt: d.RetryAt}, err
-}
-
-// Confirm, Release and Resolve delegate unconditionally, including while the
-// ramp is disabled: a reservation taken before an operator turned the ramp off
-// still has to settle. With the ramp disabled no reservation is ever created,
-// so on that path the store methods find no row and write nothing.
-func (g *outboundRampGate) Confirm(ctx context.Context, messageID string) error {
- return g.store.Confirm(ctx, messageID)
-}
-
-func (g *outboundRampGate) Release(ctx context.Context, messageID string) error {
- return g.store.Release(ctx, messageID)
-}
-
-func (g *outboundRampGate) Resolve(ctx context.Context, messageID string) error {
- return g.store.Resolve(ctx, messageID)
-}
-
func (a *outboundSendStore) ClaimSend(ctx context.Context, messageID string, jobID int64) (*outboundsend.SendJob, error) {
if a.usage == nil {
return nil, fmt.Errorf("outbound usage tracker is required")
@@ -210,7 +146,7 @@ func (a *outboundSendStore) ClaimSend(ctx context.Context, messageID string, job
anchor = *p.ScheduledAt
}
if !anchor.IsZero() && time.Since(anchor) > outboundsend.SendRetryHorizon {
- if _, _, failErr := a.MarkFailed(ctx, p.ID, jobID, 0, time.Now().UTC(),
+ if _, _, _, failErr := a.MarkFailed(ctx, p.ID, jobID, 0, time.Now().UTC(),
"daily_send_cap_timeout: daily send limit still exceeded past the retry horizon",
delivery.FailureSourceLocal, messagelifecycle.ReasonSubmissionLocalRetriesExhausted, nil); failErr != nil {
return nil, failErr
@@ -228,7 +164,7 @@ func (a *outboundSendStore) ClaimSend(ctx context.Context, messageID string, job
log.Printf("[outbound-send:%s] daily send cap exhausted at fire time, deferring to %s", p.ID, retryAt.Format(time.RFC3339))
return nil, &outboundsend.DailyQuotaDeferredError{RetryAt: retryAt}
}
- if _, _, failErr := a.MarkFailed(ctx, p.ID, jobID, 0, time.Now().UTC(),
+ if _, _, _, failErr := a.MarkFailed(ctx, p.ID, jobID, 0, time.Now().UTC(),
"send canceled: monthly send limit exceeded at send time",
delivery.FailureSourceLocal, messagelifecycle.ReasonSubmissionCancelled, nil); failErr != nil {
return nil, failErr
@@ -258,9 +194,24 @@ func (a *outboundSendStore) ClaimSend(ctx context.Context, messageID string, job
if p.ReviewedAt != nil {
sj.ReviewedAt = *p.ReviewedAt
}
+ sj.LocalHoldClass = outboundsend.HoldClass(p.LocalHoldClass)
+ if p.LocalHoldAnchor != nil {
+ sj.LocalHoldAnchor = *p.LocalHoldAnchor
+ }
+ if p.LastResumedAt != nil {
+ sj.LastResumedAt = *p.LastResumedAt
+ }
+ if p.TenantReadyAt != nil {
+ sj.TenantReadyAt = *p.TenantReadyAt
+ }
return sj, nil
}
+// RecordHold persists the worker's finite-hold class and anchor on the row.
+func (a *outboundSendStore) RecordHold(ctx context.Context, messageID string, class outboundsend.HoldClass, anchor time.Time) error {
+ return a.store.RecordOutboundHold(ctx, messageID, string(class), anchor)
+}
+
// SuppressedRecipients backs the SendWorker's pre-provider suppression guard:
// the effective account-wide + exact-agent subset (the store normalizes both
// sides).
@@ -416,7 +367,7 @@ func (a *outboundSendStore) FinalizeScheduledCancellationTx(
// time is the occurred_at the write actually used: the provider-accept
// evidence time on an evidence settle, the caller's occurredAt on a failure,
// zero on a no-op.
-func (a *outboundSendStore) MarkFailed(ctx context.Context, messageID string, jobID int64, attempt int, occurredAt time.Time, detail string, source delivery.FailureSource, reason messagelifecycle.ReasonCode, blockedRecipients []string) (delivery.Status, time.Time, error) {
+func (a *outboundSendStore) MarkFailed(ctx context.Context, messageID string, jobID int64, attempt int, occurredAt time.Time, detail string, source delivery.FailureSource, reason messagelifecycle.ReasonCode, blockedRecipients []string) (delivery.Status, time.Time, string, error) {
detail = messagelifecycle.SafeDiagnostic(detail)
blockedRecipients = normalizeBlockedRecipients(blockedRecipients)
var settled delivery.Status
@@ -464,12 +415,12 @@ func (a *outboundSendStore) MarkFailed(ctx context.Context, messageID string, jo
e.ID = webhookpub.DeterministicEventID(messageID, webhookpub.EventEmailFailed)
return a.outbox.PublishTx(ctx, tx, e)
}); err != nil {
- return "", time.Time{}, err
+ return "", time.Time{}, "", err
}
if resolved != nil {
log.Printf("[outbound-send] %s: terminal-failure guard settled as sent on provider evidence (provider id %q)", messageID, resolvedProviderID)
}
- return settled, settledAt, nil
+ return settled, settledAt, resolvedProviderID, nil
}
func (a *outboundSendStore) PreserveTerminalFailure(ctx context.Context, messageID string, jobID int64, attempt int, occurredAt time.Time, detail string, source delivery.FailureSource, reason messagelifecycle.ReasonCode, blockedRecipients []string) error {
@@ -615,30 +566,39 @@ func buildEmailFailedEventFromRow(info *identity.OutboundSentInfo, detail string
}
}
-// outboundDeliverer implements outboundsend.Deliverer over Sender.SubmitOnce — a
-// single SMTP submit of the persisted Sent-folder bytes (River owns retries).
+// outboundDeliverer implements outboundsend.Deliverer over the authorized
+// provider seam (outbound.ProviderSubmitter): one token-redeeming SMTP submit
+// of the persisted Sent-folder bytes (River owns retries). There is no
+// tokenless path through here.
type outboundDeliverer struct {
- sender *outbound.Sender
+ submitter *outbound.ProviderSubmitter
}
// NewOutboundDeliverer builds the outboundsend.Deliverer adapter for main.go.
-func NewOutboundDeliverer(sender *outbound.Sender) outboundsend.Deliverer {
- return &outboundDeliverer{sender: sender}
+func NewOutboundDeliverer(submitter *outbound.ProviderSubmitter) outboundsend.Deliverer {
+ return &outboundDeliverer{submitter: submitter}
}
-func (d *outboundDeliverer) Deliver(ctx context.Context, j *outboundsend.SendJob) outboundsend.DeliverOutcome {
- providerID, err := d.sender.SubmitOnceContext(ctx, j.MessageID, j.EnvelopeFrom, j.Recipients, j.RawMessage)
+func (d *outboundDeliverer) Deliver(ctx context.Context, j *outboundsend.SendJob, auth sendingpolicy.ProviderAuthorization) outboundsend.DeliverOutcome {
+ res, err := d.submitter.SubmitOnce(ctx, auth, outbound.Envelope{
+ From: j.EnvelopeFrom,
+ Recipients: j.Recipients,
+ Message: j.RawMessage,
+ })
if err != nil {
// Classify (design §8): a definitely-permanent 5xx is terminal (JobCancel);
// a provider-connection failure (relay unreachable/misconfigured) is an
- // outage → snooze without burning an attempt; everything else (4xx/unknown)
- // takes the bounded retry. Terminal-failing a send that could still succeed
- // would violate at-least-once.
+ // outage → snooze without burning an attempt; a failure after the body
+ // was handed over is acceptance-unknown; everything else (4xx/unknown)
+ // takes the bounded retry. Terminal-failing a send that could still
+ // succeed would violate at-least-once.
+ unknown := errors.Is(err, outbound.ErrProviderAcceptanceUnknown)
return outboundsend.DeliverOutcome{
- Err: err,
- Permanent: outbound.IsPermanentSMTPError(err),
- Outage: outbound.IsConnectionError(err),
+ Err: err,
+ Permanent: outbound.IsPermanentSMTPError(err),
+ Outage: !unknown && outbound.IsConnectionError(err),
+ AcceptanceUnknown: unknown,
}
}
- return outboundsend.DeliverOutcome{ProviderMessageID: providerID, SentAs: j.SentAs}
+ return outboundsend.DeliverOutcome{ProviderMessageID: res.ProviderMessageID, SentAs: j.SentAs, SettlementErr: res.SettlementErr}
}
diff --git a/internal/agent/outbound_async_test.go b/internal/agent/outbound_async_test.go
index 6b645dd24..574fd3409 100644
--- a/internal/agent/outbound_async_test.go
+++ b/internal/agent/outbound_async_test.go
@@ -22,6 +22,7 @@ import (
"github.com/tokencanopy/e2a/internal/messagelifecycle"
"github.com/tokencanopy/e2a/internal/outbound"
"github.com/tokencanopy/e2a/internal/outboundsend"
+ "github.com/tokencanopy/e2a/internal/sendingpolicy"
"github.com/tokencanopy/e2a/internal/testutil"
"github.com/tokencanopy/e2a/internal/usage"
"github.com/tokencanopy/e2a/internal/webhookpub"
@@ -122,13 +123,13 @@ func (f *fakeNotifyEnqueuer) EnqueueNotifyTx(_ context.Context, _ pgx.Tx, _ stri
// fakeAsyncDeliverer is the SMTP submit the SendWorker calls — no network.
type fakeAsyncDeliverer struct{ out outboundsend.DeliverOutcome }
-func (f fakeAsyncDeliverer) Deliver(_ context.Context, _ *outboundsend.SendJob) outboundsend.DeliverOutcome {
+func (f fakeAsyncDeliverer) Deliver(_ context.Context, _ *outboundsend.SendJob, _ sendingpolicy.ProviderAuthorization) outboundsend.DeliverOutcome {
return f.out
}
type countingAsyncDeliverer struct{ calls int }
-func (d *countingAsyncDeliverer) Deliver(context.Context, *outboundsend.SendJob) outboundsend.DeliverOutcome {
+func (d *countingAsyncDeliverer) Deliver(context.Context, *outboundsend.SendJob, sendingpolicy.ProviderAuthorization) outboundsend.DeliverOutcome {
d.calls++
return outboundsend.DeliverOutcome{ProviderMessageID: "unexpected"}
}
@@ -138,7 +139,7 @@ type timedAsyncDeliverer struct {
returnedAt time.Time
}
-func (d *timedAsyncDeliverer) Deliver(context.Context, *outboundsend.SendJob) outboundsend.DeliverOutcome {
+func (d *timedAsyncDeliverer) Deliver(context.Context, *outboundsend.SendJob, sendingpolicy.ProviderAuthorization) outboundsend.DeliverOutcome {
d.returnedAt = time.Now().UTC()
return d.out
}
@@ -155,7 +156,7 @@ type blockingAsyncDeliverer struct {
out outboundsend.DeliverOutcome
}
-func (d *blockingAsyncDeliverer) Deliver(_ context.Context, _ *outboundsend.SendJob) outboundsend.DeliverOutcome {
+func (d *blockingAsyncDeliverer) Deliver(_ context.Context, _ *outboundsend.SendJob, _ sendingpolicy.ProviderAuthorization) outboundsend.DeliverOutcome {
close(d.entered)
<-d.release
return d.out
@@ -1270,7 +1271,7 @@ func TestOutboundSendStore_MarkFailed(t *testing.T) {
adapter := agent.NewOutboundSendStore(store, outbox, usage.NewNoopUsageTracker())
occurredAt := time.Now().UTC()
- settled, settledAt, err := adapter.MarkFailed(ctx, res.MessageID, 999, 6, occurredAt, "550 mailbox unavailable", delivery.FailureSourceProvider, messagelifecycle.ReasonSubmissionProviderRejected, nil)
+ settled, settledAt, _, err := adapter.MarkFailed(ctx, res.MessageID, 999, 6, occurredAt, "550 mailbox unavailable", delivery.FailureSourceProvider, messagelifecycle.ReasonSubmissionProviderRejected, nil)
if err != nil {
t.Fatalf("MarkFailed: %v", err)
}
diff --git a/internal/agent/outbound_ramp_test.go b/internal/agent/outbound_ramp_test.go
deleted file mode 100644
index c909a5dd6..000000000
--- a/internal/agent/outbound_ramp_test.go
+++ /dev/null
@@ -1,152 +0,0 @@
-package agent_test
-
-import (
- "context"
- "testing"
- "time"
-
- "github.com/jackc/pgx/v5/pgxpool"
- "github.com/tokencanopy/e2a/internal/agent"
- "github.com/tokencanopy/e2a/internal/identity"
- "github.com/tokencanopy/e2a/internal/outbound"
- "github.com/tokencanopy/e2a/internal/outboundsend"
- "github.com/tokencanopy/e2a/internal/sendramp"
- "github.com/tokencanopy/e2a/internal/testutil"
- "github.com/tokencanopy/e2a/internal/usage"
-)
-
-func seedOutboundRampAdapter(t *testing.T, suffix string) (*pgxpool.Pool, *sendramp.Store, string, string, string) {
- t.Helper()
- pool := testutil.TestDB(t)
- ctx := context.Background()
- ids := identity.NewStore(pool)
- user, err := ids.CreateOrGetUser(ctx, "adapter-"+suffix+"@example.com", "Adapter", "adapter-"+suffix)
- if err != nil {
- t.Fatal(err)
- }
- domain := "adapter-" + suffix + ".example.com"
- if _, err := ids.ClaimOrCreateDomain(ctx, domain, user.ID); err != nil {
- t.Fatal(err)
- }
- if _, err := pool.Exec(ctx, `UPDATE domains SET sending_status='verified' WHERE domain=$1`, domain); err != nil {
- t.Fatal(err)
- }
- ag, err := ids.CreateAgent(ctx, "agent@"+domain, domain, "", "", "local", user.ID)
- if err != nil {
- t.Fatal(err)
- }
- msg, err := ids.CreateOutboundMessage(ctx, ag.ID, []string{"one@example.net"}, nil, nil, "subject", "send", "smtp", "", "", []byte("raw"))
- if err != nil {
- t.Fatal(err)
- }
- return pool, sendramp.NewStore(pool), user.ID, domain, msg.ID
-}
-
-// assertNoRampState asserts the full "the ramp wrote nothing" contract for one
-// account: the domain never left 'inactive' and no ledger row was created.
-func assertNoRampState(t *testing.T, pool *pgxpool.Pool, userID, domain, messageID string) {
- t.Helper()
- ctx := context.Background()
- var status string
- if err := pool.QueryRow(ctx, `SELECT sending_ramp_status FROM domains WHERE domain=$1 AND user_id=$2`, domain, userID).Scan(&status); err != nil {
- t.Fatalf("read sending_ramp_status: %v", err)
- }
- if status != sendramp.StatusInactive {
- t.Fatalf("sending_ramp_status = %q, want %q: a disabled ramp must not grandfather a domain from the send path", status, sendramp.StatusInactive)
- }
- for _, q := range []struct{ table, sql string }{
- {"sending_ramp_scopes", `SELECT count(*) FROM sending_ramp_scopes WHERE user_id=$1`},
- {"domain_send_counters", `SELECT count(*) FROM domain_send_counters WHERE user_id=$1`},
- } {
- var n int
- if err := pool.QueryRow(ctx, q.sql, userID).Scan(&n); err != nil {
- t.Fatalf("count %s: %v", q.table, err)
- }
- if n != 0 {
- t.Fatalf("%s has %d rows, want 0", q.table, n)
- }
- }
- var reservations int
- if err := pool.QueryRow(ctx, `SELECT count(*) FROM sending_ramp_reservations WHERE message_id=$1`, messageID).Scan(&reservations); err != nil {
- t.Fatalf("count sending_ramp_reservations: %v", err)
- }
- if reservations != 0 {
- t.Fatalf("sending_ramp_reservations has %d rows, want 0", reservations)
- }
-}
-
-// TestOutboundRampGateDisabledIsPassThrough pins the disabled contract: allow
-// the send and write NOTHING. The gate used to stamp the domain 'exempt' on
-// every eligible send, which permanently grandfathered any domain that sent
-// while the ramp was off — pre-empting the audited one-shot in sendingpolicy
-// and, because 'exempt' also reads as "established" to the shared probation
-// pool, handing away an abuse bound. Delete-and-re-register made it a repeatable
-// reset primitive on top.
-func TestOutboundRampGateDisabledIsPassThrough(t *testing.T) {
- pool, store, userID, domain, messageID := seedOutboundRampAdapter(t, "disabled")
- gate := agent.NewOutboundRampGate(store, sendramp.DefaultSchedule, false)
- d, err := gate.Reserve(context.Background(), outboundsend.RampRequest{MessageID: messageID, UserID: userID, Domain: domain, Units: 1})
- if err != nil || !d.Allowed {
- t.Fatalf("Reserve = %+v, %v", d, err)
- }
- assertNoRampState(t, pool, userID, domain, messageID)
-
- // The read surface (GET /v1/domains/{domain}.sending_ramp.status) therefore
- // reports 'inactive', not 'exempt', for a domain sending under a disabled ramp.
- snap, err := store.Snapshot(context.Background(), userID, domain, time.Now())
- if err != nil || snap.Status != sendramp.StatusInactive {
- t.Fatalf("Snapshot = %+v, %v, want status %q", snap, err, sendramp.StatusInactive)
- }
-}
-
-// TestSendWorkerDisabledRampWritesNoRampState is the same contract one level
-// out: a real ramp-eligible send (own_address, message_type send) driven
-// through the send worker with the ramp disabled must leave the domain
-// 'inactive' and the ramp ledger empty.
-func TestSendWorkerDisabledRampWritesNoRampState(t *testing.T) {
- api, store, outbox, _, pool := setupAsyncAPIWithPool(t)
- ctx := context.Background()
- user, ag := selfAgent(t, store, "rampdisabled")
- if err := store.SetSendingStatus(ctx, ag.RegisteredDomain, "verified", "verified", "verified", "", nil); err != nil {
- t.Fatalf("SetSendingStatus: %v", err)
- }
- res, oerr := api.DeliverOutbound(ctx, user, ag, outbound.SendRequest{
- To: []string{"recipient@external.test"}, Subject: "disabled ramp send", Body: "x",
- }, "send", "", nil, nil)
- if oerr != nil {
- t.Fatalf("DeliverOutbound: %+v", oerr)
- }
-
- ramp := agent.NewOutboundRampGate(sendramp.NewStore(pool), sendramp.DefaultSchedule, false)
- deliverer := &countingDeliverer{out: outboundsend.DeliverOutcome{ProviderMessageID: "provider-disabled-ramp", SentAs: "own_address"}}
- worker := outboundsend.NewSendWorker(
- agent.NewOutboundSendStore(store, outbox, usage.NewNoopUsageTracker()), deliverer, ramp)
-
- if err := worker.Work(ctx, workerJobWithID(res.MessageID, 999, 1)); err != nil {
- t.Fatalf("worker.Work: %v", err)
- }
- if deliverer.calls != 1 {
- t.Fatalf("deliverer calls = %d, want 1: the disabled ramp must still allow the send", deliverer.calls)
- }
- assertNoRampState(t, pool, user.ID, ag.RegisteredDomain, res.MessageID)
-}
-
-func TestOutboundRampGateInjectsDayAndDelegatesLifecycle(t *testing.T) {
- _, store, userID, domain, messageID := seedOutboundRampAdapter(t, "enabled")
- day := time.Date(2026, 7, 2, 23, 30, 0, 0, time.FixedZone("west", -7*60*60))
- gate := agent.NewOutboundRampGate(store, sendramp.NewSchedule(50, 100, 2), true, func() time.Time { return day })
- d, err := gate.Reserve(context.Background(), outboundsend.RampRequest{MessageID: messageID, UserID: userID, Domain: domain, Units: 25})
- if err != nil || !d.Allowed {
- t.Fatalf("Reserve = %+v, %v", d, err)
- }
- if err := gate.Confirm(context.Background(), messageID); err != nil {
- t.Fatal(err)
- }
- snap, err := store.Snapshot(context.Background(), userID, domain, day)
- if err != nil {
- t.Fatal(err)
- }
- if snap.ActiveDays != 1 || snap.UsedToday != 25 {
- t.Fatalf("Snapshot = %+v", snap)
- }
-}
diff --git a/internal/agent/outbound_suppression_guard_test.go b/internal/agent/outbound_suppression_guard_test.go
index 07d4a85cb..03536e5f5 100644
--- a/internal/agent/outbound_suppression_guard_test.go
+++ b/internal/agent/outbound_suppression_guard_test.go
@@ -9,7 +9,6 @@ import (
"context"
"errors"
"strings"
- "sync"
"testing"
"time"
@@ -22,38 +21,11 @@ import (
"github.com/tokencanopy/e2a/internal/messagelifecycle"
"github.com/tokencanopy/e2a/internal/outbound"
"github.com/tokencanopy/e2a/internal/outboundsend"
- "github.com/tokencanopy/e2a/internal/sendramp"
+ "github.com/tokencanopy/e2a/internal/sendingpolicy"
"github.com/tokencanopy/e2a/internal/usage"
"github.com/tokencanopy/e2a/internal/webhookpub"
)
-type blockingRampGate struct {
- entered chan struct{}
- resume chan struct{}
- mu sync.Mutex
- released []string
-}
-
-func (g *blockingRampGate) Reserve(context.Context, outboundsend.RampRequest) (outboundsend.RampDecision, error) {
- close(g.entered)
- <-g.resume
- return outboundsend.RampDecision{Allowed: true}, nil
-}
-func (*blockingRampGate) Confirm(context.Context, string) error { return nil }
-func (g *blockingRampGate) Release(_ context.Context, messageID string) error {
- g.mu.Lock()
- defer g.mu.Unlock()
- g.released = append(g.released, messageID)
- return nil
-}
-func (*blockingRampGate) Resolve(context.Context, string) error { return nil }
-
-func (g *blockingRampGate) releasedIDs() []string {
- g.mu.Lock()
- defer g.mu.Unlock()
- return append([]string(nil), g.released...)
-}
-
// countingDeliverer records provider submits so the guard can assert zero I/O.
type countingDeliverer struct {
calls int
@@ -73,7 +45,7 @@ func (s *failOnceSuppressionStore) SuppressedRecipients(ctx context.Context, use
return s.Store.SuppressedRecipients(ctx, userID, agentID, recipients)
}
-func (d *countingDeliverer) Deliver(_ context.Context, _ *outboundsend.SendJob) outboundsend.DeliverOutcome {
+func (d *countingDeliverer) Deliver(_ context.Context, _ *outboundsend.SendJob, _ sendingpolicy.ProviderAuthorization) outboundsend.DeliverOutcome {
d.calls++
return d.out
}
@@ -378,100 +350,6 @@ func TestSendWorker_ProviderEvidenceCorrectionRetainsFallbackSuppression(t *test
}
}
-func TestSendWorker_SuppressionAddedDuringRampReservePreventsProviderIO(t *testing.T) {
- api, store, outbox, _ := setupAsyncAPI(t)
- ctx := context.Background()
- user, ag := selfAgent(t, store, "suppduringramp")
- if err := store.SetSendingStatus(ctx, ag.RegisteredDomain, "verified", "verified", "verified", "", nil); err != nil {
- t.Fatalf("SetSendingStatus: %v", err)
- }
- res, oerr := api.DeliverOutbound(ctx, user, ag, outbound.SendRequest{
- To: []string{"late@external.test"}, Subject: "ramp race", Body: "x",
- }, "send", "", nil, nil)
- if oerr != nil {
- t.Fatalf("DeliverOutbound: %+v", oerr)
- }
-
- gate := &blockingRampGate{entered: make(chan struct{}), resume: make(chan struct{})}
- deliverer := &countingDeliverer{out: outboundsend.DeliverOutcome{ProviderMessageID: "must-not-happen"}}
- worker := outboundsend.NewSendWorker(agent.NewOutboundSendStore(store, outbox, usage.NewNoopUsageTracker()), deliverer, gate)
- done := make(chan error, 1)
- go func() { done <- worker.Work(ctx, workerJob(res.MessageID, 1)) }()
- <-gate.entered
- if _, _, err := store.AddAgentSuppression(ctx, user.ID, ag.ID, "late@external.test", "opted out", "unsubscribe", nil); err != nil {
- t.Fatal(err)
- }
- close(gate.resume)
- if err := <-done; err == nil {
- t.Fatal("suppression created during ramp reservation must cancel the send")
- }
- if deliverer.calls != 0 {
- t.Fatalf("provider calls = %d, want zero", deliverer.calls)
- }
- if got := gate.releasedIDs(); len(got) != 1 || got[0] != res.MessageID {
- t.Fatalf("released reservations = %v, want [%s]", got, res.MessageID)
- }
- var status, detail string
- if err := store.WithTx(ctx, func(tx pgx.Tx) error {
- return tx.QueryRow(ctx, `SELECT delivery_status, COALESCE(delivery_detail,'') FROM messages WHERE id=$1`, res.MessageID).Scan(&status, &detail)
- }); err != nil {
- t.Fatal(err)
- }
- if status != "failed" || !strings.Contains(detail, "recipient_suppressed") {
- t.Fatalf("status/detail = %q/%q, want failed recipient_suppressed", status, detail)
- }
-}
-
-func TestSendWorker_TransientSuppressionFailureReusesRealRampReservation(t *testing.T) {
- api, store, outbox, _, pool := setupAsyncAPIWithPool(t)
- ctx := context.Background()
- user, ag := selfAgent(t, store, "rampretryreal")
- if err := store.SetSendingStatus(ctx, ag.RegisteredDomain, "verified", "verified", "verified", "", nil); err != nil {
- t.Fatalf("SetSendingStatus: %v", err)
- }
- res, oerr := api.DeliverOutbound(ctx, user, ag, outbound.SendRequest{
- To: []string{"recipient@external.test"}, Subject: "retry after suppression lookup", Body: "x",
- }, "send", "", nil, nil)
- if oerr != nil {
- t.Fatalf("DeliverOutbound: %+v", oerr)
- }
-
- baseStore := agent.NewOutboundSendStore(store, outbox, usage.NewNoopUsageTracker())
- failingStore := &failOnceSuppressionStore{Store: baseStore}
- day := time.Date(2026, 7, 19, 12, 0, 0, 0, time.UTC)
- ramp := agent.NewOutboundRampGate(sendramp.NewStore(pool), sendramp.NewSchedule(50, 100, 2), true, func() time.Time { return day })
- deliverer := &countingDeliverer{out: outboundsend.DeliverOutcome{ProviderMessageID: "ses-after-retry", SentAs: "own_address"}}
- worker := outboundsend.NewSendWorker(failingStore, deliverer, ramp)
-
- if err := worker.Work(ctx, workerJobWithID(res.MessageID, 999, 1)); err == nil {
- t.Fatal("first worker attempt must return the injected transient error")
- }
- var firstState string
- if err := pool.QueryRow(ctx, `SELECT state FROM sending_ramp_reservations WHERE message_id=$1`, res.MessageID).Scan(&firstState); err != nil {
- t.Fatalf("read first reservation: %v", err)
- }
- if firstState != "reserved" {
- t.Fatalf("reservation after transient error = %q, want reserved", firstState)
- }
- if deliverer.calls != 0 {
- t.Fatalf("provider calls after transient error = %d, want zero", deliverer.calls)
- }
-
- if err := worker.Work(ctx, workerJobWithID(res.MessageID, 999, 2)); err != nil {
- t.Fatalf("retry worker attempt: %v", err)
- }
- var finalState, status string
- if err := pool.QueryRow(ctx, `SELECT state FROM sending_ramp_reservations WHERE message_id=$1`, res.MessageID).Scan(&finalState); err != nil {
- t.Fatalf("read final reservation: %v", err)
- }
- if err := pool.QueryRow(ctx, `SELECT delivery_status FROM messages WHERE id=$1`, res.MessageID).Scan(&status); err != nil {
- t.Fatalf("read final message: %v", err)
- }
- if finalState != "confirmed" || status != "sent" || deliverer.calls != 1 {
- t.Fatalf("final reservation/status/provider calls = %q/%q/%d, want confirmed/sent/1", finalState, status, deliverer.calls)
- }
-}
-
func TestAccountSuppressionFromBounceBlocksEveryAgentSend(t *testing.T) {
api, store, _, _ := setupAsyncAPI(t)
ctx := context.Background()
diff --git a/internal/agent/test_send_async_test.go b/internal/agent/test_send_async_test.go
index 64ba85368..83117d0dd 100644
--- a/internal/agent/test_send_async_test.go
+++ b/internal/agent/test_send_async_test.go
@@ -13,6 +13,7 @@ import (
"github.com/tokencanopy/e2a/internal/identity"
"github.com/tokencanopy/e2a/internal/outbound"
"github.com/tokencanopy/e2a/internal/outboundsend"
+ "github.com/tokencanopy/e2a/internal/sendingpolicy"
"github.com/tokencanopy/e2a/internal/testutil"
"github.com/tokencanopy/e2a/internal/usage"
"github.com/tokencanopy/e2a/internal/webhookpub"
@@ -26,7 +27,7 @@ type captureDeliverer struct {
out outboundsend.DeliverOutcome
}
-func (c *captureDeliverer) Deliver(_ context.Context, j *outboundsend.SendJob) outboundsend.DeliverOutcome {
+func (c *captureDeliverer) Deliver(_ context.Context, j *outboundsend.SendJob, _ sendingpolicy.ProviderAuthorization) outboundsend.DeliverOutcome {
c.jobs = append(c.jobs, j)
return c.out
}
diff --git a/internal/e2e/email_eval_runner_e2e_test.go b/internal/e2e/email_eval_runner_e2e_test.go
index eaf48b538..559fc6527 100644
--- a/internal/e2e/email_eval_runner_e2e_test.go
+++ b/internal/e2e/email_eval_runner_e2e_test.go
@@ -1490,9 +1490,18 @@ func waitForOutboundJobsTerminal(
func outboundJobMessageID(job outboundJobRecord) (string, error) {
var args map[string]json.RawMessage
- if json.Unmarshal([]byte(job.Args), &args) != nil || len(args) != 1 {
+ if json.Unmarshal([]byte(job.Args), &args) != nil {
return "", errors.New("invalid outbound job args")
}
+ // The accept transaction stamps the durable sending operation reference
+ // beside the message id (sending abuse prevention, slice B6). Nothing
+ // else may appear: the eval's safety claim is that the queue holds only
+ // the jobs it knows the shape of.
+ for key := range args {
+ if key != "message_id" && key != "operation_ref" {
+ return "", errors.New("invalid outbound job args")
+ }
+ }
var messageID string
if json.Unmarshal(args["message_id"], &messageID) != nil || messageID == "" {
return "", errors.New("invalid outbound job message identity")
diff --git a/internal/hitlworker/async_approve_test.go b/internal/hitlworker/async_approve_test.go
index 65ddf1d0f..962308d72 100644
--- a/internal/hitlworker/async_approve_test.go
+++ b/internal/hitlworker/async_approve_test.go
@@ -8,6 +8,7 @@ import (
"github.com/jackc/pgx/v5"
"github.com/tokencanopy/e2a/internal/identity"
+ "github.com/tokencanopy/e2a/internal/outboundsend"
)
// fakeEnq records EnqueueSendTx / EnqueueScheduledSendTx calls (the outbound_send
@@ -16,10 +17,14 @@ import (
type fakeEnq struct {
calls []string
scheduledCalls map[string]time.Time
+ err error
}
func (f *fakeEnq) EnqueueSendTx(_ context.Context, _ pgx.Tx, messageID string) (int64, error) {
f.calls = append(f.calls, messageID)
+ if f.err != nil {
+ return 0, f.err
+ }
return 7777, nil
}
@@ -155,3 +160,46 @@ func TestWorkerAutoApproveAsync_SelfSendStaysLoopback(t *testing.T) {
t.Errorf("self-send status = %q, want %q (resolved via loopback)", status, identity.MessageStatusReviewExpiredApproved)
}
}
+
+// TestWorkerAutoApprovePausedAccountDefersWithoutBlocking: a TTL-expired hold on
+// an account paused for sending stays pending — held, as the pause promises —
+// but its TTL is pushed forward so it does not sit at the head of the sweep and
+// starve every other expired review, and it is not retried every cycle.
+func TestWorkerAutoApprovePausedAccountDefersWithoutBlocking(t *testing.T) {
+ w, store, pool, smtpDone := setupWorker(t)
+ ctx := context.Background()
+ agent := prepareAgent(t, store, "approve-paused", identity.HITLExpirationApprove)
+ enq := &fakeEnq{err: outboundsend.ErrSendingPaused}
+ w.SetOutboundEnqueuer(enq)
+ msg, err := store.CreatePendingOutboundMessage(ctx, agent.ID,
+ []string{"alice@external.test"}, nil, nil,
+ "Held", "body", "html
", nil, "send", "", "", "", 60)
+ if err != nil {
+ t.Fatal(err)
+ }
+ backdateExpiry(t, pool, msg.ID)
+
+ w.RunOnce(ctx)
+ if msgs := smtpDone(); len(msgs) != 0 {
+ t.Fatalf("paused account must not send inline, got %d SMTP messages", len(msgs))
+ }
+ if len(enq.calls) != 1 {
+ t.Fatalf("enqueue attempts = %v, want exactly one", enq.calls)
+ }
+ var status string
+ var expiresAt time.Time
+ if err := pool.QueryRow(ctx, `SELECT status, approval_expires_at FROM messages WHERE id=$1`, msg.ID).Scan(&status, &expiresAt); err != nil {
+ t.Fatal(err)
+ }
+ if status != identity.MessageStatusPendingReview {
+ t.Fatalf("status = %q, want pending_review (held, not rejected)", status)
+ }
+ if expiresAt.Before(time.Now().Add(50 * time.Minute)) {
+ t.Fatalf("approval_expires_at = %v, want deferred about an hour ahead", expiresAt)
+ }
+ // Deferred out of the window: the next sweep leaves it alone.
+ w.RunOnce(ctx)
+ if len(enq.calls) != 1 {
+ t.Fatalf("enqueue attempts after deferral = %v, want still one", enq.calls)
+ }
+}
diff --git a/internal/hitlworker/worker.go b/internal/hitlworker/worker.go
index ca4fbe5ec..00a1467e2 100644
--- a/internal/hitlworker/worker.go
+++ b/internal/hitlworker/worker.go
@@ -27,6 +27,7 @@ import (
"github.com/tokencanopy/e2a/internal/loopback"
"github.com/tokencanopy/e2a/internal/messagelifecycle"
"github.com/tokencanopy/e2a/internal/outbound"
+ "github.com/tokencanopy/e2a/internal/outboundsend"
"github.com/tokencanopy/e2a/internal/piguard"
"github.com/tokencanopy/e2a/internal/usage"
"github.com/tokencanopy/e2a/internal/webhookpub"
@@ -61,6 +62,11 @@ const DefaultBatchSize = 100
// Worker runs the TTL sweep. Construct with New; its RunOnce is driven on a
// schedule by the River maintenance periodic (see maintenance.go).
+// pausedReviewRetry is how far a TTL-expired review on a paused account is
+// deferred before the sweep looks at it again. Long enough not to churn, short
+// enough that a resume is picked up within the hour.
+const pausedReviewRetry = time.Hour
+
type Worker struct {
store *identity.Store
sender *outbound.Sender
@@ -406,6 +412,17 @@ func (w *Worker) autoApproveAsync(ctx context.Context, agent *identity.AgentIden
if errors.Is(err, identity.ErrNotPendingApproval) {
return true // resolved between load and transition
}
+ if errors.Is(err, outboundsend.ErrSendingPaused) {
+ // The account is paused for sending. The draft stays pending_review
+ // — that is the held queue the pause promises — but it must not
+ // stay the sweep's oldest candidate, or it is re-picked first every
+ // cycle and starves every other expired review. Defer its TTL; the
+ // sweep after resume resolves it.
+ if derr := w.store.DeferReviewExpiry(ctx, c.MessageID, time.Now().Add(pausedReviewRetry)); derr != nil {
+ log.Printf("[hitl-worker] auto-approve %s: defer while account is paused: %v", c.MessageID, derr)
+ }
+ return true
+ }
// Transient tx/enqueue failure: leave the row pending_review for the next
// cycle. Do NOT autoReject — no send happened, so this is not a "stuck" send.
log.Printf("[hitl-worker] auto-approve %s: accept+enqueue: %v", c.MessageID, err)
diff --git a/internal/httpapi/error_catalog.go b/internal/httpapi/error_catalog.go
index 558bab232..e71b00587 100644
--- a/internal/httpapi/error_catalog.go
+++ b/internal/httpapi/error_catalog.go
@@ -21,6 +21,7 @@ var errorCodeCatalog = []errorCodeContract{
{Code: "unauthorized", Status: "401", Family: "auth"},
{Code: "forbidden", Status: "403", Family: "auth"},
{Code: "blocked_by_policy", Status: "403", Family: "auth"},
+ {Code: "sending_paused", Status: "403", Family: "auth"},
{Code: "invalid_request", Status: "400 / 422", Family: "validation", DetailsSchema: "ValidationErrorDetails"},
{Code: "invalid_cursor", Status: "400", Family: "validation"},
{Code: "invalid_filter", Status: "400", Family: "validation"},
diff --git a/internal/httpapi/errors.go b/internal/httpapi/errors.go
index 50b3c6470..049c48b54 100644
--- a/internal/httpapi/errors.go
+++ b/internal/httpapi/errors.go
@@ -54,7 +54,7 @@ type ErrorEnvelope struct {
// ErrorBody is the inner object of the envelope.
type ErrorBody struct {
- Code string `json:"code" doc:"Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status."`
+ Code string `json:"code" doc:"Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, sending_paused, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental), sending_paused (403, outbound sending is paused for the account by the platform abuse controls; queued mail is held, new sends are refused until an operator resumes; experimental). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status."`
Message string `json:"message" doc:"Human-readable explanation. Not for branching — use code."`
Details any `json:"details,omitempty" doc:"Optional structured context, polymorphic by code. Treat it as an open object keyed off code; unknown codes and fields must be preserved."`
RequestID string `json:"request_id" doc:"Echoes the X-Request-Id response header so a failing call is greppable in logs."`
diff --git a/internal/httpapi/spec_review_test.go b/internal/httpapi/spec_review_test.go
index abf340c3f..13a9c62f7 100644
--- a/internal/httpapi/spec_review_test.go
+++ b/internal/httpapi/spec_review_test.go
@@ -263,6 +263,8 @@ func assertMessageLifecycleContractSchema(t *testing.T, doc map[string]any) {
"suppression.recipient_blocked", "suppression.hard_bounce_applied", "suppression.complaint_applied",
"queue.inbound_processing", "queue.outbound_submission",
"submission.upstream_accepted", "submission.local_loopback_accepted", "submission.temporary_failure", "submission.provider_rejected", "submission.local_retries_exhausted", "submission.cancelled",
+ "submission.policy_budget_expired",
+ "submission.sending_setup_expired",
"delivery.recipient_server_accepted", "delivery.temporary_delay", "delivery.permanent_bounce", "delivery.transient_bounce", "delivery.undetermined_bounce",
"complaint.recipient_reported",
},
diff --git a/internal/httpapi/stability.go b/internal/httpapi/stability.go
index 7eea7ad91..7d30d3b26 100644
--- a/internal/httpapi/stability.go
+++ b/internal/httpapi/stability.go
@@ -244,9 +244,10 @@ func (s *Server) applyEvolutionStance() {
for _, schema := range []string{"HoldReasonView", "ProtectionFindingView", "ThreatCategoryView"} {
markSchema(schemas, schema, extStabilityLevel, stabilityBeta)
}
- // ErrorBody.code is a stable open discriminator; only the outbound
- // gate-policy value remains experimental.
- markProperty(schemas, "ErrorBody", "code", extExperimentalValues, []string{"blocked_by_policy"})
+ // ErrorBody.code is a stable open discriminator; the outbound gate-policy
+ // value and the sending-abuse pause value remain experimental — both are
+ // produced by controls that ship disabled.
+ markProperty(schemas, "ErrorBody", "code", extExperimentalValues, []string{"blocked_by_policy", "sending_paused"})
//
// The template hooks on send are beta (templates are beta) even though
// sendMessage itself is stable.
diff --git a/internal/httpapi/stability_test.go b/internal/httpapi/stability_test.go
index dfdb3dd0f..1f37e285d 100644
--- a/internal/httpapi/stability_test.go
+++ b/internal/httpapi/stability_test.go
@@ -436,12 +436,13 @@ func TestSpecBetaMarkers(t *testing.T) {
}
}
- // The error discriminator remains stable; only the gate-policy value is
- // experimental.
+ // The error discriminator remains stable; only the two values produced by
+ // controls that ship disabled — the outbound gate policy and the sending
+ // abuse pause — are experimental.
errorCode, _ := schemaProps(t, doc, "ErrorBody")["code"].(map[string]any)
rawErrorValues, _ := errorCode["x-experimental-values"].([]any)
- if len(rawErrorValues) != 1 || rawErrorValues[0] != "blocked_by_policy" {
- t.Errorf("ErrorBody.code x-experimental-values = %v, want [blocked_by_policy]", rawErrorValues)
+ if len(rawErrorValues) != 2 || rawErrorValues[0] != "blocked_by_policy" || rawErrorValues[1] != "sending_paused" {
+ t.Errorf("ErrorBody.code x-experimental-values = %v, want [blocked_by_policy sending_paused]", rawErrorValues)
}
// Managed unsubscribe is a beta opt-in nested inside otherwise-stable
diff --git a/internal/identity/delivery_store.go b/internal/identity/delivery_store.go
index 3be22a369..4d53d0c8d 100644
--- a/internal/identity/delivery_store.go
+++ b/internal/identity/delivery_store.go
@@ -319,6 +319,7 @@ func (s *Store) RecordDeliveryOutcomeTx(ctx context.Context, tx pgx.Tx, messageI
if _, err := tx.Exec(ctx,
`UPDATE messages
SET delivery_status = 'failed',
+ local_hold_class = NULL, local_hold_anchor = NULL,
delivery_failure_source = COALESCE(delivery_failure_source, 'provider')
WHERE id = $1`, messageID,
); err != nil {
@@ -373,7 +374,7 @@ func (s *Store) MarkMessageSent(ctx context.Context, messageID, sentAs string, t
defer tx.Rollback(ctx)
if _, err := tx.Exec(ctx,
- `UPDATE messages SET delivery_status = 'sent', sent_as = $2 WHERE id = $1`,
+ `UPDATE messages SET delivery_status = 'sent', sent_as = $2, local_hold_class = NULL, local_hold_anchor = NULL WHERE id = $1`,
messageID, nullIfEmpty(sentAs),
); err != nil {
return err
@@ -444,6 +445,18 @@ type OutboundSendPayload struct {
ReviewedAt *time.Time
// ProviderMessageID is the evidence-repaired provider id ('' when none).
ProviderMessageID string
+ // LocalHoldClass / LocalHoldAnchor are the durable finite-hold state the
+ // worker persisted on an earlier execution ('' / nil when the message has
+ // never entered a finite hold). The absolute deadline is always derived
+ // from this pair, never stored.
+ LocalHoldClass string
+ LocalHoldAnchor *time.Time
+ // LastResumedAt is account_sending_controls.last_resumed_at for the owning
+ // account; TenantReadyAt is its ses_tenant_ready_at (nil until the SES
+ // tenant is ready). Both feed the worker's hold-anchor and setup→rate
+ // transition rules. nil when the account has no control row yet.
+ LastResumedAt *time.Time
+ TenantReadyAt *time.Time
}
// OutboundSentInfo carries the fields the async worker's MarkSent/MarkFailed
@@ -590,6 +603,10 @@ func (s *Store) ClaimOutboundForSend(ctx context.Context, messageID string, jobI
failureAttempt *int
scheduledAt *time.Time
reviewedAt *time.Time
+ holdClass string
+ holdAnchor *time.Time
+ lastResumedAt *time.Time
+ tenantReadyAt *time.Time
)
var userID, registeredDomain string
// Lock agent first to match permanent agent deletion's lock order, then
@@ -613,14 +630,18 @@ func (s *Store) ClaimOutboundForSend(ctx context.Context, messageID string, jobI
m.to_recipients, m.cc, m.bcc, m.raw_message, m.created_at,
m.deleted_at, m.send_job_id, m.provider_accepted_at, COALESCE(m.provider_message_id,''),
COALESCE(m.delivery_failure_source,''),COALESCE(m.delivery_failure_reason_code,''),
- m.delivery_failure_occurred_at,m.delivery_failure_attempt,m.scheduled_at,m.reviewed_at
+ m.delivery_failure_occurred_at,m.delivery_failure_attempt,m.scheduled_at,m.reviewed_at,
+ COALESCE(m.local_hold_class,''), m.local_hold_anchor,
+ c.last_resumed_at, c.ses_tenant_ready_at
FROM messages m
+ LEFT JOIN account_sending_controls c ON c.user_id = $3
WHERE m.id = $1 AND m.agent_id = $2 AND m.direction = 'outbound'
FOR UPDATE OF m`,
- messageID, agentID,
+ messageID, agentID, userID,
).Scan(&deliveryStatus, &envelopeFrom, &sentAs, &messageType, &to, &cc, &bcc, &raw, &createdAt,
&deletedAt, &stampedJobID, &providerAcceptedAt, &providerMessageID,
- &failureSource, &failureReason, &failureOccurredAt, &failureAttempt, &scheduledAt, &reviewedAt)
+ &failureSource, &failureReason, &failureOccurredAt, &failureAttempt, &scheduledAt, &reviewedAt,
+ &holdClass, &holdAnchor, &lastResumedAt, &tenantReadyAt)
if errors.Is(err, pgx.ErrNoRows) {
if err := tx.Commit(ctx); err != nil {
return nil, err
@@ -665,6 +686,7 @@ func (s *Store) ClaimOutboundForSend(ctx context.Context, messageID string, jobI
if _, err := tx.Exec(ctx,
`UPDATE messages
SET delivery_status = 'failed',
+ local_hold_class = NULL, local_hold_anchor = NULL,
delivery_detail = 'send canceled because the message or agent is in trash',
delivery_failure_source = 'local',
delivery_failure_reason_code = 'submission.cancelled',
@@ -710,6 +732,10 @@ func (s *Store) ClaimOutboundForSend(ctx context.Context, messageID string, jobI
ProviderMessageID: providerMessageID,
ScheduledAt: scheduledAt,
ReviewedAt: reviewedAt,
+ LocalHoldClass: holdClass,
+ LocalHoldAnchor: holdAnchor,
+ LastResumedAt: lastResumedAt,
+ TenantReadyAt: tenantReadyAt,
}
if err := tx.Commit(ctx); err != nil {
return nil, err
@@ -717,6 +743,28 @@ func (s *Store) ClaimOutboundForSend(ctx context.Context, messageID string, jobI
return p, nil
}
+// RecordOutboundHold persists a message's finite-hold class and anchor.
+//
+// The worker owns the transition rules (first finite hold, setup→rate,
+// monotonic promotion to policy_budget); this writes exactly the pair it was
+// given and only while the message is still pre-terminal. Terminal writes
+// clear the pair, so a stale hold can never outlive its message's outcome.
+func (s *Store) RecordOutboundHold(ctx context.Context, messageID, class string, anchor time.Time) error {
+ if class == "" || anchor.IsZero() {
+ return fmt.Errorf("record outbound hold: class and anchor are required")
+ }
+ _, err := s.pool.Exec(ctx, `
+ UPDATE messages
+ SET local_hold_class = $2, local_hold_anchor = $3
+ WHERE id = $1 AND direction = 'outbound'
+ AND delivery_status IN ('accepted', 'sending')`,
+ messageID, class, anchor.UTC())
+ if err != nil {
+ return fmt.Errorf("record outbound hold: %w", err)
+ }
+ return nil
+}
+
func isCompleteTerminalFallback(source, reason string, occurredAt *time.Time, attempt *int) bool {
if occurredAt == nil || occurredAt.IsZero() || attempt == nil || *attempt < 0 {
return false
@@ -724,7 +772,8 @@ func isCompleteTerminalFallback(source, reason string, occurredAt *time.Time, at
switch messagelifecycle.ReasonCode(reason) {
case messagelifecycle.ReasonSubmissionProviderRejected:
return delivery.FailureSource(source) == delivery.FailureSourceProvider
- case messagelifecycle.ReasonSubmissionLocalRetriesExhausted, messagelifecycle.ReasonSubmissionCancelled:
+ case messagelifecycle.ReasonSubmissionLocalRetriesExhausted, messagelifecycle.ReasonSubmissionCancelled,
+ messagelifecycle.ReasonSubmissionPolicyBudgetExpired, messagelifecycle.ReasonSubmissionSendingSetupExpired:
return delivery.FailureSource(source) == delivery.FailureSourceLocal
default:
return false
@@ -808,6 +857,7 @@ func (s *Store) MarkOutboundSentTx(ctx context.Context, tx pgx.Tx, messageID, pr
err := tx.QueryRow(ctx,
`UPDATE messages m
SET delivery_status = 'sent', provider_message_id = $2, send_claimed_at = NULL,
+ local_hold_class = NULL, local_hold_anchor = NULL,
rfc_message_id_key = CASE
WHEN rfc_message_id_key IS NULL AND $3 <> '' THEN $3
ELSE rfc_message_id_key
@@ -891,7 +941,7 @@ func (s *Store) ResolveOutboundProviderAcceptedTx(ctx context.Context, tx pgx.Tx
m := &Message{ID: messageID, Direction: "outbound", DeliveryStatus: "sent"}
err = tx.QueryRow(ctx,
`UPDATE messages m
- SET delivery_status = 'sent', send_claimed_at = NULL, delivery_failure_source = NULL, delivery_failure_reason_code = NULL, delivery_detail = NULL,
+ SET delivery_status = 'sent', send_claimed_at = NULL, local_hold_class = NULL, local_hold_anchor = NULL, delivery_failure_source = NULL, delivery_failure_reason_code = NULL, delivery_detail = NULL,
delivery_failure_occurred_at=NULL, delivery_failure_attempt=NULL, delivery_failure_blocked_recipients=NULL
FROM agent_identities a
WHERE m.id = $1 AND m.direction = 'outbound'
@@ -981,6 +1031,7 @@ func (s *Store) MarkOutboundFailedTx(ctx context.Context, tx pgx.Tx, messageID,
err := tx.QueryRow(ctx,
`UPDATE messages m
SET delivery_status = 'failed',
+ local_hold_class = NULL, local_hold_anchor = NULL,
delivery_detail = COALESCE(NULLIF(m.delivery_detail, ''), $2),
delivery_failure_source = $3,
send_claimed_at = NULL
diff --git a/internal/identity/outbound_hold_test.go b/internal/identity/outbound_hold_test.go
new file mode 100644
index 000000000..88f8df06d
--- /dev/null
+++ b/internal/identity/outbound_hold_test.go
@@ -0,0 +1,110 @@
+package identity_test
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/jackc/pgx/v5"
+
+ "github.com/tokencanopy/e2a/internal/identity"
+ "github.com/tokencanopy/e2a/internal/testutil"
+)
+
+// The finite-hold pair rides the claim payload so every worker execution
+// re-derives the same deadline, and it is cleared by the terminal write so a
+// stale hold can never outlive its message's outcome.
+func TestOutboundHoldRidesTheClaimAndClearsOnTerminal(t *testing.T) {
+ pool := testutil.TestDB(t)
+ store := identity.NewStore(pool)
+ ctx := context.Background()
+ agentID := convoTestSetup(t, store, "hold-claim")
+
+ var userID string
+ if err := pool.QueryRow(ctx, `SELECT user_id FROM agent_identities WHERE id = $1`, agentID).Scan(&userID); err != nil {
+ t.Fatal(err)
+ }
+ resumed := time.Date(2026, 9, 1, 8, 0, 0, 0, time.UTC)
+ ready := time.Date(2026, 9, 2, 9, 30, 0, 0, time.UTC)
+ if _, err := pool.Exec(ctx, `
+ INSERT INTO account_sending_controls (user_id, last_resumed_at, ses_tenant_name, ses_tenant_ready, ses_tenant_ready_at)
+ VALUES ($1, $2, 'tenant_hold_test', true, $3)
+ ON CONFLICT (user_id) DO UPDATE SET last_resumed_at = $2, ses_tenant_ready = true, ses_tenant_ready_at = $3`,
+ userID, resumed, ready,
+ ); err != nil {
+ t.Fatal(err)
+ }
+
+ var msgID string
+ if err := store.WithTx(ctx, func(tx pgx.Tx) error {
+ m, err := store.CreateOutboundMessageTx(ctx, tx, agentID,
+ []string{"one@example.test"}, nil, nil, "Hold", "send", "smtp", "", "conv-hold",
+ []byte("From: bot\r\n\r\nbody"), "accepted", "agent@test.e2a.dev", "relay")
+ if err != nil {
+ return err
+ }
+ msgID = m.ID
+ return store.StampSendJobIDTx(ctx, tx, m.ID, 4242)
+ }); err != nil {
+ t.Fatalf("seed: %v", err)
+ }
+
+ p, err := store.ClaimOutboundForSend(ctx, msgID, 4242)
+ if err != nil || p == nil {
+ t.Fatalf("claim: payload=%v err=%v", p, err)
+ }
+ if p.LocalHoldClass != "" || p.LocalHoldAnchor != nil {
+ t.Fatalf("fresh claim carries a hold: %q %v", p.LocalHoldClass, p.LocalHoldAnchor)
+ }
+ if p.LastResumedAt == nil || !p.LastResumedAt.Equal(resumed) || p.TenantReadyAt == nil || !p.TenantReadyAt.Equal(ready) {
+ t.Fatalf("control timestamps = %v / %v, want %v / %v", p.LastResumedAt, p.TenantReadyAt, resumed, ready)
+ }
+ if err := store.ReleaseOutboundSendClaim(ctx, msgID, 4242); err != nil {
+ t.Fatal(err)
+ }
+
+ anchor := time.Date(2026, 9, 3, 10, 0, 0, 0, time.UTC)
+ if err := store.RecordOutboundHold(ctx, msgID, "policy_budget", anchor); err != nil {
+ t.Fatalf("record hold: %v", err)
+ }
+ p, err = store.ClaimOutboundForSend(ctx, msgID, 4242)
+ if err != nil || p == nil {
+ t.Fatalf("re-claim: payload=%v err=%v", p, err)
+ }
+ if p.LocalHoldClass != "policy_budget" || p.LocalHoldAnchor == nil || !p.LocalHoldAnchor.Equal(anchor) {
+ t.Fatalf("hold on re-claim = %q %v, want policy_budget @ %v", p.LocalHoldClass, p.LocalHoldAnchor, anchor)
+ }
+
+ if err := store.WithTx(ctx, func(tx pgx.Tx) error {
+ _, err := store.MarkOutboundSentTx(ctx, tx, msgID, "")
+ return err
+ }); err != nil {
+ t.Fatalf("mark sent: %v", err)
+ }
+ var class *string
+ var holdAnchor *time.Time
+ if err := pool.QueryRow(ctx, `SELECT local_hold_class, local_hold_anchor FROM messages WHERE id = $1`, msgID).Scan(&class, &holdAnchor); err != nil {
+ t.Fatal(err)
+ }
+ if class != nil || holdAnchor != nil {
+ t.Fatalf("hold survived the terminal write: %v %v", class, holdAnchor)
+ }
+ // A terminal row refuses a late hold write.
+ if err := store.RecordOutboundHold(ctx, msgID, "policy_budget", anchor); err != nil {
+ t.Fatalf("late hold write errored: %v", err)
+ }
+ if err := pool.QueryRow(ctx, `SELECT local_hold_class FROM messages WHERE id = $1`, msgID).Scan(&class); err != nil {
+ t.Fatal(err)
+ }
+ if class != nil {
+ t.Fatalf("hold written on a sent row: %q", *class)
+ }
+}
+
+func TestOutboundHoldRejectsAnEmptyPair(t *testing.T) {
+ pool := testutil.TestDB(t)
+ store := identity.NewStore(pool)
+ if err := store.RecordOutboundHold(context.Background(), "msg_none", "", time.Time{}); err == nil {
+ t.Fatal("empty class and anchor accepted")
+ }
+}
diff --git a/internal/identity/review.go b/internal/identity/review.go
index dc627786f..9fd35c958 100644
--- a/internal/identity/review.go
+++ b/internal/identity/review.go
@@ -349,6 +349,23 @@ func (s *Store) ExpireApproveReviewWithTransition(ctx context.Context, messageID
return s.transitionReview(ctx, messageID, "", MessageStatusReviewExpiredApproved, nil, "")
}
+// DeferReviewExpiry pushes a pending review's TTL forward without resolving
+// it. The expiration sweep orders candidates by approval_expires_at, so a
+// hold that cannot resolve yet — its account is paused for sending — would
+// otherwise stay the oldest candidate and be re-picked first every cycle,
+// starving every other expired review once enough of them accumulate.
+// Deferring it yields the slot; when the account resumes, the next sweep
+// after the deferred instant resolves it normally.
+func (s *Store) DeferReviewExpiry(ctx context.Context, messageID string, until time.Time) error {
+ _, err := s.pool.Exec(ctx,
+ `UPDATE messages SET approval_expires_at = $2 WHERE id = $1 AND status = 'pending_review'`,
+ messageID, until.UTC())
+ if err != nil {
+ return fmt.Errorf("defer review expiry: %w", err)
+ }
+ return nil
+}
+
// ExpireRejectReview is the worker-side TTL auto-reject: drops the message
// (status review_expired_rejected) with no human reviewer. System-scoped.
func (s *Store) ExpireRejectReview(ctx context.Context, messageID, reason string) error {
diff --git a/internal/messagelifecycle/catalog.go b/internal/messagelifecycle/catalog.go
index b7f71272d..17ccd5b68 100644
--- a/internal/messagelifecycle/catalog.go
+++ b/internal/messagelifecycle/catalog.go
@@ -66,12 +66,20 @@ const (
ReasonSubmissionProviderRejected ReasonCode = "submission.provider_rejected"
ReasonSubmissionLocalRetriesExhausted ReasonCode = "submission.local_retries_exhausted"
ReasonSubmissionCancelled ReasonCode = "submission.cancelled"
- ReasonDeliveryRecipientServerAccepted ReasonCode = "delivery.recipient_server_accepted"
- ReasonDeliveryTemporaryDelay ReasonCode = "delivery.temporary_delay"
- ReasonDeliveryPermanentBounce ReasonCode = "delivery.permanent_bounce"
- ReasonDeliveryTransientBounce ReasonCode = "delivery.transient_bounce"
- ReasonDeliveryUndeterminedBounce ReasonCode = "delivery.undetermined_bounce"
- ReasonComplaintRecipientReported ReasonCode = "complaint.recipient_reported"
+ // ReasonSubmissionPolicyBudgetExpired means a sending-budget hold reached
+ // its seven-day deadline without capacity freeing. It is a local policy
+ // outcome, never a recipient rejection or a provider outage.
+ ReasonSubmissionPolicyBudgetExpired ReasonCode = "submission.policy_budget_expired"
+ // ReasonSubmissionSendingSetupExpired means the account's provider-side
+ // sending setup (SES tenant readiness) did not complete within the
+ // 72-hour setup deadline.
+ ReasonSubmissionSendingSetupExpired ReasonCode = "submission.sending_setup_expired"
+ ReasonDeliveryRecipientServerAccepted ReasonCode = "delivery.recipient_server_accepted"
+ ReasonDeliveryTemporaryDelay ReasonCode = "delivery.temporary_delay"
+ ReasonDeliveryPermanentBounce ReasonCode = "delivery.permanent_bounce"
+ ReasonDeliveryTransientBounce ReasonCode = "delivery.transient_bounce"
+ ReasonDeliveryUndeterminedBounce ReasonCode = "delivery.undetermined_bounce"
+ ReasonComplaintRecipientReported ReasonCode = "complaint.recipient_reported"
)
// Definition is the fixed meaning of a reason code.
@@ -106,6 +114,8 @@ var canonicalCatalog = map[ReasonCode]Definition{
ReasonSubmissionProviderRejected: {StageSubmission, OutcomeFailed, false},
ReasonSubmissionLocalRetriesExhausted: {StageSubmission, OutcomeFailed, true},
ReasonSubmissionCancelled: {StageSubmission, OutcomeFailed, false},
+ ReasonSubmissionPolicyBudgetExpired: {StageSubmission, OutcomeFailed, true},
+ ReasonSubmissionSendingSetupExpired: {StageSubmission, OutcomeFailed, true},
ReasonDeliveryRecipientServerAccepted: {StageDelivery, OutcomeDelivered, false},
ReasonDeliveryTemporaryDelay: {StageDelivery, OutcomeDeferred, true},
ReasonDeliveryPermanentBounce: {StageDelivery, OutcomeBounced, false},
diff --git a/internal/messagelifecycle/model.go b/internal/messagelifecycle/model.go
index 783187f82..9c2c2b0ab 100644
--- a/internal/messagelifecycle/model.go
+++ b/internal/messagelifecycle/model.go
@@ -69,7 +69,7 @@ type MessageLifecycleTransition struct {
Recipient string `json:"recipient,omitempty" nullable:"true"`
Stage Stage `json:"stage" enum:"accepted,authentication,review,suppression,queued,submission,delivery,complaint"`
Outcome Outcome `json:"outcome" enum:"accepted,passed,failed,indeterminate,pending,approved,rejected,blocked,applied,enqueued,deferred,delivered,bounced,reported"`
- ReasonCode ReasonCode `json:"reason_code" enum:"acceptance.inbound_smtp,acceptance.outbound_api,acceptance.local_loopback,authentication.dmarc_pass,authentication.dmarc_fail,authentication.dmarc_none,authentication.dmarc_temporary_error,authentication.dmarc_permanent_error,review.hold_created,review.approved,review.rejected,review.expired_approved,review.expired_rejected,suppression.recipient_blocked,suppression.hard_bounce_applied,suppression.complaint_applied,queue.inbound_processing,queue.outbound_submission,submission.upstream_accepted,submission.local_loopback_accepted,submission.temporary_failure,submission.provider_rejected,submission.local_retries_exhausted,submission.cancelled,delivery.recipient_server_accepted,delivery.temporary_delay,delivery.permanent_bounce,delivery.transient_bounce,delivery.undetermined_bounce,complaint.recipient_reported"`
+ ReasonCode ReasonCode `json:"reason_code" enum:"acceptance.inbound_smtp,acceptance.outbound_api,acceptance.local_loopback,authentication.dmarc_pass,authentication.dmarc_fail,authentication.dmarc_none,authentication.dmarc_temporary_error,authentication.dmarc_permanent_error,review.hold_created,review.approved,review.rejected,review.expired_approved,review.expired_rejected,suppression.recipient_blocked,suppression.hard_bounce_applied,suppression.complaint_applied,queue.inbound_processing,queue.outbound_submission,submission.upstream_accepted,submission.local_loopback_accepted,submission.temporary_failure,submission.provider_rejected,submission.local_retries_exhausted,submission.cancelled,submission.policy_budget_expired,submission.sending_setup_expired,delivery.recipient_server_accepted,delivery.temporary_delay,delivery.permanent_bounce,delivery.transient_bounce,delivery.undetermined_bounce,complaint.recipient_reported"`
Retryable bool `json:"retryable"`
Evidence map[string]any `json:"evidence"`
CorrelationIDs map[string]string `json:"correlation_ids"`
diff --git a/internal/messagelifecycle/model_test.go b/internal/messagelifecycle/model_test.go
index 47ef4fdcb..21295bc6e 100644
--- a/internal/messagelifecycle/model_test.go
+++ b/internal/messagelifecycle/model_test.go
@@ -42,6 +42,8 @@ func TestCatalogIsExhaustive(t *testing.T) {
{ReasonSubmissionProviderRejected, StageSubmission, OutcomeFailed, false},
{ReasonSubmissionLocalRetriesExhausted, StageSubmission, OutcomeFailed, true},
{ReasonSubmissionCancelled, StageSubmission, OutcomeFailed, false},
+ {ReasonSubmissionPolicyBudgetExpired, StageSubmission, OutcomeFailed, true},
+ {ReasonSubmissionSendingSetupExpired, StageSubmission, OutcomeFailed, true},
{ReasonDeliveryRecipientServerAccepted, StageDelivery, OutcomeDelivered, false},
{ReasonDeliveryTemporaryDelay, StageDelivery, OutcomeDeferred, true},
{ReasonDeliveryPermanentBounce, StageDelivery, OutcomeBounced, false},
@@ -51,7 +53,7 @@ func TestCatalogIsExhaustive(t *testing.T) {
}
catalog := Catalog()
- if got, want := len(catalog), 30; got != want {
+ if got, want := len(catalog), 32; got != want {
t.Fatalf("Catalog() length = %d, want %d", got, want)
}
seen := make(map[ReasonCode]bool, len(tests))
@@ -93,7 +95,7 @@ func TestCatalogRejectsUnknownAndCannotBeMutated(t *testing.T) {
if !ok || got != (Definition{Stage: StageAccepted, Outcome: OutcomeAccepted}) {
t.Fatalf("caller mutation changed canonical lookup: %+v, %v", got, ok)
}
- if got := len(Catalog()); got != 30 {
+ if got := len(Catalog()); got != 32 {
t.Fatalf("caller mutation changed canonical catalog length to %d", got)
}
}
@@ -500,7 +502,7 @@ func TestNewTransitionSchemaEnumTags(t *testing.T) {
assertTag("Direction", "enum", "inbound,outbound")
assertTag("Stage", "enum", "accepted,authentication,review,suppression,queued,submission,delivery,complaint")
assertTag("Outcome", "enum", "accepted,passed,failed,indeterminate,pending,approved,rejected,blocked,applied,enqueued,deferred,delivered,bounced,reported")
- assertTag("ReasonCode", "enum", "acceptance.inbound_smtp,acceptance.outbound_api,acceptance.local_loopback,authentication.dmarc_pass,authentication.dmarc_fail,authentication.dmarc_none,authentication.dmarc_temporary_error,authentication.dmarc_permanent_error,review.hold_created,review.approved,review.rejected,review.expired_approved,review.expired_rejected,suppression.recipient_blocked,suppression.hard_bounce_applied,suppression.complaint_applied,queue.inbound_processing,queue.outbound_submission,submission.upstream_accepted,submission.local_loopback_accepted,submission.temporary_failure,submission.provider_rejected,submission.local_retries_exhausted,submission.cancelled,delivery.recipient_server_accepted,delivery.temporary_delay,delivery.permanent_bounce,delivery.transient_bounce,delivery.undetermined_bounce,complaint.recipient_reported")
+ assertTag("ReasonCode", "enum", "acceptance.inbound_smtp,acceptance.outbound_api,acceptance.local_loopback,authentication.dmarc_pass,authentication.dmarc_fail,authentication.dmarc_none,authentication.dmarc_temporary_error,authentication.dmarc_permanent_error,review.hold_created,review.approved,review.rejected,review.expired_approved,review.expired_rejected,suppression.recipient_blocked,suppression.hard_bounce_applied,suppression.complaint_applied,queue.inbound_processing,queue.outbound_submission,submission.upstream_accepted,submission.local_loopback_accepted,submission.temporary_failure,submission.provider_rejected,submission.local_retries_exhausted,submission.cancelled,submission.policy_budget_expired,submission.sending_setup_expired,delivery.recipient_server_accepted,delivery.temporary_delay,delivery.permanent_bounce,delivery.transient_bounce,delivery.undetermined_bounce,complaint.recipient_reported")
}
func validAppendInput() AppendInput {
diff --git a/internal/outbound/provider_submit.go b/internal/outbound/provider_submit.go
index 79668a82b..fa01de163 100644
--- a/internal/outbound/provider_submit.go
+++ b/internal/outbound/provider_submit.go
@@ -139,6 +139,10 @@ func NewProviderSubmitter(relay *SMTPRelay, gate sendingpolicy.Gate) *ProviderSu
// tagged with. Empty means no header (dev/self-host without SES).
func (s *ProviderSubmitter) SetSESConfigurationSet(name string) { s.sesConfigSet = name }
+// SESConfigurationSet reports the configured configuration set, for wiring
+// tests that must prove delivery feedback stayed switched on.
+func (s *ProviderSubmitter) SESConfigurationSet() string { return s.sesConfigSet }
+
// SubmitOnce makes exactly one provider call for one authorized attempt.
//
// The sequence is fixed and every early exit is I/O-free: prove the envelope is
diff --git a/internal/outboundsend/gate_worker_test.go b/internal/outboundsend/gate_worker_test.go
new file mode 100644
index 000000000..78d9ec641
--- /dev/null
+++ b/internal/outboundsend/gate_worker_test.go
@@ -0,0 +1,526 @@
+package outboundsend_test
+
+import (
+ "context"
+ "errors"
+ "testing"
+ "time"
+
+ "github.com/riverqueue/river"
+
+ "github.com/tokencanopy/e2a/internal/delivery"
+ "github.com/tokencanopy/e2a/internal/messagelifecycle"
+ "github.com/tokencanopy/e2a/internal/outboundsend"
+ "github.com/tokencanopy/e2a/internal/sendingpolicy"
+)
+
+// These tests pin the fixed worker order over the sending-protection gate:
+// Reserve → rate → suppression → ConsumeAttempt → authorized submit, with
+// every hold snoozing without provider I/O, every deferral/cancellation
+// returning the right ledger, and every finite hold persisting a class whose
+// derived deadline decides expiry and its lifecycle reason.
+
+func isSnooze(err error) bool {
+ var snooze *river.JobSnoozeError
+ return errors.As(err, &snooze)
+}
+
+func isCancel(err error) bool {
+ var cancel *river.JobCancelError
+ return errors.As(err, &cancel)
+}
+
+func TestGatedWorker_AllowedPathAuthorizesThenSubmits(t *testing.T) {
+ st := &fakeStore{job: acceptedJob("msg_1")}
+ dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{ProviderMessageID: "ses-1", SentAs: "relay"}}
+ g := allowAll()
+ if err := outboundsend.NewSendWorker(st, dl).WithGate(g).Work(context.Background(), gatedJob("msg_1", 1)); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+ if g.reserves != 1 || g.consumes != 1 || dl.calls != 1 || len(st.sent) != 1 {
+ t.Fatalf("reserves=%d consumes=%d delivers=%d sent=%d, want 1/1/1/1", g.reserves, g.consumes, dl.calls, len(st.sent))
+ }
+ if len(g.deferred)+len(g.cancelled) != 0 {
+ t.Fatalf("deferred=%v cancelled=%v on an allowed path", g.deferred, g.cancelled)
+ }
+}
+
+func TestGatedWorker_EarlyHoldSnoozesWithoutProviderIOAndPersistsClass(t *testing.T) {
+ for reason, want := range map[string]outboundsend.HoldClass{
+ sendingpolicy.ReasonAccountDailyBudget: outboundsend.HoldPolicyBudget,
+ sendingpolicy.ReasonGlobalProbation: outboundsend.HoldPolicyBudget,
+ sendingpolicy.ReasonTenantNotReady: outboundsend.HoldTenantSetup,
+ sendingpolicy.ReasonTenantUnnamed: outboundsend.HoldTenantSetup,
+ sendingpolicy.ReasonRampCapacity: outboundsend.HoldRateRampOrProvider,
+ sendingpolicy.ReasonSendingIdentityUnverified: outboundsend.HoldRateRampOrProvider,
+ } {
+ j := acceptedJob("msg_hold")
+ j.AcceptedAt = time.Now().Add(-time.Hour)
+ st := &fakeStore{job: j}
+ dl := &fakeDeliverer{}
+ g := &fakeGate{reserve: sendingpolicy.Decision{Allow: false, Reason: reason, RetryAt: time.Now().Add(2 * time.Hour)}}
+ err := outboundsend.NewSendWorker(st, dl).WithGate(g).Work(context.Background(), gatedJob("msg_hold", 1))
+ if !isSnooze(err) {
+ t.Fatalf("%s: err = %v, want snooze", reason, err)
+ }
+ if dl.calls != 0 || len(st.failed) != 0 || g.consumes != 0 {
+ t.Fatalf("%s: delivers=%d failed=%d consumes=%d, want no I/O and no terminal", reason, dl.calls, len(st.failed), g.consumes)
+ }
+ if len(st.holds) != 1 || st.holds[0].class != want {
+ t.Fatalf("%s: holds = %+v, want one %s hold", reason, st.holds, want)
+ }
+ // A first-observed tenant-setup hold starts its clock at the
+ // observation; every other class starts at the latest of the
+ // message's own timestamps.
+ if want == outboundsend.HoldTenantSetup {
+ if st.holds[0].anchor.Before(j.AcceptedAt.Add(time.Hour - time.Minute)) {
+ t.Fatalf("%s: anchor = %v, want the observation time, not accept", reason, st.holds[0].anchor)
+ }
+ } else if !st.holds[0].anchor.Equal(j.AcceptedAt) {
+ t.Fatalf("%s: anchor = %v, want accept %v", reason, st.holds[0].anchor, j.AcceptedAt)
+ }
+ if len(st.released) != 1 {
+ t.Fatalf("%s: claim releases = %v, want one", reason, st.released)
+ }
+ }
+}
+
+func TestGatedWorker_PauseHoldIsIndefiniteAndPersistsNothing(t *testing.T) {
+ j := acceptedJob("msg_paused")
+ j.AcceptedAt = time.Now().Add(-30 * 24 * time.Hour) // far past every finite horizon
+ st := &fakeStore{job: j}
+ g := &fakeGate{reserve: sendingpolicy.Decision{Allow: false, Reason: sendingpolicy.ReasonAccountPaused}}
+ err := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(g).Work(context.Background(), gatedJob("msg_paused", 1))
+ if !isSnooze(err) {
+ t.Fatalf("err = %v, want snooze — a pause waits for an operator", err)
+ }
+ if len(st.holds) != 0 || len(st.failed) != 0 {
+ t.Fatalf("holds=%+v failed=%+v, want neither for a pause", st.holds, st.failed)
+ }
+}
+
+func TestGatedWorker_PauseNeverEvaluatesADeadlineButNeverExtendsIt(t *testing.T) {
+ // Paused with a budget deadline already eight days gone: the paused job
+ // only waits. Nothing is failed, nothing rewritten.
+ j := acceptedJob("msg_paused_budget")
+ j.LocalHoldClass, j.LocalHoldAnchor = outboundsend.HoldPolicyBudget, time.Now().Add(-8*24*time.Hour)
+ st := &fakeStore{job: j}
+ g := &fakeGate{reserve: sendingpolicy.Decision{Allow: false, Reason: sendingpolicy.ReasonAccountPaused}}
+ if err := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(g).Work(context.Background(), gatedJob("msg_paused_budget", 1)); !isSnooze(err) {
+ t.Fatalf("paused err = %v, want snooze — a paused job evaluates no deadline", err)
+ }
+ if len(st.failed) != 0 || len(st.holds) != 0 {
+ t.Fatalf("failed=%+v holds=%+v, want nothing touched while paused", st.failed, st.holds)
+ }
+ // After resume the first hold it meets applies the unextended deadline.
+ g = &fakeGate{reserve: sendingpolicy.Decision{Allow: false, Reason: sendingpolicy.ReasonGlobalAllBudget, RetryAt: time.Now().Add(time.Hour)}}
+ err := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(g).Work(context.Background(), gatedJob("msg_paused_budget", 2))
+ if !isCancel(err) || len(st.failed) != 1 || st.failed[0].reason != messagelifecycle.ReasonSubmissionPolicyBudgetExpired {
+ t.Fatalf("after resume err=%v failed=%+v, want the budget deadline to fire with its own reason", err, st.failed)
+ }
+}
+
+func TestGatedWorker_BudgetHoldPromotesAnyClassAndKeepsTheAnchor(t *testing.T) {
+ anchor := time.Now().Add(-2 * time.Hour)
+ for _, existing := range []outboundsend.HoldClass{outboundsend.HoldRateRampOrProvider, outboundsend.HoldTenantSetup} {
+ j := acceptedJob("msg_promote")
+ j.LocalHoldClass, j.LocalHoldAnchor = existing, anchor
+ st := &fakeStore{job: j}
+ g := &fakeGate{reserve: sendingpolicy.Decision{Allow: false, Reason: sendingpolicy.ReasonGlobalAllBudget, RetryAt: time.Now().Add(time.Hour)}}
+ if err := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(g).Work(context.Background(), gatedJob("msg_promote", 1)); !isSnooze(err) {
+ t.Fatalf("%s: err = %v, want snooze", existing, err)
+ }
+ if len(st.holds) != 1 || st.holds[0].class != outboundsend.HoldPolicyBudget || !st.holds[0].anchor.Equal(anchor) {
+ t.Fatalf("%s: holds = %+v, want promotion to policy_budget with the anchor kept", existing, st.holds)
+ }
+ }
+ // And policy_budget never changes again, even under a later setup hold.
+ j := acceptedJob("msg_sticky")
+ j.LocalHoldClass, j.LocalHoldAnchor = outboundsend.HoldPolicyBudget, anchor
+ st := &fakeStore{job: j}
+ g := &fakeGate{reserve: sendingpolicy.Decision{Allow: false, Reason: sendingpolicy.ReasonTenantNotReady}}
+ if err := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(g).Work(context.Background(), gatedJob("msg_sticky", 1)); !isSnooze(err) {
+ t.Fatalf("err = %v, want snooze", err)
+ }
+ if len(st.holds) != 0 {
+ t.Fatalf("holds = %+v, want no rewrite of a policy_budget hold", st.holds)
+ }
+}
+
+func TestGatedWorker_ExpiryReasonFollowsTheClass(t *testing.T) {
+ for _, tc := range []struct {
+ class outboundsend.HoldClass
+ age time.Duration
+ reason messagelifecycle.ReasonCode
+ hold string
+ }{
+ {outboundsend.HoldPolicyBudget, 7*24*time.Hour + time.Minute, messagelifecycle.ReasonSubmissionPolicyBudgetExpired, sendingpolicy.ReasonGlobalAllBudget},
+ {outboundsend.HoldTenantSetup, 72*time.Hour + time.Minute, messagelifecycle.ReasonSubmissionSendingSetupExpired, sendingpolicy.ReasonTenantNotReady},
+ {outboundsend.HoldRateRampOrProvider, 72*time.Hour + time.Minute, messagelifecycle.ReasonSubmissionLocalRetriesExhausted, sendingpolicy.ReasonRampCapacity},
+ } {
+ j := acceptedJob("msg_expire")
+ j.LocalHoldClass, j.LocalHoldAnchor = tc.class, time.Now().Add(-tc.age)
+ st := &fakeStore{job: j}
+ g := &fakeGate{reserve: sendingpolicy.Decision{Allow: false, Reason: tc.hold, RetryAt: time.Now().Add(time.Hour)}}
+ err := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(g).Work(context.Background(), gatedJob("msg_expire", 1))
+ if !isCancel(err) {
+ t.Fatalf("%s: err = %v, want cancel", tc.class, err)
+ }
+ if len(st.failed) != 1 || st.failed[0].reason != tc.reason || st.failed[0].source != delivery.FailureSourceLocal {
+ t.Fatalf("%s: failed = %+v, want one local failure with reason %s", tc.class, st.failed, tc.reason)
+ }
+ if len(g.cancelled) != 1 {
+ t.Fatalf("%s: cancelled = %v, want the attempt given back", tc.class, g.cancelled)
+ }
+ }
+ // One minute short of the deadline still snoozes.
+ j := acceptedJob("msg_almost")
+ j.LocalHoldClass, j.LocalHoldAnchor = outboundsend.HoldPolicyBudget, time.Now().Add(-7*24*time.Hour+time.Minute)
+ st := &fakeStore{job: j}
+ g := &fakeGate{reserve: sendingpolicy.Decision{Allow: false, Reason: sendingpolicy.ReasonGlobalAllBudget, RetryAt: time.Now().Add(time.Hour)}}
+ if err := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(g).Work(context.Background(), gatedJob("msg_almost", 1)); !isSnooze(err) {
+ t.Fatalf("err = %v, want snooze one minute before the deadline", err)
+ }
+}
+
+func TestGatedWorker_TerminalHoldCancelsNow(t *testing.T) {
+ for _, reason := range []string{sendingpolicy.ReasonAccountDeleted, sendingpolicy.ReasonClassChanged, sendingpolicy.ReasonRampUnavailable} {
+ st := &fakeStore{job: acceptedJob("msg_terminal")}
+ g := &fakeGate{reserve: sendingpolicy.Decision{Allow: false, Reason: reason, Terminal: true}}
+ err := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(g).Work(context.Background(), gatedJob("msg_terminal", 1))
+ if !isCancel(err) {
+ t.Fatalf("%s: err = %v, want cancel", reason, err)
+ }
+ if len(st.failed) != 1 || st.failed[0].reason != messagelifecycle.ReasonSubmissionCancelled {
+ t.Fatalf("%s: failed = %+v, want one local cancellation", reason, st.failed)
+ }
+ }
+}
+
+func TestGatedWorker_RateDeferralDefersTheAttempt(t *testing.T) {
+ st := &fakeStore{job: acceptedJob("msg_rate")}
+ g := allowAll()
+ gate := &fakeRateGate{decision: outboundsend.RateDecision{Allowed: false, RetryAt: time.Now().Add(30 * time.Second)}, window: time.Minute}
+ err := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(g).WithRateGate(gate).Work(context.Background(), gatedJob("msg_rate", 1))
+ if !isSnooze(err) {
+ t.Fatalf("err = %v, want snooze", err)
+ }
+ if len(g.deferred) != 1 || g.consumes != 0 {
+ t.Fatalf("deferred=%v consumes=%d, want the attempt deferred before final authorization", g.deferred, g.consumes)
+ }
+ if len(st.holds) != 1 || st.holds[0].class != outboundsend.HoldRateRampOrProvider {
+ t.Fatalf("holds = %+v, want a rate/ramp/provider hold", st.holds)
+ }
+}
+
+func TestGatedWorker_SuppressionCancelsTheAttempt(t *testing.T) {
+ st := &fakeStore{job: acceptedJob("msg_sup"), suppressed: []string{"b@y.com"}}
+ g := allowAll()
+ dl := &fakeDeliverer{}
+ err := outboundsend.NewSendWorker(st, dl).WithGate(g).Work(context.Background(), gatedJob("msg_sup", 1))
+ if !isCancel(err) || dl.calls != 0 {
+ t.Fatalf("err=%v delivers=%d, want cancel with no I/O", err, dl.calls)
+ }
+ if len(g.cancelled) != 1 || g.consumes != 0 {
+ t.Fatalf("cancelled=%v consumes=%d, want the attempt cancelled before final authorization", g.cancelled, g.consumes)
+ }
+}
+
+func TestGatedWorker_FinalAuthorizationHoldSnoozesWithoutProviderIO(t *testing.T) {
+ j := acceptedJob("msg_late_hold")
+ j.AcceptedAt = time.Now().Add(-time.Hour)
+ st := &fakeStore{job: j}
+ dl := &fakeDeliverer{}
+ g := &fakeGate{reserve: sendingpolicy.Decision{Allow: true}, consume: sendingpolicy.Decision{Allow: false, Reason: sendingpolicy.ReasonAccountSharedBudget, RetryAt: time.Now().Add(time.Hour)}}
+ err := outboundsend.NewSendWorker(st, dl).WithGate(g).Work(context.Background(), gatedJob("msg_late_hold", 1))
+ if !isSnooze(err) || dl.calls != 0 {
+ t.Fatalf("err=%v delivers=%d, want snooze with no I/O", err, dl.calls)
+ }
+ if len(st.holds) != 1 || st.holds[0].class != outboundsend.HoldPolicyBudget {
+ t.Fatalf("holds = %+v, want a policy_budget hold from the late gate", st.holds)
+ }
+}
+
+func TestGatedWorker_GateOutageSnoozesWithoutBurningAnAttempt(t *testing.T) {
+ for name, g := range map[string]*fakeGate{
+ "reserve": {reserveErr: errors.New("policy db down")},
+ "authorize": {reserve: sendingpolicy.Decision{Allow: true}, consumeErr: errors.New("policy db down")},
+ } {
+ st := &fakeStore{job: acceptedJob("msg_gate_down")}
+ dl := &fakeDeliverer{}
+ err := outboundsend.NewSendWorker(st, dl).WithGate(g).Work(context.Background(), gatedJob("msg_gate_down", 1))
+ if !isSnooze(err) || dl.calls != 0 || len(st.failed) != 0 {
+ t.Fatalf("%s: err=%v delivers=%d failed=%d, want snooze, no I/O, no terminal", name, err, dl.calls, len(st.failed))
+ }
+ if len(st.released) != 1 {
+ t.Fatalf("%s: claim releases = %v, want one", name, st.released)
+ }
+ }
+}
+
+func TestGatedWorker_ProviderEvidenceSettlesTheOperation(t *testing.T) {
+ j := acceptedJob("msg_evidence")
+ j.ProviderAccepted, j.ProviderMessageID = true, "ses-evidence"
+ st := &fakeStore{job: j}
+ dl := &fakeDeliverer{}
+ g := allowAll()
+ if err := outboundsend.NewSendWorker(st, dl).WithGate(g).Work(context.Background(), gatedJob("msg_evidence", 2)); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+ if dl.calls != 0 || len(st.sent) != 1 || g.reserves != 0 {
+ t.Fatalf("delivers=%d sent=%d reserves=%d, want settle without resubmit or a new reservation", dl.calls, len(st.sent), g.reserves)
+ }
+ if g.lookupCalls != 1 || len(g.settled) != 1 || g.settled[0] != sendingpolicy.SettlementProviderAccepted {
+ t.Fatalf("lookups=%d settled=%v, want the operation settled as accepted", g.lookupCalls, g.settled)
+ }
+ if len(g.settledIDs) != 1 || g.settledIDs[0] != "ses-evidence" {
+ t.Fatalf("settled ids = %v, want the evidence's provider id carried into the settlement", g.settledIDs)
+ }
+}
+
+func TestGatedWorker_LegacyJobResolvesThroughTheAcceptPath(t *testing.T) {
+ st := &fakeStore{job: acceptedJob("msg_legacy")}
+ dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{ProviderMessageID: "ses-legacy"}}
+ g := allowAll()
+ resolved := 0
+ w := outboundsend.NewSendWorker(st, dl).WithGate(g).WithOperationResolver(func(_ context.Context, id string) (sendingpolicy.AcceptanceDecision, sendingpolicy.OperationRef, error) {
+ resolved++
+ return sendingpolicy.AcceptanceAccept, refFor(id), nil
+ })
+ if err := w.Work(context.Background(), job("msg_legacy", 1)); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+ if resolved != 1 || g.reserves != 1 || dl.calls != 1 {
+ t.Fatalf("resolved=%d reserves=%d delivers=%d, want the legacy job authorized like a new one", resolved, g.reserves, dl.calls)
+ }
+
+ // A paused account at resolution holds; an orphan source cancels; no
+ // resolver at all fails closed.
+ st = &fakeStore{job: acceptedJob("msg_legacy_paused")}
+ w = outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(allowAll()).WithOperationResolver(func(context.Context, string) (sendingpolicy.AcceptanceDecision, sendingpolicy.OperationRef, error) {
+ return sendingpolicy.AcceptanceSendingPaused, sendingpolicy.OperationRef{}, nil
+ })
+ if err := w.Work(context.Background(), job("msg_legacy_paused", 1)); !isSnooze(err) {
+ t.Fatalf("paused legacy: err = %v, want snooze", err)
+ }
+ st = &fakeStore{job: acceptedJob("msg_legacy_orphan")}
+ w = outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(allowAll()).WithOperationResolver(func(context.Context, string) (sendingpolicy.AcceptanceDecision, sendingpolicy.OperationRef, error) {
+ return "", sendingpolicy.OperationRef{}, sendingpolicy.ErrSourceUnavailable
+ })
+ if err := w.Work(context.Background(), job("msg_legacy_orphan", 1)); !isCancel(err) || len(st.failed) != 1 {
+ t.Fatalf("orphan legacy: err=%v failed=%d, want cancel with one local failure", err, len(st.failed))
+ }
+ st = &fakeStore{job: acceptedJob("msg_legacy_unwired")}
+ dl = &fakeDeliverer{}
+ if err := outboundsend.NewSendWorker(st, dl).WithGate(allowAll()).Work(context.Background(), job("msg_legacy_unwired", 1)); !isCancel(err) || dl.calls != 0 {
+ t.Fatalf("unwired resolver: err=%v delivers=%d, want cancel with no I/O", err, dl.calls)
+ }
+}
+
+func TestGatedWorker_TenantReadinessMovesSetupHoldToRateClassOnce(t *testing.T) {
+ anchor := time.Now().Add(-70 * time.Hour)
+ ready := anchor.Add(60 * time.Hour) // inside the 72h setup deadline
+ j := acceptedJob("msg_ready")
+ j.LocalHoldClass, j.LocalHoldAnchor, j.TenantReadyAt = outboundsend.HoldTenantSetup, anchor, ready
+ st := &fakeStore{job: j}
+ dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{ProviderMessageID: "ses-ready"}}
+ if err := outboundsend.NewSendWorker(st, dl).WithGate(allowAll()).Work(context.Background(), gatedJob("msg_ready", 1)); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+ if len(st.holds) != 1 || st.holds[0].class != outboundsend.HoldRateRampOrProvider || !st.holds[0].anchor.Equal(ready) {
+ t.Fatalf("holds = %+v, want the one-way move to rate_ramp_or_provider anchored at readiness", st.holds)
+ }
+
+ // Readiness that landed AFTER the setup deadline does not rescue the
+ // message: it expires as setup on its next hold.
+ late := acceptedJob("msg_late_ready")
+ late.LocalHoldClass, late.LocalHoldAnchor, late.TenantReadyAt = outboundsend.HoldTenantSetup, time.Now().Add(-80*time.Hour), time.Now().Add(-time.Hour)
+ st = &fakeStore{job: late}
+ g := &fakeGate{reserve: sendingpolicy.Decision{Allow: false, Reason: sendingpolicy.ReasonRampCapacity, RetryAt: time.Now().Add(time.Hour)}}
+ err := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(g).Work(context.Background(), gatedJob("msg_late_ready", 1))
+ if !isCancel(err) || len(st.failed) != 1 || st.failed[0].reason != messagelifecycle.ReasonSubmissionSendingSetupExpired {
+ t.Fatalf("late readiness: err=%v failed=%+v, want setup expiry", err, st.failed)
+ }
+}
+
+func TestGatedWorker_FirstHoldAnchorsAtTheLatestOfAcceptScheduleReviewResume(t *testing.T) {
+ base := time.Now().Add(-10 * 24 * time.Hour)
+ j := acceptedJob("msg_anchor")
+ j.AcceptedAt = base
+ j.ScheduledAt = base.Add(24 * time.Hour)
+ j.ReviewedAt = base.Add(48 * time.Hour)
+ j.LastResumedAt = base.Add(9*24*time.Hour + 23*time.Hour) // an hour ago: the latest
+ st := &fakeStore{job: j}
+ g := &fakeGate{reserve: sendingpolicy.Decision{Allow: false, Reason: sendingpolicy.ReasonRampCapacity, RetryAt: time.Now().Add(time.Hour)}}
+ if err := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(g).Work(context.Background(), gatedJob("msg_anchor", 1)); !isSnooze(err) {
+ t.Fatalf("err = %v, want snooze — a ten-day-old accept is not the clock, the resume an hour ago is", err)
+ }
+ if len(st.holds) != 1 || !st.holds[0].anchor.Equal(j.LastResumedAt) {
+ t.Fatalf("holds = %+v, want anchored at the last resume", st.holds)
+ }
+}
+
+func TestGatedWorker_AcceptanceUnknownIsRetriedAsANewOrdinalNotSettled(t *testing.T) {
+ st := &fakeStore{job: acceptedJob("msg_unknown")}
+ dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{Err: errors.New("data final: acceptance unknown"), AcceptanceUnknown: true}}
+ g := allowAll()
+ err := outboundsend.NewSendWorker(st, dl).WithGate(g).Work(context.Background(), gatedJob("msg_unknown", 1))
+ if err == nil || isSnooze(err) || isCancel(err) {
+ t.Fatalf("err = %v, want a plain retryable error (River's next attempt returns to Reserve)", err)
+ }
+ if len(st.temporary) != 1 || len(st.failed) != 0 || len(g.settled) != 0 {
+ t.Fatalf("temporary=%d failed=%d settled=%v, want a temporary record and nothing settled", len(st.temporary), len(st.failed), g.settled)
+ }
+}
+
+func TestGatedWorker_HoldConstantsMatchThePolicyDefault(t *testing.T) {
+ if got := time.Duration(sendingpolicy.DisabledPolicy().BudgetHoldMaxDays) * 24 * time.Hour; got != outboundsend.PolicyBudgetHoldHorizon {
+ t.Fatalf("PolicyBudgetHoldHorizon = %s, policy budget_hold_max_days default = %s", outboundsend.PolicyBudgetHoldHorizon, got)
+ }
+}
+
+func TestGatedWorker_GateOutageIsBoundedByTheHoldDeadline(t *testing.T) {
+ j := acceptedJob("msg_gate_down_long")
+ j.AcceptedAt = time.Now().Add(-73 * time.Hour)
+ st := &fakeStore{job: j}
+ dl := &fakeDeliverer{}
+ g := &fakeGate{reserveErr: errors.New("policy db down")}
+ err := outboundsend.NewSendWorker(st, dl).WithGate(g).Work(context.Background(), gatedJob("msg_gate_down_long", 1))
+ if !isCancel(err) || dl.calls != 0 {
+ t.Fatalf("err=%v delivers=%d, want the 72-hour expiry with no I/O", err, dl.calls)
+ }
+ if len(st.failed) != 1 || st.failed[0].reason != messagelifecycle.ReasonSubmissionLocalRetriesExhausted {
+ t.Fatalf("failed = %+v, want local_retries_exhausted", st.failed)
+ }
+ // Inside the horizon it holds as rate/ramp/provider and snoozes.
+ j = acceptedJob("msg_gate_down_short")
+ j.AcceptedAt = time.Now().Add(-time.Hour)
+ st = &fakeStore{job: j}
+ if err := outboundsend.NewSendWorker(st, dl).WithGate(g).Work(context.Background(), gatedJob("msg_gate_down_short", 1)); !isSnooze(err) {
+ t.Fatalf("err = %v, want snooze", err)
+ }
+ if len(st.holds) != 1 || st.holds[0].class != outboundsend.HoldRateRampOrProvider {
+ t.Fatalf("holds = %+v, want a rate/ramp/provider hold", st.holds)
+ }
+}
+
+func TestGatedWorker_ReadinessLossDoesNotReplaceARateClass(t *testing.T) {
+ anchor := time.Now().Add(-time.Hour)
+ j := acceptedJob("msg_keep_rate")
+ j.LocalHoldClass, j.LocalHoldAnchor = outboundsend.HoldRateRampOrProvider, anchor
+ st := &fakeStore{job: j}
+ g := &fakeGate{reserve: sendingpolicy.Decision{Allow: false, Reason: sendingpolicy.ReasonTenantNotReady}}
+ if err := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(g).Work(context.Background(), gatedJob("msg_keep_rate", 1)); !isSnooze(err) {
+ t.Fatalf("err = %v, want snooze", err)
+ }
+ if len(st.holds) != 0 {
+ t.Fatalf("holds = %+v, want the persisted rate class left alone", st.holds)
+ }
+}
+
+func TestGatedWorker_ProviderOutagePersistsTheHoldAndHonorsTheBudgetClock(t *testing.T) {
+ // First outage: enters the rate/ramp/provider class anchored at accept.
+ j := acceptedJob("msg_outage")
+ j.AcceptedAt = time.Now().Add(-time.Hour)
+ st := &fakeStore{job: j}
+ dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{Err: errors.New("connection refused"), Outage: true}}
+ if err := outboundsend.NewSendWorker(st, dl).WithGate(allowAll()).Work(context.Background(), gatedJob("msg_outage", 1)); !isSnooze(err) {
+ t.Fatalf("err = %v, want snooze", err)
+ }
+ if len(st.holds) != 1 || st.holds[0].class != outboundsend.HoldRateRampOrProvider || !st.holds[0].anchor.Equal(j.AcceptedAt) {
+ t.Fatalf("holds = %+v, want rate/ramp/provider anchored at accept", st.holds)
+ }
+ // Under a policy_budget hold four days old, an outage keeps waiting on
+ // the seven-day clock instead of the 72-hour one.
+ j = acceptedJob("msg_outage_budget")
+ j.AcceptedAt = time.Now().Add(-5 * 24 * time.Hour)
+ j.LocalHoldClass, j.LocalHoldAnchor = outboundsend.HoldPolicyBudget, time.Now().Add(-4*24*time.Hour)
+ st = &fakeStore{job: j}
+ if err := outboundsend.NewSendWorker(st, dl).WithGate(allowAll()).Work(context.Background(), gatedJob("msg_outage_budget", 1)); !isSnooze(err) {
+ t.Fatalf("budget-held outage err = %v, want snooze on the seven-day clock", err)
+ }
+ if len(st.holds) != 0 || len(st.failed) != 0 {
+ t.Fatalf("holds=%+v failed=%+v, want the budget class untouched", st.holds, st.failed)
+ }
+ // An outage that expires a tenant_setup class never emits the setup
+ // reason: setup was not what blocked the send at the end.
+ j = acceptedJob("msg_outage_setup")
+ j.LocalHoldClass, j.LocalHoldAnchor = outboundsend.HoldTenantSetup, time.Now().Add(-73*time.Hour)
+ st = &fakeStore{job: j}
+ err := outboundsend.NewSendWorker(st, dl).WithGate(allowAll()).Work(context.Background(), gatedJob("msg_outage_setup", 1))
+ if err == nil || len(st.failed) != 1 || st.failed[0].reason != messagelifecycle.ReasonSubmissionLocalRetriesExhausted {
+ t.Fatalf("err=%v failed=%+v, want local_retries_exhausted, never sending_setup_expired", err, st.failed)
+ }
+}
+
+func TestGatedWorker_EvidenceSettleUnderATerminalWriteSettlesTheOperation(t *testing.T) {
+ // A suppression arrives for a message whose earlier attempt dialed and
+ // whose provider evidence has since landed: the guarded terminal write
+ // settles the row as SENT, and the dialed attempt must be settled too.
+ st := &fakeStore{job: acceptedJob("msg_late_evidence"), suppressed: []string{"b@y.com"}, settleStatus: delivery.StatusSent, settleProviderID: "ses-under-terminal"}
+ g := allowAll()
+ if err := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(g).Work(context.Background(), gatedJob("msg_late_evidence", 2)); !isCancel(err) {
+ t.Fatalf("err = %v, want cancel", err)
+ }
+ if g.lookupCalls != 1 || len(g.settled) != 1 || g.settled[0] != sendingpolicy.SettlementProviderAccepted {
+ t.Fatalf("lookups=%d settled=%v, want the operation settled as accepted from the evidence", g.lookupCalls, g.settled)
+ }
+ if len(g.settledIDs) != 1 || g.settledIDs[0] != "ses-under-terminal" {
+ t.Fatalf("settled ids = %v, want the store's resolved provider id carried into the settlement", g.settledIDs)
+ }
+}
+
+func TestHoldClassForNamesEveryReasonExplicitly(t *testing.T) {
+ // Every hold reason the gate can emit decides a horizon; the mapping is
+ // by name, and an unknown name takes the shorter clock.
+ cases := map[string]outboundsend.HoldClass{
+ sendingpolicy.ReasonAccountPaused: "",
+ sendingpolicy.ReasonAccountDailyBudget: outboundsend.HoldPolicyBudget,
+ sendingpolicy.ReasonAccountSharedBudget: outboundsend.HoldPolicyBudget,
+ sendingpolicy.ReasonGlobalAllBudget: outboundsend.HoldPolicyBudget,
+ sendingpolicy.ReasonGlobalProbation: outboundsend.HoldPolicyBudget,
+ sendingpolicy.ReasonGlobalCritical: outboundsend.HoldPolicyBudget,
+ sendingpolicy.ReasonGlobalViolation: outboundsend.HoldPolicyBudget,
+ sendingpolicy.ReasonTenantNotReady: outboundsend.HoldTenantSetup,
+ sendingpolicy.ReasonTenantUnnamed: outboundsend.HoldTenantSetup,
+ sendingpolicy.ReasonRampCapacity: outboundsend.HoldRateRampOrProvider,
+ sendingpolicy.ReasonSendingIdentityUnverified: outboundsend.HoldRateRampOrProvider,
+ "some_future_budget_exhausted": outboundsend.HoldRateRampOrProvider,
+ }
+ for reason, want := range cases {
+ if got := outboundsend.HoldClassFor(reason); got != want {
+ t.Errorf("HoldClassFor(%q) = %q, want %q", reason, got, want)
+ }
+ }
+}
+
+func TestGatedWorker_OperationReferenceMustNameThisMessage(t *testing.T) {
+ st := &fakeStore{job: acceptedJob("msg_a")}
+ dl := &fakeDeliverer{}
+ g := allowAll()
+ rj := job("msg_a", 1)
+ other := refFor("msg_b")
+ rj.Args.OperationRef = &other
+ err := outboundsend.NewSendWorker(st, dl).WithGate(g).Work(context.Background(), rj)
+ if !isCancel(err) || dl.calls != 0 || g.reserves != 0 {
+ t.Fatalf("err=%v delivers=%d reserves=%d, want cancel before any ledger call", err, dl.calls, g.reserves)
+ }
+ if len(st.failed) != 1 || st.failed[0].reason != messagelifecycle.ReasonSubmissionCancelled {
+ t.Fatalf("failed = %+v, want one local cancellation", st.failed)
+ }
+}
+
+func TestGatedWorker_FailedSettlementAfterAcceptanceIsRetriedNotResent(t *testing.T) {
+ st := &fakeStore{job: acceptedJob("msg_resettle")}
+ dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{ProviderMessageID: "ses-resettle", SettlementErr: errors.New("settle: db blip")}}
+ g := allowAll()
+ if err := outboundsend.NewSendWorker(st, dl).WithGate(g).Work(context.Background(), gatedJob("msg_resettle", 1)); err != nil {
+ t.Fatalf("Work: %v — an accepted send must never surface a settlement failure as a send error", err)
+ }
+ if dl.calls != 1 || len(st.sent) != 1 {
+ t.Fatalf("delivers=%d sent=%d, want exactly one of each", dl.calls, len(st.sent))
+ }
+ if len(g.settled) != 1 || g.settled[0] != sendingpolicy.SettlementProviderAccepted || g.settledIDs[0] != "ses-resettle" {
+ t.Fatalf("settlements = %v / %v, want one retried acceptance carrying the provider id", g.settled, g.settledIDs)
+ }
+}
diff --git a/internal/outboundsend/jobs.go b/internal/outboundsend/jobs.go
index 8636ba6d9..2819eb3f1 100644
--- a/internal/outboundsend/jobs.go
+++ b/internal/outboundsend/jobs.go
@@ -2,6 +2,7 @@ package outboundsend
import (
"context"
+ "fmt"
"time"
"github.com/jackc/pgx/v5"
@@ -9,6 +10,7 @@ import (
"github.com/riverqueue/river"
"github.com/tokencanopy/e2a/internal/jobs"
+ "github.com/tokencanopy/e2a/internal/sendingpolicy"
)
// Jobs is the outbound-send integration on the shared River client: a
@@ -19,23 +21,57 @@ import (
type Jobs struct {
store Store
deliverer Deliverer
- ramp RampGate
+ gate sendingpolicy.Gate
rate RateGate
pool *pgxpool.Pool
enq jobs.Enqueuer
metrics Metrics
+
+ // registered is the send worker the last RegisterJobs call handed to River.
+ registered *SendWorker
}
// NewJobs builds the integration with its dependencies (no client yet). pool
-// backs the periodic terminal-state reconciler's scan.
-func NewJobs(store Store, deliverer Deliverer, pool *pgxpool.Pool, ramp ...RampGate) *Jobs {
- j := &Jobs{store: store, deliverer: deliverer, pool: pool, metrics: noopMetrics{}}
- if len(ramp) > 0 {
- j.ramp = ramp[0]
+// backs the periodic terminal-state reconciler's scan and the legacy-argument
+// resolver's transaction.
+func NewJobs(store Store, deliverer Deliverer, pool *pgxpool.Pool) *Jobs {
+ return &Jobs{store: store, deliverer: deliverer, pool: pool, metrics: noopMetrics{}}
+}
+
+// WithGate injects the sending-protection gate. Every enqueue then prepares a
+// durable operation in the accept transaction, and every worker execution
+// authorizes through it. Chainable; nil keeps the gateless default (unit
+// tests only — see NewSendWorker).
+func (j *Jobs) WithGate(g sendingpolicy.Gate) *Jobs {
+ if g != nil {
+ j.gate = g
}
return j
}
+// SendWorker builds the fully armed send worker RegisterJobs registers: the
+// gate, the legacy resolver, the rate gate, and metrics. It is the one place
+// those are wired, and the composition root's test inspects its result.
+func (j *Jobs) SendWorker() *SendWorker {
+ return NewSendWorker(j.store, j.deliverer).WithMetrics(j.metrics).WithRateGate(j.rate).WithGate(j.gate).WithOperationResolver(j.ResolveLegacyOperation)
+}
+
+// TerminalReconcileWorker builds the reconciler RegisterJobs registers.
+func (j *Jobs) TerminalReconcileWorker() *TerminalReconcileWorker {
+ return NewTerminalReconcileWorker(j.pool, j.store).WithMetrics(j.metrics).WithGate(j.gate)
+}
+
+// RegisteredSendWorker returns the send worker the last RegisterJobs call
+// registered with River, or nil before any registration.
+func (j *Jobs) RegisteredSendWorker() *SendWorker { return j.registered }
+
+// Gate exposes the wired sending-protection gate, for the composition root's
+// wiring test. nil when none is wired.
+func (j *Jobs) Gate() sendingpolicy.Gate { return j.gate }
+
+// Deliverer exposes the wired provider deliverer, for the same test.
+func (j *Jobs) Deliverer() Deliverer { return j.deliverer }
+
// SetEnqueuer injects the shared client so EnqueueSendTx can insert jobs.
func (j *Jobs) SetEnqueuer(e jobs.Enqueuer) { j.enq = e }
@@ -62,8 +98,12 @@ func (j *Jobs) WithRateGate(g RateGate) *Jobs {
// RegisterJobs adds the SendWorker and terminal-state safety net to the shared
// client's bundle. Implements jobs.Registrar.
func (j *Jobs) RegisterJobs(w *river.Workers) []*river.PeriodicJob {
- river.AddWorker(w, NewSendWorker(j.store, j.deliverer, j.ramp).WithMetrics(j.metrics).WithRateGate(j.rate))
- river.AddWorker(w, NewTerminalReconcileWorker(j.pool, j.store, j.ramp).WithMetrics(j.metrics))
+ // The worker registered here is recorded so the composition root's
+ // wiring test can inspect the exact object River will run, not merely
+ // what a constructor would produce.
+ j.registered = j.SendWorker()
+ river.AddWorker(w, j.registered)
+ river.AddWorker(w, j.TerminalReconcileWorker())
return []*river.PeriodicJob{
river.NewPeriodicJob(
river.PeriodicInterval(terminalReconcileInterval),
@@ -119,7 +159,31 @@ func (j *Jobs) EnqueueScheduledSendTx(ctx context.Context, tx pgx.Tx, messageID
// enqueueSendTx is the shared outbox insert behind the immediate and scheduled
// entry points. A non-zero `at` sets InsertOpts.ScheduledAt; a zero value omits
// it (River defaults ScheduledAt to now, i.e. immediately available).
+//
+// With a gate wired, the durable provider operation is prepared HERE, after
+// the message insert and before the River insert, in the caller's transaction:
+// a paused account is refused at the door (ErrSendingPaused) rather than
+// queueing mail that can never leave, and the job carries the operation
+// reference so the worker never derives purpose or attribution on its own.
func (j *Jobs) enqueueSendTx(ctx context.Context, tx pgx.Tx, messageID string, at time.Time) (int64, error) {
+ args := OutboundSendArgs{MessageID: messageID}
+ if j.gate != nil {
+ decision, ref, err := j.gate.PrepareExternalTx(ctx, tx, messageID)
+ if err != nil {
+ return 0, fmt.Errorf("prepare sending operation: %w", err)
+ }
+ if decision == sendingpolicy.AcceptanceSendingPaused {
+ return 0, ErrSendingPaused
+ }
+ if ref.IsZero() {
+ // The only accepted shape without an operation is an exact
+ // self-send, and those never enqueue. Refusing here keeps a
+ // prepared-but-operationless job from masquerading as a legacy
+ // one that the worker would then kill.
+ return 0, fmt.Errorf("prepare sending operation: message %s has no provider operation", messageID)
+ }
+ args.OperationRef = &ref
+ }
opts := &river.InsertOpts{
Queue: jobs.QueueOutbound,
MaxAttempts: MaxSendAttempts,
@@ -127,9 +191,34 @@ func (j *Jobs) enqueueSendTx(ctx context.Context, tx pgx.Tx, messageID string, a
if !at.IsZero() {
opts.ScheduledAt = at
}
- res, err := j.enq.InsertTx(ctx, tx, OutboundSendArgs{MessageID: messageID}, opts)
+ res, err := j.enq.InsertTx(ctx, tx, args, opts)
if err != nil {
return 0, err
}
return res.Job.ID, nil
}
+
+// ResolveLegacyOperation is the compatibility resolver for a job enqueued by
+// a pre-floor slot with no operation reference. It runs the same
+// PrepareExternalTx an accept transaction runs — idempotent on the durable
+// operation row — in its own committed transaction, so an old job and a new
+// one authorize identically. There is deliberately no other way to obtain an
+// operation from a bare message id.
+func (j *Jobs) ResolveLegacyOperation(ctx context.Context, messageID string) (sendingpolicy.AcceptanceDecision, sendingpolicy.OperationRef, error) {
+ if j.gate == nil || j.pool == nil {
+ return "", sendingpolicy.OperationRef{}, fmt.Errorf("legacy operation resolver is not wired")
+ }
+ tx, err := j.pool.Begin(ctx)
+ if err != nil {
+ return "", sendingpolicy.OperationRef{}, fmt.Errorf("begin legacy resolve: %w", err)
+ }
+ defer func() { _ = tx.Rollback(ctx) }()
+ decision, ref, err := j.gate.PrepareExternalTx(ctx, tx, messageID)
+ if err != nil {
+ return "", sendingpolicy.OperationRef{}, err
+ }
+ if err := tx.Commit(ctx); err != nil {
+ return "", sendingpolicy.OperationRef{}, fmt.Errorf("commit legacy resolve: %w", err)
+ }
+ return decision, ref, nil
+}
diff --git a/internal/outboundsend/jobs_gate_test.go b/internal/outboundsend/jobs_gate_test.go
new file mode 100644
index 000000000..0c73f4add
--- /dev/null
+++ b/internal/outboundsend/jobs_gate_test.go
@@ -0,0 +1,338 @@
+package outboundsend_test
+
+import (
+ "context"
+ "errors"
+ "testing"
+ "time"
+
+ "github.com/jackc/pgx/v5"
+ "github.com/jackc/pgx/v5/pgxpool"
+ "github.com/riverqueue/river"
+ "github.com/riverqueue/river/rivertype"
+
+ "github.com/tokencanopy/e2a/internal/agent"
+ "github.com/tokencanopy/e2a/internal/identity"
+ "github.com/tokencanopy/e2a/internal/jobs"
+ "github.com/tokencanopy/e2a/internal/outboundsend"
+ "github.com/tokencanopy/e2a/internal/sendingpolicy"
+ "github.com/tokencanopy/e2a/internal/testutil"
+ "github.com/tokencanopy/e2a/internal/usage"
+ "github.com/tokencanopy/e2a/internal/webhookpub"
+)
+
+// These tests drive the real gate against real Postgres through the jobs
+// bundle: the accept transaction prepares the operation, a paused account is
+// refused at the door, and a legacy job with no reference authorizes through
+// the same path as a new one.
+
+type gateFixture struct {
+ t *testing.T
+ ctx context.Context
+ pool *pgxpool.Pool
+ store *identity.Store
+ adapter outboundsend.Store
+ gate sendingpolicy.Gate
+ userID string
+ agentID string
+ client jobs.Enqueuer
+ gated *outboundsend.Jobs
+ legacy *outboundsend.Jobs
+}
+
+func newGateFixture(t *testing.T) *gateFixture {
+ t.Helper()
+ ctx := context.Background()
+ pool := testutil.TestDB(t)
+ store := identity.NewStore(pool)
+ if err := jobs.Migrate(ctx, pool); err != nil {
+ t.Fatalf("jobs.Migrate: %v", err)
+ }
+ user, err := store.CreateOrGetUser(ctx, "owner-gate@example.test", "Owner", "google-gate")
+ if err != nil {
+ t.Fatalf("CreateOrGetUser: %v", err)
+ }
+ domain := "gate.example.test"
+ if _, err := store.ClaimOrCreateDomain(ctx, domain, user.ID); err != nil {
+ t.Fatalf("ClaimOrCreateDomain: %v", err)
+ }
+ if err := store.VerifyDomain(ctx, domain, user.ID); err != nil {
+ t.Fatalf("VerifyDomain: %v", err)
+ }
+ ag, err := store.CreateAgent(ctx, "bot@"+domain, domain, "", "", "local", user.ID)
+ if err != nil {
+ t.Fatalf("CreateAgent: %v", err)
+ }
+ adapter := agent.NewOutboundSendStore(store, webhookpub.NewOutbox(pool, webhookpub.StaticFlag(true)), usage.NewNoopUsageTracker())
+ gate := sendingpolicy.NewGate(pool, sendingpolicy.Secrets{}, sendingpolicy.PolicySourceConfig, sendingpolicy.DisabledPolicy())
+ gated := outboundsend.NewJobs(adapter, &fakeDeliverer{}, pool).WithGate(gate)
+ legacy := outboundsend.NewJobs(adapter, &fakeDeliverer{}, pool)
+ client, err := jobs.New(pool, jobs.Config{}, gated)
+ if err != nil {
+ t.Fatalf("jobs.New: %v", err)
+ }
+ gated.SetEnqueuer(client)
+ legacy.SetEnqueuer(client)
+ return &gateFixture{t: t, ctx: ctx, pool: pool, store: store, adapter: adapter, gate: gate, userID: user.ID, agentID: ag.ID, client: client, gated: gated, legacy: legacy}
+}
+
+// accept runs the accept transaction the API runs, through the given bundle.
+func (f *gateFixture) accept(bundle *outboundsend.Jobs, label string) (messageID string, jobID int64, err error) {
+ f.t.Helper()
+ err = f.store.WithTx(f.ctx, func(tx pgx.Tx) error {
+ m, err := f.store.CreateOutboundMessageTx(f.ctx, tx, f.agentID,
+ []string{label + "@example.test"}, nil, nil, label, "send", "smtp", "", "conv-"+label,
+ []byte("From: bot\r\n\r\nbody"), "accepted", "bot@gate.example.test", "relay")
+ if err != nil {
+ return err
+ }
+ messageID = m.ID
+ jobID, err = bundle.EnqueueSendTx(f.ctx, tx, messageID)
+ if err != nil {
+ return err
+ }
+ return f.store.StampSendJobIDTx(f.ctx, tx, messageID, jobID)
+ })
+ return messageID, jobID, err
+}
+
+func (f *gateFixture) operationExists(messageID string) bool {
+ f.t.Helper()
+ var n int
+ if err := f.pool.QueryRow(f.ctx, `SELECT count(*) FROM sending_provider_operations WHERE operation_id = $1 AND purpose = 'customer_message'`, messageID).Scan(&n); err != nil {
+ f.t.Fatal(err)
+ }
+ return n == 1
+}
+
+func TestJobs_EnqueuePreparesTheOperationInTheAcceptTransaction(t *testing.T) {
+ f := newGateFixture(t)
+ messageID, jobID, err := f.accept(f.gated, "prepared")
+ if err != nil {
+ t.Fatalf("accept: %v", err)
+ }
+ var refID string
+ if err := f.pool.QueryRow(f.ctx, `SELECT args->'operation_ref'->>'id' FROM river_job WHERE id = $1`, jobID).Scan(&refID); err != nil {
+ t.Fatal(err)
+ }
+ if refID != messageID {
+ t.Fatalf("job carries operation_ref id %q, want the message id %q", refID, messageID)
+ }
+ if !f.operationExists(messageID) {
+ t.Fatal("no customer_message operation was prepared in the accept transaction")
+ }
+}
+
+func TestJobs_EnqueueRefusesAPausedAccountAndRollsBack(t *testing.T) {
+ f := newGateFixture(t)
+ if _, err := f.pool.Exec(f.ctx, `
+ INSERT INTO account_sending_controls (user_id, state, reason, actor) VALUES ($1, 'paused', 'test', 'test')
+ ON CONFLICT (user_id) DO UPDATE SET state = 'paused'`, f.userID); err != nil {
+ t.Fatal(err)
+ }
+ messageID, _, err := f.accept(f.gated, "paused")
+ if !errors.Is(err, outboundsend.ErrSendingPaused) {
+ t.Fatalf("accept on a paused account err = %v, want ErrSendingPaused", err)
+ }
+ var rows int
+ if err := f.pool.QueryRow(f.ctx, `SELECT count(*) FROM messages WHERE id = $1`, messageID).Scan(&rows); err != nil {
+ t.Fatal(err)
+ }
+ if rows != 0 {
+ t.Fatalf("message row survived the refused accept; the transaction must roll back")
+ }
+}
+
+func TestJobs_LegacyJobResolvesAndAuthorizesThroughTheGate(t *testing.T) {
+ f := newGateFixture(t)
+ // A pre-floor slot enqueued this job: no operation reference in its args.
+ messageID, jobID, err := f.accept(f.legacy, "legacy")
+ if err != nil {
+ t.Fatalf("legacy accept: %v", err)
+ }
+ var hasRef bool
+ if err := f.pool.QueryRow(f.ctx, `SELECT args ? 'operation_ref' FROM river_job WHERE id = $1`, jobID).Scan(&hasRef); err != nil {
+ t.Fatal(err)
+ }
+ if hasRef || f.operationExists(messageID) {
+ t.Fatal("the legacy enqueue must carry no reference and prepare nothing")
+ }
+
+ dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{ProviderMessageID: ""}}
+ w := outboundsend.NewSendWorker(f.adapter, dl).WithGate(f.gate).WithOperationResolver(f.gated.ResolveLegacyOperation)
+ rj := &river.Job[outboundsend.OutboundSendArgs]{
+ JobRow: &rivertype.JobRow{ID: jobID, Attempt: 1, MaxAttempts: outboundsend.MaxSendAttempts, Kind: outboundsend.OutboundSendArgs{}.Kind()},
+ Args: outboundsend.OutboundSendArgs{MessageID: messageID},
+ }
+ if err := w.Work(f.ctx, rj); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+ if dl.calls != 1 {
+ t.Fatalf("provider calls = %d, want exactly one", dl.calls)
+ }
+ if !f.operationExists(messageID) {
+ t.Fatal("the resolver did not prepare the operation")
+ }
+ var state, callState string
+ if err := f.pool.QueryRow(f.ctx, `
+ SELECT state, call_state FROM sending_budget_reservations
+ WHERE operation_id = $1 AND submission_attempt = 1`, messageID).Scan(&state, &callState); err != nil {
+ t.Fatalf("read reservation: %v", err)
+ }
+ if state != "confirmed" {
+ t.Fatalf("attempt state = %s, want confirmed (final authorization ran)", state)
+ }
+ var status string
+ if err := f.pool.QueryRow(f.ctx, `SELECT delivery_status FROM messages WHERE id = $1`, messageID).Scan(&status); err != nil {
+ t.Fatal(err)
+ }
+ if status != "sent" {
+ t.Fatalf("delivery_status = %s, want sent", status)
+ }
+}
+
+func TestJobs_GatedWorkerAuthorizesANewJob(t *testing.T) {
+ f := newGateFixture(t)
+ messageID, jobID, err := f.accept(f.gated, "gated")
+ if err != nil {
+ t.Fatalf("accept: %v", err)
+ }
+ dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{ProviderMessageID: ""}}
+ w := outboundsend.NewSendWorker(f.adapter, dl).WithGate(f.gate)
+ ref := refFor(messageID)
+ rj := &river.Job[outboundsend.OutboundSendArgs]{
+ JobRow: &rivertype.JobRow{ID: jobID, Attempt: 1, MaxAttempts: outboundsend.MaxSendAttempts, Kind: outboundsend.OutboundSendArgs{}.Kind()},
+ Args: outboundsend.OutboundSendArgs{MessageID: messageID, OperationRef: &ref},
+ }
+ if err := w.Work(f.ctx, rj); err != nil {
+ t.Fatalf("Work: %v", err)
+ }
+ if dl.calls != 1 || len(dl.auths) != 1 || dl.auths[0].IsZero() {
+ t.Fatalf("calls=%d auths=%d, want one provider call carrying a real authorization", dl.calls, len(dl.auths))
+ }
+ // A re-drive of the sent row is a no-op: no new ordinal, no new call.
+ if err := w.Work(f.ctx, rj); err != nil {
+ t.Fatalf("re-drive: %v", err)
+ }
+ var attempts int
+ if err := f.pool.QueryRow(f.ctx, `SELECT current_attempt FROM sending_provider_operations WHERE operation_id = $1`, messageID).Scan(&attempts); err != nil {
+ t.Fatal(err)
+ }
+ if dl.calls != 1 || attempts != 1 {
+ t.Fatalf("after re-drive calls=%d current_attempt=%d, want 1/1", dl.calls, attempts)
+ }
+}
+
+// TestJobs_ReconcilerSettlesTheDialedAttemptFromEvidence: the worker dialed
+// (the token was redeemed) but lost the 250; SES's feedback later proved
+// acceptance; the job is terminal. The reconciler settles the row as sent and,
+// through the gate, settles the attempt that dialed — binding the provider id
+// to its correlation — without resubmitting or reserving anything.
+func TestJobs_ReconcilerSettlesTheDialedAttemptFromEvidence(t *testing.T) {
+ f := newGateFixture(t)
+ messageID, jobID, err := f.accept(f.gated, "evidence")
+ if err != nil {
+ t.Fatalf("accept: %v", err)
+ }
+ dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{Err: errors.New("data final: lost"), AcceptanceUnknown: true}}
+ w := outboundsend.NewSendWorker(f.adapter, dl).WithGate(f.gate)
+ ref := refFor(messageID)
+ rj := &river.Job[outboundsend.OutboundSendArgs]{
+ JobRow: &rivertype.JobRow{ID: jobID, Attempt: 1, MaxAttempts: outboundsend.MaxSendAttempts, Kind: outboundsend.OutboundSendArgs{}.Kind()},
+ Args: outboundsend.OutboundSendArgs{MessageID: messageID, OperationRef: &ref},
+ }
+ if err := w.Work(f.ctx, rj); err == nil {
+ t.Fatal("an acceptance-unknown failure must return a retryable error")
+ }
+ // The production submitter redeems before it dials; the fake did not, so
+ // redeem the token it was handed to reproduce "dialed, answer lost".
+ if len(dl.auths) != 1 {
+ t.Fatalf("auths = %d, want the one the worker handed over", len(dl.auths))
+ }
+ if err := f.gate.RedeemProviderCall(f.ctx, dl.auths[0]); err != nil {
+ t.Fatalf("redeem: %v", err)
+ }
+ // SES feedback proved acceptance; River gave up on the job.
+ if _, err := f.pool.Exec(f.ctx, `
+ UPDATE messages SET provider_accepted_at = now(), provider_message_id = ''
+ WHERE id = $1`, messageID); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := f.pool.Exec(f.ctx, `UPDATE river_job SET state = 'discarded', finalized_at = now() - interval '16 minutes' WHERE id = $1`, jobID); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := outboundsend.NewTerminalReconcileWorker(f.pool, f.adapter).WithGate(f.gate).Work(f.ctx, &river.Job[outboundsend.TerminalReconcileArgs]{}); err != nil {
+ t.Fatalf("reconcile: %v", err)
+ }
+ var status string
+ if err := f.pool.QueryRow(f.ctx, `SELECT delivery_status FROM messages WHERE id = $1`, messageID).Scan(&status); err != nil {
+ t.Fatal(err)
+ }
+ if status != "sent" {
+ t.Fatalf("delivery_status = %s, want sent from evidence", status)
+ }
+ var bound *string
+ if err := f.pool.QueryRow(f.ctx, `
+ SELECT provider_message_id FROM sending_feedback_correlations
+ WHERE operation_id = $1 AND submission_attempt = 1`, messageID).Scan(&bound); err != nil {
+ t.Fatalf("read correlation: %v", err)
+ }
+ if bound == nil || *bound != "ses-evidence-000000" {
+ t.Fatalf("correlation provider id = %v, want the bare evidence id bound to the dialed attempt", bound)
+ }
+ if dl.calls != 1 {
+ t.Fatalf("provider calls = %d, want the original one only", dl.calls)
+ }
+}
+
+func TestJobs_RateDeferralReleasesTheRealReservation(t *testing.T) {
+ f := newGateFixture(t)
+ messageID, jobID, err := f.accept(f.gated, "rate")
+ if err != nil {
+ t.Fatalf("accept: %v", err)
+ }
+ gate := &fakeRateGate{decision: outboundsend.RateDecision{Allowed: false, RetryAt: time.Now().Add(30 * time.Second)}, window: time.Minute}
+ dl := &fakeDeliverer{}
+ w := outboundsend.NewSendWorker(f.adapter, dl).WithGate(f.gate).WithRateGate(gate)
+ ref := refFor(messageID)
+ rj := &river.Job[outboundsend.OutboundSendArgs]{
+ JobRow: &rivertype.JobRow{ID: jobID, Attempt: 1, MaxAttempts: outboundsend.MaxSendAttempts, Kind: outboundsend.OutboundSendArgs{}.Kind()},
+ Args: outboundsend.OutboundSendArgs{MessageID: messageID, OperationRef: &ref},
+ }
+ if err := w.Work(f.ctx, rj); !isSnooze(err) || dl.calls != 0 {
+ t.Fatalf("err=%v delivers=%d, want snooze with no I/O", err, dl.calls)
+ }
+ var state string
+ if err := f.pool.QueryRow(f.ctx, `SELECT state FROM sending_budget_reservations WHERE operation_id = $1 AND submission_attempt = 1`, messageID).Scan(&state); err != nil {
+ t.Fatalf("read reservation: %v", err)
+ }
+ if state != "released" {
+ t.Fatalf("reservation state = %s, want released — the deferral must give the budget back", state)
+ }
+}
+
+// zeroRefGate accepts but prepares nothing — the shape only an exact
+// self-send produces, which never enqueues.
+type zeroRefGate struct{ *fakeGate }
+
+func (zeroRefGate) PrepareExternalTx(context.Context, pgx.Tx, string) (sendingpolicy.AcceptanceDecision, sendingpolicy.OperationRef, error) {
+ return sendingpolicy.AcceptanceAccept, sendingpolicy.OperationRef{}, nil
+}
+
+func TestJobs_EnqueueRefusesAnAcceptWithoutAnOperation(t *testing.T) {
+ f := newGateFixture(t)
+ bundle := outboundsend.NewJobs(f.adapter, &fakeDeliverer{}, f.pool).WithGate(zeroRefGate{allowAll()})
+ bundle.SetEnqueuer(f.client)
+ messageID, _, err := f.accept(bundle, "zero-ref")
+ if err == nil {
+ t.Fatal("an accept that prepared no operation was enqueued as a legacy-looking job")
+ }
+ var rows int
+ if err := f.pool.QueryRow(f.ctx, `SELECT count(*) FROM messages WHERE id = $1`, messageID).Scan(&rows); err != nil {
+ t.Fatal(err)
+ }
+ if rows != 0 {
+ t.Fatal("the refused accept left a message row behind")
+ }
+}
diff --git a/internal/outboundsend/rate_test.go b/internal/outboundsend/rate_test.go
index cb71b5294..f05f0c602 100644
--- a/internal/outboundsend/rate_test.go
+++ b/internal/outboundsend/rate_test.go
@@ -222,13 +222,12 @@ func TestSendWorker_RateLimitedPastRetryHorizonFailsTerminally(t *testing.T) {
j.Domain, j.MessageType, j.SentAs = "new.example.com", "send", "own_address"
st := &fakeStore{job: j}
dl := &fakeDeliverer{}
- ramp := &fakeRampGate{decision: outboundsend.RampDecision{Allowed: true}}
gate := &fakeRateGate{decision: outboundsend.RateDecision{
Allowed: false,
RetryAt: time.Now().Add(30 * time.Second),
}}
rec := &recordingMetrics{}
- w := outboundsend.NewSendWorker(st, dl, ramp).WithRateGate(gate).WithMetrics(rec)
+ w := outboundsend.NewSendWorker(st, dl).WithRateGate(gate).WithMetrics(rec)
err := w.Work(context.Background(), job("msg_1", 4))
if err == nil {
@@ -248,9 +247,6 @@ func TestSendWorker_RateLimitedPastRetryHorizonFailsTerminally(t *testing.T) {
t.Errorf("terminal = {detail %q, source %v}, want {send_rate_timeout, local}",
got.detail, got.source)
}
- if len(ramp.released) != 1 || ramp.released[0] != "msg_1" {
- t.Errorf("ramp releases = %v, want [msg_1] (timeout releases the reservation)", ramp.released)
- }
if !stringsEqual(rec.terminals, []string{"failed_local_retries"}) {
t.Errorf("terminals = %v, want [failed_local_retries]", rec.terminals)
}
@@ -266,10 +262,9 @@ func TestSendWorker_RateGateErrorPastRetryHorizonFailsTerminally(t *testing.T) {
j.AcceptedAt = time.Now().Add(-73 * time.Hour)
j.Domain, j.MessageType, j.SentAs = "new.example.com", "send", "own_address"
st := &fakeStore{job: j}
- ramp := &fakeRampGate{decision: outboundsend.RampDecision{Allowed: true}}
gate := &fakeRateGate{err: errors.New("rate store down")}
rec := &recordingMetrics{}
- w := outboundsend.NewSendWorker(st, &fakeDeliverer{}, ramp).WithRateGate(gate).WithMetrics(rec)
+ w := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithRateGate(gate).WithMetrics(rec)
err := w.Work(context.Background(), job("msg_1", 4))
if err == nil {
@@ -286,36 +281,6 @@ func TestSendWorker_RateGateErrorPastRetryHorizonFailsTerminally(t *testing.T) {
t.Errorf("terminal = {detail %q, source %v}, want {send_rate_timeout: rate store down, local}",
got.detail, got.source)
}
- if len(ramp.released) != 1 || ramp.released[0] != "msg_1" {
- t.Errorf("ramp releases = %v, want [msg_1] (timeout releases the reservation)", ramp.released)
- }
-}
-
-// TestSendWorker_RateLimitedDeferralKeepsRampReservation pins the complement
-// of the horizon path: an ordinary deferral releases the SEND CLAIM but keeps
-// the ramp reservation — same-message Reserve is idempotent, while a released
-// reservation is terminal and cannot be re-reserved.
-func TestSendWorker_RateLimitedDeferralKeepsRampReservation(t *testing.T) {
- j := acceptedJob("msg_1")
- j.Domain, j.MessageType, j.SentAs = "new.example.com", "send", "own_address"
- st := &fakeStore{job: j}
- ramp := &fakeRampGate{decision: outboundsend.RampDecision{Allowed: true}}
- gate := &fakeRateGate{decision: outboundsend.RateDecision{
- Allowed: false,
- RetryAt: time.Now().Add(30 * time.Second),
- }}
- w := outboundsend.NewSendWorker(st, &fakeDeliverer{}, ramp).WithRateGate(gate)
-
- requireSnooze(t, w.Work(context.Background(), job("msg_1", 1)))
- if len(ramp.calls) != 1 {
- t.Errorf("ramp reserves = %d, want 1 (taken before the rate gate)", len(ramp.calls))
- }
- if len(ramp.released) != 0 {
- t.Errorf("ramp releases = %v, want none — a deferral keeps the reservation", ramp.released)
- }
- if len(st.released) != 1 || st.released[0] != "msg_1" {
- t.Errorf("send-claim releases = %v, want [msg_1]", st.released)
- }
}
// TestSendWorker_RateGateAllowsSubmission: an allowed reservation falls
diff --git a/internal/outboundsend/reconcile_test.go b/internal/outboundsend/reconcile_test.go
index c81325f89..c048843bc 100644
--- a/internal/outboundsend/reconcile_test.go
+++ b/internal/outboundsend/reconcile_test.go
@@ -20,6 +20,7 @@ import (
"github.com/tokencanopy/e2a/internal/jobs"
"github.com/tokencanopy/e2a/internal/messagelifecycle"
"github.com/tokencanopy/e2a/internal/outboundsend"
+ "github.com/tokencanopy/e2a/internal/sendingpolicy"
"github.com/tokencanopy/e2a/internal/testutil"
"github.com/tokencanopy/e2a/internal/usage"
"github.com/tokencanopy/e2a/internal/webhookpub"
@@ -481,9 +482,8 @@ func TestTerminalReconcileWorker_ReconcilesOnlyTerminalJobs(t *testing.T) {
sentID := f.seed(t, "sent", "sent", "completed", false)
missingID := f.seed(t, "missing", "accepted", "", true)
- gate := &fakeRampGate{}
rec := &recordingMetrics{}
- worker := outboundsend.NewTerminalReconcileWorker(pool, adapter, gate).WithMetrics(rec)
+ worker := outboundsend.NewTerminalReconcileWorker(pool, adapter).WithMetrics(rec)
if err := worker.Work(context.Background(), &river.Job[outboundsend.TerminalReconcileArgs]{}); err != nil {
t.Fatalf("Work: %v", err)
}
@@ -541,9 +541,6 @@ func TestTerminalReconcileWorker_ReconcilesOnlyTerminalJobs(t *testing.T) {
}
f.assertEventCarriesOnly(t, tc.id, webhookpub.EventEmailFailed, tr)
}
- if len(gate.resolved) != 4 {
- t.Errorf("ramp resolutions = %v, want four terminal outcomes", gate.resolved)
- }
// One terminal metric per settled row; all four sweeps here wrote a
// locally inferred failure (no provider provenance, no suppression list).
// One terminal per settled row, labeled by provenance: the cancelled-state
@@ -572,82 +569,6 @@ func TestTerminalReconcileWorker_ReconcilesOnlyTerminalJobs(t *testing.T) {
}
}
-func TestTerminalReconcileWorker_ResolvesReservedRampForTerminalMessage(t *testing.T) {
- pool := testutil.TestDB(t)
- store := identity.NewStore(pool)
- adapter := agent.NewOutboundSendStore(store,
- webhookpub.NewOutbox(pool, webhookpub.StaticFlag(true)), usage.NewNoopUsageTracker())
- f := newTerminalFixture(t, pool, store, adapter)
- messageID := f.seed(t, "terminal-ramp-cleanup", "accepted", "cancelled", false)
-
- ctx := context.Background()
- var userID string
- if err := pool.QueryRow(ctx, `SELECT user_id FROM agent_identities WHERE id=$1`, f.agentID).Scan(&userID); err != nil {
- t.Fatalf("read agent owner: %v", err)
- }
- if _, err := pool.Exec(ctx,
- `UPDATE messages SET delivery_status='failed' WHERE id=$1`, messageID); err != nil {
- t.Fatalf("make message terminal: %v", err)
- }
- if _, err := pool.Exec(ctx,
- `INSERT INTO domain_send_counters (user_id, domain, day, reserved_count, confirmed_count, daily_limit)
- VALUES ($1, 'example.com', current_date, 1, 0, 50)`, userID); err != nil {
- t.Fatalf("seed ramp counter: %v", err)
- }
- if _, err := pool.Exec(ctx,
- `INSERT INTO sending_ramp_reservations (message_id, day, user_id, domain, units)
- VALUES ($1, current_date, $2, 'example.com', 1)`, messageID, userID); err != nil {
- t.Fatalf("seed reserved ramp: %v", err)
- }
-
- gate := &fakeRampGate{}
- worker := outboundsend.NewTerminalReconcileWorker(pool, adapter, gate)
- if err := worker.Work(ctx, &river.Job[outboundsend.TerminalReconcileArgs]{}); err != nil {
- t.Fatalf("Work: %v", err)
- }
- if len(gate.resolved) != 1 || gate.resolved[0] != messageID {
- t.Fatalf("ramp resolutions = %v, want [%s]", gate.resolved, messageID)
- }
-}
-
-func TestTerminalReconcileWorker_ResolvesReleasedRampAfterProviderCorrection(t *testing.T) {
- pool := testutil.TestDB(t)
- store := identity.NewStore(pool)
- adapter := agent.NewOutboundSendStore(store,
- webhookpub.NewOutbox(pool, webhookpub.StaticFlag(true)), usage.NewNoopUsageTracker())
- f := newTerminalFixture(t, pool, store, adapter)
- messageID := f.seed(t, "released-ramp-provider-correction", "accepted", "cancelled", false)
-
- ctx := context.Background()
- var userID string
- if err := pool.QueryRow(ctx, `SELECT user_id FROM agent_identities WHERE id=$1`, f.agentID).Scan(&userID); err != nil {
- t.Fatalf("read agent owner: %v", err)
- }
- if _, err := pool.Exec(ctx,
- `UPDATE messages SET delivery_status='delivered' WHERE id=$1`, messageID); err != nil {
- t.Fatalf("apply provider correction: %v", err)
- }
- if _, err := pool.Exec(ctx,
- `INSERT INTO domain_send_counters (user_id, domain, day, reserved_count, confirmed_count, daily_limit)
- VALUES ($1, 'example.com', current_date, 0, 0, 50)`, userID); err != nil {
- t.Fatalf("seed ramp counter: %v", err)
- }
- if _, err := pool.Exec(ctx,
- `INSERT INTO sending_ramp_reservations (message_id, day, user_id, domain, units, state)
- VALUES ($1, current_date, $2, 'example.com', 1, 'released')`, messageID, userID); err != nil {
- t.Fatalf("seed released ramp: %v", err)
- }
-
- gate := &fakeRampGate{}
- worker := outboundsend.NewTerminalReconcileWorker(pool, adapter, gate)
- if err := worker.Work(ctx, &river.Job[outboundsend.TerminalReconcileArgs]{}); err != nil {
- t.Fatalf("Work: %v", err)
- }
- if len(gate.resolved) != 1 || gate.resolved[0] != messageID {
- t.Fatalf("ramp resolutions = %v, want [%s]", gate.resolved, messageID)
- }
-}
-
// TestTerminalReconcileWorker_GraceWindowHoldsFreshTerminalJobs pins the §3.1
// grace behavior: a row whose job just reached a terminal state is NOT failed
// while provider evidence may still be arriving; it is failed once the job has
@@ -662,8 +583,7 @@ func TestTerminalReconcileWorker_GraceWindowHoldsFreshTerminalJobs(t *testing.T)
freshID := f.seed(t, "fresh-discard", "accepted", "discarded", false)
f.freshenJob(t, freshID) // terminal seconds ago — inside the grace window
- gate := &fakeRampGate{}
- worker := outboundsend.NewTerminalReconcileWorker(pool, adapter, gate)
+ worker := outboundsend.NewTerminalReconcileWorker(pool, adapter)
if err := worker.Work(context.Background(), &river.Job[outboundsend.TerminalReconcileArgs]{}); err != nil {
t.Fatalf("Work: %v", err)
}
@@ -721,8 +641,7 @@ func TestTerminalReconcileWorker_ProviderEvidenceSettlesAsSent(t *testing.T) {
t.Fatal(err)
}
- gate := &fakeRampGate{}
- worker := outboundsend.NewTerminalReconcileWorker(pool, adapter, gate)
+ worker := outboundsend.NewTerminalReconcileWorker(pool, adapter)
if err := worker.Work(context.Background(), &river.Job[outboundsend.TerminalReconcileArgs]{}); err != nil {
t.Fatalf("Work: %v", err)
}
@@ -784,9 +703,6 @@ func TestTerminalReconcileWorker_ProviderEvidenceSettlesAsSent(t *testing.T) {
if got := f.failedEventCount(t, evidenceID); got != 0 {
t.Errorf("email.failed count = %d, want 0 — evidence must suppress the false failure", got)
}
- if len(gate.resolved) != 1 || gate.resolved[0] != evidenceID {
- t.Errorf("ramp resolutions = %v, want evidence message", gate.resolved)
- }
// Idempotent: a second pass no-ops (the row left accepted/sending).
if err := worker.Work(context.Background(), &river.Job[outboundsend.TerminalReconcileArgs]{}); err != nil {
@@ -938,14 +854,19 @@ func testLocalFallbackReason(t *testing.T, label string, want messagelifecycle.R
if _, err := pool.Exec(context.Background(), `UPDATE messages SET sent_as='own_address' WHERE id=$1`, messageID); err != nil {
t.Fatal(err)
}
- worker = outboundsend.NewSendWorker(adapter, &fakeDeliverer{}, &fakeRampGate{err: permanentRampError{msg: "invalid ramp"}})
+ // A terminal gate hold (the account is gone) is the local cancellation
+ // this reason describes.
+ worker = outboundsend.NewSendWorker(adapter, &fakeDeliverer{}).WithGate(&fakeGate{
+ reserve: sendingpolicy.Decision{Allow: false, Reason: sendingpolicy.ReasonAccountDeleted, Terminal: true},
+ })
} else {
if _, err := pool.Exec(context.Background(), `UPDATE messages SET created_at=now()-interval '73 hours' WHERE id=$1`, messageID); err != nil {
t.Fatal(err)
}
worker = outboundsend.NewSendWorker(adapter, &fakeDeliverer{out: outboundsend.DeliverOutcome{Err: errors.New("provider unavailable"), Outage: true}})
}
- rj := &river.Job[outboundsend.OutboundSendArgs]{JobRow: &rivertype.JobRow{ID: jobID, Attempt: 3, CreatedAt: time.Now().UTC()}, Args: outboundsend.OutboundSendArgs{MessageID: messageID}}
+ ref := refFor(messageID)
+ rj := &river.Job[outboundsend.OutboundSendArgs]{JobRow: &rivertype.JobRow{ID: jobID, Attempt: 3, CreatedAt: time.Now().UTC()}, Args: outboundsend.OutboundSendArgs{MessageID: messageID, OperationRef: &ref}}
if err := worker.Work(context.Background(), rj); err == nil {
t.Fatal("terminal branch must return cancellation/error")
}
@@ -988,8 +909,7 @@ func testProviderRejectionAtomicFailure(t *testing.T, label, install, uninstall
}
t.Cleanup(func() { _, _ = pool.Exec(context.Background(), uninstall) })
deliverer := &fakeDeliverer{out: outboundsend.DeliverOutcome{Err: errors.New("550 explicit rejection"), Permanent: true}}
- ramp := &fakeRampGate{decision: outboundsend.RampDecision{Allowed: true}}
- w := outboundsend.NewSendWorker(adapter, deliverer, ramp)
+ w := outboundsend.NewSendWorker(adapter, deliverer)
rj := &river.Job[outboundsend.OutboundSendArgs]{JobRow: &rivertype.JobRow{ID: jobID, Attempt: 2, CreatedAt: time.Now().UTC()}, Args: outboundsend.OutboundSendArgs{MessageID: messageID}}
if err := w.Work(context.Background(), rj); err == nil {
t.Fatal("provider rejection must cancel")
@@ -1018,9 +938,6 @@ func testProviderRejectionAtomicFailure(t *testing.T, label, install, uninstall
if deliverer.calls != 1 {
t.Fatalf("fallback re-drive provider calls=%d, want exactly the original call", deliverer.calls)
}
- if len(ramp.calls) != 1 || len(ramp.released) != 1 || len(ramp.resolved) != 1 {
- t.Fatalf("fallback ramp reserve=%d release=%v resolve=%v, want one of each without re-reserve", len(ramp.calls), ramp.released, ramp.resolved)
- }
if _, err := pool.Exec(context.Background(), uninstall); err != nil {
t.Fatal(err)
}
@@ -1132,14 +1049,17 @@ func (s failingTerminalStore) ClaimSend(context.Context, string, int64) (*outbou
return nil, nil
}
func (s failingTerminalStore) ReleaseSend(context.Context, string, int64) error { return nil }
+func (s failingTerminalStore) RecordHold(context.Context, string, outboundsend.HoldClass, time.Time) error {
+ return nil
+}
func (s failingTerminalStore) MarkSent(context.Context, string, int64, int, time.Time, string, string) error {
return nil
}
-func (s failingTerminalStore) MarkFailed(_ context.Context, _ string, _ int64, _ int, occurredAt time.Time, _ string, _ delivery.FailureSource, _ messagelifecycle.ReasonCode, _ []string) (delivery.Status, time.Time, error) {
+func (s failingTerminalStore) MarkFailed(_ context.Context, _ string, _ int64, _ int, occurredAt time.Time, _ string, _ delivery.FailureSource, _ messagelifecycle.ReasonCode, _ []string) (delivery.Status, time.Time, string, error) {
if s.err != nil {
- return "", time.Time{}, s.err
+ return "", time.Time{}, "", s.err
}
- return delivery.StatusFailed, occurredAt, nil
+ return delivery.StatusFailed, occurredAt, "", nil
}
func (s failingTerminalStore) PreserveTerminalFailure(context.Context, string, int64, int, time.Time, string, delivery.FailureSource, messagelifecycle.ReasonCode, []string) error {
return nil
diff --git a/internal/outboundsend/suppression_test.go b/internal/outboundsend/suppression_test.go
index cb535827e..80bd3803e 100644
--- a/internal/outboundsend/suppression_test.go
+++ b/internal/outboundsend/suppression_test.go
@@ -17,12 +17,13 @@ import (
"testing"
"github.com/tokencanopy/e2a/internal/outboundsend"
+ "github.com/tokencanopy/e2a/internal/sendingpolicy"
)
// trippingDeliverer fails the test if any provider I/O is attempted.
type trippingDeliverer struct{ t *testing.T }
-func (d trippingDeliverer) Deliver(_ context.Context, j *outboundsend.SendJob) outboundsend.DeliverOutcome {
+func (d trippingDeliverer) Deliver(_ context.Context, j *outboundsend.SendJob, _ sendingpolicy.ProviderAuthorization) outboundsend.DeliverOutcome {
d.t.Errorf("provider Deliver called for %s despite suppression guard", j.MessageID)
return outboundsend.DeliverOutcome{}
}
@@ -31,8 +32,7 @@ func TestSendWorker_SuppressedRecipientFailsTerminallyWithoutProviderIO(t *testi
j := acceptedJob("msg_1")
j.Domain, j.MessageType, j.SentAs = "new.example.com", "send", "own_address"
st := &fakeStore{job: j, suppressed: []string{"b@y.com"}}
- gate := &fakeRampGate{decision: outboundsend.RampDecision{Allowed: true}}
- w := outboundsend.NewSendWorker(st, trippingDeliverer{t}, gate)
+ w := outboundsend.NewSendWorker(st, trippingDeliverer{t})
err := w.Work(context.Background(), job("msg_1", 1))
if err == nil {
@@ -57,9 +57,6 @@ func TestSendWorker_SuppressedRecipientFailsTerminallyWithoutProviderIO(t *testi
if st.suppressionAgentID != st.job.AgentID {
t.Errorf("suppression check agent = %q, want %q", st.suppressionAgentID, st.job.AgentID)
}
- if len(gate.released) != 1 || gate.released[0] != "msg_1" {
- t.Errorf("ramp releases = %v, want [msg_1]", gate.released)
- }
}
// A store error on the guard is conservative: no provider I/O, no terminal
@@ -83,45 +80,6 @@ func TestSendWorker_SuppressionCheckErrorFailsClosed(t *testing.T) {
}
}
-func TestSendWorker_SuppressionCheckErrorAfterRampPreservesReservation(t *testing.T) {
- j := acceptedJob("msg_1")
- j.Domain, j.MessageType, j.SentAs = "new.example.com", "send", "own_address"
- st := &fakeStore{job: j, suppressedErr: errors.New("suppression store down")}
- gate := &fakeRampGate{decision: outboundsend.RampDecision{Allowed: true}}
- w := outboundsend.NewSendWorker(st, trippingDeliverer{t}, gate)
-
- if err := w.Work(context.Background(), job("msg_1", 1)); err == nil {
- t.Fatal("suppression-store error must retry")
- }
- if len(gate.released) != 0 {
- t.Fatalf("ramp releases = %v, want none so same-day retry stays idempotent", gate.released)
- }
- if len(st.released) != 1 || st.released[0] != "msg_1" {
- t.Fatalf("claim releases = %v, want [msg_1]", st.released)
- }
-}
-
-func TestSendWorker_SuppressionCheckErrorKeepsRampReservationWhenClaimReleaseFails(t *testing.T) {
- lookupErr := errors.New("suppression store down")
- claimErr := errors.New("claim release down")
- j := acceptedJob("msg_1")
- j.Domain, j.MessageType, j.SentAs = "new.example.com", "send", "own_address"
- st := &fakeStore{job: j, suppressedErr: lookupErr, releaseErr: claimErr}
- gate := &fakeRampGate{decision: outboundsend.RampDecision{Allowed: true}}
- w := outboundsend.NewSendWorker(st, trippingDeliverer{t}, gate)
-
- err := w.Work(context.Background(), job("msg_1", 1))
- if !errors.Is(err, lookupErr) || !errors.Is(err, claimErr) {
- t.Fatalf("error = %v, want joined lookup and claim-release causes", err)
- }
- if len(st.released) != 1 {
- t.Fatalf("claim release calls = %v, want one attempt", st.released)
- }
- if len(gate.released) != 0 {
- t.Fatalf("ramp releases = %v, want none while claim remains held", gate.released)
- }
-}
-
func TestSendWorker_UnsuppressedRecipientStillSends(t *testing.T) {
st := &fakeStore{job: acceptedJob("msg_1")} // no suppressions
dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{ProviderMessageID: "ses-ok", SentAs: "relay"}}
diff --git a/internal/outboundsend/terminal_reconcile.go b/internal/outboundsend/terminal_reconcile.go
index 584361204..e212c7477 100644
--- a/internal/outboundsend/terminal_reconcile.go
+++ b/internal/outboundsend/terminal_reconcile.go
@@ -2,6 +2,7 @@ package outboundsend
import (
"context"
+ "errors"
"fmt"
"log"
"time"
@@ -12,6 +13,7 @@ import (
"github.com/tokencanopy/e2a/internal/delivery"
"github.com/tokencanopy/e2a/internal/jobs"
"github.com/tokencanopy/e2a/internal/messagelifecycle"
+ "github.com/tokencanopy/e2a/internal/sendingpolicy"
)
const terminalReconcileInterval = time.Minute
@@ -49,15 +51,21 @@ type TerminalReconcileWorker struct {
river.WorkerDefaults[TerminalReconcileArgs]
pool *pgxpool.Pool
store Store
- ramp RampGate
+ gate sendingpolicy.Gate
metrics Metrics
}
// NewTerminalReconcileWorker builds the periodic safety-net worker.
-func NewTerminalReconcileWorker(pool *pgxpool.Pool, store Store, ramps ...RampGate) *TerminalReconcileWorker {
- w := &TerminalReconcileWorker{pool: pool, store: store, metrics: noopMetrics{}}
- if len(ramps) > 0 {
- w.ramp = ramps[0]
+func NewTerminalReconcileWorker(pool *pgxpool.Pool, store Store) *TerminalReconcileWorker {
+ return &TerminalReconcileWorker{pool: pool, store: store, metrics: noopMetrics{}}
+}
+
+// WithGate injects the sending-protection gate so an evidence-settled row can
+// also settle its provider attempt (ramp progress, provider-id binding).
+// Reconciliation is settlement-only: it never resubmits and never reserves.
+func (w *TerminalReconcileWorker) WithGate(g sendingpolicy.Gate) *TerminalReconcileWorker {
+ if g != nil {
+ w.gate = g
}
return w
}
@@ -86,6 +94,7 @@ type terminalCandidate struct {
failureOccurredAt *time.Time
failureAttempt *int
failureBlockedRecipients []string
+ providerMessageID string
}
// submissionAnchor is this candidate's acceptance→terminal SLI baseline — the
@@ -116,7 +125,8 @@ func (w *TerminalReconcileWorker) Work(ctx context.Context, _ *river.Job[Termina
r.finalized_at,
m.created_at, m.scheduled_at, m.reviewed_at,
COALESCE(m.delivery_failure_source,''),COALESCE(m.delivery_detail,''),COALESCE(m.delivery_failure_reason_code,''),
- m.delivery_failure_occurred_at,m.delivery_failure_attempt,m.delivery_failure_blocked_recipients
+ m.delivery_failure_occurred_at,m.delivery_failure_attempt,m.delivery_failure_blocked_recipients,
+ COALESCE(m.provider_message_id,'')
FROM messages m
LEFT JOIN river_job r ON r.id = m.send_job_id
WHERE m.direction = 'outbound'
@@ -138,7 +148,7 @@ func (w *TerminalReconcileWorker) Work(ctx context.Context, _ *river.Job[Termina
candidates := make([]terminalCandidate, 0)
for rows.Next() {
var candidate terminalCandidate
- if err := rows.Scan(&candidate.messageID, &candidate.jobID, &candidate.attempt, &candidate.state, &candidate.finalizedAt, &candidate.acceptedAt, &candidate.scheduledAt, &candidate.reviewedAt, &candidate.failureSource, &candidate.detail, &candidate.failureReason, &candidate.failureOccurredAt, &candidate.failureAttempt, &candidate.failureBlockedRecipients); err != nil {
+ if err := rows.Scan(&candidate.messageID, &candidate.jobID, &candidate.attempt, &candidate.state, &candidate.finalizedAt, &candidate.acceptedAt, &candidate.scheduledAt, &candidate.reviewedAt, &candidate.failureSource, &candidate.detail, &candidate.failureReason, &candidate.failureOccurredAt, &candidate.failureAttempt, &candidate.failureBlockedRecipients, &candidate.providerMessageID); err != nil {
return err
}
candidates = append(candidates, candidate)
@@ -184,7 +194,7 @@ func (w *TerminalReconcileWorker) Work(ctx context.Context, _ *river.Job[Termina
// fails it with provenance 'local' so later authoritative evidence can
// still correct it. The stored detail of a deferred final attempt is
// preferred over this generic sweep detail.
- settled, settledAt, err := w.store.MarkFailed(ctx, candidate.messageID, candidate.jobID, attempt, occurredAt, detail, source, reason, candidate.failureBlockedRecipients)
+ settled, settledAt, providerID, err := w.store.MarkFailed(ctx, candidate.messageID, candidate.jobID, attempt, occurredAt, detail, source, reason, candidate.failureBlockedRecipients)
if err != nil {
if processed > 0 {
log.Printf("[outbound-terminal-reconcile] processed %d candidates", processed)
@@ -204,69 +214,41 @@ func (w *TerminalReconcileWorker) Work(ctx context.Context, _ *river.Job[Termina
emitTerminal(w.metrics, terminalOutcome(source, reason, candidate.failureBlockedRecipients), candidate.submissionAnchor(), settledAt)
case delivery.StatusSent:
emitTerminal(w.metrics, terminalSent, candidate.submissionAnchor(), settledAt)
- }
- if w.ramp != nil {
- if err := w.ramp.Resolve(ctx, candidate.messageID); err != nil {
- return fmt.Errorf("resolve sending ramp for %s: %w", candidate.messageID, err)
+ // Provider evidence settled the row; settle the attempt that
+ // dialed, so ramp progress and the provider-id binding catch up.
+ // Best effort and idempotent — an attempt that predates the gate
+ // has nothing to settle.
+ if providerID == "" {
+ providerID = candidate.providerMessageID
}
+ w.settleFromEvidence(ctx, candidate.messageID, providerID)
}
processed++
}
if processed > 0 {
log.Printf("[outbound-terminal-reconcile] processed %d candidates", processed)
}
- return w.resolveTerminalRampReservations(ctx)
+ return nil
}
-// resolveTerminalRampReservations is the durable safety net for the narrow
-// window where a worker commits a terminal message outcome, then cannot settle
-// its sending-ramp reservation. That worker returns an error and normally fixes
-// the reservation on its next (unclaimable-message) retry, but its last River
-// attempt can be discarded before another retry. The sweep also revisits a
-// released reservation when authoritative provider feedback later corrects a
-// locally inferred failure. The reservation table's state/updated_at index
-// makes this bounded sweep cheap; Resolve derives confirm versus release from
-// the message's durable delivery status.
-func (w *TerminalReconcileWorker) resolveTerminalRampReservations(ctx context.Context) error {
- if w.ramp == nil {
- return nil
+func (w *TerminalReconcileWorker) settleFromEvidence(ctx context.Context, messageID, providerMessageID string) {
+ if w.gate == nil {
+ return
}
- rows, err := w.pool.Query(ctx,
- `SELECT r.message_id
- FROM sending_ramp_reservations r
- JOIN messages m ON m.id = r.message_id
- WHERE (r.state = 'reserved'
- AND m.delivery_status IN ('sent', 'failed', 'deferred', 'delivered', 'bounced', 'complained'))
- OR (r.state = 'released'
- AND m.delivery_status IN ('sent', 'deferred', 'delivered', 'bounced', 'complained'))
- ORDER BY r.updated_at ASC, r.message_id ASC
- LIMIT $1`,
- jobs.DefaultReconcileBatch,
- )
+ ref, err := w.gate.LookupOperation(ctx, messageID)
if err != nil {
- return err
- }
- messageIDs := make([]string, 0)
- for rows.Next() {
- var messageID string
- if err := rows.Scan(&messageID); err != nil {
- rows.Close()
- return err
+ if !errors.Is(err, sendingpolicy.ErrSourceUnavailable) {
+ log.Printf("[outbound-terminal-reconcile] lookup operation for %s: %v", messageID, err)
}
- messageIDs = append(messageIDs, messageID)
- }
- if err := rows.Err(); err != nil {
- rows.Close()
- return err
+ return
}
- rows.Close()
-
- for _, messageID := range messageIDs {
- if err := w.ramp.Resolve(ctx, messageID); err != nil {
- return fmt.Errorf("resolve terminal sending ramp for %s: %w", messageID, err)
+ if err := w.gate.SettleOperation(ctx, ref, sendingpolicy.SettlementProviderAccepted, providerMessageID); err != nil && !errors.Is(err, sendingpolicy.ErrAttemptStale) {
+ if errors.Is(err, sendingpolicy.ErrProviderMessageIDConflict) {
+ log.Printf("[outbound-terminal-reconcile] CRITICAL: provider id conflict settling %s from evidence: %v", messageID, err)
+ return
}
+ log.Printf("[outbound-terminal-reconcile] settle %s from provider evidence: %v", messageID, err)
}
- return nil
}
func terminalReconcilePeriodicConstructor() (river.JobArgs, *river.InsertOpts) {
diff --git a/internal/outboundsend/worker.go b/internal/outboundsend/worker.go
index a1bd86181..c43a612ee 100644
--- a/internal/outboundsend/worker.go
+++ b/internal/outboundsend/worker.go
@@ -15,6 +15,15 @@
// ambiguously defers its terminal write to the reconciler's provider-evidence
// grace window rather than firing an immediate — possibly false — email.failed.
//
+// Every provider call passes through the sending-protection Gate
+// (internal/sendingpolicy). The worker order is fixed: Reserve the durable
+// attempt; on a hold, snooze without provider I/O; on a rate deferral,
+// DeferAttempt; on a final suppression match, CancelAttempt; ConsumeAttempt is
+// the last serialized decision; the authorized submitter redeems the token
+// immediately before the socket opens and settles the provider's answer. A
+// later execution after a confirmed attempt returns to Reserve, which
+// allocates the next ordinal — the worker never chooses one.
+//
// One SMTP attempt per job attempt — River owns the multi-attempt envelope via
// NextRetry, so Work() stays short (the deliverer does a single submit, not an
// internal retry loop). See the design's "claim + rescue, not a lease" note.
@@ -35,6 +44,7 @@ import (
"github.com/tokencanopy/e2a/internal/delivery"
"github.com/tokencanopy/e2a/internal/messagelifecycle"
"github.com/tokencanopy/e2a/internal/outbound"
+ "github.com/tokencanopy/e2a/internal/sendingpolicy"
"github.com/tokencanopy/e2a/internal/sendrate"
)
@@ -57,34 +67,92 @@ const MaxSendAttempts = 6
// MaxSendAttempts (design §8 circuit breaker).
const outageSnoozeInterval = 5 * time.Minute
-// rampErrorSnoozeInterval keeps a durable message queued when the ramp store is
-// temporarily unavailable. JobSnooze does not consume a River attempt.
-const rampErrorSnoozeInterval = time.Minute
+// gateErrorSnoozeInterval keeps a durable message queued when the sending
+// protection gate is temporarily unavailable. JobSnooze does not consume a
+// River attempt: fail toward retry, never toward an unauthorized submit.
+const gateErrorSnoozeInterval = time.Minute
// rateErrorSnoozeInterval keeps a durable message queued when the fire-time
-// rate store is temporarily unavailable — mirroring rampErrorSnoozeInterval:
-// fail toward retry, never toward an unthrottled submit.
+// rate store is temporarily unavailable — fail toward retry, never toward an
+// unthrottled submit.
const rateErrorSnoozeInterval = time.Minute
// rateMinSnooze floors a rate deferral so a RetryAt at (or just past) now —
// the window-boundary race — cannot hot-loop the queue.
const rateMinSnooze = 250 * time.Millisecond
-// SendRetryHorizon bounds the outage-tolerant tail: past this age (from accept) an
-// outage-snoozing job stops deferring and is declared terminally failed. 72h matches
-// the industry MTA retry horizon (and the webhook deliverer's envelope) — long enough
-// to ride out a multi-hour regional SES incident, not forever.
+// indefiniteHoldSnooze paces a hold that has no clock of its own — an account
+// pause waits for an operator, not for midnight.
+const indefiniteHoldSnooze = time.Hour
+
+// SendRetryHorizon bounds the outage-tolerant tail: past this age a message in a
+// rate/ramp/provider or tenant-setup hold is declared terminally failed. 72h
+// matches the industry MTA retry horizon (and the webhook deliverer's envelope)
+// — long enough to ride out a multi-hour regional SES incident, not forever.
const SendRetryHorizon = 72 * time.Hour
-// OutboundSendArgs drives one outbound send. Args carry only the message id; the
-// worker re-reads the messages row (the source of truth) each attempt.
+// PolicyBudgetHoldHorizon bounds a sending-budget hold: a message may wait
+// through several UTC days for capacity, but not forever. Seven days is the
+// policy's budget_hold_max_days default; the worker holds it as a constant
+// because the deadline is derived, never stored, and every execution must
+// derive the same one.
+const PolicyBudgetHoldHorizon = 7 * 24 * time.Hour
+
+// HoldClass is the durable finite-hold classification persisted on the message
+// the first time it waits for something with a clock.
+type HoldClass string
+
+const (
+ // HoldRateRampOrProvider: per-agent rate, custom-domain ramp, or provider
+ // outage. 72-hour deadline; expiry reason submission.local_retries_exhausted.
+ HoldRateRampOrProvider HoldClass = "rate_ramp_or_provider"
+ // HoldTenantSetup: the account's SES tenant is not ready. 72-hour deadline;
+ // expiry reason submission.sending_setup_expired. Transitions exactly once
+ // to HoldRateRampOrProvider when readiness lands before the setup deadline.
+ HoldTenantSetup HoldClass = "tenant_setup"
+ // HoldPolicyBudget: a sending-budget pool is exhausted. Seven-day deadline
+ // from the existing anchor; every finite class promotes to it and nothing
+ // moves it afterwards. Expiry reason submission.policy_budget_expired.
+ HoldPolicyBudget HoldClass = "policy_budget"
+)
+
+// horizon is the class's absolute deadline measured from its anchor.
+func (c HoldClass) horizon() time.Duration {
+ if c == HoldPolicyBudget {
+ return PolicyBudgetHoldHorizon
+ }
+ return SendRetryHorizon
+}
+
+// expiryReason is the lifecycle reason a class emits when its deadline passes.
+func (c HoldClass) expiryReason() messagelifecycle.ReasonCode {
+ switch c {
+ case HoldPolicyBudget:
+ return messagelifecycle.ReasonSubmissionPolicyBudgetExpired
+ case HoldTenantSetup:
+ return messagelifecycle.ReasonSubmissionSendingSetupExpired
+ }
+ return messagelifecycle.ReasonSubmissionLocalRetriesExhausted
+}
+
+// ErrSendingPaused is returned by the enqueue entry points when the owning
+// account is paused: the acceptance surface must reject the request rather
+// than queue mail that can never leave.
+var ErrSendingPaused = errors.New("outboundsend: account sending is paused")
+
+// OutboundSendArgs drives one outbound send. Args carry the message id and the
+// durable operation reference the accept transaction prepared; the worker
+// re-reads the messages row (the source of truth) each attempt. A job enqueued
+// before the reference existed (a pre-floor slot) carries none and is resolved
+// at fire time through the same Prepare path.
type OutboundSendArgs struct {
- MessageID string `json:"message_id"`
+ MessageID string `json:"message_id"`
+ OperationRef *sendingpolicy.OperationRef `json:"operation_ref,omitempty"`
}
func (OutboundSendArgs) Kind() string { return "outbound_send" }
-// SendJob is the send payload the worker loads from the messages row (Store.LoadForSend).
+// SendJob is the send payload the worker loads from the messages row (Store.ClaimSend).
type SendJob struct {
MessageID string
// UserID is the owning account — the tenant scope for the pre-provider
@@ -98,46 +166,34 @@ type SendJob struct {
Recipients []string
RawMessage []byte // composed MIME
SentAs string // From identity decided at accept ("own_address"|"relay")
- // AcceptedAt is messages.created_at — the outage tail's clock, so a job that has
- // been snoozing through an outage past SendRetryHorizon can be terminated.
+ // AcceptedAt is messages.created_at.
AcceptedAt time.Time
// ScheduledAt is messages.scheduled_at for a scheduled send (zero for an
- // immediate one). The retry horizon is measured from max(AcceptedAt,
- // ScheduledAt): a send scheduled far past accept still gets the full
- // outage-tolerant tail from its fire time, instead of a horizon already blown
- // the moment it first runs.
+ // immediate one).
ScheduledAt time.Time
// ReviewedAt is messages.reviewed_at — when a HITL hold was resolved into the
- // send pipeline (human approve or TTL auto-approve), zero for a message that
- // was never held. Consumed ONLY by submissionAnchor for the latency SLI; the
- // retry horizon deliberately still measures from AcceptedAt, so the F2
- // limitation in docs/design/hitl-ttl-async-send.md is unchanged by this field.
+ // send pipeline, zero for a message that was never held.
ReviewedAt time.Time
// ProviderAccepted is set when authoritatively correlated provider-accept
- // evidence (an SNS-verified, header- or provider-id-matched SES
- // notification) has been recorded for this message: the provider already
- // has it — an earlier attempt's submit landed in the SMTP-accept↔mark-sent
- // crash window — so the worker settles the row as sent instead of
- // re-submitting a duplicate.
+ // evidence has been recorded for this message: the provider already has it,
+ // so the worker settles the row as sent instead of re-submitting a duplicate.
ProviderAccepted bool
ProviderAcceptedAt *time.Time
// ProviderMessageID is the evidence-repaired provider id accompanying
// ProviderAccepted ('' when no evidence).
ProviderMessageID string
-}
-
-// pastRetryHorizon reports whether the accept is older than the outage-tolerant
-// retry horizon. Zero AcceptedAt (unknown) is treated as not-past so an outage keeps
-// deferring rather than being falsely terminated on a missing timestamp.
-func (j *SendJob) pastRetryHorizon() bool {
- // Measure from max(accept, scheduled): a scheduled send's outage tail starts
- // when it fires, not when it was accepted, so a >72h-out schedule isn't
- // terminally failed on its very first attempt.
- start := j.AcceptedAt
- if j.ScheduledAt.After(start) {
- start = j.ScheduledAt
- }
- return !start.IsZero() && time.Since(start) > SendRetryHorizon
+ // LocalHoldClass / LocalHoldAnchor are the durable finite-hold pair a
+ // previous execution persisted (empty/zero when never held). The deadline
+ // is derived from them on every execution and never stored.
+ LocalHoldClass HoldClass
+ LocalHoldAnchor time.Time
+ // LastResumedAt is the owning account's last pause→active transition; a
+ // first finite hold anchors no earlier than it, so a pause that preceded
+ // the hold does not consume its horizon. Zero when unknown.
+ LastResumedAt time.Time
+ // TenantReadyAt is when the account's SES tenant became ready (zero until
+ // it is). Drives the one-way tenant_setup → rate_ramp_or_provider move.
+ TenantReadyAt time.Time
}
// submissionAnchor is this job's acceptance→terminal SLI baseline — see the
@@ -160,7 +216,20 @@ func (j *SendJob) alreadyDone() bool {
return s != delivery.StatusAccepted && s != delivery.StatusSending
}
-// DeliverOutcome is the result of one SMTP submit attempt.
+// initialHoldAnchor is where a message's first finite hold starts its clock:
+// the latest of accept, schedule, review, and the account's last resume, so
+// time spent in review or under an earlier pause is not charged to the hold.
+func (j *SendJob) initialHoldAnchor() time.Time {
+ anchor := j.AcceptedAt
+ for _, t := range []time.Time{j.ScheduledAt, j.ReviewedAt, j.LastResumedAt} {
+ if t.After(anchor) {
+ anchor = t
+ }
+ }
+ return anchor
+}
+
+// DeliverOutcome is the result of one authorized provider submission.
type DeliverOutcome struct {
ProviderMessageID string
SentAs string
@@ -172,33 +241,23 @@ type DeliverOutcome struct {
// the worker snoozes without burning an attempt (design §8), up to the retry
// horizon. Mutually exclusive with Permanent in practice.
Outage bool
+ // AcceptanceUnknown marks a failure AFTER the whole body was handed to the
+ // provider (the 250 never came): the provider may hold the message. Never
+ // permanent; the next attempt is a new ordinal, and provider feedback
+ // carrying the attempt header is the only authoritative answer.
+ AcceptanceUnknown bool
+ // SettlementErr reports that the provider ACCEPTED the message but the
+ // local settlement did not commit. The send happened; the caller must not
+ // resubmit.
+ SettlementErr error
}
-// Deliverer performs a SINGLE SMTP submit — River owns re-attempts. Implemented in
-// the binary over internal/outbound's single-attempt path.
+// Deliverer performs a SINGLE authorized SMTP submit — River owns re-attempts.
+// The token is the authorization for exactly this call; the production
+// implementation (the outbound.ProviderSubmitter) redeems it immediately before
+// the socket opens and refuses to dial without it.
type Deliverer interface {
- Deliver(ctx context.Context, j *SendJob) DeliverOutcome
-}
-
-type RampRequest struct {
- MessageID string
- UserID string
- Domain string
- Units int
-}
-
-type RampDecision struct {
- Allowed bool
- RetryAt time.Time
-}
-
-// RampGate reserves recipient capacity for an eligible custom-domain send.
-// Implementations must make a same-message/day call idempotent.
-type RampGate interface {
- Reserve(ctx context.Context, req RampRequest) (RampDecision, error)
- Confirm(ctx context.Context, messageID string) error
- Release(ctx context.Context, messageID string) error
- Resolve(ctx context.Context, messageID string) error
+ Deliver(ctx context.Context, j *SendJob, auth sendingpolicy.ProviderAuthorization) DeliverOutcome
}
// RateDecision is the fire-time rate gate's answer for one submission slot:
@@ -210,21 +269,20 @@ type RateDecision = sendrate.Decision
// RateGate reserves one slot in the per-agent fire-time submission budget
// (internal/sendrate) — the durable counterpart to the acceptance-time
// in-memory send limit, enforced immediately before provider submission so
-// scheduled-send bursts and multi-replica deployments cannot exceed it.
-// Unlike RampGate there is no Confirm/Release: the slot is consumed at
-// Reserve and ages out of the sliding window on its own (see the sendrate
-// package doc for the crash semantics). A nil gate allows everything.
-// Window exposes the gate's sliding window so the deferral snooze clamp
-// cannot diverge from the limiter's real window.
+// scheduled-send bursts and multi-replica deployments cannot exceed it. It
+// stays separate from the sending-protection gate because it controls provider
+// throughput, not reputation admission. A nil gate allows everything.
type RateGate interface {
Reserve(ctx context.Context, agentID string) (RateDecision, error)
Window() time.Duration
}
-// Store is the messages-store surface the worker needs. Implemented over
-// internal/identity in the binary. ClaimSend atomically checks that the message
-// and agent are live and persists delivery_status='sending' for the stamped River
-// job before provider I/O begins.
+// OperationResolver recovers the durable operation for a job that carries no
+// reference — a legacy argument shape from a pre-floor slot. It runs the same
+// Prepare path an accept transaction runs, idempotently, so an old job and a
+// new one authorize identically.
+type OperationResolver func(ctx context.Context, messageID string) (sendingpolicy.AcceptanceDecision, sendingpolicy.OperationRef, error)
+
// DailyQuotaDeferredError is returned by Store.ClaimSend when the owning
// account's per-day send cap is exhausted at fire time. The store has already
// released the send claim; the worker snoozes the job until RetryAt (the next
@@ -237,6 +295,10 @@ func (e *DailyQuotaDeferredError) Error() string {
return fmt.Sprintf("daily send cap exhausted; deferred until %s", e.RetryAt.Format(time.RFC3339))
}
+// Store is the messages-store surface the worker needs. Implemented over
+// internal/identity in the binary. ClaimSend atomically checks that the message
+// and agent are live and persists delivery_status='sending' for the stamped River
+// job before provider I/O begins.
type Store interface {
// ClaimSend returns nil when the message is gone, trashed, terminal, or owned
// by a different River job. It returns *DailyQuotaDeferredError (claim
@@ -245,6 +307,9 @@ type Store interface {
ClaimSend(ctx context.Context, messageID string, jobID int64) (*SendJob, error)
// ReleaseSend clears a side-effect-free attempt before River backoff.
ReleaseSend(ctx context.Context, messageID string, jobID int64) error
+ // RecordHold persists the message's finite-hold class and anchor. Terminal
+ // writes clear the pair.
+ RecordHold(ctx context.Context, messageID string, class HoldClass, anchor time.Time) error
// MarkSent records the provider outcome monotonically from a pre-terminal
// state, including when trash won after ClaimSend.
MarkSent(ctx context.Context, messageID string, jobID int64, attempt int, occurredAt time.Time, providerMessageID, sentAs string) error
@@ -256,11 +321,11 @@ type Store interface {
// state", not to unconditionally fail.
// The returned status reports what the guarded write actually did:
// StatusFailed, StatusSent (evidence settle), or "" (no-op). The returned
- // time is the occurred_at the write actually used — the provider-accept
- // evidence time on an evidence settle, the passed occurredAt on a
- // failure, zero on a no-op — so observability reports what the write
- // did, not what the caller asked for.
- MarkFailed(ctx context.Context, messageID string, jobID int64, attempt int, occurredAt time.Time, detail string, source delivery.FailureSource, reason messagelifecycle.ReasonCode, blockedRecipients []string) (delivery.Status, time.Time, error)
+ // time is the occurred_at the write actually used, and the returned
+ // provider id is the evidence's provider message id on an evidence
+ // settle ('' otherwise), so the attempt that dialed can be settled with
+ // it.
+ MarkFailed(ctx context.Context, messageID string, jobID int64, attempt int, occurredAt time.Time, detail string, source delivery.FailureSource, reason messagelifecycle.ReasonCode, blockedRecipients []string) (delivery.Status, time.Time, string, error)
PreserveTerminalFailure(ctx context.Context, messageID string, jobID int64, attempt int, occurredAt time.Time, detail string, source delivery.FailureSource, reason messagelifecycle.ReasonCode, blockedRecipients []string) error
// DeferTerminalFailure records a final attempt's diagnostic + releases the
// I/O claim WITHOUT declaring failed: the terminal reconciler declares the
@@ -281,17 +346,19 @@ type SendWorker struct {
river.WorkerDefaults[OutboundSendArgs]
store Store
deliverer Deliverer
- ramp RampGate
+ gate sendingpolicy.Gate
+ resolve OperationResolver
rate RateGate
metrics Metrics
+ now func() time.Time
}
-func NewSendWorker(store Store, deliverer Deliverer, ramp ...RampGate) *SendWorker {
- w := &SendWorker{store: store, deliverer: deliverer, metrics: noopMetrics{}}
- if len(ramp) > 0 {
- w.ramp = ramp[0]
- }
- return w
+// NewSendWorker builds a worker with no sending-protection gate. Without a
+// gate every provider call is made with an empty authorization, which the
+// production submitter refuses before it dials; the composition root always
+// installs one via WithGate, and its wiring test proves it.
+func NewSendWorker(store Store, deliverer Deliverer) *SendWorker {
+ return &SendWorker{store: store, deliverer: deliverer, metrics: noopMetrics{}, now: time.Now}
}
// WithMetrics injects the SLI recorder. Chainable; nil keeps the no-op
@@ -312,6 +379,41 @@ func (w *SendWorker) WithRateGate(g RateGate) *SendWorker {
return w
}
+// WithGate injects the sending-protection gate every provider call must pass.
+// Chainable; nil keeps the gateless default described on NewSendWorker.
+func (w *SendWorker) WithGate(g sendingpolicy.Gate) *SendWorker {
+ if g != nil {
+ w.gate = g
+ }
+ return w
+}
+
+// WithOperationResolver injects the legacy-argument resolver. Chainable; nil
+// leaves a legacy job failing closed.
+func (w *SendWorker) WithOperationResolver(r OperationResolver) *SendWorker {
+ if r != nil {
+ w.resolve = r
+ }
+ return w
+}
+
+// WithClock overrides the worker's clock for deadline tests. Chainable.
+func (w *SendWorker) WithClock(now func() time.Time) *SendWorker {
+ if now != nil {
+ w.now = now
+ }
+ return w
+}
+
+// Gate exposes the wired sending-protection gate (nil when none), for the
+// composition root's wiring test.
+func (w *SendWorker) Gate() sendingpolicy.Gate { return w.gate }
+
+// HasOperationResolver reports whether a legacy-argument resolver is wired.
+// Without one every job from a pre-floor slot fails closed, so the wiring
+// test insists on it.
+func (w *SendWorker) HasOperationResolver() bool { return w.resolve != nil }
+
// NextRetry overrides River's default backoff with the decided send envelope.
func (w *SendWorker) NextRetry(job *river.Job[OutboundSendArgs]) time.Time {
i := job.Attempt
@@ -325,12 +427,10 @@ func (w *SendWorker) NextRetry(job *river.Job[OutboundSendArgs]) time.Time {
// River's 60s default JobTimeout. (Contrast the maintenance/sweep workers, which
// override it because they can run for minutes.)
func (w *SendWorker) Work(ctx context.Context, job *river.Job[OutboundSendArgs]) error {
- // Queue-wait SLI: due→pickup latency for THIS attempt (River stamps
- // scheduled_at at enqueue, at each retry's backoff target, and on snooze;
- // attempted_at at claim). scheduled_at — NOT created_at — is the baseline:
- // a retried/snoozed/ramp-deferred message would otherwise record its entire
- // cumulative age as "queue wait" on every pass, poisoning the p95. Guarded
- // against zero/negative deltas (clock skew, hand-built rows).
+ // Queue-wait SLI: due→pickup latency for THIS attempt. scheduled_at — NOT
+ // created_at — is the baseline: a retried/snoozed/deferred message would
+ // otherwise record its entire cumulative age as "queue wait" on every
+ // pass, poisoning the p95. Guarded against zero/negative deltas.
if job.AttemptedAt != nil && !job.ScheduledAt.IsZero() {
if wait := job.AttemptedAt.Sub(job.ScheduledAt); wait > 0 {
w.metrics.OutboundQueueWait(wait.Seconds())
@@ -353,23 +453,9 @@ func (w *SendWorker) Work(ctx context.Context, job *river.Job[OutboundSendArgs])
return err // DB error — retryable
}
if j == nil {
- // A previous terminal attempt may have committed the durable message
- // outcome before ramp cleanup failed. Terminal rows cannot be claimed on
- // retry, so resolve any reservation from that durable outcome here. Resolve
- // is also safe for deleted, non-ramped, and missing messages.
- if w.ramp != nil {
- if err := w.ramp.Resolve(ctx, job.Args.MessageID); err != nil {
- return fmt.Errorf("resolve sending ramp for unclaimable message: %w", err)
- }
- }
return nil // message gone or already terminal — nothing to provider-submit
}
if j.alreadyDone() {
- if w.ramp != nil && j.rampEligible() {
- if err := w.ramp.Resolve(ctx, j.MessageID); err != nil {
- return fmt.Errorf("resolve sending ramp for completed message: %w", err)
- }
- }
return nil // already submitted (sent+) — idempotent re-drive
}
if j.ProviderAccepted {
@@ -384,138 +470,80 @@ func (w *SendWorker) Work(ctx context.Context, job *river.Job[OutboundSendArgs])
return err
}
// Terminal 'sent', but NOT an attempt — the submit happened on an
- // earlier attempt; only the settle lands here. occurredAt is the
- // provider-accept evidence time, so the latency measures
- // acceptance→provider-accept, not acceptance→settle.
+ // earlier attempt; only the settle lands here.
emitTerminal(w.metrics, terminalSent, j.submissionAnchor(), observedAt)
- if w.ramp != nil && j.rampEligible() {
- return w.ramp.Confirm(ctx, j.MessageID)
- }
+ w.settleFromEvidence(ctx, j.MessageID, j.ProviderMessageID)
return nil
}
- // Ramp only mail that uses a verified customer identity. Platform-originated
- // test mail uses the relay identity and remains exempt; loopback never enters
- // this worker. Reserve after the provider-evidence guard. The final suppression
- // check deliberately follows an allowed reservation, closing the policy window
- // while Reserve waits on shared capacity. Retryable work after Reserve keeps
- // that reservation: same-message/day Reserve is idempotent, while a released
- // reservation is terminal and cannot be re-reserved.
- if w.ramp != nil && j.rampEligible() {
- decision, rerr := w.ramp.Reserve(ctx, RampRequest{
- MessageID: j.MessageID,
- UserID: j.UserID,
- Domain: j.Domain,
- Units: uniqueRecipientCount(j.Recipients),
- })
- observedAt = time.Now().UTC()
- if rerr != nil {
- if isPermanentRampError(rerr) {
- if err := w.markFailed(ctx, j.MessageID, job.ID, job.Attempt, j.submissionAnchor(), observedAt, "sending_ramp_invalid: "+rerr.Error(), delivery.FailureSourceLocal, messagelifecycle.ReasonSubmissionCancelled, nil); err != nil {
- return err
- }
- return river.JobCancel(rerr)
- }
- if j.pastRetryHorizon() {
- if err := w.markFailed(ctx, j.MessageID, job.ID, job.Attempt, j.submissionAnchor(), observedAt, "ramp_capacity_timeout: "+rerr.Error(), delivery.FailureSourceLocal, messagelifecycle.ReasonSubmissionLocalRetriesExhausted, nil); err != nil {
- return err
- }
- _ = w.ramp.Release(ctx, j.MessageID)
- return river.JobCancel(fmt.Errorf("sending ramp unavailable past %s horizon: %w", SendRetryHorizon, rerr))
- }
- if err := w.store.ReleaseSend(ctx, j.MessageID, job.ID); err != nil {
- return fmt.Errorf("release outbound send claim after ramp-check failure: %w", err)
- }
- log.Printf("[outbound-send] ramp reservation failed for %s (snoozing): %v", j.MessageID, rerr)
- return river.JobSnooze(rampErrorSnoozeInterval)
- }
- if !decision.Allowed {
- if j.pastRetryHorizon() {
- if err := w.markFailed(ctx, j.MessageID, job.ID, job.Attempt, j.submissionAnchor(), observedAt, "ramp_capacity_timeout", delivery.FailureSourceLocal, messagelifecycle.ReasonSubmissionLocalRetriesExhausted, nil); err != nil {
- return err
- }
- if err := w.ramp.Release(ctx, j.MessageID); err != nil {
- return fmt.Errorf("release ramp reservation after timeout: %w", err)
- }
- return river.JobCancel(fmt.Errorf("sending ramp deferred past %s horizon", SendRetryHorizon))
- }
- if err := w.store.ReleaseSend(ctx, j.MessageID, job.ID); err != nil {
- return fmt.Errorf("release outbound send claim after ramp deferral: %w", err)
- }
- delay := time.Until(decision.RetryAt)
- if delay < time.Minute {
- delay = time.Minute
+ // A message whose SES tenant became ready in time leaves the setup class
+ // before any later gate is consulted, so the setup deadline it has already
+ // escaped cannot fail it and the new 72-hour horizon starts at readiness.
+ if err := w.applyTenantReadiness(ctx, j); err != nil {
+ return err
+ }
+
+ // Without a gate (unit tests only) the gate steps are skipped and every
+ // other guard still runs; the production deliverer refuses the empty
+ // authorization that results, so a deployment that reaches the provider
+ // this way sends nothing. The composition root's wiring test proves
+ // production never builds this shape.
+ var attempt sendingpolicy.AttemptRef
+ if w.gate != nil {
+ ref, holdErr := w.operationFor(ctx, job, j)
+ if holdErr != nil {
+ return holdErr
+ }
+ // 1. Reserve the durable attempt. Reserve is idempotent per ordinal,
+ // so a re-driven execution that never reached ConsumeAttempt finds
+ // its own reservation, and a confirmed one is followed by a fresh
+ // ordinal.
+ early, reserved, err := w.gate.Reserve(ctx, ref)
+ observedAt = w.now().UTC()
+ if err != nil {
+ if errors.Is(err, sendingpolicy.ErrSourceUnavailable) {
+ return w.cancelTerminally(ctx, job, j, reserved, observedAt, "sending_policy: operation unavailable: "+err.Error())
}
- return river.JobSnooze(delay)
+ return w.snoozeOnGateError(ctx, job, j, reserved, "reserve", err)
+ }
+ if !early.Allow {
+ return w.hold(ctx, job, j, reserved, early, observedAt)
}
+ attempt = reserved
}
- // Fire-time per-agent rate gate (internal/sendrate): the durable,
- // cross-replica counterpart of the acceptance-time in-memory send limit —
- // scheduled sends accumulate as River jobs and would otherwise burst past
- // the advertised 60/min/agent at the provider when they fire. Grouped with
- // the other wait-gates: after the ramp reservation, before the final
- // suppression check. A deferral RELEASES the send claim but KEEPS the ramp
- // reservation (same invariant as the outage snooze above — same-message
- // Reserve is idempotent, a released reservation is terminal), and snoozes
- // WITHOUT burning an attempt, metering, or emitting lifecycle/terminal
- // events: the message simply fires when the window frees capacity.
+ // 2. Fire-time per-agent rate gate: a deferral DeferAttempts (the budget
+ // is given back; the ramp reservation is kept) and snoozes WITHOUT
+ // burning an attempt, metering, or emitting lifecycle/terminal events.
if w.rate != nil {
decision, rerr := w.rate.Reserve(ctx, j.AgentID)
- observedAt = time.Now().UTC()
- if rerr != nil {
- // Fail toward retry, never toward an unthrottled submit: the
- // provider is never exposed because the limiter is down.
- if j.pastRetryHorizon() {
- if err := w.markFailed(ctx, j.MessageID, job.ID, job.Attempt, j.submissionAnchor(), observedAt, "send_rate_timeout: "+rerr.Error(), delivery.FailureSourceLocal, messagelifecycle.ReasonSubmissionLocalRetriesExhausted, nil); err != nil {
- return err
- }
- if w.ramp != nil && j.rampEligible() {
- _ = w.ramp.Release(ctx, j.MessageID)
- }
- return river.JobCancel(fmt.Errorf("send rate gate unavailable past %s horizon: %w", SendRetryHorizon, rerr))
- }
- if err := w.store.ReleaseSend(ctx, j.MessageID, job.ID); err != nil {
- return fmt.Errorf("release outbound send claim after rate-gate failure: %w", err)
- }
- log.Printf("[outbound-send] rate gate unavailable for %s (snoozing): %v", j.MessageID, rerr)
- return river.JobSnooze(rateErrorSnoozeInterval)
- }
- if !decision.Allowed {
- if j.pastRetryHorizon() {
- if err := w.markFailed(ctx, j.MessageID, job.ID, job.Attempt, j.submissionAnchor(), observedAt, "send_rate_timeout", delivery.FailureSourceLocal, messagelifecycle.ReasonSubmissionLocalRetriesExhausted, nil); err != nil {
- return err
- }
- if w.ramp != nil && j.rampEligible() {
- if err := w.ramp.Release(ctx, j.MessageID); err != nil {
- return fmt.Errorf("release ramp reservation after send-rate timeout: %w", err)
- }
- }
- return river.JobCancel(fmt.Errorf("send rate deferred past %s horizon", SendRetryHorizon))
- }
- if err := w.store.ReleaseSend(ctx, j.MessageID, job.ID); err != nil {
- return fmt.Errorf("release outbound send claim after rate deferral: %w", err)
+ observedAt = w.now().UTC()
+ if rerr != nil || !decision.Allowed {
+ w.deferAttempt(ctx, attempt, "rate")
+ if rerr != nil {
+ log.Printf("[outbound-send] rate gate unavailable for %s (snoozing): %v", j.MessageID, rerr)
+ return w.holdFinite(ctx, job, j, attempt, HoldRateRampOrProvider, "send_rate_timeout: "+rerr.Error(), rateErrorSnoozeInterval, observedAt)
}
delay := clampRateSnooze(time.Until(decision.RetryAt), w.rate.Window()) + rateJitter(j.MessageID, w.rate.Window())
- w.metrics.OutboundRateDeferred()
- // IDs only — never recipient data.
- log.Printf("[outbound-send] rate_limited agent=%s msg=%s retry_in=%s", j.AgentID, j.MessageID, delay)
- return river.JobSnooze(delay)
+ if !w.holdExpired(j, HoldRateRampOrProvider, observedAt) {
+ // A deferral is counted only when it defers; an expiry is a
+ // terminal outcome and is counted as one by markFailed.
+ w.metrics.OutboundRateDeferred()
+ // IDs only — never recipient data.
+ log.Printf("[outbound-send] rate_limited agent=%s msg=%s retry_in=%s", j.AgentID, j.MessageID, delay)
+ }
+ return w.holdFinite(ctx, job, j, attempt, HoldRateRampOrProvider, "send_rate_timeout", delay, observedAt)
}
}
- // Final suppression guard immediately before provider I/O: a suppression
- // added after acceptance or while an allowed ramp reservation was in flight
- // must still prevent delivery. A match is terminal; a store error fails
- // closed, releasing the side-effect-free claim while preserving an allowed
- // ramp reservation for the idempotent River retry.
+ // 3. Final suppression guard immediately before authorization: a
+ // suppression added after acceptance must still prevent delivery. A
+ // match is terminal and cancels the attempt (both ledgers); a store
+ // error fails closed, releasing the side-effect-free claim.
suppressed, serr := w.store.SuppressedRecipients(ctx, j.UserID, j.AgentID, j.Recipients)
- observedAt = time.Now().UTC()
+ observedAt = w.now().UTC()
if serr != nil {
if err := w.store.ReleaseSend(ctx, j.MessageID, job.ID); err != nil {
- // Keep the idempotent ramp reservation while the message claim remains
- // held. Releasing capacity first would let another message consume it,
- // then a retry could reserve the same message a second time.
return fmt.Errorf("suppression check and claim cleanup before outbound send: %w",
errors.Join(serr, fmt.Errorf("release outbound send claim: %w", err)))
}
@@ -526,17 +554,40 @@ func (w *SendWorker) Work(ctx context.Context, job *river.Job[OutboundSendArgs])
if err := w.markFailed(ctx, j.MessageID, job.ID, job.Attempt, j.submissionAnchor(), observedAt, supErr.Error(), delivery.FailureSourceLocal, messagelifecycle.ReasonSubmissionCancelled, suppressed); err != nil {
return err
}
- if w.ramp != nil && j.rampEligible() {
- if err := w.ramp.Release(ctx, j.MessageID); err != nil {
- return fmt.Errorf("release ramp reservation after suppression: %w", err)
- }
- }
+ w.cancelAttempt(ctx, attempt, "suppression")
return river.JobCancel(supErr)
}
+ if w.gate == nil {
+ return w.submit(ctx, job, j, sendingpolicy.ProviderAuthorization{}, observedAt)
+ }
+
+ // 4. Final authorization. ConsumeAttempt re-checks account state, tenant
+ // readiness, both ledgers, and the post-lock UTC day under lock; a hold
+ // here is handled exactly like an early one, and an error leaves the
+ // reservation standing for the idempotent retry.
+ decision, auth, err := w.gate.ConsumeAttempt(ctx, attempt)
+ observedAt = w.now().UTC()
+ if err != nil {
+ if errors.Is(err, sendingpolicy.ErrSourceUnavailable) {
+ return w.cancelTerminally(ctx, job, j, attempt, observedAt, "sending_policy: operation unavailable: "+err.Error())
+ }
+ return w.snoozeOnGateError(ctx, job, j, attempt, "authorize", err)
+ }
+ if !decision.Allow || auth == nil {
+ return w.hold(ctx, job, j, attempt, decision, observedAt)
+ }
+
+ // 5-6. The authorized submitter redeems the token immediately before the
+ // socket opens and settles the provider's answer.
+ return w.submit(ctx, job, j, *auth, observedAt)
+}
+
+// submit makes the single authorized provider call and records its outcome.
+func (w *SendWorker) submit(ctx context.Context, job *river.Job[OutboundSendArgs], j *SendJob, auth sendingpolicy.ProviderAuthorization, observedAt time.Time) error {
deliverStart := time.Now()
- out := w.deliverer.Deliver(ctx, j)
- observedAt = time.Now().UTC()
+ out := w.deliverer.Deliver(ctx, j, auth)
+ observedAt = w.now().UTC()
// Every Deliver call is exactly one submission attempt; classify it here
// so no downstream branch (outage, horizon, deferral) can drop the sample.
deliverSeconds := time.Since(deliverStart).Seconds()
@@ -556,49 +607,44 @@ func (w *SendWorker) Work(ctx context.Context, job *river.Job[OutboundSendArgs])
// Emitted even when MarkSent was a no-op (the row was already
// finalized sent by a racing SNS delivery notification): that path is
// NOT instrumented, so this is still the message's ONLY sent count.
- // If FinalizeProviderAcceptedTx is ever given its own emission, this
- // site must become status-aware (like MarkFailed) or the race
- // double-counts. The latency observation shares this exactly-once
- // contract — emitTerminal emits count and latency together, here and
- // everywhere else, and the SNS-feedback path stays uninstrumented
- // for both.
emitTerminal(w.metrics, terminalSent, j.submissionAnchor(), observedAt)
- if w.ramp != nil && j.rampEligible() {
- if err := w.ramp.Confirm(ctx, j.MessageID); err != nil {
- return fmt.Errorf("confirm sending ramp: %w", err)
- }
+ if out.SettlementErr != nil {
+ // The provider has the message; only the local settlement (ramp
+ // progress, provider-id binding) is behind. Never a resend: retry
+ // the settlement itself, idempotently, and leave the delayed
+ // feedback path to finish it if that fails too.
+ w.resettle(ctx, j.MessageID, out.ProviderMessageID, out.SettlementErr)
}
return nil
}
// Permanent failure (validation / permanent 5xx) — terminal now, no retries.
// Provenance 'provider': SES itself refused this submission, so the §3.1
- // correction never revives it.
+ // correction never revives it. The submitter has already settled it.
if out.Permanent {
if err := w.markFailed(ctx, j.MessageID, job.ID, job.Attempt, j.submissionAnchor(), observedAt, out.Err.Error(), delivery.FailureSourceProvider, messagelifecycle.ReasonSubmissionProviderRejected, nil); err != nil {
return err
}
- if w.ramp != nil && j.rampEligible() {
- if err := w.ramp.Release(ctx, j.MessageID); err != nil {
- return fmt.Errorf("release ramp reservation after provider rejection: %w", err)
- }
- }
return river.JobCancel(out.Err)
}
// Provider outage (relay unreachable) — snooze WITHOUT burning an attempt so a
// multi-hour SES incident defers instead of exhausting MaxSendAttempts and
- // mass-firing false email.failed (§8 circuit breaker). Bounded by the retry
- // horizon: once the accept is older than SendRetryHorizon, give up terminally
- // (provenance 'local': the provider never confirmed a rejection).
+ // mass-firing false email.failed (§8 circuit breaker). Bounded by the hold
+ // deadline: a message under a policy_budget hold keeps its seven-day
+ // clock; any other message gets the 72-hour provider horizon.
if out.Outage {
- if j.pastRetryHorizon() {
- if err := w.markFailed(ctx, j.MessageID, job.ID, job.Attempt, j.submissionAnchor(), observedAt, out.Err.Error(), delivery.FailureSourceLocal, messagelifecycle.ReasonSubmissionLocalRetriesExhausted, nil); err != nil {
- return err
+ class, anchor, changed := w.nextHoldState(j, HoldRateRampOrProvider, observedAt)
+ if changed {
+ if err := w.store.RecordHold(ctx, j.MessageID, class, anchor); err != nil {
+ return fmt.Errorf("record outbound hold: %w", err)
}
- if w.ramp != nil && j.rampEligible() {
- _ = w.ramp.Release(ctx, j.MessageID)
+ j.LocalHoldClass, j.LocalHoldAnchor = class, anchor
+ }
+ if !observedAt.Before(anchor.Add(class.horizon())) {
+ if err := w.markFailed(ctx, j.MessageID, job.ID, job.Attempt, j.submissionAnchor(), observedAt, out.Err.Error(), delivery.FailureSourceLocal, expiryReasonFor(class, true), nil); err != nil {
+ return err
}
- return fmt.Errorf("outbound send failed (provider outage past %s horizon): %w", SendRetryHorizon, out.Err)
+ return fmt.Errorf("outbound send failed (provider outage past %s horizon): %w", class.horizon(), out.Err)
}
if err := w.store.RecordTemporaryFailure(ctx, j.MessageID, job.ID, job.Attempt, observedAt, out.Err.Error()); err != nil {
return fmt.Errorf("record outbound provider outage and release claim: %w", err)
@@ -615,21 +661,301 @@ func (w *SendWorker) Work(ctx context.Context, job *river.Job[OutboundSendArgs])
if err := w.store.DeferTerminalFailure(ctx, j.MessageID, job.ID, job.Attempt, observedAt, out.Err.Error()); err != nil {
log.Printf("[outbound-send] defer terminal failure for %s: %v", j.MessageID, err)
}
- // Not counted as terminal: the reconciler declares the real outcome
- // (sent on evidence, failed otherwise) after the grace window and
- // emits it then — counting the deferral too would double-count the
- // message in e2a_outbound_terminal_total.
return fmt.Errorf("outbound send failed (final attempt %d; outcome deferred to terminal reconciler): %w", job.Attempt, out.Err)
}
- // Retryable — River reschedules per NextRetry.
+ // Retryable — River reschedules per NextRetry. The next execution returns
+ // to Reserve, which allocates the next ordinal; an acceptance-unknown
+ // failure takes the same path because only provider feedback can say
+ // whether the body was kept.
if err := w.store.RecordTemporaryFailure(ctx, j.MessageID, job.ID, job.Attempt, observedAt, out.Err.Error()); err != nil {
return fmt.Errorf("record outbound temporary failure and release claim: %w", err)
}
return fmt.Errorf("outbound send attempt %d failed: %w", job.Attempt, out.Err)
}
-func (j *SendJob) rampEligible() bool {
- return j.SentAs == "own_address" && j.MessageType != "test"
+// operationFor returns the job's durable operation, resolving a legacy job
+// through the accept path. It returns a River verdict (snooze/cancel) as its
+// error when the message cannot proceed.
+func (w *SendWorker) operationFor(ctx context.Context, job *river.Job[OutboundSendArgs], j *SendJob) (sendingpolicy.OperationRef, error) {
+ observedAt := w.now().UTC()
+ if job.Args.OperationRef != nil && !job.Args.OperationRef.IsZero() {
+ // A customer message's operation IS its message id. A job whose
+ // reference names another operation would charge that operation's
+ // account and route this message's feedback to that message; the
+ // gate cannot tell, because every reference reloads its row. This is
+ // the one place the two ids meet, so this is where they must agree.
+ if job.Args.OperationRef.ID() != j.MessageID {
+ return sendingpolicy.OperationRef{}, w.cancelTerminally(ctx, job, j, sendingpolicy.AttemptRef{}, observedAt, "sending_policy: job operation reference does not name this message")
+ }
+ return *job.Args.OperationRef, nil
+ }
+ if w.resolve == nil {
+ return sendingpolicy.OperationRef{}, w.cancelTerminally(ctx, job, j, sendingpolicy.AttemptRef{}, observedAt, "sending_policy: legacy job carries no operation and no resolver is wired")
+ }
+ decision, ref, err := w.resolve(ctx, j.MessageID)
+ if err != nil {
+ if errors.Is(err, sendingpolicy.ErrSourceUnavailable) {
+ return sendingpolicy.OperationRef{}, w.cancelTerminally(ctx, job, j, sendingpolicy.AttemptRef{}, observedAt, "sending_policy: legacy source unavailable: "+err.Error())
+ }
+ return sendingpolicy.OperationRef{}, w.snoozeOnGateError(ctx, job, j, sendingpolicy.AttemptRef{}, "resolve", err)
+ }
+ if decision == sendingpolicy.AcceptanceSendingPaused {
+ return sendingpolicy.OperationRef{}, w.hold(ctx, job, j, sendingpolicy.AttemptRef{}, sendingpolicy.Decision{Reason: sendingpolicy.ReasonAccountPaused}, observedAt)
+ }
+ if ref.IsZero() {
+ // The only accepted shape with no operation is an exact self-send,
+ // which never enqueues. A queued message that resolves to nothing is
+ // not something this worker can authorize.
+ return sendingpolicy.OperationRef{}, w.cancelTerminally(ctx, job, j, sendingpolicy.AttemptRef{}, observedAt, "sending_policy: message has no provider operation")
+ }
+ return ref, nil
+}
+
+// hold handles a gate hold: a terminal one fails the message now; a pause
+// waits for an operator; every other one is a finite hold with a clock.
+func (w *SendWorker) hold(ctx context.Context, job *river.Job[OutboundSendArgs], j *SendJob, attempt sendingpolicy.AttemptRef, d sendingpolicy.Decision, observedAt time.Time) error {
+ if d.Terminal {
+ return w.cancelTerminally(ctx, job, j, attempt, observedAt, "sending_policy: "+d.Reason)
+ }
+ class := HoldClassFor(d.Reason)
+ delay := indefiniteHoldSnooze
+ if !d.RetryAt.IsZero() {
+ delay = time.Until(d.RetryAt)
+ if delay < time.Minute {
+ delay = time.Minute
+ }
+ }
+ if class == "" {
+ // An account pause has no clock of its own. It starts no finite hold
+ // and evaluates none: a paused job only waits. A deadline persisted
+ // before the pause is not extended either — after resume the job
+ // either continues within its remaining time or expires with its
+ // class's own reason on the next hold it meets.
+ if err := w.store.ReleaseSend(ctx, j.MessageID, job.ID); err != nil {
+ return fmt.Errorf("release outbound send claim during account pause: %w", err)
+ }
+ return river.JobSnooze(delay)
+ }
+ return w.holdFinite(ctx, job, j, attempt, class, "sending_policy_hold: "+d.Reason, delay, observedAt)
+}
+
+// holdFinite persists the hold state, expires the message when its derived
+// deadline has passed, and otherwise releases the claim and snoozes.
+func (w *SendWorker) holdFinite(ctx context.Context, job *river.Job[OutboundSendArgs], j *SendJob, attempt sendingpolicy.AttemptRef, requested HoldClass, detail string, delay time.Duration, observedAt time.Time) error {
+ class, anchor, changed := w.nextHoldState(j, requested, observedAt)
+ if changed {
+ if err := w.store.RecordHold(ctx, j.MessageID, class, anchor); err != nil {
+ return fmt.Errorf("record outbound hold: %w", err)
+ }
+ j.LocalHoldClass, j.LocalHoldAnchor = class, anchor
+ }
+ if !observedAt.Before(anchor.Add(class.horizon())) {
+ if err := w.markFailed(ctx, j.MessageID, job.ID, job.Attempt, j.submissionAnchor(), observedAt, detail, delivery.FailureSourceLocal, expiryReasonFor(class, false), nil); err != nil {
+ return err
+ }
+ w.cancelAttempt(ctx, attempt, "hold expiry")
+ return river.JobCancel(fmt.Errorf("%s: %s hold expired after %s", detail, class, class.horizon()))
+ }
+ if err := w.store.ReleaseSend(ctx, j.MessageID, job.ID); err != nil {
+ return fmt.Errorf("release outbound send claim during hold: %w", err)
+ }
+ return river.JobSnooze(delay)
+}
+
+// holdExpired reports whether encountering `requested` now would find the
+// message past its derived deadline, without persisting anything.
+func (w *SendWorker) holdExpired(j *SendJob, requested HoldClass, observedAt time.Time) bool {
+ class, anchor, _ := w.nextHoldState(j, requested, observedAt)
+ return !observedAt.Before(anchor.Add(class.horizon()))
+}
+
+// nextHoldState applies the durable hold rules to the message's persisted pair
+// and the class it is now encountering, reporting whether anything changed.
+//
+// - First finite hold: the requested class, anchored at the latest of accept,
+// schedule, review, and last resume — or at the observation time for a
+// tenant-setup hold observed later than that.
+// - A budget hold promotes any class to policy_budget, keeping the anchor.
+// - policy_budget never changes again.
+// - Otherwise the persisted class stands: a later readiness loss does not
+// replace a rate class, and a rate hold does not replace a setup class.
+func (w *SendWorker) nextHoldState(j *SendJob, requested HoldClass, observedAt time.Time) (HoldClass, time.Time, bool) {
+ if j.LocalHoldClass == "" {
+ anchor := j.initialHoldAnchor()
+ // A tenant-setup hold observed later than the anchor starts its
+ // clock at the observation; so does a message whose timestamps are
+ // unknown, which must never be treated as already expired.
+ if (requested == HoldTenantSetup && observedAt.After(anchor)) || anchor.IsZero() {
+ anchor = observedAt
+ }
+ return requested, anchor, true
+ }
+ if j.LocalHoldClass == HoldPolicyBudget {
+ return HoldPolicyBudget, j.LocalHoldAnchor, false
+ }
+ if requested == HoldPolicyBudget {
+ return HoldPolicyBudget, j.LocalHoldAnchor, true
+ }
+ return j.LocalHoldClass, j.LocalHoldAnchor, false
+}
+
+// applyTenantReadiness performs the one-way setup→rate transition when the
+// tenant became ready on or before the setup deadline. The comparison uses the
+// stored readiness time, so a worker waking after the old deadline still
+// honors readiness that committed in time.
+func (w *SendWorker) applyTenantReadiness(ctx context.Context, j *SendJob) error {
+ if j.LocalHoldClass != HoldTenantSetup || j.TenantReadyAt.IsZero() {
+ return nil
+ }
+ if j.TenantReadyAt.After(j.LocalHoldAnchor.Add(HoldTenantSetup.horizon())) {
+ return nil
+ }
+ if err := w.store.RecordHold(ctx, j.MessageID, HoldRateRampOrProvider, j.TenantReadyAt); err != nil {
+ return fmt.Errorf("record tenant readiness transition: %w", err)
+ }
+ j.LocalHoldClass, j.LocalHoldAnchor = HoldRateRampOrProvider, j.TenantReadyAt
+ return nil
+}
+
+// expiryReasonFor picks the lifecycle reason for a hold that expired. The
+// persisted class decides, with the one exception the design names: a later
+// provider outage cannot emit the setup reason, because the provider — not
+// setup — is what blocked the send at the end. A rate or ramp wait met by a
+// setup-class message still expires as setup: missing or late readiness is
+// the story of that message.
+func expiryReasonFor(class HoldClass, providerOutage bool) messagelifecycle.ReasonCode {
+ if class == HoldTenantSetup && providerOutage {
+ return HoldRateRampOrProvider.expiryReason()
+ }
+ return class.expiryReason()
+}
+
+// HoldClassFor maps a gate hold reason to its finite-hold class; "" means the
+// hold has no clock (an account pause).
+func HoldClassFor(reason string) HoldClass {
+ switch reason {
+ case sendingpolicy.ReasonAccountPaused:
+ return ""
+ case sendingpolicy.ReasonAccountDailyBudget, sendingpolicy.ReasonAccountSharedBudget,
+ sendingpolicy.ReasonGlobalAllBudget, sendingpolicy.ReasonGlobalProbation,
+ sendingpolicy.ReasonGlobalCritical, sendingpolicy.ReasonGlobalViolation:
+ return HoldPolicyBudget
+ case sendingpolicy.ReasonTenantNotReady, sendingpolicy.ReasonTenantUnnamed:
+ return HoldTenantSetup
+ }
+ // Ramp capacity, an unverified sending identity, and any hold reason this
+ // worker does not know by name all wait on the 72-hour clock: unknown is
+ // the shorter horizon, never the longer one.
+ return HoldRateRampOrProvider
+}
+
+// cancelTerminally fails the message for a reason no retry can change and
+// gives its attempt back where the gate still allows it.
+func (w *SendWorker) cancelTerminally(ctx context.Context, job *river.Job[OutboundSendArgs], j *SendJob, attempt sendingpolicy.AttemptRef, observedAt time.Time, detail string) error {
+ if err := w.markFailed(ctx, j.MessageID, job.ID, job.Attempt, j.submissionAnchor(), observedAt, detail, delivery.FailureSourceLocal, messagelifecycle.ReasonSubmissionCancelled, nil); err != nil {
+ return err
+ }
+ w.cancelAttempt(ctx, attempt, "terminal")
+ return river.JobCancel(errors.New(detail))
+}
+
+// snoozeOnGateError releases the claim and snoozes when the gate itself is
+// unavailable: fail toward retry, never toward an unauthorized submit, and
+// never burn a River attempt on infrastructure.
+//
+// It is a bounded wait like every other one: the message enters (or stays
+// in) the rate/ramp/provider class and expires at that class's deadline, so a
+// gate that is down for days does not park mail forever.
+func (w *SendWorker) snoozeOnGateError(ctx context.Context, job *river.Job[OutboundSendArgs], j *SendJob, attempt sendingpolicy.AttemptRef, step string, gerr error) error {
+ log.Printf("[outbound-send] sending policy %s failed for %s (snoozing): %v", step, j.MessageID, gerr)
+ return w.holdFinite(ctx, job, j, attempt, HoldRateRampOrProvider, "sending_policy_unavailable: "+step+": "+gerr.Error(), gateErrorSnoozeInterval, w.now().UTC())
+}
+
+// deferAttempt gives the budget back for a rate deferral; a stale or already
+// released attempt is not an error here — the next Reserve is idempotent.
+func (w *SendWorker) deferAttempt(ctx context.Context, attempt sendingpolicy.AttemptRef, why string) {
+ if w.gate == nil {
+ return
+ }
+ if err := w.gate.DeferAttempt(ctx, attempt); err != nil &&
+ !errors.Is(err, sendingpolicy.ErrAttemptStale) && !errors.Is(err, sendingpolicy.ErrSourceUnavailable) {
+ log.Printf("[outbound-send] defer attempt (%s) for %s: %v", why, attempt.OperationID(), err)
+ }
+}
+
+// cancelAttempt gives both ledgers back for a terminal local outcome. A
+// started attempt cannot be refunded and says so; that is expected on a
+// terminal path reached after a socket opened.
+func (w *SendWorker) cancelAttempt(ctx context.Context, attempt sendingpolicy.AttemptRef, why string) {
+ if w.gate == nil {
+ return
+ }
+ // A zero attempt (no reservation was ever made) has nothing to give back;
+ // the gate says so with ErrSourceUnavailable and that is not worth a log.
+ if err := w.gate.CancelAttempt(ctx, attempt); err != nil &&
+ !errors.Is(err, sendingpolicy.ErrAttemptStale) && !errors.Is(err, sendingpolicy.ErrProviderCallStarted) &&
+ !errors.Is(err, sendingpolicy.ErrSourceUnavailable) {
+ log.Printf("[outbound-send] cancel attempt (%s) for %s: %v", why, attempt.OperationID(), err)
+ }
+}
+
+// resettle retries a settlement that failed after the provider accepted the
+// message. It mirrors markFailed's bounded retry: a transient database error
+// should not cost a domain its ramp progress for the day.
+func (w *SendWorker) resettle(ctx context.Context, messageID, providerMessageID string, first error) {
+ if w.gate == nil {
+ return
+ }
+ err := first
+ for i := 0; i < terminalWriteRetries; i++ {
+ select {
+ case <-ctx.Done():
+ // Shutdown or the job timeout: the one moment a lost settlement
+ // is likeliest, so it must not also be the one that goes unlogged.
+ log.Printf("[outbound-send] CRITICAL: %s accepted by provider but not settled (context ended before retry): %v", messageID, err)
+ return
+ case <-time.After(time.Duration(i+1) * terminalWriteBackoff):
+ }
+ ref, lerr := w.gate.LookupOperation(ctx, messageID)
+ if lerr != nil {
+ err = lerr
+ continue
+ }
+ err = w.gate.SettleOperation(ctx, ref, sendingpolicy.SettlementProviderAccepted, providerMessageID)
+ if err == nil || errors.Is(err, sendingpolicy.ErrAttemptStale) {
+ return
+ }
+ if errors.Is(err, sendingpolicy.ErrProviderMessageIDConflict) {
+ break
+ }
+ }
+ log.Printf("[outbound-send] CRITICAL: %s accepted by provider but not settled after retries: %v", messageID, err)
+}
+
+// settleFromEvidence applies provider-accept evidence to the operation's
+// latest dialed attempt. Best effort: the row is already settled as sent, and
+// an attempt that predates the gate has nothing to settle.
+func (w *SendWorker) settleFromEvidence(ctx context.Context, messageID, providerMessageID string) {
+ if w.gate == nil {
+ return
+ }
+ ref, err := w.gate.LookupOperation(ctx, messageID)
+ if err != nil {
+ if !errors.Is(err, sendingpolicy.ErrSourceUnavailable) {
+ log.Printf("[outbound-send] lookup operation for evidence settle of %s: %v", messageID, err)
+ }
+ return
+ }
+ if err := w.gate.SettleOperation(ctx, ref, sendingpolicy.SettlementProviderAccepted, providerMessageID); err != nil && !errors.Is(err, sendingpolicy.ErrAttemptStale) {
+ if errors.Is(err, sendingpolicy.ErrProviderMessageIDConflict) {
+ // Two physical sends for one charged attempt, or evidence
+ // attributed to the wrong attempt: an invariant violation, never
+ // a transient. Surface it as such.
+ log.Printf("[outbound-send] CRITICAL: provider id conflict settling %s from evidence: %v", messageID, err)
+ return
+ }
+ log.Printf("[outbound-send] settle %s from provider evidence: %v", messageID, err)
+ }
}
// clampRateSnooze bounds a rate deferral to [rateMinSnooze, window]: the floor
@@ -672,11 +998,6 @@ func rateJitter(messageID string, window time.Duration) time.Duration {
return time.Duration(h.Sum32()%uint32(ms)) * time.Millisecond
}
-func isPermanentRampError(err error) bool {
- var permanent interface{ Permanent() bool }
- return errors.As(err, &permanent) && permanent.Permanent()
-}
-
func uniqueRecipientCount(recipients []string) int {
seen := make(map[string]struct{}, len(recipients))
for _, recipient := range recipients {
@@ -704,7 +1025,8 @@ func (w *SendWorker) markFailed(ctx context.Context, messageID string, jobID int
for i := 0; i < terminalWriteRetries; i++ {
var settled delivery.Status
var settledAt time.Time
- if settled, settledAt, err = w.store.MarkFailed(ctx, messageID, jobID, attempt, occurredAt, detail, source, reason, blockedRecipients); err == nil {
+ var providerID string
+ if settled, settledAt, providerID, err = w.store.MarkFailed(ctx, messageID, jobID, attempt, occurredAt, detail, source, reason, blockedRecipients); err == nil {
// Emit what the guarded write actually did, exactly once, only
// after the durable write: a failure with the caller's provenance,
// or "sent" when provider evidence settled the row. A no-op write
@@ -719,6 +1041,11 @@ func (w *SendWorker) markFailed(ctx context.Context, messageID string, jobID int
emitTerminal(w.metrics, terminalOutcome(source, reason, blockedRecipients), anchorAt, settledAt)
case delivery.StatusSent:
emitTerminal(w.metrics, terminalSent, anchorAt, settledAt)
+ // Provider evidence settled the row under a terminal write that
+ // expected to fail it. The attempt that dialed still needs
+ // settling — ramp progress and the correlation binding — and
+ // only the operation, not this call's attempt, names it.
+ w.settleFromEvidence(ctx, messageID, providerID)
}
return nil
}
diff --git a/internal/outboundsend/worker_test.go b/internal/outboundsend/worker_test.go
index 4ca28b6ac..1bcd55426 100644
--- a/internal/outboundsend/worker_test.go
+++ b/internal/outboundsend/worker_test.go
@@ -2,16 +2,19 @@ package outboundsend_test
import (
"context"
+ "encoding/json"
"errors"
"testing"
"time"
+ "github.com/jackc/pgx/v5"
"github.com/riverqueue/river"
"github.com/riverqueue/river/rivertype"
"github.com/tokencanopy/e2a/internal/delivery"
"github.com/tokencanopy/e2a/internal/messagelifecycle"
"github.com/tokencanopy/e2a/internal/outboundsend"
+ "github.com/tokencanopy/e2a/internal/sendingpolicy"
)
type fakeStore struct {
@@ -24,6 +27,8 @@ type fakeStore struct {
// occurred_at to the provider-accept evidence time for the durable write.
settleStatus delivery.Status
settleAt time.Time
+ // settleProviderID is the evidence's provider id an evidence settle reports.
+ settleProviderID string
// terminalAfterFailure mirrors the production store: once MarkFailed commits,
// a retry can no longer claim the terminal message and ClaimSend returns nil.
terminalAfterFailure bool
@@ -32,6 +37,7 @@ type fakeStore struct {
suppressedErr error
sent []sentCall
+ holds []holdCall
failed []failedCall
deferred []failedCall
temporary []failedCall
@@ -43,12 +49,18 @@ type fakeStore struct {
}
type sentCall struct{ id, provider, sentAs string }
+type holdCall struct {
+ id string
+ class outboundsend.HoldClass
+ anchor time.Time
+}
type failedCall struct {
id string
attempt int
occurredAt time.Time
detail string
source delivery.FailureSource
+ reason messagelifecycle.ReasonCode
blockedRecipients []string
}
@@ -62,8 +74,8 @@ func (f *fakeStore) MarkSent(_ context.Context, id string, _ int64, _ int, _ tim
f.sent = append(f.sent, sentCall{id, provider, sentAs})
return f.markSentErr
}
-func (f *fakeStore) MarkFailed(_ context.Context, id string, _ int64, attempt int, occurredAt time.Time, detail string, source delivery.FailureSource, _ messagelifecycle.ReasonCode, blockedRecipients []string) (delivery.Status, time.Time, error) {
- f.failed = append(f.failed, failedCall{id: id, attempt: attempt, occurredAt: occurredAt, detail: detail, source: source, blockedRecipients: blockedRecipients})
+func (f *fakeStore) MarkFailed(_ context.Context, id string, _ int64, attempt int, occurredAt time.Time, detail string, source delivery.FailureSource, reason messagelifecycle.ReasonCode, blockedRecipients []string) (delivery.Status, time.Time, string, error) {
+ f.failed = append(f.failed, failedCall{id: id, attempt: attempt, occurredAt: occurredAt, detail: detail, source: source, reason: reason, blockedRecipients: blockedRecipients})
status := f.settleStatus
if status == "" {
status = delivery.StatusFailed
@@ -72,7 +84,7 @@ func (f *fakeStore) MarkFailed(_ context.Context, id string, _ int64, attempt in
if at.IsZero() {
at = occurredAt
}
- return status, at, nil
+ return status, at, f.settleProviderID, nil
}
func (f *fakeStore) PreserveTerminalFailure(context.Context, string, int64, int, time.Time, string, delivery.FailureSource, messagelifecycle.ReasonCode, []string) error {
return nil
@@ -86,6 +98,13 @@ func (f *fakeStore) RecordTemporaryFailure(_ context.Context, id string, _ int64
f.temporary = append(f.temporary, failedCall{id: id})
return f.releaseErr
}
+func (f *fakeStore) RecordHold(_ context.Context, id string, class outboundsend.HoldClass, anchor time.Time) error {
+ f.holds = append(f.holds, holdCall{id: id, class: class, anchor: anchor})
+ if f.job != nil && f.job.MessageID == id {
+ f.job.LocalHoldClass, f.job.LocalHoldAnchor = class, anchor
+ }
+ return nil
+}
func (f *fakeStore) ReleaseSend(_ context.Context, id string, _ int64) error {
f.released = append(f.released, id)
return f.releaseErr
@@ -101,45 +120,11 @@ type fakeDeliverer struct {
out outboundsend.DeliverOutcome
calls int
returnedAt time.Time
+ auths []sendingpolicy.ProviderAuthorization
}
-type fakeRampGate struct {
- decision outboundsend.RampDecision
- err error
- calls []outboundsend.RampRequest
- confirmed []string
- released []string
- resolved []string
- confirmErr error
- releaseErr error
-}
-
-func (f *fakeRampGate) Reserve(_ context.Context, req outboundsend.RampRequest) (outboundsend.RampDecision, error) {
- f.calls = append(f.calls, req)
- return f.decision, f.err
-}
-
-func (f *fakeRampGate) Confirm(_ context.Context, messageID string) error {
- f.confirmed = append(f.confirmed, messageID)
- return f.confirmErr
-}
-
-func (f *fakeRampGate) Release(_ context.Context, messageID string) error {
- f.released = append(f.released, messageID)
- return f.releaseErr
-}
-
-func (f *fakeRampGate) Resolve(_ context.Context, messageID string) error {
- f.resolved = append(f.resolved, messageID)
- return nil
-}
-
-type permanentRampError struct{ msg string }
-
-func (e permanentRampError) Error() string { return e.msg }
-func (e permanentRampError) Permanent() bool { return true }
-
-func (f *fakeDeliverer) Deliver(_ context.Context, _ *outboundsend.SendJob) outboundsend.DeliverOutcome {
+func (f *fakeDeliverer) Deliver(_ context.Context, _ *outboundsend.SendJob, auth sendingpolicy.ProviderAuthorization) outboundsend.DeliverOutcome {
+ f.auths = append(f.auths, auth)
f.calls++
f.returnedAt = time.Now().UTC()
return f.out
@@ -286,172 +271,6 @@ func TestSendWorker_SuppressionObservationTimeFollowsDecision(t *testing.T) {
}
}
-func TestSendWorker_RampLimitedReleasesAndSnoozesWithoutProviderIO(t *testing.T) {
- j := acceptedJob("msg_1")
- j.Domain = "new.example.com"
- j.MessageType = "send"
- j.SentAs = "own_address"
- j.Recipients = []string{"One@example.net", "one@example.net", "two@example.net"}
- st := &fakeStore{job: j}
- dl := &fakeDeliverer{}
- gate := &fakeRampGate{decision: outboundsend.RampDecision{
- Allowed: false,
- RetryAt: time.Now().Add(6 * time.Hour),
- }}
-
- err := outboundsend.NewSendWorker(st, dl, gate).Work(context.Background(), job("msg_1", 5))
- if err == nil {
- t.Fatal("limited send should snooze")
- }
- if dl.calls != 0 {
- t.Fatalf("provider calls = %d, want 0", dl.calls)
- }
- if len(st.released) != 1 || st.released[0] != "msg_1" {
- t.Fatalf("released = %v, want msg_1", st.released)
- }
- if len(gate.calls) != 1 || gate.calls[0].Units != 2 || gate.calls[0].Domain != "new.example.com" {
- t.Fatalf("gate calls = %+v, want two deduplicated recipients", gate.calls)
- }
-}
-
-func TestSendWorker_RampErrorFailsClosedAndSnoozes(t *testing.T) {
- j := acceptedJob("msg_1")
- j.Domain, j.MessageType, j.SentAs = "new.example.com", "send", "own_address"
- st := &fakeStore{job: j}
- dl := &fakeDeliverer{}
- gate := &fakeRampGate{err: errors.New("database unavailable")}
-
- if err := outboundsend.NewSendWorker(st, dl, gate).Work(context.Background(), job("msg_1", 1)); err == nil {
- t.Fatal("ramp storage error should snooze")
- }
- if dl.calls != 0 || len(st.released) != 1 {
- t.Fatalf("gate error must release without provider I/O: calls=%d released=%v", dl.calls, st.released)
- }
-}
-
-func TestSendWorker_RampExemptsPlatformTest(t *testing.T) {
- j := acceptedJob("msg_test")
- j.Domain, j.MessageType, j.SentAs = "new.example.com", "test", "relay"
- st := &fakeStore{job: j}
- dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{ProviderMessageID: "ses-test"}}
- gate := &fakeRampGate{decision: outboundsend.RampDecision{Allowed: false}}
-
- if err := outboundsend.NewSendWorker(st, dl, gate).Work(context.Background(), job("msg_test", 1)); err != nil {
- t.Fatalf("Work: %v", err)
- }
- if len(gate.calls) != 0 || dl.calls != 1 {
- t.Fatalf("platform test should bypass ramp: gate=%d provider=%d", len(gate.calls), dl.calls)
- }
-}
-
-func TestSendWorker_ProviderEvidencePrecedesRamp(t *testing.T) {
- j := acceptedJob("msg_1")
- j.Domain, j.MessageType, j.SentAs = "new.example.com", "send", "own_address"
- j.ProviderAccepted, j.ProviderMessageID = true, "ses-evidence"
- st := &fakeStore{job: j}
- gate := &fakeRampGate{decision: outboundsend.RampDecision{Allowed: false}}
-
- if err := outboundsend.NewSendWorker(st, &fakeDeliverer{}, gate).Work(context.Background(), job("msg_1", 2)); err != nil {
- t.Fatalf("Work: %v", err)
- }
- if len(gate.calls) != 0 {
- t.Fatalf("provider evidence must settle before ramp reservation, got %+v", gate.calls)
- }
-}
-
-func TestSendWorker_ConfirmsRampAfterMarkSent(t *testing.T) {
- j := acceptedJob("msg_confirm")
- j.Domain, j.MessageType, j.SentAs = "new.example.com", "send", "own_address"
- st := &fakeStore{job: j}
- gate := &fakeRampGate{decision: outboundsend.RampDecision{Allowed: true}}
- dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{ProviderMessageID: "ses-confirm", SentAs: "own_address"}}
- if err := outboundsend.NewSendWorker(st, dl, gate).Work(context.Background(), job(j.MessageID, 1)); err != nil {
- t.Fatalf("Work: %v", err)
- }
- if len(st.sent) != 1 || len(gate.confirmed) != 1 || gate.confirmed[0] != j.MessageID {
- t.Fatalf("sent=%v confirmed=%v", st.sent, gate.confirmed)
- }
-}
-
-func TestSendWorker_RepairsRampConfirmationForAlreadySentMessage(t *testing.T) {
- j := acceptedJob("msg_repair")
- j.Domain, j.MessageType, j.SentAs, j.Status = "new.example.com", "send", "own_address", "sent"
- gate := &fakeRampGate{}
- dl := &fakeDeliverer{}
- if err := outboundsend.NewSendWorker(&fakeStore{job: j}, dl, gate).Work(context.Background(), job(j.MessageID, 2)); err != nil {
- t.Fatalf("Work: %v", err)
- }
- if dl.calls != 0 || len(gate.resolved) != 1 {
- t.Fatalf("deliver=%d resolved=%v", dl.calls, gate.resolved)
- }
-}
-
-func TestSendWorker_ReleasesRampOnPermanentProviderFailure(t *testing.T) {
- j := acceptedJob("msg_release")
- j.Domain, j.MessageType, j.SentAs = "new.example.com", "send", "own_address"
- gate := &fakeRampGate{decision: outboundsend.RampDecision{Allowed: true}}
- dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{Err: errors.New("rejected"), Permanent: true}}
- _ = outboundsend.NewSendWorker(&fakeStore{job: j}, dl, gate).Work(context.Background(), job(j.MessageID, 1))
- if len(gate.released) != 1 || gate.released[0] != j.MessageID {
- t.Fatalf("released=%v", gate.released)
- }
-}
-
-func TestSendWorker_RetainsRampOnAmbiguousFailure(t *testing.T) {
- j := acceptedJob("msg_ambiguous")
- j.Domain, j.MessageType, j.SentAs = "new.example.com", "send", "own_address"
- gate := &fakeRampGate{decision: outboundsend.RampDecision{Allowed: true}}
- dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{Err: errors.New("connection reset")}}
- _ = outboundsend.NewSendWorker(&fakeStore{job: j}, dl, gate).Work(context.Background(), job(j.MessageID, 1))
- if len(gate.released) != 0 {
- t.Fatalf("ambiguous failure released ramp: %v", gate.released)
- }
-}
-
-func TestSendWorker_FailsPermanentRampInvariant(t *testing.T) {
- j := acceptedJob("msg_bad_ramp")
- j.Domain, j.MessageType, j.SentAs = "new.example.com", "send", "own_address"
- st := &fakeStore{job: j}
- gate := &fakeRampGate{err: permanentRampError{"domain missing"}}
- if err := outboundsend.NewSendWorker(st, &fakeDeliverer{}, gate).Work(context.Background(), job(j.MessageID, 1)); err == nil {
- t.Fatal("permanent ramp invariant should terminate")
- }
- if len(st.failed) != 1 {
- t.Fatalf("failed=%v", st.failed)
- }
-}
-
-func TestSendWorker_FailsRampDeferredMessagePastHorizon(t *testing.T) {
- j := acceptedJob("msg_ramp_timeout")
- j.Domain, j.MessageType, j.SentAs = "new.example.com", "send", "own_address"
- j.AcceptedAt = time.Now().Add(-73 * time.Hour)
- st := &fakeStore{job: j}
- gate := &fakeRampGate{decision: outboundsend.RampDecision{Allowed: false, RetryAt: time.Now().Add(time.Hour)}}
- if err := outboundsend.NewSendWorker(st, &fakeDeliverer{}, gate).Work(context.Background(), job(j.MessageID, 1)); err == nil {
- t.Fatal("past-horizon ramp deferral should terminate")
- }
- if len(st.failed) != 1 || len(gate.released) != 1 {
- t.Fatalf("failed=%v released=%v", st.failed, gate.released)
- }
-}
-
-// A scheduled send measures its retry horizon from scheduled_at, not accept:
-// accepted 10 days ago but firing ~now, a ramp deferral must snooze/retry — NOT
-// terminally fail as the immediate-send case above does at the same accept age.
-// Guards the fix for the long-scheduled-send false-failure blocker.
-func TestSendWorker_ScheduledSendHorizonMeasuredFromScheduledAt(t *testing.T) {
- j := acceptedJob("msg_sched_horizon")
- j.Domain, j.MessageType, j.SentAs = "new.example.com", "send", "own_address"
- j.AcceptedAt = time.Now().Add(-10 * 24 * time.Hour) // long before fire
- j.ScheduledAt = time.Now() // just fired — inside the horizon
- st := &fakeStore{job: j}
- gate := &fakeRampGate{decision: outboundsend.RampDecision{Allowed: false, RetryAt: time.Now().Add(time.Hour)}}
- err := outboundsend.NewSendWorker(st, &fakeDeliverer{}, gate).Work(context.Background(), job(j.MessageID, 1))
- if len(st.failed) != 0 {
- t.Fatalf("a just-fired long-scheduled send must not be terminated on a ramp deferral; failed=%v err=%v", st.failed, err)
- }
-}
-
func TestSendWorker_RetryableFailureDoesNotMarkFailed(t *testing.T) {
st := &fakeStore{job: acceptedJob("msg_1")}
dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{Err: errors.New("transient 421")}}
@@ -483,34 +302,6 @@ func TestSendWorker_RetryableFailureReleaseErrorRetries(t *testing.T) {
}
}
-func TestSendWorker_TerminalRampReleaseFailureResolvesOnRetry(t *testing.T) {
- j := acceptedJob("msg_1")
- j.Domain, j.MessageType, j.SentAs = "new.example.com", "send", "own_address"
- st := &fakeStore{job: j, terminalAfterFailure: true}
- dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{Err: errors.New("provider rejected message"), Permanent: true}}
- gate := &fakeRampGate{
- decision: outboundsend.RampDecision{Allowed: true},
- releaseErr: errors.New("ramp database unavailable"),
- }
- w := outboundsend.NewSendWorker(st, dl, gate)
-
- if err := w.Work(context.Background(), job(j.MessageID, 1)); err == nil || !errors.Is(err, gate.releaseErr) {
- t.Fatalf("first Work error = %v, want ramp release failure", err)
- }
- if len(st.failed) != 1 || len(gate.released) != 1 {
- t.Fatalf("first Work failed/released = %v/%v, want one each", st.failed, gate.released)
- }
-
- // MarkFailed made the message terminal, so the retry cannot claim it. The
- // worker must still settle the orphaned reservation from the durable outcome.
- if err := w.Work(context.Background(), job(j.MessageID, 2)); err != nil {
- t.Fatalf("retry Work: %v", err)
- }
- if len(gate.resolved) != 1 || gate.resolved[0] != j.MessageID {
- t.Fatalf("resolved reservations = %v, want [%s]", gate.resolved, j.MessageID)
- }
-}
-
func TestSendWorker_OutageSnoozesWithoutBurningAttempt(t *testing.T) {
j := acceptedJob("msg_1")
j.AcceptedAt = time.Now() // fresh accept — within the retry horizon
@@ -561,3 +352,94 @@ func TestSendWorker_NextRetryMatchesEnvelope(t *testing.T) {
}
}
}
+
+// fakeGate is a scriptable sendingpolicy.Gate. Its references and tokens are
+// zero values — the worker never inspects them beyond nil/zero checks — and it
+// records every ledger call so tests can assert the fixed worker order.
+type fakeGate struct {
+ reserve sendingpolicy.Decision
+ reserveErr error
+ consume sendingpolicy.Decision
+ consumeErr error
+ deferred []string
+ cancelled []string
+ settled []sendingpolicy.SettlementOutcome
+ settledIDs []string
+ reserves int
+ consumes int
+ lookupErr error
+ lookupCalls int
+}
+
+func allowAll() *fakeGate {
+ return &fakeGate{reserve: sendingpolicy.Decision{Allow: true}, consume: sendingpolicy.Decision{Allow: true}}
+}
+
+func (g *fakeGate) PrepareExternalTx(context.Context, pgx.Tx, string) (sendingpolicy.AcceptanceDecision, sendingpolicy.OperationRef, error) {
+ return sendingpolicy.AcceptanceAccept, refFor("msg_prepared"), nil
+}
+func (g *fakeGate) PrepareNotificationTx(context.Context, pgx.Tx, sendingpolicy.NotificationRef) (sendingpolicy.OperationRef, error) {
+ return sendingpolicy.OperationRef{}, nil
+}
+func (g *fakeGate) PrepareProtectionNoticeTx(context.Context, pgx.Tx, sendingpolicy.ProtectionNoticeRef) (sendingpolicy.OperationRef, error) {
+ return sendingpolicy.OperationRef{}, nil
+}
+func (g *fakeGate) PreparePublicFeedback(context.Context, sendingpolicy.PublicFeedbackRef) (sendingpolicy.OperationRef, error) {
+ return sendingpolicy.OperationRef{}, nil
+}
+func (g *fakeGate) Reserve(context.Context, sendingpolicy.OperationRef) (sendingpolicy.Decision, sendingpolicy.AttemptRef, error) {
+ g.reserves++
+ return g.reserve, sendingpolicy.AttemptRef{}, g.reserveErr
+}
+func (g *fakeGate) ConsumeAttempt(context.Context, sendingpolicy.AttemptRef) (sendingpolicy.Decision, *sendingpolicy.ProviderAuthorization, error) {
+ g.consumes++
+ if g.consumeErr != nil || !g.consume.Allow {
+ return g.consume, nil, g.consumeErr
+ }
+ return g.consume, &sendingpolicy.ProviderAuthorization{}, nil
+}
+func (g *fakeGate) RedeemProviderCall(context.Context, sendingpolicy.ProviderAuthorization) error {
+ return nil
+}
+func (g *fakeGate) DeferAttempt(_ context.Context, a sendingpolicy.AttemptRef) error {
+ g.deferred = append(g.deferred, a.OperationID())
+ return nil
+}
+func (g *fakeGate) CancelAttempt(_ context.Context, a sendingpolicy.AttemptRef) error {
+ g.cancelled = append(g.cancelled, a.OperationID())
+ return nil
+}
+func (g *fakeGate) SettleProvider(_ context.Context, s sendingpolicy.ProviderSettlement) error {
+ g.settled = append(g.settled, s.Outcome)
+ return nil
+}
+func (g *fakeGate) SettleOperation(_ context.Context, _ sendingpolicy.OperationRef, o sendingpolicy.SettlementOutcome, id string) error {
+ g.settled = append(g.settled, o)
+ g.settledIDs = append(g.settledIDs, id)
+ return nil
+}
+func (g *fakeGate) LookupOperation(_ context.Context, id string) (sendingpolicy.OperationRef, error) {
+ g.lookupCalls++
+ if g.lookupErr != nil {
+ return sendingpolicy.OperationRef{}, g.lookupErr
+ }
+ return refFor(id), nil
+}
+
+// refFor builds an operation reference the way a River job carries one: the
+// versioned wire form holding only the id.
+func refFor(id string) sendingpolicy.OperationRef {
+ var ref sendingpolicy.OperationRef
+ if err := json.Unmarshal([]byte(`{"v":1,"id":"`+id+`"}`), &ref); err != nil {
+ panic(err)
+ }
+ return ref
+}
+
+// gatedJob is job() with the operation reference the accept path would stamp.
+func gatedJob(id string, attempt int) *river.Job[outboundsend.OutboundSendArgs] {
+ j := job(id, attempt)
+ ref := refFor(id)
+ j.Args.OperationRef = &ref
+ return j
+}
diff --git a/internal/sendingpolicy/gate.go b/internal/sendingpolicy/gate.go
index 1bf71b0d8..15534515c 100644
--- a/internal/sendingpolicy/gate.go
+++ b/internal/sendingpolicy/gate.go
@@ -33,6 +33,8 @@ type Gate interface {
DeferAttempt(context.Context, AttemptRef) error
CancelAttempt(context.Context, AttemptRef) error
SettleProvider(context.Context, ProviderSettlement) error
+ SettleOperation(context.Context, OperationRef, SettlementOutcome, string) error
+ LookupOperation(context.Context, string) (OperationRef, error)
}
var _ Gate = (*Module)(nil)
@@ -1809,6 +1811,55 @@ func (m *Module) SettleProvider(ctx context.Context, settlement ProviderSettleme
if settlement.Attempt.IsZero() {
return ErrSourceUnavailable
}
+ return m.settle(ctx, settlement.Attempt.operationID, settlement.Attempt.attempt, settlement)
+}
+
+// LookupOperation recovers a reference to an operation that already exists.
+//
+// This is not a constructor: it returns a reference only for a durable
+// operation row, and the reference carries an id and advisory fields exactly
+// as a deserialized River argument does — every Gate method reloads the row
+// under lock, so recovering a reference grants nothing. It exists for the
+// reconciler, which learns of provider evidence by message id long after the
+// worker and its token are gone.
+func (m *Module) LookupOperation(ctx context.Context, operationID string) (OperationRef, error) {
+ if strings.TrimSpace(operationID) == "" {
+ return OperationRef{}, ErrSourceUnavailable
+ }
+ var row operationRow
+ err := m.pool.QueryRow(ctx, `
+ SELECT operation_id, source_account_ref, policy_subject_ref, purpose, shared_reputation
+ FROM sending_provider_operations
+ WHERE operation_id = $1`, operationID,
+ ).Scan(&row.OperationID, &row.SourceAccountRef, &row.PolicySubjectRef, &row.Purpose, &row.Shared)
+ if errors.Is(err, pgx.ErrNoRows) {
+ return OperationRef{}, ErrSourceUnavailable
+ }
+ if err != nil {
+ return OperationRef{}, fmt.Errorf("sendingpolicy: lookup operation: %w", err)
+ }
+ return row.ref(), nil
+}
+
+// SettleOperation applies a delayed authoritative provider outcome to the
+// attempt of an operation that most recently opened a socket.
+//
+// It exists for the two callers that hold evidence but no token: the worker
+// that finds provider-accept evidence already recorded on a row it is about to
+// re-drive, and the terminal reconciler settling a stranded row from that same
+// evidence. Neither can name an ordinal — the token that could is gone with the
+// process that held it — but both know which OPERATION the evidence belongs to,
+// and the only attempt evidence can describe is the latest one that dialed.
+func (m *Module) SettleOperation(ctx context.Context, ref OperationRef, outcome SettlementOutcome, providerMessageID string) error {
+ if ref.IsZero() {
+ return ErrSourceUnavailable
+ }
+ return m.settle(ctx, ref.id, 0, ProviderSettlement{Outcome: outcome, ProviderMessageID: providerMessageID})
+}
+
+// settle is the shared settlement body. attempt 0 means "the latest attempt
+// whose provider call started", resolved under the operation lock.
+func (m *Module) settle(ctx context.Context, operationID string, attempt int, settlement ProviderSettlement) error {
if !settlement.Outcome.valid() {
return fmt.Errorf("sendingpolicy: unsupported settlement outcome %q", settlement.Outcome)
}
@@ -1819,11 +1870,46 @@ func (m *Module) SettleProvider(ctx context.Context, settlement ProviderSettleme
}
defer func() { _ = tx.Rollback(ctx) }()
- op, err := lockOperation(ctx, tx, settlement.Attempt.operationID)
+ op, err := lockOperation(ctx, tx, operationID)
if err != nil {
return err
}
- stored, err := lockReservation(ctx, tx, settlement.Attempt.operationID, settlement.Attempt.attempt)
+ if attempt == 0 {
+ // Evidence without a token names an operation, not an ordinal. The
+ // attempt is chosen in this order: one already bound to this exact
+ // provider id (a replay, which must be idempotent and must not spill
+ // onto a later attempt); else the oldest dialed attempt with no
+ // provider id yet, because feedback arrives in send order far more
+ // often than not and each binding retires its attempt from this
+ // choice; else the latest dialed attempt, whose bind refuses a
+ // different id rather than absorb it.
+ if err := tx.QueryRow(ctx, `
+ SELECT COALESCE(
+ (SELECT MIN(r.submission_attempt)
+ FROM sending_budget_reservations r
+ JOIN sending_feedback_correlations c
+ ON c.operation_id = r.operation_id AND c.submission_attempt = r.submission_attempt
+ WHERE r.operation_id = $1 AND r.call_state = 'started'
+ AND $2 <> '' AND c.provider_message_id = $2),
+ (SELECT MIN(r.submission_attempt)
+ FROM sending_budget_reservations r
+ LEFT JOIN sending_feedback_correlations c
+ ON c.operation_id = r.operation_id AND c.submission_attempt = r.submission_attempt
+ WHERE r.operation_id = $1 AND r.call_state = 'started'
+ AND c.provider_message_id IS NULL),
+ (SELECT MAX(submission_attempt)
+ FROM sending_budget_reservations
+ WHERE operation_id = $1 AND call_state = 'started'),
+ 0)`, operationID, NormalizeProviderMessageID(settlement.ProviderMessageID),
+ ).Scan(&attempt); err != nil {
+ return fmt.Errorf("sendingpolicy: find started attempt: %w", err)
+ }
+ if attempt == 0 {
+ return ErrAttemptStale
+ }
+ }
+ settlement.Attempt = AttemptRef{operationID: operationID, attempt: attempt}
+ stored, err := lockReservation(ctx, tx, operationID, attempt)
if err != nil {
return err
}
diff --git a/internal/sendingpolicy/provider_token_test.go b/internal/sendingpolicy/provider_token_test.go
index 1e3d2d696..721eb4d5b 100644
--- a/internal/sendingpolicy/provider_token_test.go
+++ b/internal/sendingpolicy/provider_token_test.go
@@ -269,3 +269,139 @@ func TestProviderTokenSettlementComparesNormalizedProviderMessageID(t *testing.T
t.Fatalf("different id err = %v, want ErrProviderMessageIDConflict", err)
}
}
+
+// TestProviderTokenSettleOperationTargetsTheLatestDialedAttempt: evidence that
+// arrives without a token settles the most recent attempt that opened a
+// socket — not a later ordinal that was only reserved, and nothing at all when
+// no attempt ever dialed.
+func TestProviderTokenSettleOperationTargetsTheLatestDialedAttempt(t *testing.T) {
+ f := newFixture(t)
+ g := f.gate(enforcingPolicy(nil))
+ agent := f.agent(f.user("standard"))
+ ref, attempt := f.prepareAndReserve(g, agent, 1)
+
+ err := g.SettleOperation(f.ctx, ref, sendingpolicy.SettlementProviderAccepted, "ses-early")
+ if !errors.Is(err, sendingpolicy.ErrAttemptStale) {
+ t.Fatalf("settle before any dial err = %v, want ErrAttemptStale", err)
+ }
+
+ _, auth, err := g.ConsumeAttempt(f.ctx, attempt)
+ if err != nil || auth == nil {
+ t.Fatalf("authorize: auth=%v err=%v", auth, err)
+ }
+ if err := g.RedeemProviderCall(f.ctx, *auth); err != nil {
+ t.Fatalf("redeem: %v", err)
+ }
+ // The worker died after the socket opened; a later execution re-reserved
+ // ordinal two but never consumed it. Delayed evidence belongs to ordinal one.
+ if _, next, err := g.Reserve(f.ctx, ref); err != nil || next.Attempt() != 2 {
+ t.Fatalf("re-reserve: attempt=%v err=%v", next, err)
+ }
+ if err := g.SettleOperation(f.ctx, ref, sendingpolicy.SettlementProviderAccepted, ""); err != nil {
+ t.Fatalf("settle by operation: %v", err)
+ }
+ if got := f.providerMessageID(ref.ID(), 1); got == nil || *got != "ses-late" {
+ t.Fatalf("attempt one bound = %v, want ses-late", got)
+ }
+ var bound int
+ if err := f.pool.QueryRow(f.ctx, `
+ SELECT count(*) FROM sending_feedback_correlations
+ WHERE operation_id = $1 AND provider_message_id IS NOT NULL`, ref.ID()).Scan(&bound); err != nil {
+ t.Fatal(err)
+ }
+ if bound != 1 {
+ t.Fatalf("%d attempts carry a provider id, want exactly the dialed one", bound)
+ }
+ if err := g.SettleProvider(f.ctx, sendingpolicy.ProviderSettlement{
+ Attempt: auth.Attempt(), Outcome: sendingpolicy.SettlementProviderAccepted, ProviderMessageID: "ses-late",
+ }); err != nil {
+ t.Fatalf("replay by token: %v", err)
+ }
+}
+
+// TestProviderTokenLookupOperationResolvesOnlyDurableOperations: a reference
+// can be recovered for an operation that exists, and for nothing else.
+func TestProviderTokenLookupOperationResolvesOnlyDurableOperations(t *testing.T) {
+ f := newFixture(t)
+ g := f.gate(enforcingPolicy(nil))
+ agent := f.agent(f.user("standard"))
+ _, ref := f.prepareMessage(g, f.message(agent, "own_address", 1))
+
+ got, err := g.LookupOperation(f.ctx, ref.ID())
+ if err != nil || got.ID() != ref.ID() || got.Purpose() != sendingpolicy.PurposeCustomerMessage {
+ t.Fatalf("lookup = %+v err=%v, want the prepared operation", got, err)
+ }
+ if _, err := g.LookupOperation(f.ctx, "msg_never_prepared"); !errors.Is(err, sendingpolicy.ErrSourceUnavailable) {
+ t.Fatalf("lookup of an unknown operation err = %v, want ErrSourceUnavailable", err)
+ }
+ if _, err := g.LookupOperation(f.ctx, ""); !errors.Is(err, sendingpolicy.ErrSourceUnavailable) {
+ t.Fatalf("lookup of an empty id err = %v, want ErrSourceUnavailable", err)
+ }
+}
+
+// TestProviderTokenSettleOperationPrefersTheOldestUnboundDialedAttempt: two
+// attempts dialed and both lost their 250. Evidence arriving in send order
+// binds attempt one first, then attempt two — neither steals the other's id.
+func TestProviderTokenSettleOperationPrefersTheOldestUnboundDialedAttempt(t *testing.T) {
+ f := newFixture(t)
+ g := f.gate(enforcingPolicy(nil))
+ agent := f.agent(f.user("standard"))
+ ref, attempt := f.prepareAndReserve(g, agent, 1)
+ for i := 1; i <= 2; i++ {
+ if i == 2 {
+ var err error
+ if _, attempt, err = g.Reserve(f.ctx, ref); err != nil || attempt.Attempt() != 2 {
+ t.Fatalf("reserve ordinal two: attempt=%v err=%v", attempt, err)
+ }
+ }
+ _, auth, err := g.ConsumeAttempt(f.ctx, attempt)
+ if err != nil || auth == nil {
+ t.Fatalf("authorize %d: auth=%v err=%v", i, auth, err)
+ }
+ if err := g.RedeemProviderCall(f.ctx, *auth); err != nil {
+ t.Fatalf("redeem %d: %v", i, err)
+ }
+ }
+ if err := g.SettleOperation(f.ctx, ref, sendingpolicy.SettlementProviderAccepted, "ses-first"); err != nil {
+ t.Fatalf("settle first evidence: %v", err)
+ }
+ if err := g.SettleOperation(f.ctx, ref, sendingpolicy.SettlementProviderAccepted, "ses-second"); err != nil {
+ t.Fatalf("settle second evidence: %v", err)
+ }
+ if got := f.providerMessageID(ref.ID(), 1); got == nil || *got != "ses-first" {
+ t.Fatalf("attempt one bound = %v, want ses-first", got)
+ }
+ if got := f.providerMessageID(ref.ID(), 2); got == nil || *got != "ses-second" {
+ t.Fatalf("attempt two bound = %v, want ses-second", got)
+ }
+ // A replay for attempt one arriving while a LATER attempt is still
+ // unbound must return to attempt one, never spill onto the unbound one.
+ // Set that shape up on ordinal three.
+ if _, third, err := g.Reserve(f.ctx, ref); err != nil || third.Attempt() != 3 {
+ t.Fatalf("reserve ordinal three: attempt=%v err=%v", third, err)
+ } else {
+ _, auth, err := g.ConsumeAttempt(f.ctx, third)
+ if err != nil || auth == nil {
+ t.Fatalf("authorize 3: auth=%v err=%v", auth, err)
+ }
+ if err := g.RedeemProviderCall(f.ctx, *auth); err != nil {
+ t.Fatalf("redeem 3: %v", err)
+ }
+ }
+ if err := g.SettleOperation(f.ctx, ref, sendingpolicy.SettlementProviderAccepted, "ses-first"); err != nil {
+ t.Fatalf("replay of attempt one with attempt three unbound: %v", err)
+ }
+ if got := f.providerMessageID(ref.ID(), 3); got != nil {
+ t.Fatalf("attempt three bound = %q by a replay of attempt one's id", *got)
+ }
+ if err := g.SettleOperation(f.ctx, ref, sendingpolicy.SettlementProviderAccepted, "ses-third"); err != nil {
+ t.Fatalf("attempt three's own evidence: %v", err)
+ }
+ // Everything bound: a replay of any id is idempotent, a fourth id is a conflict.
+ if err := g.SettleOperation(f.ctx, ref, sendingpolicy.SettlementProviderAccepted, "ses-second"); err != nil {
+ t.Fatalf("replay: %v", err)
+ }
+ if err := g.SettleOperation(f.ctx, ref, sendingpolicy.SettlementProviderAccepted, "ses-fourth"); !errors.Is(err, sendingpolicy.ErrProviderMessageIDConflict) {
+ t.Fatalf("fourth id err = %v, want ErrProviderMessageIDConflict", err)
+ }
+}
diff --git a/internal/testutil/contract_server.go b/internal/testutil/contract_server.go
index 86a3e3835..63ae44ad5 100644
--- a/internal/testutil/contract_server.go
+++ b/internal/testutil/contract_server.go
@@ -18,6 +18,7 @@ import (
"github.com/tokencanopy/e2a/internal/outbound"
"github.com/tokencanopy/e2a/internal/outboundsend"
"github.com/tokencanopy/e2a/internal/relay"
+ "github.com/tokencanopy/e2a/internal/sendingpolicy"
"github.com/tokencanopy/e2a/internal/testutil/testdb"
"github.com/tokencanopy/e2a/internal/unsubscribe"
"github.com/tokencanopy/e2a/internal/usage"
@@ -117,11 +118,15 @@ func StartContractServer(ctx context.Context, dbURL string) (*ContractServer, er
// River enqueue semantics without submitting external email.
outboundSendStore := agent.NewOutboundSendStore(store, outbox, noopUsage)
store.SetScheduledSendFinalizer(outboundSendStore)
+ // The same composition production uses: a config-source gate running the
+ // disabled policy (pass-through admission, every attempt still durable)
+ // and the authorized submitter that refuses to dial without its token.
+ sendingGate := sendingpolicy.NewGate(pool, sendingpolicy.Secrets{}, sendingpolicy.PolicySourceConfig, sendingpolicy.DisabledPolicy())
outboundJobs := outboundsend.NewJobs(
outboundSendStore,
- agent.NewOutboundDeliverer(sender),
+ agent.NewOutboundDeliverer(outbound.NewProviderSubmitter(smtpRelay, sendingGate)),
pool,
- )
+ ).WithGate(sendingGate)
jobsClient, err := jobs.New(pool, jobs.Config{OutboundWorkers: 1}, outboundJobs)
if err != nil {
pool.Close()
diff --git a/internal/testutil/server.go b/internal/testutil/server.go
index c96a3d686..22d37e500 100644
--- a/internal/testutil/server.go
+++ b/internal/testutil/server.go
@@ -27,6 +27,7 @@ import (
"github.com/tokencanopy/e2a/internal/outbound"
"github.com/tokencanopy/e2a/internal/outboundsend"
"github.com/tokencanopy/e2a/internal/relay"
+ "github.com/tokencanopy/e2a/internal/sendingpolicy"
"github.com/tokencanopy/e2a/internal/usage"
"github.com/tokencanopy/e2a/internal/webhook"
"github.com/tokencanopy/e2a/internal/webhookdelivery"
@@ -217,11 +218,15 @@ func TestServer(t *testing.T, pool *pgxpool.Pool, opts ...TestServerOption) *E2A
}
outboundSendStore := agent.NewOutboundSendStore(store, outbox, noopUsage)
store.SetScheduledSendFinalizer(outboundSendStore)
+ // The same composition production uses: a config-source gate running the
+ // disabled policy (pass-through admission, every attempt still durable)
+ // and the authorized submitter that refuses to dial without its token.
+ sendingGate := sendingpolicy.NewGate(pool, sendingpolicy.Secrets{}, sendingpolicy.PolicySourceConfig, sendingpolicy.DisabledPolicy())
outboundJobs := outboundsend.NewJobs(
outboundSendStore,
- agent.NewOutboundDeliverer(sender),
+ agent.NewOutboundDeliverer(outbound.NewProviderSubmitter(smtpRelay, sendingGate)),
pool,
- )
+ ).WithGate(sendingGate)
jobsClient, err := jobs.New(pool, jobs.Config{OutboundWorkers: 2}, outboundJobs)
if err != nil {
t.Fatalf("build River client: %v", err)
diff --git a/sdks/python/src/e2a/v1/errors.py b/sdks/python/src/e2a/v1/errors.py
index 700fa6658..f7abe5e1d 100644
--- a/sdks/python/src/e2a/v1/errors.py
+++ b/sdks/python/src/e2a/v1/errors.py
@@ -193,6 +193,7 @@ def is_retryable_status(status: int) -> bool:
# 403 family
"forbidden": (E2APermissionError, False),
"blocked_by_policy": (E2APermissionError, False),
+ "sending_paused": (E2APermissionError, False),
# 404/410 family — also covers *_not_found via the suffix check in _resolve.
"not_found": (E2ANotFoundError, False),
"gone": (E2ANotFoundError, False),
diff --git a/sdks/python/src/e2a/v1/generated/models/error_body.py b/sdks/python/src/e2a/v1/generated/models/error_body.py
index fc57b1a03..8e60232ef 100644
--- a/sdks/python/src/e2a/v1/generated/models/error_body.py
+++ b/sdks/python/src/e2a/v1/generated/models/error_body.py
@@ -26,7 +26,7 @@ class ErrorBody(BaseModel):
"""
ErrorBody
""" # noqa: E501
- code: StrictStr = Field(description="Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status.")
+ code: StrictStr = Field(description="Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, sending_paused, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental), sending_paused (403, outbound sending is paused for the account by the platform abuse controls; queued mail is held, new sends are refused until an operator resumes; experimental). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status.")
details: Optional[Dict[str, Any]] = Field(default=None, description="Optional structured context, polymorphic by code. Treat it as an open object keyed off code; unknown codes and fields must be preserved.")
message: StrictStr = Field(description="Human-readable explanation. Not for branching — use code.")
request_id: StrictStr = Field(description="Echoes the X-Request-Id response header so a failing call is greppable in logs.")
diff --git a/sdks/python/src/e2a/v1/generated/models/message_lifecycle_transition.py b/sdks/python/src/e2a/v1/generated/models/message_lifecycle_transition.py
index edc2fac22..094aa1b2d 100644
--- a/sdks/python/src/e2a/v1/generated/models/message_lifecycle_transition.py
+++ b/sdks/python/src/e2a/v1/generated/models/message_lifecycle_transition.py
@@ -59,8 +59,8 @@ def outcome_validate_enum(cls, value):
@field_validator('reason_code')
def reason_code_validate_enum(cls, value):
"""Validates the enum"""
- if value not in set(['acceptance.inbound_smtp', 'acceptance.outbound_api', 'acceptance.local_loopback', 'authentication.dmarc_pass', 'authentication.dmarc_fail', 'authentication.dmarc_none', 'authentication.dmarc_temporary_error', 'authentication.dmarc_permanent_error', 'review.hold_created', 'review.approved', 'review.rejected', 'review.expired_approved', 'review.expired_rejected', 'suppression.recipient_blocked', 'suppression.hard_bounce_applied', 'suppression.complaint_applied', 'queue.inbound_processing', 'queue.outbound_submission', 'submission.upstream_accepted', 'submission.local_loopback_accepted', 'submission.temporary_failure', 'submission.provider_rejected', 'submission.local_retries_exhausted', 'submission.cancelled', 'delivery.recipient_server_accepted', 'delivery.temporary_delay', 'delivery.permanent_bounce', 'delivery.transient_bounce', 'delivery.undetermined_bounce', 'complaint.recipient_reported']):
- raise ValueError("must be one of enum values ('acceptance.inbound_smtp', 'acceptance.outbound_api', 'acceptance.local_loopback', 'authentication.dmarc_pass', 'authentication.dmarc_fail', 'authentication.dmarc_none', 'authentication.dmarc_temporary_error', 'authentication.dmarc_permanent_error', 'review.hold_created', 'review.approved', 'review.rejected', 'review.expired_approved', 'review.expired_rejected', 'suppression.recipient_blocked', 'suppression.hard_bounce_applied', 'suppression.complaint_applied', 'queue.inbound_processing', 'queue.outbound_submission', 'submission.upstream_accepted', 'submission.local_loopback_accepted', 'submission.temporary_failure', 'submission.provider_rejected', 'submission.local_retries_exhausted', 'submission.cancelled', 'delivery.recipient_server_accepted', 'delivery.temporary_delay', 'delivery.permanent_bounce', 'delivery.transient_bounce', 'delivery.undetermined_bounce', 'complaint.recipient_reported')")
+ if value not in set(['acceptance.inbound_smtp', 'acceptance.outbound_api', 'acceptance.local_loopback', 'authentication.dmarc_pass', 'authentication.dmarc_fail', 'authentication.dmarc_none', 'authentication.dmarc_temporary_error', 'authentication.dmarc_permanent_error', 'review.hold_created', 'review.approved', 'review.rejected', 'review.expired_approved', 'review.expired_rejected', 'suppression.recipient_blocked', 'suppression.hard_bounce_applied', 'suppression.complaint_applied', 'queue.inbound_processing', 'queue.outbound_submission', 'submission.upstream_accepted', 'submission.local_loopback_accepted', 'submission.temporary_failure', 'submission.provider_rejected', 'submission.local_retries_exhausted', 'submission.cancelled', 'submission.policy_budget_expired', 'submission.sending_setup_expired', 'delivery.recipient_server_accepted', 'delivery.temporary_delay', 'delivery.permanent_bounce', 'delivery.transient_bounce', 'delivery.undetermined_bounce', 'complaint.recipient_reported']):
+ raise ValueError("must be one of enum values ('acceptance.inbound_smtp', 'acceptance.outbound_api', 'acceptance.local_loopback', 'authentication.dmarc_pass', 'authentication.dmarc_fail', 'authentication.dmarc_none', 'authentication.dmarc_temporary_error', 'authentication.dmarc_permanent_error', 'review.hold_created', 'review.approved', 'review.rejected', 'review.expired_approved', 'review.expired_rejected', 'suppression.recipient_blocked', 'suppression.hard_bounce_applied', 'suppression.complaint_applied', 'queue.inbound_processing', 'queue.outbound_submission', 'submission.upstream_accepted', 'submission.local_loopback_accepted', 'submission.temporary_failure', 'submission.provider_rejected', 'submission.local_retries_exhausted', 'submission.cancelled', 'submission.policy_budget_expired', 'submission.sending_setup_expired', 'delivery.recipient_server_accepted', 'delivery.temporary_delay', 'delivery.permanent_bounce', 'delivery.transient_bounce', 'delivery.undetermined_bounce', 'complaint.recipient_reported')")
return value
@field_validator('stage')
diff --git a/sdks/python/tests/test_enum_forward_compat.py b/sdks/python/tests/test_enum_forward_compat.py
index 1275ef8f0..3c47279dc 100644
--- a/sdks/python/tests/test_enum_forward_compat.py
+++ b/sdks/python/tests/test_enum_forward_compat.py
@@ -76,6 +76,8 @@
"submission.provider_rejected",
"submission.local_retries_exhausted",
"submission.cancelled",
+ "submission.policy_budget_expired",
+ "submission.sending_setup_expired",
"delivery.recipient_server_accepted",
"delivery.temporary_delay",
"delivery.permanent_bounce",
diff --git a/sdks/python/tests/test_v1_errors.py b/sdks/python/tests/test_v1_errors.py
index c7be267c9..a1c12658d 100644
--- a/sdks/python/tests/test_v1_errors.py
+++ b/sdks/python/tests/test_v1_errors.py
@@ -231,6 +231,11 @@ def test_catalog_family_overrides():
),
E2APermissionError,
)
+ paused = from_api_exception(
+ _exc(403, body='{"error":{"code":"sending_paused","message":"x"}}')
+ )
+ assert isinstance(paused, E2APermissionError)
+ assert paused.retryable is False
assert isinstance(
from_api_exception(
_exc(409, body='{"error":{"code":"message_not_pending","message":"x"}}')
diff --git a/sdks/typescript/src/v1/errors.ts b/sdks/typescript/src/v1/errors.ts
index bed04c315..f56e9c0c5 100644
--- a/sdks/typescript/src/v1/errors.ts
+++ b/sdks/typescript/src/v1/errors.ts
@@ -108,6 +108,7 @@ const CODE_TABLE: Record = {
// 403
forbidden: { make: mkPermission, retryable: false },
blocked_by_policy: { make: mkPermission, retryable: false },
+ sending_paused: { make: mkPermission, retryable: false },
// 404 / 410 — the *_not_found suffix family resolves in resolve() below.
not_found: { make: mkNotFound, retryable: false },
gone: { make: mkNotFound, retryable: false },
diff --git a/sdks/typescript/src/v1/generated/models/ErrorBody.ts b/sdks/typescript/src/v1/generated/models/ErrorBody.ts
index efb982e43..ba6f627b2 100644
--- a/sdks/typescript/src/v1/generated/models/ErrorBody.ts
+++ b/sdks/typescript/src/v1/generated/models/ErrorBody.ts
@@ -14,7 +14,7 @@ import { HttpFile } from '../http/http.js';
export class ErrorBody {
/**
- * Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status.
+ * Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, sending_paused, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental), sending_paused (403, outbound sending is paused for the account by the platform abuse controls; queued mail is held, new sends are refused until an operator resumes; experimental). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status.
*/
'code': string;
/**
diff --git a/sdks/typescript/src/v1/generated/models/MessageLifecycleTransition.ts b/sdks/typescript/src/v1/generated/models/MessageLifecycleTransition.ts
index cfd726c5e..4c59b8575 100644
--- a/sdks/typescript/src/v1/generated/models/MessageLifecycleTransition.ts
+++ b/sdks/typescript/src/v1/generated/models/MessageLifecycleTransition.ts
@@ -157,6 +157,8 @@ export enum MessageLifecycleTransitionReasonCodeEnum {
SubmissionProviderRejected = 'submission.provider_rejected',
SubmissionLocalRetriesExhausted = 'submission.local_retries_exhausted',
SubmissionCancelled = 'submission.cancelled',
+ SubmissionPolicyBudgetExpired = 'submission.policy_budget_expired',
+ SubmissionSendingSetupExpired = 'submission.sending_setup_expired',
DeliveryRecipientServerAccepted = 'delivery.recipient_server_accepted',
DeliveryTemporaryDelay = 'delivery.temporary_delay',
DeliveryPermanentBounce = 'delivery.permanent_bounce',
diff --git a/sdks/typescript/test/v1/errors.test.ts b/sdks/typescript/test/v1/errors.test.ts
index cb1dfdf72..d1a249da8 100644
--- a/sdks/typescript/test/v1/errors.test.ts
+++ b/sdks/typescript/test/v1/errors.test.ts
@@ -168,6 +168,10 @@ describe("code-first class selection (F2)", () => {
expect(toE2AError({ status: 403, code: "blocked_by_policy", message: "x" })).toBeInstanceOf(
E2APermissionError,
);
+ expect(toE2AError({ status: 403, code: "sending_paused", message: "x" })).toBeInstanceOf(
+ E2APermissionError,
+ );
+ expect(toE2AError({ status: 403, code: "sending_paused", message: "x" }).retryable).toBe(false);
expect(toE2AError({ status: 409, code: "message_not_pending", message: "x" })).toBeInstanceOf(
E2AConflictError,
);
diff --git a/web/src/app/components/messages/MessageLifecycleTimeline.tsx b/web/src/app/components/messages/MessageLifecycleTimeline.tsx
index 2b599ed6c..4aa74f51f 100644
--- a/web/src/app/components/messages/MessageLifecycleTimeline.tsx
+++ b/web/src/app/components/messages/MessageLifecycleTimeline.tsx
@@ -40,6 +40,8 @@ export const LIFECYCLE_PRESENTATION: Record =
"submission.provider_rejected": { title: "Delivery provider rejected message", description: "The delivery provider refused the message, so it was not handed off." },
"submission.local_retries_exhausted": { title: "Delivery failed", description: "e2a could not hand off the message after repeated attempts." },
"submission.cancelled": { title: "Delivery cancelled", description: "Delivery was stopped before the message was handed off." },
+ "submission.policy_budget_expired": { title: "Delivery failed", description: "The message waited for sending capacity for seven days and was not handed off." },
+ "submission.sending_setup_expired": { title: "Delivery failed", description: "Sending setup for this account did not complete in time, so the message was not handed off." },
"delivery.recipient_server_accepted": { title: "Accepted by recipient server", description: "The recipient's mail server accepted the message. This does not confirm inbox placement." },
"delivery.temporary_delay": { title: "Delivery delayed", description: "The delivery provider reported a temporary delay." },
"delivery.permanent_bounce": { title: "Delivery failed permanently", description: "The recipient's mail server permanently rejected the message." },
@@ -79,6 +81,8 @@ function lifecycleSummary(last: MessageLifecycleTransitionWire): string {
case "submission.provider_rejected":
case "submission.local_retries_exhausted":
case "submission.cancelled":
+ case "submission.policy_budget_expired":
+ case "submission.sending_setup_expired":
case "suppression.recipient_blocked":
return "Failed";
default:
diff --git a/web/src/lib/messageLifecycle.ts b/web/src/lib/messageLifecycle.ts
index 8b6fae737..96d361e43 100644
--- a/web/src/lib/messageLifecycle.ts
+++ b/web/src/lib/messageLifecycle.ts
@@ -19,6 +19,7 @@ export const MESSAGE_LIFECYCLE_REASON_CODES = [
"submission.upstream_accepted", "submission.local_loopback_accepted",
"submission.temporary_failure", "submission.provider_rejected",
"submission.local_retries_exhausted", "submission.cancelled",
+ "submission.policy_budget_expired", "submission.sending_setup_expired",
"delivery.recipient_server_accepted", "delivery.temporary_delay",
"delivery.permanent_bounce", "delivery.transient_bounce",
"delivery.undetermined_bounce", "complaint.recipient_reported",