From 63171ffb36a9dc84afea068cd446248a81b1bb30 Mon Sep 17 00:00:00 2001 From: Josh Zhang <39790535+jiashuoz@users.noreply.github.com> Date: Fri, 4 Sep 2026 22:01:41 -0700 Subject: [PATCH 01/12] feat(messagelifecycle): add policy-budget and sending-setup expiry reasons Two additive local failure reasons for the sending-protection holds: submission.policy_budget_expired (a sending-budget hold reached its seven-day deadline) and submission.sending_setup_expired (SES tenant readiness did not land within the 72-hour setup deadline). Both are local, correctable outcomes like submission.local_retries_exhausted, and neither may ever be reported as a recipient rejection or a provider outage. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01AjfGxvXW6fNKWGFHuo68yX --- internal/agent/outbound_ramp_test.go | 152 ------------------------ internal/messagelifecycle/catalog.go | 10 ++ internal/messagelifecycle/model_test.go | 6 +- 3 files changed, 14 insertions(+), 154 deletions(-) delete mode 100644 internal/agent/outbound_ramp_test.go diff --git a/internal/agent/outbound_ramp_test.go b/internal/agent/outbound_ramp_test.go deleted file mode 100644 index c909a5dd6..000000000 --- a/internal/agent/outbound_ramp_test.go +++ /dev/null @@ -1,152 +0,0 @@ -package agent_test - -import ( - "context" - "testing" - "time" - - "github.com/jackc/pgx/v5/pgxpool" - "github.com/tokencanopy/e2a/internal/agent" - "github.com/tokencanopy/e2a/internal/identity" - "github.com/tokencanopy/e2a/internal/outbound" - "github.com/tokencanopy/e2a/internal/outboundsend" - "github.com/tokencanopy/e2a/internal/sendramp" - "github.com/tokencanopy/e2a/internal/testutil" - "github.com/tokencanopy/e2a/internal/usage" -) - -func seedOutboundRampAdapter(t *testing.T, suffix string) (*pgxpool.Pool, *sendramp.Store, string, string, string) { - t.Helper() - pool := testutil.TestDB(t) - ctx := context.Background() - ids := identity.NewStore(pool) - user, err := ids.CreateOrGetUser(ctx, "adapter-"+suffix+"@example.com", "Adapter", "adapter-"+suffix) - if err != nil { - t.Fatal(err) - } - domain := "adapter-" + suffix + ".example.com" - if _, err := ids.ClaimOrCreateDomain(ctx, domain, user.ID); err != nil { - t.Fatal(err) - } - if _, err := pool.Exec(ctx, `UPDATE domains SET sending_status='verified' WHERE domain=$1`, domain); err != nil { - t.Fatal(err) - } - ag, err := ids.CreateAgent(ctx, "agent@"+domain, domain, "", "", "local", user.ID) - if err != nil { - t.Fatal(err) - } - msg, err := ids.CreateOutboundMessage(ctx, ag.ID, []string{"one@example.net"}, nil, nil, "subject", "send", "smtp", "", "", []byte("raw")) - if err != nil { - t.Fatal(err) - } - return pool, sendramp.NewStore(pool), user.ID, domain, msg.ID -} - -// assertNoRampState asserts the full "the ramp wrote nothing" contract for one -// account: the domain never left 'inactive' and no ledger row was created. -func assertNoRampState(t *testing.T, pool *pgxpool.Pool, userID, domain, messageID string) { - t.Helper() - ctx := context.Background() - var status string - if err := pool.QueryRow(ctx, `SELECT sending_ramp_status FROM domains WHERE domain=$1 AND user_id=$2`, domain, userID).Scan(&status); err != nil { - t.Fatalf("read sending_ramp_status: %v", err) - } - if status != sendramp.StatusInactive { - t.Fatalf("sending_ramp_status = %q, want %q: a disabled ramp must not grandfather a domain from the send path", status, sendramp.StatusInactive) - } - for _, q := range []struct{ table, sql string }{ - {"sending_ramp_scopes", `SELECT count(*) FROM sending_ramp_scopes WHERE user_id=$1`}, - {"domain_send_counters", `SELECT count(*) FROM domain_send_counters WHERE user_id=$1`}, - } { - var n int - if err := pool.QueryRow(ctx, q.sql, userID).Scan(&n); err != nil { - t.Fatalf("count %s: %v", q.table, err) - } - if n != 0 { - t.Fatalf("%s has %d rows, want 0", q.table, n) - } - } - var reservations int - if err := pool.QueryRow(ctx, `SELECT count(*) FROM sending_ramp_reservations WHERE message_id=$1`, messageID).Scan(&reservations); err != nil { - t.Fatalf("count sending_ramp_reservations: %v", err) - } - if reservations != 0 { - t.Fatalf("sending_ramp_reservations has %d rows, want 0", reservations) - } -} - -// TestOutboundRampGateDisabledIsPassThrough pins the disabled contract: allow -// the send and write NOTHING. The gate used to stamp the domain 'exempt' on -// every eligible send, which permanently grandfathered any domain that sent -// while the ramp was off — pre-empting the audited one-shot in sendingpolicy -// and, because 'exempt' also reads as "established" to the shared probation -// pool, handing away an abuse bound. Delete-and-re-register made it a repeatable -// reset primitive on top. -func TestOutboundRampGateDisabledIsPassThrough(t *testing.T) { - pool, store, userID, domain, messageID := seedOutboundRampAdapter(t, "disabled") - gate := agent.NewOutboundRampGate(store, sendramp.DefaultSchedule, false) - d, err := gate.Reserve(context.Background(), outboundsend.RampRequest{MessageID: messageID, UserID: userID, Domain: domain, Units: 1}) - if err != nil || !d.Allowed { - t.Fatalf("Reserve = %+v, %v", d, err) - } - assertNoRampState(t, pool, userID, domain, messageID) - - // The read surface (GET /v1/domains/{domain}.sending_ramp.status) therefore - // reports 'inactive', not 'exempt', for a domain sending under a disabled ramp. - snap, err := store.Snapshot(context.Background(), userID, domain, time.Now()) - if err != nil || snap.Status != sendramp.StatusInactive { - t.Fatalf("Snapshot = %+v, %v, want status %q", snap, err, sendramp.StatusInactive) - } -} - -// TestSendWorkerDisabledRampWritesNoRampState is the same contract one level -// out: a real ramp-eligible send (own_address, message_type send) driven -// through the send worker with the ramp disabled must leave the domain -// 'inactive' and the ramp ledger empty. -func TestSendWorkerDisabledRampWritesNoRampState(t *testing.T) { - api, store, outbox, _, pool := setupAsyncAPIWithPool(t) - ctx := context.Background() - user, ag := selfAgent(t, store, "rampdisabled") - if err := store.SetSendingStatus(ctx, ag.RegisteredDomain, "verified", "verified", "verified", "", nil); err != nil { - t.Fatalf("SetSendingStatus: %v", err) - } - res, oerr := api.DeliverOutbound(ctx, user, ag, outbound.SendRequest{ - To: []string{"recipient@external.test"}, Subject: "disabled ramp send", Body: "x", - }, "send", "", nil, nil) - if oerr != nil { - t.Fatalf("DeliverOutbound: %+v", oerr) - } - - ramp := agent.NewOutboundRampGate(sendramp.NewStore(pool), sendramp.DefaultSchedule, false) - deliverer := &countingDeliverer{out: outboundsend.DeliverOutcome{ProviderMessageID: "provider-disabled-ramp", SentAs: "own_address"}} - worker := outboundsend.NewSendWorker( - agent.NewOutboundSendStore(store, outbox, usage.NewNoopUsageTracker()), deliverer, ramp) - - if err := worker.Work(ctx, workerJobWithID(res.MessageID, 999, 1)); err != nil { - t.Fatalf("worker.Work: %v", err) - } - if deliverer.calls != 1 { - t.Fatalf("deliverer calls = %d, want 1: the disabled ramp must still allow the send", deliverer.calls) - } - assertNoRampState(t, pool, user.ID, ag.RegisteredDomain, res.MessageID) -} - -func TestOutboundRampGateInjectsDayAndDelegatesLifecycle(t *testing.T) { - _, store, userID, domain, messageID := seedOutboundRampAdapter(t, "enabled") - day := time.Date(2026, 7, 2, 23, 30, 0, 0, time.FixedZone("west", -7*60*60)) - gate := agent.NewOutboundRampGate(store, sendramp.NewSchedule(50, 100, 2), true, func() time.Time { return day }) - d, err := gate.Reserve(context.Background(), outboundsend.RampRequest{MessageID: messageID, UserID: userID, Domain: domain, Units: 25}) - if err != nil || !d.Allowed { - t.Fatalf("Reserve = %+v, %v", d, err) - } - if err := gate.Confirm(context.Background(), messageID); err != nil { - t.Fatal(err) - } - snap, err := store.Snapshot(context.Background(), userID, domain, day) - if err != nil { - t.Fatal(err) - } - if snap.ActiveDays != 1 || snap.UsedToday != 25 { - t.Fatalf("Snapshot = %+v", snap) - } -} diff --git a/internal/messagelifecycle/catalog.go b/internal/messagelifecycle/catalog.go index b7f71272d..a85c270ea 100644 --- a/internal/messagelifecycle/catalog.go +++ b/internal/messagelifecycle/catalog.go @@ -66,6 +66,14 @@ const ( ReasonSubmissionProviderRejected ReasonCode = "submission.provider_rejected" ReasonSubmissionLocalRetriesExhausted ReasonCode = "submission.local_retries_exhausted" ReasonSubmissionCancelled ReasonCode = "submission.cancelled" + // ReasonSubmissionPolicyBudgetExpired means a sending-budget hold reached + // its seven-day deadline without capacity freeing. It is a local policy + // outcome, never a recipient rejection or a provider outage. + ReasonSubmissionPolicyBudgetExpired ReasonCode = "submission.policy_budget_expired" + // ReasonSubmissionSendingSetupExpired means the account's provider-side + // sending setup (SES tenant readiness) did not complete within the + // 72-hour setup deadline. + ReasonSubmissionSendingSetupExpired ReasonCode = "submission.sending_setup_expired" ReasonDeliveryRecipientServerAccepted ReasonCode = "delivery.recipient_server_accepted" ReasonDeliveryTemporaryDelay ReasonCode = "delivery.temporary_delay" ReasonDeliveryPermanentBounce ReasonCode = "delivery.permanent_bounce" @@ -106,6 +114,8 @@ var canonicalCatalog = map[ReasonCode]Definition{ ReasonSubmissionProviderRejected: {StageSubmission, OutcomeFailed, false}, ReasonSubmissionLocalRetriesExhausted: {StageSubmission, OutcomeFailed, true}, ReasonSubmissionCancelled: {StageSubmission, OutcomeFailed, false}, + ReasonSubmissionPolicyBudgetExpired: {StageSubmission, OutcomeFailed, true}, + ReasonSubmissionSendingSetupExpired: {StageSubmission, OutcomeFailed, true}, ReasonDeliveryRecipientServerAccepted: {StageDelivery, OutcomeDelivered, false}, ReasonDeliveryTemporaryDelay: {StageDelivery, OutcomeDeferred, true}, ReasonDeliveryPermanentBounce: {StageDelivery, OutcomeBounced, false}, diff --git a/internal/messagelifecycle/model_test.go b/internal/messagelifecycle/model_test.go index 47ef4fdcb..7e741d490 100644 --- a/internal/messagelifecycle/model_test.go +++ b/internal/messagelifecycle/model_test.go @@ -42,6 +42,8 @@ func TestCatalogIsExhaustive(t *testing.T) { {ReasonSubmissionProviderRejected, StageSubmission, OutcomeFailed, false}, {ReasonSubmissionLocalRetriesExhausted, StageSubmission, OutcomeFailed, true}, {ReasonSubmissionCancelled, StageSubmission, OutcomeFailed, false}, + {ReasonSubmissionPolicyBudgetExpired, StageSubmission, OutcomeFailed, true}, + {ReasonSubmissionSendingSetupExpired, StageSubmission, OutcomeFailed, true}, {ReasonDeliveryRecipientServerAccepted, StageDelivery, OutcomeDelivered, false}, {ReasonDeliveryTemporaryDelay, StageDelivery, OutcomeDeferred, true}, {ReasonDeliveryPermanentBounce, StageDelivery, OutcomeBounced, false}, @@ -51,7 +53,7 @@ func TestCatalogIsExhaustive(t *testing.T) { } catalog := Catalog() - if got, want := len(catalog), 30; got != want { + if got, want := len(catalog), 32; got != want { t.Fatalf("Catalog() length = %d, want %d", got, want) } seen := make(map[ReasonCode]bool, len(tests)) @@ -93,7 +95,7 @@ func TestCatalogRejectsUnknownAndCannotBeMutated(t *testing.T) { if !ok || got != (Definition{Stage: StageAccepted, Outcome: OutcomeAccepted}) { t.Fatalf("caller mutation changed canonical lookup: %+v, %v", got, ok) } - if got := len(Catalog()); got != 30 { + if got := len(Catalog()); got != 32 { t.Fatalf("caller mutation changed canonical catalog length to %d", got) } } From d9bc2b9a1709eb5818ad59a28c61e9077efc2498 Mon Sep 17 00:00:00 2001 From: Josh Zhang <39790535+jiashuoz@users.noreply.github.com> Date: Fri, 4 Sep 2026 22:01:41 -0700 Subject: [PATCH 02/12] feat(sendingpolicy): settle and look up operations by id for evidence paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two callers hold provider evidence but no token: the worker that finds provider-accept evidence already recorded on a row it is about to re-drive, and the terminal reconciler settling a stranded row from that same evidence. Neither can name an ordinal. SettleOperation applies the outcome to the latest attempt whose provider call started — never a later ordinal that was only reserved, and nothing when no attempt ever dialed — through the same body SettleProvider uses. LookupOperation recovers a reference for an operation that already exists; it is not a constructor, and every Gate method still reloads the row under lock. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01AjfGxvXW6fNKWGFHuo68yX --- internal/sendingpolicy/gate.go | 68 +++++++++++++++++- internal/sendingpolicy/provider_token_test.go | 69 +++++++++++++++++++ 2 files changed, 135 insertions(+), 2 deletions(-) diff --git a/internal/sendingpolicy/gate.go b/internal/sendingpolicy/gate.go index 1bf71b0d8..2a47826fa 100644 --- a/internal/sendingpolicy/gate.go +++ b/internal/sendingpolicy/gate.go @@ -33,6 +33,8 @@ type Gate interface { DeferAttempt(context.Context, AttemptRef) error CancelAttempt(context.Context, AttemptRef) error SettleProvider(context.Context, ProviderSettlement) error + SettleOperation(context.Context, OperationRef, SettlementOutcome, string) error + LookupOperation(context.Context, string) (OperationRef, error) } var _ Gate = (*Module)(nil) @@ -1809,6 +1811,55 @@ func (m *Module) SettleProvider(ctx context.Context, settlement ProviderSettleme if settlement.Attempt.IsZero() { return ErrSourceUnavailable } + return m.settle(ctx, settlement.Attempt.operationID, settlement.Attempt.attempt, settlement) +} + +// LookupOperation recovers a reference to an operation that already exists. +// +// This is not a constructor: it returns a reference only for a durable +// operation row, and the reference carries an id and advisory fields exactly +// as a deserialized River argument does — every Gate method reloads the row +// under lock, so recovering a reference grants nothing. It exists for the +// reconciler, which learns of provider evidence by message id long after the +// worker and its token are gone. +func (m *Module) LookupOperation(ctx context.Context, operationID string) (OperationRef, error) { + if strings.TrimSpace(operationID) == "" { + return OperationRef{}, ErrSourceUnavailable + } + var row operationRow + err := m.pool.QueryRow(ctx, ` + SELECT operation_id, source_account_ref, policy_subject_ref, purpose, shared_reputation + FROM sending_provider_operations + WHERE operation_id = $1`, operationID, + ).Scan(&row.OperationID, &row.SourceAccountRef, &row.PolicySubjectRef, &row.Purpose, &row.Shared) + if errors.Is(err, pgx.ErrNoRows) { + return OperationRef{}, ErrSourceUnavailable + } + if err != nil { + return OperationRef{}, fmt.Errorf("sendingpolicy: lookup operation: %w", err) + } + return row.ref(), nil +} + +// SettleOperation applies a delayed authoritative provider outcome to the +// attempt of an operation that most recently opened a socket. +// +// It exists for the two callers that hold evidence but no token: the worker +// that finds provider-accept evidence already recorded on a row it is about to +// re-drive, and the terminal reconciler settling a stranded row from that same +// evidence. Neither can name an ordinal — the token that could is gone with the +// process that held it — but both know which OPERATION the evidence belongs to, +// and the only attempt evidence can describe is the latest one that dialed. +func (m *Module) SettleOperation(ctx context.Context, ref OperationRef, outcome SettlementOutcome, providerMessageID string) error { + if ref.IsZero() { + return ErrSourceUnavailable + } + return m.settle(ctx, ref.id, 0, ProviderSettlement{Outcome: outcome, ProviderMessageID: providerMessageID}) +} + +// settle is the shared settlement body. attempt 0 means "the latest attempt +// whose provider call started", resolved under the operation lock. +func (m *Module) settle(ctx context.Context, operationID string, attempt int, settlement ProviderSettlement) error { if !settlement.Outcome.valid() { return fmt.Errorf("sendingpolicy: unsupported settlement outcome %q", settlement.Outcome) } @@ -1819,11 +1870,24 @@ func (m *Module) SettleProvider(ctx context.Context, settlement ProviderSettleme } defer func() { _ = tx.Rollback(ctx) }() - op, err := lockOperation(ctx, tx, settlement.Attempt.operationID) + op, err := lockOperation(ctx, tx, operationID) if err != nil { return err } - stored, err := lockReservation(ctx, tx, settlement.Attempt.operationID, settlement.Attempt.attempt) + if attempt == 0 { + if err := tx.QueryRow(ctx, ` + SELECT COALESCE(MAX(submission_attempt), 0) + FROM sending_budget_reservations + WHERE operation_id = $1 AND call_state = 'started'`, operationID, + ).Scan(&attempt); err != nil { + return fmt.Errorf("sendingpolicy: find started attempt: %w", err) + } + if attempt == 0 { + return ErrAttemptStale + } + } + settlement.Attempt = AttemptRef{operationID: operationID, attempt: attempt} + stored, err := lockReservation(ctx, tx, operationID, attempt) if err != nil { return err } diff --git a/internal/sendingpolicy/provider_token_test.go b/internal/sendingpolicy/provider_token_test.go index 1e3d2d696..9c0a3085e 100644 --- a/internal/sendingpolicy/provider_token_test.go +++ b/internal/sendingpolicy/provider_token_test.go @@ -269,3 +269,72 @@ func TestProviderTokenSettlementComparesNormalizedProviderMessageID(t *testing.T t.Fatalf("different id err = %v, want ErrProviderMessageIDConflict", err) } } + +// TestProviderTokenSettleOperationTargetsTheLatestDialedAttempt: evidence that +// arrives without a token settles the most recent attempt that opened a +// socket — not a later ordinal that was only reserved, and nothing at all when +// no attempt ever dialed. +func TestProviderTokenSettleOperationTargetsTheLatestDialedAttempt(t *testing.T) { + f := newFixture(t) + g := f.gate(enforcingPolicy(nil)) + agent := f.agent(f.user("standard")) + ref, attempt := f.prepareAndReserve(g, agent, 1) + + err := g.SettleOperation(f.ctx, ref, sendingpolicy.SettlementProviderAccepted, "ses-early") + if !errors.Is(err, sendingpolicy.ErrAttemptStale) { + t.Fatalf("settle before any dial err = %v, want ErrAttemptStale", err) + } + + _, auth, err := g.ConsumeAttempt(f.ctx, attempt) + if err != nil || auth == nil { + t.Fatalf("authorize: auth=%v err=%v", auth, err) + } + if err := g.RedeemProviderCall(f.ctx, *auth); err != nil { + t.Fatalf("redeem: %v", err) + } + // The worker died after the socket opened; a later execution re-reserved + // ordinal two but never consumed it. Delayed evidence belongs to ordinal one. + if _, next, err := g.Reserve(f.ctx, ref); err != nil || next.Attempt() != 2 { + t.Fatalf("re-reserve: attempt=%v err=%v", next, err) + } + if err := g.SettleOperation(f.ctx, ref, sendingpolicy.SettlementProviderAccepted, ""); err != nil { + t.Fatalf("settle by operation: %v", err) + } + if got := f.providerMessageID(ref.ID(), 1); got == nil || *got != "ses-late" { + t.Fatalf("attempt one bound = %v, want ses-late", got) + } + var bound int + if err := f.pool.QueryRow(f.ctx, ` + SELECT count(*) FROM sending_feedback_correlations + WHERE operation_id = $1 AND provider_message_id IS NOT NULL`, ref.ID()).Scan(&bound); err != nil { + t.Fatal(err) + } + if bound != 1 { + t.Fatalf("%d attempts carry a provider id, want exactly the dialed one", bound) + } + if err := g.SettleProvider(f.ctx, sendingpolicy.ProviderSettlement{ + Attempt: auth.Attempt(), Outcome: sendingpolicy.SettlementProviderAccepted, ProviderMessageID: "ses-late", + }); err != nil { + t.Fatalf("replay by token: %v", err) + } +} + +// TestProviderTokenLookupOperationResolvesOnlyDurableOperations: a reference +// can be recovered for an operation that exists, and for nothing else. +func TestProviderTokenLookupOperationResolvesOnlyDurableOperations(t *testing.T) { + f := newFixture(t) + g := f.gate(enforcingPolicy(nil)) + agent := f.agent(f.user("standard")) + _, ref := f.prepareMessage(g, f.message(agent, "own_address", 1)) + + got, err := g.LookupOperation(f.ctx, ref.ID()) + if err != nil || got.ID() != ref.ID() || got.Purpose() != sendingpolicy.PurposeCustomerMessage { + t.Fatalf("lookup = %+v err=%v, want the prepared operation", got, err) + } + if _, err := g.LookupOperation(f.ctx, "msg_never_prepared"); !errors.Is(err, sendingpolicy.ErrSourceUnavailable) { + t.Fatalf("lookup of an unknown operation err = %v, want ErrSourceUnavailable", err) + } + if _, err := g.LookupOperation(f.ctx, ""); !errors.Is(err, sendingpolicy.ErrSourceUnavailable) { + t.Fatalf("lookup of an empty id err = %v, want ErrSourceUnavailable", err) + } +} From f1fb9fc95ae3c43ac2c731daefca1b2a2e1ce1a1 Mon Sep 17 00:00:00 2001 From: Josh Zhang <39790535+jiashuoz@users.noreply.github.com> Date: Fri, 4 Sep 2026 22:01:41 -0700 Subject: [PATCH 03/12] feat(identity): carry finite-hold state on the send claim MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The send claim now returns the message's persisted hold class and anchor (migration 116) plus the owning account's last_resumed_at and ses_tenant_ready_at, so every worker execution can re-derive the same deadline. RecordOutboundHold writes the pair only while the message is pre-terminal; every terminal write — sent, failed, evidence-settled, trash-cancelled — clears it, so a stale hold can never outlive its message's outcome. The two new local expiry reasons are recognized as complete terminal fallbacks. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01AjfGxvXW6fNKWGFHuo68yX --- internal/identity/delivery_store.go | 63 ++++++++++++-- internal/identity/outbound_hold_test.go | 110 ++++++++++++++++++++++++ 2 files changed, 167 insertions(+), 6 deletions(-) create mode 100644 internal/identity/outbound_hold_test.go diff --git a/internal/identity/delivery_store.go b/internal/identity/delivery_store.go index 3be22a369..4d53d0c8d 100644 --- a/internal/identity/delivery_store.go +++ b/internal/identity/delivery_store.go @@ -319,6 +319,7 @@ func (s *Store) RecordDeliveryOutcomeTx(ctx context.Context, tx pgx.Tx, messageI if _, err := tx.Exec(ctx, `UPDATE messages SET delivery_status = 'failed', + local_hold_class = NULL, local_hold_anchor = NULL, delivery_failure_source = COALESCE(delivery_failure_source, 'provider') WHERE id = $1`, messageID, ); err != nil { @@ -373,7 +374,7 @@ func (s *Store) MarkMessageSent(ctx context.Context, messageID, sentAs string, t defer tx.Rollback(ctx) if _, err := tx.Exec(ctx, - `UPDATE messages SET delivery_status = 'sent', sent_as = $2 WHERE id = $1`, + `UPDATE messages SET delivery_status = 'sent', sent_as = $2, local_hold_class = NULL, local_hold_anchor = NULL WHERE id = $1`, messageID, nullIfEmpty(sentAs), ); err != nil { return err @@ -444,6 +445,18 @@ type OutboundSendPayload struct { ReviewedAt *time.Time // ProviderMessageID is the evidence-repaired provider id ('' when none). ProviderMessageID string + // LocalHoldClass / LocalHoldAnchor are the durable finite-hold state the + // worker persisted on an earlier execution ('' / nil when the message has + // never entered a finite hold). The absolute deadline is always derived + // from this pair, never stored. + LocalHoldClass string + LocalHoldAnchor *time.Time + // LastResumedAt is account_sending_controls.last_resumed_at for the owning + // account; TenantReadyAt is its ses_tenant_ready_at (nil until the SES + // tenant is ready). Both feed the worker's hold-anchor and setup→rate + // transition rules. nil when the account has no control row yet. + LastResumedAt *time.Time + TenantReadyAt *time.Time } // OutboundSentInfo carries the fields the async worker's MarkSent/MarkFailed @@ -590,6 +603,10 @@ func (s *Store) ClaimOutboundForSend(ctx context.Context, messageID string, jobI failureAttempt *int scheduledAt *time.Time reviewedAt *time.Time + holdClass string + holdAnchor *time.Time + lastResumedAt *time.Time + tenantReadyAt *time.Time ) var userID, registeredDomain string // Lock agent first to match permanent agent deletion's lock order, then @@ -613,14 +630,18 @@ func (s *Store) ClaimOutboundForSend(ctx context.Context, messageID string, jobI m.to_recipients, m.cc, m.bcc, m.raw_message, m.created_at, m.deleted_at, m.send_job_id, m.provider_accepted_at, COALESCE(m.provider_message_id,''), COALESCE(m.delivery_failure_source,''),COALESCE(m.delivery_failure_reason_code,''), - m.delivery_failure_occurred_at,m.delivery_failure_attempt,m.scheduled_at,m.reviewed_at + m.delivery_failure_occurred_at,m.delivery_failure_attempt,m.scheduled_at,m.reviewed_at, + COALESCE(m.local_hold_class,''), m.local_hold_anchor, + c.last_resumed_at, c.ses_tenant_ready_at FROM messages m + LEFT JOIN account_sending_controls c ON c.user_id = $3 WHERE m.id = $1 AND m.agent_id = $2 AND m.direction = 'outbound' FOR UPDATE OF m`, - messageID, agentID, + messageID, agentID, userID, ).Scan(&deliveryStatus, &envelopeFrom, &sentAs, &messageType, &to, &cc, &bcc, &raw, &createdAt, &deletedAt, &stampedJobID, &providerAcceptedAt, &providerMessageID, - &failureSource, &failureReason, &failureOccurredAt, &failureAttempt, &scheduledAt, &reviewedAt) + &failureSource, &failureReason, &failureOccurredAt, &failureAttempt, &scheduledAt, &reviewedAt, + &holdClass, &holdAnchor, &lastResumedAt, &tenantReadyAt) if errors.Is(err, pgx.ErrNoRows) { if err := tx.Commit(ctx); err != nil { return nil, err @@ -665,6 +686,7 @@ func (s *Store) ClaimOutboundForSend(ctx context.Context, messageID string, jobI if _, err := tx.Exec(ctx, `UPDATE messages SET delivery_status = 'failed', + local_hold_class = NULL, local_hold_anchor = NULL, delivery_detail = 'send canceled because the message or agent is in trash', delivery_failure_source = 'local', delivery_failure_reason_code = 'submission.cancelled', @@ -710,6 +732,10 @@ func (s *Store) ClaimOutboundForSend(ctx context.Context, messageID string, jobI ProviderMessageID: providerMessageID, ScheduledAt: scheduledAt, ReviewedAt: reviewedAt, + LocalHoldClass: holdClass, + LocalHoldAnchor: holdAnchor, + LastResumedAt: lastResumedAt, + TenantReadyAt: tenantReadyAt, } if err := tx.Commit(ctx); err != nil { return nil, err @@ -717,6 +743,28 @@ func (s *Store) ClaimOutboundForSend(ctx context.Context, messageID string, jobI return p, nil } +// RecordOutboundHold persists a message's finite-hold class and anchor. +// +// The worker owns the transition rules (first finite hold, setup→rate, +// monotonic promotion to policy_budget); this writes exactly the pair it was +// given and only while the message is still pre-terminal. Terminal writes +// clear the pair, so a stale hold can never outlive its message's outcome. +func (s *Store) RecordOutboundHold(ctx context.Context, messageID, class string, anchor time.Time) error { + if class == "" || anchor.IsZero() { + return fmt.Errorf("record outbound hold: class and anchor are required") + } + _, err := s.pool.Exec(ctx, ` + UPDATE messages + SET local_hold_class = $2, local_hold_anchor = $3 + WHERE id = $1 AND direction = 'outbound' + AND delivery_status IN ('accepted', 'sending')`, + messageID, class, anchor.UTC()) + if err != nil { + return fmt.Errorf("record outbound hold: %w", err) + } + return nil +} + func isCompleteTerminalFallback(source, reason string, occurredAt *time.Time, attempt *int) bool { if occurredAt == nil || occurredAt.IsZero() || attempt == nil || *attempt < 0 { return false @@ -724,7 +772,8 @@ func isCompleteTerminalFallback(source, reason string, occurredAt *time.Time, at switch messagelifecycle.ReasonCode(reason) { case messagelifecycle.ReasonSubmissionProviderRejected: return delivery.FailureSource(source) == delivery.FailureSourceProvider - case messagelifecycle.ReasonSubmissionLocalRetriesExhausted, messagelifecycle.ReasonSubmissionCancelled: + case messagelifecycle.ReasonSubmissionLocalRetriesExhausted, messagelifecycle.ReasonSubmissionCancelled, + messagelifecycle.ReasonSubmissionPolicyBudgetExpired, messagelifecycle.ReasonSubmissionSendingSetupExpired: return delivery.FailureSource(source) == delivery.FailureSourceLocal default: return false @@ -808,6 +857,7 @@ func (s *Store) MarkOutboundSentTx(ctx context.Context, tx pgx.Tx, messageID, pr err := tx.QueryRow(ctx, `UPDATE messages m SET delivery_status = 'sent', provider_message_id = $2, send_claimed_at = NULL, + local_hold_class = NULL, local_hold_anchor = NULL, rfc_message_id_key = CASE WHEN rfc_message_id_key IS NULL AND $3 <> '' THEN $3 ELSE rfc_message_id_key @@ -891,7 +941,7 @@ func (s *Store) ResolveOutboundProviderAcceptedTx(ctx context.Context, tx pgx.Tx m := &Message{ID: messageID, Direction: "outbound", DeliveryStatus: "sent"} err = tx.QueryRow(ctx, `UPDATE messages m - SET delivery_status = 'sent', send_claimed_at = NULL, delivery_failure_source = NULL, delivery_failure_reason_code = NULL, delivery_detail = NULL, + SET delivery_status = 'sent', send_claimed_at = NULL, local_hold_class = NULL, local_hold_anchor = NULL, delivery_failure_source = NULL, delivery_failure_reason_code = NULL, delivery_detail = NULL, delivery_failure_occurred_at=NULL, delivery_failure_attempt=NULL, delivery_failure_blocked_recipients=NULL FROM agent_identities a WHERE m.id = $1 AND m.direction = 'outbound' @@ -981,6 +1031,7 @@ func (s *Store) MarkOutboundFailedTx(ctx context.Context, tx pgx.Tx, messageID, err := tx.QueryRow(ctx, `UPDATE messages m SET delivery_status = 'failed', + local_hold_class = NULL, local_hold_anchor = NULL, delivery_detail = COALESCE(NULLIF(m.delivery_detail, ''), $2), delivery_failure_source = $3, send_claimed_at = NULL diff --git a/internal/identity/outbound_hold_test.go b/internal/identity/outbound_hold_test.go new file mode 100644 index 000000000..88f8df06d --- /dev/null +++ b/internal/identity/outbound_hold_test.go @@ -0,0 +1,110 @@ +package identity_test + +import ( + "context" + "testing" + "time" + + "github.com/jackc/pgx/v5" + + "github.com/tokencanopy/e2a/internal/identity" + "github.com/tokencanopy/e2a/internal/testutil" +) + +// The finite-hold pair rides the claim payload so every worker execution +// re-derives the same deadline, and it is cleared by the terminal write so a +// stale hold can never outlive its message's outcome. +func TestOutboundHoldRidesTheClaimAndClearsOnTerminal(t *testing.T) { + pool := testutil.TestDB(t) + store := identity.NewStore(pool) + ctx := context.Background() + agentID := convoTestSetup(t, store, "hold-claim") + + var userID string + if err := pool.QueryRow(ctx, `SELECT user_id FROM agent_identities WHERE id = $1`, agentID).Scan(&userID); err != nil { + t.Fatal(err) + } + resumed := time.Date(2026, 9, 1, 8, 0, 0, 0, time.UTC) + ready := time.Date(2026, 9, 2, 9, 30, 0, 0, time.UTC) + if _, err := pool.Exec(ctx, ` + INSERT INTO account_sending_controls (user_id, last_resumed_at, ses_tenant_name, ses_tenant_ready, ses_tenant_ready_at) + VALUES ($1, $2, 'tenant_hold_test', true, $3) + ON CONFLICT (user_id) DO UPDATE SET last_resumed_at = $2, ses_tenant_ready = true, ses_tenant_ready_at = $3`, + userID, resumed, ready, + ); err != nil { + t.Fatal(err) + } + + var msgID string + if err := store.WithTx(ctx, func(tx pgx.Tx) error { + m, err := store.CreateOutboundMessageTx(ctx, tx, agentID, + []string{"one@example.test"}, nil, nil, "Hold", "send", "smtp", "", "conv-hold", + []byte("From: bot\r\n\r\nbody"), "accepted", "agent@test.e2a.dev", "relay") + if err != nil { + return err + } + msgID = m.ID + return store.StampSendJobIDTx(ctx, tx, m.ID, 4242) + }); err != nil { + t.Fatalf("seed: %v", err) + } + + p, err := store.ClaimOutboundForSend(ctx, msgID, 4242) + if err != nil || p == nil { + t.Fatalf("claim: payload=%v err=%v", p, err) + } + if p.LocalHoldClass != "" || p.LocalHoldAnchor != nil { + t.Fatalf("fresh claim carries a hold: %q %v", p.LocalHoldClass, p.LocalHoldAnchor) + } + if p.LastResumedAt == nil || !p.LastResumedAt.Equal(resumed) || p.TenantReadyAt == nil || !p.TenantReadyAt.Equal(ready) { + t.Fatalf("control timestamps = %v / %v, want %v / %v", p.LastResumedAt, p.TenantReadyAt, resumed, ready) + } + if err := store.ReleaseOutboundSendClaim(ctx, msgID, 4242); err != nil { + t.Fatal(err) + } + + anchor := time.Date(2026, 9, 3, 10, 0, 0, 0, time.UTC) + if err := store.RecordOutboundHold(ctx, msgID, "policy_budget", anchor); err != nil { + t.Fatalf("record hold: %v", err) + } + p, err = store.ClaimOutboundForSend(ctx, msgID, 4242) + if err != nil || p == nil { + t.Fatalf("re-claim: payload=%v err=%v", p, err) + } + if p.LocalHoldClass != "policy_budget" || p.LocalHoldAnchor == nil || !p.LocalHoldAnchor.Equal(anchor) { + t.Fatalf("hold on re-claim = %q %v, want policy_budget @ %v", p.LocalHoldClass, p.LocalHoldAnchor, anchor) + } + + if err := store.WithTx(ctx, func(tx pgx.Tx) error { + _, err := store.MarkOutboundSentTx(ctx, tx, msgID, "") + return err + }); err != nil { + t.Fatalf("mark sent: %v", err) + } + var class *string + var holdAnchor *time.Time + if err := pool.QueryRow(ctx, `SELECT local_hold_class, local_hold_anchor FROM messages WHERE id = $1`, msgID).Scan(&class, &holdAnchor); err != nil { + t.Fatal(err) + } + if class != nil || holdAnchor != nil { + t.Fatalf("hold survived the terminal write: %v %v", class, holdAnchor) + } + // A terminal row refuses a late hold write. + if err := store.RecordOutboundHold(ctx, msgID, "policy_budget", anchor); err != nil { + t.Fatalf("late hold write errored: %v", err) + } + if err := pool.QueryRow(ctx, `SELECT local_hold_class FROM messages WHERE id = $1`, msgID).Scan(&class); err != nil { + t.Fatal(err) + } + if class != nil { + t.Fatalf("hold written on a sent row: %q", *class) + } +} + +func TestOutboundHoldRejectsAnEmptyPair(t *testing.T) { + pool := testutil.TestDB(t) + store := identity.NewStore(pool) + if err := store.RecordOutboundHold(context.Background(), "msg_none", "", time.Time{}); err == nil { + t.Fatal("empty class and anchor accepted") + } +} From 857ee344ef667ea97a261fe899065925e1d56b3a Mon Sep 17 00:00:00 2001 From: Josh Zhang <39790535+jiashuoz@users.noreply.github.com> Date: Fri, 4 Sep 2026 22:01:41 -0700 Subject: [PATCH 04/12] feat(outbound): enforce sending policy at fire time The outbound send worker now authorizes every provider call through the sending-protection Gate, in the fixed order the design names: Reserve the durable attempt; snooze on an early hold without provider I/O; DeferAttempt on a rate deferral and CancelAttempt on a final suppression match; ConsumeAttempt as the last serialized decision; then the authorized submitter, which redeems the token immediately before the socket opens and settles the provider's answer. A later execution after a confirmed attempt returns to Reserve, which allocates the next ordinal. The worker-owned RampGate and agent.NewOutboundRampGate are removed: the ramp is composed inside the gate and its progress moves only through settlement. The Deliverer contract carries the token; the production deliverer is outbound.ProviderSubmitter and refuses to dial without one. A lost 250 (ErrProviderAcceptanceUnknown) is retried as a new ordinal and never settled. Enqueue prepares the operation in the accept transaction, between the message insert and the River insert; a paused account is refused there (ErrSendingPaused, HTTP 403 sending_paused) rather than queued. Jobs from a pre-floor slot carry no reference and resolve at fire time through the same Prepare path. Finite holds persist a class and anchor on the message and derive the deadline every execution: 72 hours for rate/ramp/provider and tenant setup, seven days for policy budget. The first finite hold anchors at the latest of accept, schedule, review, and last resume; a budget hold promotes any class and keeps the anchor; policy_budget never changes again; tenant readiness landing inside the setup deadline moves the class to rate/ramp/provider exactly once; a pause has no clock but a running deadline keeps running. Expiry emits the class's own reason. Terminal reconciliation is settlement-only: an evidence-settled row also settles the attempt that dialed through Gate.SettleOperation. cmd/e2a gains one composition root (newOutboundSending) and a wiring test that proves the registered send path holds the concrete gate and the ProviderSubmitter-backed deliverer. The test servers build the same composition with the disabled policy. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01AjfGxvXW6fNKWGFHuo68yX --- cmd/e2a/main.go | 32 +- cmd/e2a/outbound_wiring.go | 51 ++ cmd/e2a/sending_policy_wiring_test.go | 54 ++ docs/design/async-message-pipeline.md | 36 + internal/agent/api.go | 7 + internal/agent/outbound_async.go | 120 +-- internal/agent/outbound_async_test.go | 9 +- .../agent/outbound_suppression_guard_test.go | 126 +-- internal/agent/test_send_async_test.go | 3 +- internal/outboundsend/gate_worker_test.go | 370 +++++++++ internal/outboundsend/jobs.go | 79 +- internal/outboundsend/jobs_gate_test.go | 286 +++++++ internal/outboundsend/rate_test.go | 39 +- internal/outboundsend/reconcile_test.go | 110 +-- internal/outboundsend/suppression_test.go | 48 +- internal/outboundsend/terminal_reconcile.go | 89 +- internal/outboundsend/worker.go | 770 ++++++++++++------ internal/outboundsend/worker_test.go | 344 +++----- internal/testutil/contract_server.go | 9 +- internal/testutil/server.go | 9 +- 20 files changed, 1620 insertions(+), 971 deletions(-) create mode 100644 cmd/e2a/outbound_wiring.go create mode 100644 cmd/e2a/sending_policy_wiring_test.go create mode 100644 internal/outboundsend/gate_worker_test.go create mode 100644 internal/outboundsend/jobs_gate_test.go diff --git a/cmd/e2a/main.go b/cmd/e2a/main.go index aa8ec21a7..7e13c1726 100644 --- a/cmd/e2a/main.go +++ b/cmd/e2a/main.go @@ -38,7 +38,6 @@ import ( "github.com/tokencanopy/e2a/internal/limits" "github.com/tokencanopy/e2a/internal/oauth" "github.com/tokencanopy/e2a/internal/outbound" - "github.com/tokencanopy/e2a/internal/outboundsend" "github.com/tokencanopy/e2a/internal/relay" "github.com/tokencanopy/e2a/internal/senderidentity" "github.com/tokencanopy/e2a/internal/sendingpolicy" @@ -343,29 +342,28 @@ func main() { // Outbound delivery is queue-first and at-least-once for GA. The accept-tx // enqueues an outbound_send job in the same transaction as the message row; - // there is no submit-inline fallback. + // there is no submit-inline fallback. Every provider call passes through + // the sending-protection gate and the authorized submitter — see + // newOutboundSending, whose wiring test pins that composition. rampStore := sendramp.NewStore(pool) - outboundRamp := agent.NewOutboundRampGate( - rampStore, - sendramp.NewSchedule(cfg.SendingRamp.StartDaily, cfg.SendingRamp.TargetDaily, cfg.SendingRamp.RampDays), - cfg.SendingRamp.Enabled, - ) - if cfg.SendingRamp.Enabled { - log.Printf("Outbound sending ramp enabled: %d→%d recipients over %d qualified days", cfg.SendingRamp.StartDaily, cfg.SendingRamp.TargetDaily, cfg.SendingRamp.RampDays) - } outboundSendStore := agent.NewOutboundSendStore(store, webhookOutbox, usageTracker) store.SetScheduledSendFinalizer(outboundSendStore) - outboundJobs := outboundsend.NewJobs( - outboundSendStore, - agent.NewOutboundDeliverer(sender), - pool, - outboundRamp, - ).WithMetrics(metrics). + outboundSending := newOutboundSending(outboundSendingDeps{ + pool: pool, + store: outboundSendStore, + relay: smtpRelay, + secrets: spSecrets, + source: spSource, + policy: spPolicy, + sesConfigSet: cfg.DeliveryFeedback.SESConfigurationSet, + metrics: metrics, // Fire-time per-agent rate limit (60 submissions/min/agent sliding // window, durable in Postgres): the cross-replica counterpart of the // acceptance-time in-memory limiter, enforced immediately before // provider submission so scheduled-send bursts can't exceed it. - WithRateGate(sendrate.NewStore(pool, time.Minute, 60)) + rate: sendrate.NewStore(pool, time.Minute, 60), + }) + outboundJobs := outboundSending.jobs registrars = append(registrars, outboundJobs) registrars = append(registrars, sendramp.NewMaintenanceJobs(rampStore)) // Queue depth/age gauges: a 30s maintenance periodic sampling river_job diff --git a/cmd/e2a/outbound_wiring.go b/cmd/e2a/outbound_wiring.go new file mode 100644 index 000000000..462cdea66 --- /dev/null +++ b/cmd/e2a/outbound_wiring.go @@ -0,0 +1,51 @@ +package main + +import ( + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/tokencanopy/e2a/internal/agent" + "github.com/tokencanopy/e2a/internal/outbound" + "github.com/tokencanopy/e2a/internal/outboundsend" + "github.com/tokencanopy/e2a/internal/sendingpolicy" +) + +// outboundSendingDeps is everything the outbound composition root needs. It +// is a struct rather than positional arguments so the wiring test can build +// the production composition from synthetic inputs and inspect the result. +type outboundSendingDeps struct { + pool *pgxpool.Pool + store outboundsend.Store + relay *outbound.SMTPRelay + secrets sendingpolicy.Secrets + source sendingpolicy.PolicySource + policy sendingpolicy.RuntimePolicy + sesConfigSet string + metrics outboundsend.Metrics + rate outboundsend.RateGate +} + +// outboundSending is the composed outbound send path. +type outboundSending struct { + gate sendingpolicy.Gate + submitter *outbound.ProviderSubmitter + jobs *outboundsend.Jobs +} + +// newOutboundSending is the ONE composition root for provider-bound customer +// mail. The gate is the deployment's policy authority; the submitter is the +// only object that opens a socket to the provider and it refuses to do so +// without a token from that gate; the jobs bundle prepares an operation at +// enqueue and authorizes every worker execution through the same gate. No +// raw sender and no direct ramp store reach the worker from here. +func newOutboundSending(d outboundSendingDeps) outboundSending { + gate := sendingpolicy.NewGate(d.pool, d.secrets, d.source, d.policy) + submitter := outbound.NewProviderSubmitter(d.relay, gate) + // Delivery feedback: tag outbound with the SES configuration set so SES + // publishes delivery/bounce/complaint events. Empty = off. + submitter.SetSESConfigurationSet(d.sesConfigSet) + jobs := outboundsend.NewJobs(d.store, agent.NewOutboundDeliverer(submitter), d.pool). + WithGate(gate). + WithMetrics(d.metrics). + WithRateGate(d.rate) + return outboundSending{gate: gate, submitter: submitter, jobs: jobs} +} diff --git a/cmd/e2a/sending_policy_wiring_test.go b/cmd/e2a/sending_policy_wiring_test.go new file mode 100644 index 000000000..45da21822 --- /dev/null +++ b/cmd/e2a/sending_policy_wiring_test.go @@ -0,0 +1,54 @@ +package main + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/tokencanopy/e2a/internal/config" + "github.com/tokencanopy/e2a/internal/outbound" + "github.com/tokencanopy/e2a/internal/sendingpolicy" + "github.com/tokencanopy/e2a/internal/testutil/testdb" +) + +// TestSendingPolicyWiring builds the production outbound composition from +// synthetic inputs and proves the registered send path holds the concrete +// Gate and the authorized submitter. It exists so that a refactor that +// reintroduced a raw sender or a direct ramp gate in the worker's path could +// not pass CI: the only deliverer the composition root may produce is the one +// over outbound.ProviderSubmitter, and the only admission authority is the +// sendingpolicy module. +func TestSendingPolicyWiring(t *testing.T) { + pool := testdb.TestDB(t) + relay := outbound.NewSMTPRelay(&config.OutboundSMTPConfig{Host: "relay.invalid", Port: 587, FromDomain: "test.e2a.dev"}) + + composed := newOutboundSending(outboundSendingDeps{ + pool: pool, + store: nil, // the store is not exercised by construction + relay: relay, + secrets: sendingpolicy.Secrets{}, + source: sendingpolicy.PolicySourceConfig, + policy: sendingpolicy.DisabledPolicy(), + sesConfigSet: "e2a-delivery-test", + }) + + if _, ok := composed.gate.(*sendingpolicy.Module); !ok { + t.Fatalf("gate is %T, want the concrete *sendingpolicy.Module", composed.gate) + } + if composed.submitter == nil { + t.Fatal("no authorized submitter composed") + } + if composed.jobs.Gate() != composed.gate { + t.Fatal("the jobs bundle does not hold the composed gate") + } + if got := fmt.Sprintf("%T", composed.jobs.Deliverer()); !strings.HasSuffix(got, "agent.outboundDeliverer") { + t.Fatalf("worker deliverer is %s, want the ProviderSubmitter-backed agent.outboundDeliverer", got) + } + + // The composed gate is live: a config-source module answers policy reads + // against the real database, which is what the worker will do. + if _, err := composed.gate.LookupOperation(context.Background(), "op_wiring_probe"); err == nil { + t.Fatal("a never-prepared operation resolved") + } +} diff --git a/docs/design/async-message-pipeline.md b/docs/design/async-message-pipeline.md index 1a27614d0..7aff7c3de 100644 --- a/docs/design/async-message-pipeline.md +++ b/docs/design/async-message-pipeline.md @@ -256,3 +256,39 @@ Still open: 6. **Residual-window reconciler** (header-tagged SNS feedback vs a `sending` row): ~~alert-only v1, auto-heal later~~ **shipped as auto-heal (2026-07-16)**: header-tagged evidence is recorded on the row, the re-driven worker/terminal reconciler settles evidence-bearing `accepted`/`sending` rows as sent, and the §3.1 correction rule heals an already-written local `failed` when correlated delivery feedback arrives. 7. **Inbound (I2): raw-blob retention** — `river_job.args` holds full raw messages for pending inbound jobs; cap size / age-out policy for a backlog. 8. **`email.accepted` event — emit or not?** Currently **not** emitted: the caller learns `accepted` synchronously (the 200 body + `delivery_status='accepted'` on the row), and contract §4's *push* vocabulary is deliberately terminal-only (`sent`/`failed`/`deferred`). Optional addition: a one-line `PublishTx` of `email.accepted` in the accept-tx would populate the `webhook_events` log (visible in `GET /v1/events`) and deliver only to anyone who *explicitly* subscribes — harmless, but it widens the event vocabulary. Decide: accept-time event-log entry for observability vs. keep the push vocabulary terminal-only. (Leaning: skip at GA — the sync 200 already carries `accepted`; revisit if subscribers ask for an accept-time signal.) + +## Addendum (2026-09-05): the sending-protection gate owns admission + +Slice B6 of the sending abuse prevention plan (`e2a-ops` docs/superpowers) moved +every provider-bound decision behind `internal/sendingpolicy`'s `Gate`. The +worker-owned `RampGate` and `agent.NewOutboundRampGate` are gone; the +custom-domain ramp is composed inside the gate (B4) and the SMTP seam is the +token-requiring `outbound.ProviderSubmitter` (B5). The worker order is now +fixed: + +1. `Reserve` the durable attempt (idempotent per ordinal; a confirmed ordinal + is followed by a fresh one, allocated by the gate, never by the worker); +2. an early hold snoozes without provider I/O; +3. the per-agent rate gate `DeferAttempt`s and snoozes; a final suppression + match `CancelAttempt`s and fails; +4. `ConsumeAttempt` is the last serialized decision; a hold here is handled + like an early one; +5. the authorized submitter redeems the token immediately before the socket + opens and settles the provider's answer (`SettleProvider`); a lost 250 is + `ErrProviderAcceptanceUnknown` — retried as a new ordinal, never settled. + +The accept transaction prepares the operation (`PrepareExternalTx`) between the +message insert and the River insert; a paused account is refused at the door +(`ErrSendingPaused` → HTTP 403 `sending_paused`). Jobs enqueued by a pre-floor +slot carry no reference and are resolved at fire time through the same path +(`Jobs.ResolveLegacyOperation`). + +Finite holds persist `messages.local_hold_class` / `local_hold_anchor` +(migration 116); the deadline is always derived — 72 hours for +`rate_ramp_or_provider` and `tenant_setup`, seven days for `policy_budget` — +and expiry emits `submission.local_retries_exhausted`, +`submission.sending_setup_expired`, or `submission.policy_budget_expired` +respectively. An account pause has no clock and starts no hold, but a deadline +already running keeps running. Terminal reconciliation is settlement-only: an +evidence-settled row also settles the attempt that dialed +(`Gate.SettleOperation`). diff --git a/internal/agent/api.go b/internal/agent/api.go index 42bf91119..c34180b7b 100644 --- a/internal/agent/api.go +++ b/internal/agent/api.go @@ -34,6 +34,7 @@ import ( "github.com/tokencanopy/e2a/internal/logredact" "github.com/tokencanopy/e2a/internal/oauth" "github.com/tokencanopy/e2a/internal/outbound" + "github.com/tokencanopy/e2a/internal/outboundsend" "github.com/tokencanopy/e2a/internal/piguard" "github.com/tokencanopy/e2a/internal/ratelimit" "github.com/tokencanopy/e2a/internal/telemetry" @@ -1607,6 +1608,12 @@ func (a *API) DeliverOutbound(ctx context.Context, user *identity.User, agent *i accepted = msg return nil }); txErr != nil { + if errors.Is(txErr, outboundsend.ErrSendingPaused) { + // The account is paused for sending abuse: refuse at the door + // rather than queue mail that can never leave. Nothing was + // committed — the message row rolled back with the job. + return nil, &OutboundError{Status: http.StatusForbidden, Code: "sending_paused", Msg: "sending is paused for this account"} + } log.Printf("[api] async accept tx failed: agent=%s to_count=%d to_domains=%v error=%v", agent.Domain, len(req.To), logredact.AddressDomains(req.To), txErr) return nil, &OutboundError{Status: http.StatusInternalServerError, Code: "internal_error", Msg: "failed to accept message for send"} } diff --git a/internal/agent/outbound_async.go b/internal/agent/outbound_async.go index 4408be993..2fa40cb85 100644 --- a/internal/agent/outbound_async.go +++ b/internal/agent/outbound_async.go @@ -2,6 +2,7 @@ package agent import ( "context" + "errors" "fmt" "hash/fnv" "log" @@ -17,7 +18,7 @@ import ( "github.com/tokencanopy/e2a/internal/messagelifecycle" "github.com/tokencanopy/e2a/internal/outbound" "github.com/tokencanopy/e2a/internal/outboundsend" - "github.com/tokencanopy/e2a/internal/sendramp" + "github.com/tokencanopy/e2a/internal/sendingpolicy" "github.com/tokencanopy/e2a/internal/usage" "github.com/tokencanopy/e2a/internal/webhookpub" ) @@ -106,71 +107,6 @@ func NewOutboundSendStore(store *identity.Store, outbox webhookpub.Outbox, usage return &outboundSendStore{store: store, outbox: outbox, usage: usageTracker} } -type outboundRampGate struct { - store *sendramp.Store - schedule sendramp.Schedule - enabled bool - now func() time.Time -} - -// NewOutboundRampGate adapts the durable sendramp store to the worker-owned -// gate contract. The schedule is snapshotted by Store on the first eligible -// send; config changes therefore affect only domains that have not armed yet. -func NewOutboundRampGate(store *sendramp.Store, schedule sendramp.Schedule, enabled bool, clocks ...func() time.Time) outboundsend.RampGate { - now := time.Now - if len(clocks) > 0 && clocks[0] != nil { - now = clocks[0] - } - return &outboundRampGate{store: store, schedule: schedule, enabled: enabled, now: now} -} - -func (g *outboundRampGate) Reserve(ctx context.Context, req outboundsend.RampRequest) (outboundsend.RampDecision, error) { - if !g.enabled { - // Disabled is pass-through: reserve nothing, count nothing, stamp - // nothing. The domain stays 'inactive'. - // - // An earlier revision stamped the domain 'exempt' here, reasoning that - // a sender allowed to send unthrottled must not be re-throttled if the - // ramp is later enabled. That turned every eligible send into a silent, - // unmarked grandfathering decision, and 'exempt' has since grown - // meaning beyond "skip the ramp": an exempt domain reads as an - // established sender, so it also stops consuming the shared probation - // pool that bounds Sybil abuse. Widening that set from the send path, - // once per send, is not a decision this gate gets to make. - // - // Grandfathering belongs to the audited one-shot that already exists - // for it: sendingpolicy's ActivationRequest.GrandfatherCurrentSendingDomains, - // which writes a replay marker, locks the domains table against - // concurrent sender transitions, and can never widen its set twice. - return outboundsend.RampDecision{Allowed: true}, nil - } - d, err := g.store.Reserve(ctx, sendramp.ReserveRequest{ - MessageID: req.MessageID, - UserID: req.UserID, - Domain: req.Domain, - Units: req.Units, - Day: g.now().UTC(), - Schedule: g.schedule, - }) - return outboundsend.RampDecision{Allowed: d.Allowed, RetryAt: d.RetryAt}, err -} - -// Confirm, Release and Resolve delegate unconditionally, including while the -// ramp is disabled: a reservation taken before an operator turned the ramp off -// still has to settle. With the ramp disabled no reservation is ever created, -// so on that path the store methods find no row and write nothing. -func (g *outboundRampGate) Confirm(ctx context.Context, messageID string) error { - return g.store.Confirm(ctx, messageID) -} - -func (g *outboundRampGate) Release(ctx context.Context, messageID string) error { - return g.store.Release(ctx, messageID) -} - -func (g *outboundRampGate) Resolve(ctx context.Context, messageID string) error { - return g.store.Resolve(ctx, messageID) -} - func (a *outboundSendStore) ClaimSend(ctx context.Context, messageID string, jobID int64) (*outboundsend.SendJob, error) { if a.usage == nil { return nil, fmt.Errorf("outbound usage tracker is required") @@ -258,9 +194,24 @@ func (a *outboundSendStore) ClaimSend(ctx context.Context, messageID string, job if p.ReviewedAt != nil { sj.ReviewedAt = *p.ReviewedAt } + sj.LocalHoldClass = outboundsend.HoldClass(p.LocalHoldClass) + if p.LocalHoldAnchor != nil { + sj.LocalHoldAnchor = *p.LocalHoldAnchor + } + if p.LastResumedAt != nil { + sj.LastResumedAt = *p.LastResumedAt + } + if p.TenantReadyAt != nil { + sj.TenantReadyAt = *p.TenantReadyAt + } return sj, nil } +// RecordHold persists the worker's finite-hold class and anchor on the row. +func (a *outboundSendStore) RecordHold(ctx context.Context, messageID string, class outboundsend.HoldClass, anchor time.Time) error { + return a.store.RecordOutboundHold(ctx, messageID, string(class), anchor) +} + // SuppressedRecipients backs the SendWorker's pre-provider suppression guard: // the effective account-wide + exact-agent subset (the store normalizes both // sides). @@ -615,30 +566,39 @@ func buildEmailFailedEventFromRow(info *identity.OutboundSentInfo, detail string } } -// outboundDeliverer implements outboundsend.Deliverer over Sender.SubmitOnce — a -// single SMTP submit of the persisted Sent-folder bytes (River owns retries). +// outboundDeliverer implements outboundsend.Deliverer over the authorized +// provider seam (outbound.ProviderSubmitter): one token-redeeming SMTP submit +// of the persisted Sent-folder bytes (River owns retries). There is no +// tokenless path through here. type outboundDeliverer struct { - sender *outbound.Sender + submitter *outbound.ProviderSubmitter } // NewOutboundDeliverer builds the outboundsend.Deliverer adapter for main.go. -func NewOutboundDeliverer(sender *outbound.Sender) outboundsend.Deliverer { - return &outboundDeliverer{sender: sender} +func NewOutboundDeliverer(submitter *outbound.ProviderSubmitter) outboundsend.Deliverer { + return &outboundDeliverer{submitter: submitter} } -func (d *outboundDeliverer) Deliver(ctx context.Context, j *outboundsend.SendJob) outboundsend.DeliverOutcome { - providerID, err := d.sender.SubmitOnceContext(ctx, j.MessageID, j.EnvelopeFrom, j.Recipients, j.RawMessage) +func (d *outboundDeliverer) Deliver(ctx context.Context, j *outboundsend.SendJob, auth sendingpolicy.ProviderAuthorization) outboundsend.DeliverOutcome { + res, err := d.submitter.SubmitOnce(ctx, auth, outbound.Envelope{ + From: j.EnvelopeFrom, + Recipients: j.Recipients, + Message: j.RawMessage, + }) if err != nil { // Classify (design §8): a definitely-permanent 5xx is terminal (JobCancel); // a provider-connection failure (relay unreachable/misconfigured) is an - // outage → snooze without burning an attempt; everything else (4xx/unknown) - // takes the bounded retry. Terminal-failing a send that could still succeed - // would violate at-least-once. + // outage → snooze without burning an attempt; a failure after the body + // was handed over is acceptance-unknown; everything else (4xx/unknown) + // takes the bounded retry. Terminal-failing a send that could still + // succeed would violate at-least-once. + unknown := errors.Is(err, outbound.ErrProviderAcceptanceUnknown) return outboundsend.DeliverOutcome{ - Err: err, - Permanent: outbound.IsPermanentSMTPError(err), - Outage: outbound.IsConnectionError(err), + Err: err, + Permanent: outbound.IsPermanentSMTPError(err), + Outage: !unknown && outbound.IsConnectionError(err), + AcceptanceUnknown: unknown, } } - return outboundsend.DeliverOutcome{ProviderMessageID: providerID, SentAs: j.SentAs} + return outboundsend.DeliverOutcome{ProviderMessageID: res.ProviderMessageID, SentAs: j.SentAs, SettlementErr: res.SettlementErr} } diff --git a/internal/agent/outbound_async_test.go b/internal/agent/outbound_async_test.go index 6b645dd24..fc4396c43 100644 --- a/internal/agent/outbound_async_test.go +++ b/internal/agent/outbound_async_test.go @@ -22,6 +22,7 @@ import ( "github.com/tokencanopy/e2a/internal/messagelifecycle" "github.com/tokencanopy/e2a/internal/outbound" "github.com/tokencanopy/e2a/internal/outboundsend" + "github.com/tokencanopy/e2a/internal/sendingpolicy" "github.com/tokencanopy/e2a/internal/testutil" "github.com/tokencanopy/e2a/internal/usage" "github.com/tokencanopy/e2a/internal/webhookpub" @@ -122,13 +123,13 @@ func (f *fakeNotifyEnqueuer) EnqueueNotifyTx(_ context.Context, _ pgx.Tx, _ stri // fakeAsyncDeliverer is the SMTP submit the SendWorker calls — no network. type fakeAsyncDeliverer struct{ out outboundsend.DeliverOutcome } -func (f fakeAsyncDeliverer) Deliver(_ context.Context, _ *outboundsend.SendJob) outboundsend.DeliverOutcome { +func (f fakeAsyncDeliverer) Deliver(_ context.Context, _ *outboundsend.SendJob, _ sendingpolicy.ProviderAuthorization) outboundsend.DeliverOutcome { return f.out } type countingAsyncDeliverer struct{ calls int } -func (d *countingAsyncDeliverer) Deliver(context.Context, *outboundsend.SendJob) outboundsend.DeliverOutcome { +func (d *countingAsyncDeliverer) Deliver(context.Context, *outboundsend.SendJob, sendingpolicy.ProviderAuthorization) outboundsend.DeliverOutcome { d.calls++ return outboundsend.DeliverOutcome{ProviderMessageID: "unexpected"} } @@ -138,7 +139,7 @@ type timedAsyncDeliverer struct { returnedAt time.Time } -func (d *timedAsyncDeliverer) Deliver(context.Context, *outboundsend.SendJob) outboundsend.DeliverOutcome { +func (d *timedAsyncDeliverer) Deliver(context.Context, *outboundsend.SendJob, sendingpolicy.ProviderAuthorization) outboundsend.DeliverOutcome { d.returnedAt = time.Now().UTC() return d.out } @@ -155,7 +156,7 @@ type blockingAsyncDeliverer struct { out outboundsend.DeliverOutcome } -func (d *blockingAsyncDeliverer) Deliver(_ context.Context, _ *outboundsend.SendJob) outboundsend.DeliverOutcome { +func (d *blockingAsyncDeliverer) Deliver(_ context.Context, _ *outboundsend.SendJob, _ sendingpolicy.ProviderAuthorization) outboundsend.DeliverOutcome { close(d.entered) <-d.release return d.out diff --git a/internal/agent/outbound_suppression_guard_test.go b/internal/agent/outbound_suppression_guard_test.go index 07d4a85cb..03536e5f5 100644 --- a/internal/agent/outbound_suppression_guard_test.go +++ b/internal/agent/outbound_suppression_guard_test.go @@ -9,7 +9,6 @@ import ( "context" "errors" "strings" - "sync" "testing" "time" @@ -22,38 +21,11 @@ import ( "github.com/tokencanopy/e2a/internal/messagelifecycle" "github.com/tokencanopy/e2a/internal/outbound" "github.com/tokencanopy/e2a/internal/outboundsend" - "github.com/tokencanopy/e2a/internal/sendramp" + "github.com/tokencanopy/e2a/internal/sendingpolicy" "github.com/tokencanopy/e2a/internal/usage" "github.com/tokencanopy/e2a/internal/webhookpub" ) -type blockingRampGate struct { - entered chan struct{} - resume chan struct{} - mu sync.Mutex - released []string -} - -func (g *blockingRampGate) Reserve(context.Context, outboundsend.RampRequest) (outboundsend.RampDecision, error) { - close(g.entered) - <-g.resume - return outboundsend.RampDecision{Allowed: true}, nil -} -func (*blockingRampGate) Confirm(context.Context, string) error { return nil } -func (g *blockingRampGate) Release(_ context.Context, messageID string) error { - g.mu.Lock() - defer g.mu.Unlock() - g.released = append(g.released, messageID) - return nil -} -func (*blockingRampGate) Resolve(context.Context, string) error { return nil } - -func (g *blockingRampGate) releasedIDs() []string { - g.mu.Lock() - defer g.mu.Unlock() - return append([]string(nil), g.released...) -} - // countingDeliverer records provider submits so the guard can assert zero I/O. type countingDeliverer struct { calls int @@ -73,7 +45,7 @@ func (s *failOnceSuppressionStore) SuppressedRecipients(ctx context.Context, use return s.Store.SuppressedRecipients(ctx, userID, agentID, recipients) } -func (d *countingDeliverer) Deliver(_ context.Context, _ *outboundsend.SendJob) outboundsend.DeliverOutcome { +func (d *countingDeliverer) Deliver(_ context.Context, _ *outboundsend.SendJob, _ sendingpolicy.ProviderAuthorization) outboundsend.DeliverOutcome { d.calls++ return d.out } @@ -378,100 +350,6 @@ func TestSendWorker_ProviderEvidenceCorrectionRetainsFallbackSuppression(t *test } } -func TestSendWorker_SuppressionAddedDuringRampReservePreventsProviderIO(t *testing.T) { - api, store, outbox, _ := setupAsyncAPI(t) - ctx := context.Background() - user, ag := selfAgent(t, store, "suppduringramp") - if err := store.SetSendingStatus(ctx, ag.RegisteredDomain, "verified", "verified", "verified", "", nil); err != nil { - t.Fatalf("SetSendingStatus: %v", err) - } - res, oerr := api.DeliverOutbound(ctx, user, ag, outbound.SendRequest{ - To: []string{"late@external.test"}, Subject: "ramp race", Body: "x", - }, "send", "", nil, nil) - if oerr != nil { - t.Fatalf("DeliverOutbound: %+v", oerr) - } - - gate := &blockingRampGate{entered: make(chan struct{}), resume: make(chan struct{})} - deliverer := &countingDeliverer{out: outboundsend.DeliverOutcome{ProviderMessageID: "must-not-happen"}} - worker := outboundsend.NewSendWorker(agent.NewOutboundSendStore(store, outbox, usage.NewNoopUsageTracker()), deliverer, gate) - done := make(chan error, 1) - go func() { done <- worker.Work(ctx, workerJob(res.MessageID, 1)) }() - <-gate.entered - if _, _, err := store.AddAgentSuppression(ctx, user.ID, ag.ID, "late@external.test", "opted out", "unsubscribe", nil); err != nil { - t.Fatal(err) - } - close(gate.resume) - if err := <-done; err == nil { - t.Fatal("suppression created during ramp reservation must cancel the send") - } - if deliverer.calls != 0 { - t.Fatalf("provider calls = %d, want zero", deliverer.calls) - } - if got := gate.releasedIDs(); len(got) != 1 || got[0] != res.MessageID { - t.Fatalf("released reservations = %v, want [%s]", got, res.MessageID) - } - var status, detail string - if err := store.WithTx(ctx, func(tx pgx.Tx) error { - return tx.QueryRow(ctx, `SELECT delivery_status, COALESCE(delivery_detail,'') FROM messages WHERE id=$1`, res.MessageID).Scan(&status, &detail) - }); err != nil { - t.Fatal(err) - } - if status != "failed" || !strings.Contains(detail, "recipient_suppressed") { - t.Fatalf("status/detail = %q/%q, want failed recipient_suppressed", status, detail) - } -} - -func TestSendWorker_TransientSuppressionFailureReusesRealRampReservation(t *testing.T) { - api, store, outbox, _, pool := setupAsyncAPIWithPool(t) - ctx := context.Background() - user, ag := selfAgent(t, store, "rampretryreal") - if err := store.SetSendingStatus(ctx, ag.RegisteredDomain, "verified", "verified", "verified", "", nil); err != nil { - t.Fatalf("SetSendingStatus: %v", err) - } - res, oerr := api.DeliverOutbound(ctx, user, ag, outbound.SendRequest{ - To: []string{"recipient@external.test"}, Subject: "retry after suppression lookup", Body: "x", - }, "send", "", nil, nil) - if oerr != nil { - t.Fatalf("DeliverOutbound: %+v", oerr) - } - - baseStore := agent.NewOutboundSendStore(store, outbox, usage.NewNoopUsageTracker()) - failingStore := &failOnceSuppressionStore{Store: baseStore} - day := time.Date(2026, 7, 19, 12, 0, 0, 0, time.UTC) - ramp := agent.NewOutboundRampGate(sendramp.NewStore(pool), sendramp.NewSchedule(50, 100, 2), true, func() time.Time { return day }) - deliverer := &countingDeliverer{out: outboundsend.DeliverOutcome{ProviderMessageID: "ses-after-retry", SentAs: "own_address"}} - worker := outboundsend.NewSendWorker(failingStore, deliverer, ramp) - - if err := worker.Work(ctx, workerJobWithID(res.MessageID, 999, 1)); err == nil { - t.Fatal("first worker attempt must return the injected transient error") - } - var firstState string - if err := pool.QueryRow(ctx, `SELECT state FROM sending_ramp_reservations WHERE message_id=$1`, res.MessageID).Scan(&firstState); err != nil { - t.Fatalf("read first reservation: %v", err) - } - if firstState != "reserved" { - t.Fatalf("reservation after transient error = %q, want reserved", firstState) - } - if deliverer.calls != 0 { - t.Fatalf("provider calls after transient error = %d, want zero", deliverer.calls) - } - - if err := worker.Work(ctx, workerJobWithID(res.MessageID, 999, 2)); err != nil { - t.Fatalf("retry worker attempt: %v", err) - } - var finalState, status string - if err := pool.QueryRow(ctx, `SELECT state FROM sending_ramp_reservations WHERE message_id=$1`, res.MessageID).Scan(&finalState); err != nil { - t.Fatalf("read final reservation: %v", err) - } - if err := pool.QueryRow(ctx, `SELECT delivery_status FROM messages WHERE id=$1`, res.MessageID).Scan(&status); err != nil { - t.Fatalf("read final message: %v", err) - } - if finalState != "confirmed" || status != "sent" || deliverer.calls != 1 { - t.Fatalf("final reservation/status/provider calls = %q/%q/%d, want confirmed/sent/1", finalState, status, deliverer.calls) - } -} - func TestAccountSuppressionFromBounceBlocksEveryAgentSend(t *testing.T) { api, store, _, _ := setupAsyncAPI(t) ctx := context.Background() diff --git a/internal/agent/test_send_async_test.go b/internal/agent/test_send_async_test.go index 64ba85368..83117d0dd 100644 --- a/internal/agent/test_send_async_test.go +++ b/internal/agent/test_send_async_test.go @@ -13,6 +13,7 @@ import ( "github.com/tokencanopy/e2a/internal/identity" "github.com/tokencanopy/e2a/internal/outbound" "github.com/tokencanopy/e2a/internal/outboundsend" + "github.com/tokencanopy/e2a/internal/sendingpolicy" "github.com/tokencanopy/e2a/internal/testutil" "github.com/tokencanopy/e2a/internal/usage" "github.com/tokencanopy/e2a/internal/webhookpub" @@ -26,7 +27,7 @@ type captureDeliverer struct { out outboundsend.DeliverOutcome } -func (c *captureDeliverer) Deliver(_ context.Context, j *outboundsend.SendJob) outboundsend.DeliverOutcome { +func (c *captureDeliverer) Deliver(_ context.Context, j *outboundsend.SendJob, _ sendingpolicy.ProviderAuthorization) outboundsend.DeliverOutcome { c.jobs = append(c.jobs, j) return c.out } diff --git a/internal/outboundsend/gate_worker_test.go b/internal/outboundsend/gate_worker_test.go new file mode 100644 index 000000000..536c80410 --- /dev/null +++ b/internal/outboundsend/gate_worker_test.go @@ -0,0 +1,370 @@ +package outboundsend_test + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/riverqueue/river" + + "github.com/tokencanopy/e2a/internal/delivery" + "github.com/tokencanopy/e2a/internal/messagelifecycle" + "github.com/tokencanopy/e2a/internal/outboundsend" + "github.com/tokencanopy/e2a/internal/sendingpolicy" +) + +// These tests pin the fixed worker order over the sending-protection gate: +// Reserve → rate → suppression → ConsumeAttempt → authorized submit, with +// every hold snoozing without provider I/O, every deferral/cancellation +// returning the right ledger, and every finite hold persisting a class whose +// derived deadline decides expiry and its lifecycle reason. + +func isSnooze(err error) bool { + var snooze *river.JobSnoozeError + return errors.As(err, &snooze) +} + +func isCancel(err error) bool { + var cancel *river.JobCancelError + return errors.As(err, &cancel) +} + +func TestGatedWorker_AllowedPathAuthorizesThenSubmits(t *testing.T) { + st := &fakeStore{job: acceptedJob("msg_1")} + dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{ProviderMessageID: "ses-1", SentAs: "relay"}} + g := allowAll() + if err := outboundsend.NewSendWorker(st, dl).WithGate(g).Work(context.Background(), gatedJob("msg_1", 1)); err != nil { + t.Fatalf("Work: %v", err) + } + if g.reserves != 1 || g.consumes != 1 || dl.calls != 1 || len(st.sent) != 1 { + t.Fatalf("reserves=%d consumes=%d delivers=%d sent=%d, want 1/1/1/1", g.reserves, g.consumes, dl.calls, len(st.sent)) + } + if len(g.deferred)+len(g.cancelled) != 0 { + t.Fatalf("deferred=%v cancelled=%v on an allowed path", g.deferred, g.cancelled) + } +} + +func TestGatedWorker_EarlyHoldSnoozesWithoutProviderIOAndPersistsClass(t *testing.T) { + for reason, want := range map[string]outboundsend.HoldClass{ + sendingpolicy.ReasonAccountDailyBudget: outboundsend.HoldPolicyBudget, + sendingpolicy.ReasonGlobalProbation: outboundsend.HoldPolicyBudget, + sendingpolicy.ReasonTenantNotReady: outboundsend.HoldTenantSetup, + sendingpolicy.ReasonTenantUnnamed: outboundsend.HoldTenantSetup, + sendingpolicy.ReasonRampCapacity: outboundsend.HoldRateRampOrProvider, + sendingpolicy.ReasonSendingIdentityUnverified: outboundsend.HoldRateRampOrProvider, + } { + j := acceptedJob("msg_hold") + j.AcceptedAt = time.Now().Add(-time.Hour) + st := &fakeStore{job: j} + dl := &fakeDeliverer{} + g := &fakeGate{reserve: sendingpolicy.Decision{Allow: false, Reason: reason, RetryAt: time.Now().Add(2 * time.Hour)}} + err := outboundsend.NewSendWorker(st, dl).WithGate(g).Work(context.Background(), gatedJob("msg_hold", 1)) + if !isSnooze(err) { + t.Fatalf("%s: err = %v, want snooze", reason, err) + } + if dl.calls != 0 || len(st.failed) != 0 || g.consumes != 0 { + t.Fatalf("%s: delivers=%d failed=%d consumes=%d, want no I/O and no terminal", reason, dl.calls, len(st.failed), g.consumes) + } + if len(st.holds) != 1 || st.holds[0].class != want { + t.Fatalf("%s: holds = %+v, want one %s hold", reason, st.holds, want) + } + // A first-observed tenant-setup hold starts its clock at the + // observation; every other class starts at the latest of the + // message's own timestamps. + if want == outboundsend.HoldTenantSetup { + if st.holds[0].anchor.Before(j.AcceptedAt.Add(time.Hour - time.Minute)) { + t.Fatalf("%s: anchor = %v, want the observation time, not accept", reason, st.holds[0].anchor) + } + } else if !st.holds[0].anchor.Equal(j.AcceptedAt) { + t.Fatalf("%s: anchor = %v, want accept %v", reason, st.holds[0].anchor, j.AcceptedAt) + } + if len(st.released) != 1 { + t.Fatalf("%s: claim releases = %v, want one", reason, st.released) + } + } +} + +func TestGatedWorker_PauseHoldIsIndefiniteAndPersistsNothing(t *testing.T) { + j := acceptedJob("msg_paused") + j.AcceptedAt = time.Now().Add(-30 * 24 * time.Hour) // far past every finite horizon + st := &fakeStore{job: j} + g := &fakeGate{reserve: sendingpolicy.Decision{Allow: false, Reason: sendingpolicy.ReasonAccountPaused}} + err := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(g).Work(context.Background(), gatedJob("msg_paused", 1)) + if !isSnooze(err) { + t.Fatalf("err = %v, want snooze — a pause waits for an operator", err) + } + if len(st.holds) != 0 || len(st.failed) != 0 { + t.Fatalf("holds=%+v failed=%+v, want neither for a pause", st.holds, st.failed) + } +} + +func TestGatedWorker_PauseDoesNotExtendARunningBudgetDeadline(t *testing.T) { + j := acceptedJob("msg_paused_budget") + j.LocalHoldClass, j.LocalHoldAnchor = outboundsend.HoldPolicyBudget, time.Now().Add(-8*24*time.Hour) + st := &fakeStore{job: j} + g := &fakeGate{reserve: sendingpolicy.Decision{Allow: false, Reason: sendingpolicy.ReasonAccountPaused}} + err := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(g).Work(context.Background(), gatedJob("msg_paused_budget", 1)) + if !isCancel(err) { + t.Fatalf("err = %v, want cancel — the seven-day budget deadline governs every later wait", err) + } + if len(st.failed) != 1 || st.failed[0].source != delivery.FailureSourceLocal { + t.Fatalf("failed = %+v, want one local failure", st.failed) + } +} + +func TestGatedWorker_BudgetHoldPromotesAnyClassAndKeepsTheAnchor(t *testing.T) { + anchor := time.Now().Add(-2 * time.Hour) + for _, existing := range []outboundsend.HoldClass{outboundsend.HoldRateRampOrProvider, outboundsend.HoldTenantSetup} { + j := acceptedJob("msg_promote") + j.LocalHoldClass, j.LocalHoldAnchor = existing, anchor + st := &fakeStore{job: j} + g := &fakeGate{reserve: sendingpolicy.Decision{Allow: false, Reason: sendingpolicy.ReasonGlobalAllBudget, RetryAt: time.Now().Add(time.Hour)}} + if err := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(g).Work(context.Background(), gatedJob("msg_promote", 1)); !isSnooze(err) { + t.Fatalf("%s: err = %v, want snooze", existing, err) + } + if len(st.holds) != 1 || st.holds[0].class != outboundsend.HoldPolicyBudget || !st.holds[0].anchor.Equal(anchor) { + t.Fatalf("%s: holds = %+v, want promotion to policy_budget with the anchor kept", existing, st.holds) + } + } + // And policy_budget never changes again, even under a later setup hold. + j := acceptedJob("msg_sticky") + j.LocalHoldClass, j.LocalHoldAnchor = outboundsend.HoldPolicyBudget, anchor + st := &fakeStore{job: j} + g := &fakeGate{reserve: sendingpolicy.Decision{Allow: false, Reason: sendingpolicy.ReasonTenantNotReady}} + if err := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(g).Work(context.Background(), gatedJob("msg_sticky", 1)); !isSnooze(err) { + t.Fatalf("err = %v, want snooze", err) + } + if len(st.holds) != 0 { + t.Fatalf("holds = %+v, want no rewrite of a policy_budget hold", st.holds) + } +} + +func TestGatedWorker_ExpiryReasonFollowsTheClass(t *testing.T) { + for _, tc := range []struct { + class outboundsend.HoldClass + age time.Duration + reason messagelifecycle.ReasonCode + hold string + }{ + {outboundsend.HoldPolicyBudget, 7*24*time.Hour + time.Minute, messagelifecycle.ReasonSubmissionPolicyBudgetExpired, sendingpolicy.ReasonGlobalAllBudget}, + {outboundsend.HoldTenantSetup, 72*time.Hour + time.Minute, messagelifecycle.ReasonSubmissionSendingSetupExpired, sendingpolicy.ReasonTenantNotReady}, + {outboundsend.HoldRateRampOrProvider, 72*time.Hour + time.Minute, messagelifecycle.ReasonSubmissionLocalRetriesExhausted, sendingpolicy.ReasonRampCapacity}, + } { + j := acceptedJob("msg_expire") + j.LocalHoldClass, j.LocalHoldAnchor = tc.class, time.Now().Add(-tc.age) + st := &fakeStore{job: j} + g := &fakeGate{reserve: sendingpolicy.Decision{Allow: false, Reason: tc.hold, RetryAt: time.Now().Add(time.Hour)}} + err := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(g).Work(context.Background(), gatedJob("msg_expire", 1)) + if !isCancel(err) { + t.Fatalf("%s: err = %v, want cancel", tc.class, err) + } + if len(st.failed) != 1 || st.failed[0].reason != tc.reason || st.failed[0].source != delivery.FailureSourceLocal { + t.Fatalf("%s: failed = %+v, want one local failure with reason %s", tc.class, st.failed, tc.reason) + } + if len(g.cancelled) != 1 { + t.Fatalf("%s: cancelled = %v, want the attempt given back", tc.class, g.cancelled) + } + } + // One minute short of the deadline still snoozes. + j := acceptedJob("msg_almost") + j.LocalHoldClass, j.LocalHoldAnchor = outboundsend.HoldPolicyBudget, time.Now().Add(-7*24*time.Hour+time.Minute) + st := &fakeStore{job: j} + g := &fakeGate{reserve: sendingpolicy.Decision{Allow: false, Reason: sendingpolicy.ReasonGlobalAllBudget, RetryAt: time.Now().Add(time.Hour)}} + if err := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(g).Work(context.Background(), gatedJob("msg_almost", 1)); !isSnooze(err) { + t.Fatalf("err = %v, want snooze one minute before the deadline", err) + } +} + +func TestGatedWorker_TerminalHoldCancelsNow(t *testing.T) { + for _, reason := range []string{sendingpolicy.ReasonAccountDeleted, sendingpolicy.ReasonClassChanged, sendingpolicy.ReasonRampUnavailable} { + st := &fakeStore{job: acceptedJob("msg_terminal")} + g := &fakeGate{reserve: sendingpolicy.Decision{Allow: false, Reason: reason, Terminal: true}} + err := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(g).Work(context.Background(), gatedJob("msg_terminal", 1)) + if !isCancel(err) { + t.Fatalf("%s: err = %v, want cancel", reason, err) + } + if len(st.failed) != 1 || st.failed[0].reason != messagelifecycle.ReasonSubmissionCancelled { + t.Fatalf("%s: failed = %+v, want one local cancellation", reason, st.failed) + } + } +} + +func TestGatedWorker_RateDeferralDefersTheAttempt(t *testing.T) { + st := &fakeStore{job: acceptedJob("msg_rate")} + g := allowAll() + gate := &fakeRateGate{decision: outboundsend.RateDecision{Allowed: false, RetryAt: time.Now().Add(30 * time.Second)}, window: time.Minute} + err := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(g).WithRateGate(gate).Work(context.Background(), gatedJob("msg_rate", 1)) + if !isSnooze(err) { + t.Fatalf("err = %v, want snooze", err) + } + if len(g.deferred) != 1 || g.consumes != 0 { + t.Fatalf("deferred=%v consumes=%d, want the attempt deferred before final authorization", g.deferred, g.consumes) + } + if len(st.holds) != 1 || st.holds[0].class != outboundsend.HoldRateRampOrProvider { + t.Fatalf("holds = %+v, want a rate/ramp/provider hold", st.holds) + } +} + +func TestGatedWorker_SuppressionCancelsTheAttempt(t *testing.T) { + st := &fakeStore{job: acceptedJob("msg_sup"), suppressed: []string{"b@y.com"}} + g := allowAll() + dl := &fakeDeliverer{} + err := outboundsend.NewSendWorker(st, dl).WithGate(g).Work(context.Background(), gatedJob("msg_sup", 1)) + if !isCancel(err) || dl.calls != 0 { + t.Fatalf("err=%v delivers=%d, want cancel with no I/O", err, dl.calls) + } + if len(g.cancelled) != 1 || g.consumes != 0 { + t.Fatalf("cancelled=%v consumes=%d, want the attempt cancelled before final authorization", g.cancelled, g.consumes) + } +} + +func TestGatedWorker_FinalAuthorizationHoldSnoozesWithoutProviderIO(t *testing.T) { + j := acceptedJob("msg_late_hold") + j.AcceptedAt = time.Now().Add(-time.Hour) + st := &fakeStore{job: j} + dl := &fakeDeliverer{} + g := &fakeGate{reserve: sendingpolicy.Decision{Allow: true}, consume: sendingpolicy.Decision{Allow: false, Reason: sendingpolicy.ReasonAccountSharedBudget, RetryAt: time.Now().Add(time.Hour)}} + err := outboundsend.NewSendWorker(st, dl).WithGate(g).Work(context.Background(), gatedJob("msg_late_hold", 1)) + if !isSnooze(err) || dl.calls != 0 { + t.Fatalf("err=%v delivers=%d, want snooze with no I/O", err, dl.calls) + } + if len(st.holds) != 1 || st.holds[0].class != outboundsend.HoldPolicyBudget { + t.Fatalf("holds = %+v, want a policy_budget hold from the late gate", st.holds) + } +} + +func TestGatedWorker_GateOutageSnoozesWithoutBurningAnAttempt(t *testing.T) { + for name, g := range map[string]*fakeGate{ + "reserve": {reserveErr: errors.New("policy db down")}, + "authorize": {reserve: sendingpolicy.Decision{Allow: true}, consumeErr: errors.New("policy db down")}, + } { + st := &fakeStore{job: acceptedJob("msg_gate_down")} + dl := &fakeDeliverer{} + err := outboundsend.NewSendWorker(st, dl).WithGate(g).Work(context.Background(), gatedJob("msg_gate_down", 1)) + if !isSnooze(err) || dl.calls != 0 || len(st.failed) != 0 { + t.Fatalf("%s: err=%v delivers=%d failed=%d, want snooze, no I/O, no terminal", name, err, dl.calls, len(st.failed)) + } + if len(st.released) != 1 { + t.Fatalf("%s: claim releases = %v, want one", name, st.released) + } + } +} + +func TestGatedWorker_ProviderEvidenceSettlesTheOperation(t *testing.T) { + j := acceptedJob("msg_evidence") + j.ProviderAccepted, j.ProviderMessageID = true, "ses-evidence" + st := &fakeStore{job: j} + dl := &fakeDeliverer{} + g := allowAll() + if err := outboundsend.NewSendWorker(st, dl).WithGate(g).Work(context.Background(), gatedJob("msg_evidence", 2)); err != nil { + t.Fatalf("Work: %v", err) + } + if dl.calls != 0 || len(st.sent) != 1 || g.reserves != 0 { + t.Fatalf("delivers=%d sent=%d reserves=%d, want settle without resubmit or a new reservation", dl.calls, len(st.sent), g.reserves) + } + if g.lookupCalls != 1 || len(g.settled) != 1 || g.settled[0] != sendingpolicy.SettlementProviderAccepted { + t.Fatalf("lookups=%d settled=%v, want the operation settled as accepted", g.lookupCalls, g.settled) + } +} + +func TestGatedWorker_LegacyJobResolvesThroughTheAcceptPath(t *testing.T) { + st := &fakeStore{job: acceptedJob("msg_legacy")} + dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{ProviderMessageID: "ses-legacy"}} + g := allowAll() + resolved := 0 + w := outboundsend.NewSendWorker(st, dl).WithGate(g).WithOperationResolver(func(_ context.Context, id string) (sendingpolicy.AcceptanceDecision, sendingpolicy.OperationRef, error) { + resolved++ + return sendingpolicy.AcceptanceAccept, refFor(id), nil + }) + if err := w.Work(context.Background(), job("msg_legacy", 1)); err != nil { + t.Fatalf("Work: %v", err) + } + if resolved != 1 || g.reserves != 1 || dl.calls != 1 { + t.Fatalf("resolved=%d reserves=%d delivers=%d, want the legacy job authorized like a new one", resolved, g.reserves, dl.calls) + } + + // A paused account at resolution holds; an orphan source cancels; no + // resolver at all fails closed. + st = &fakeStore{job: acceptedJob("msg_legacy_paused")} + w = outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(allowAll()).WithOperationResolver(func(context.Context, string) (sendingpolicy.AcceptanceDecision, sendingpolicy.OperationRef, error) { + return sendingpolicy.AcceptanceSendingPaused, sendingpolicy.OperationRef{}, nil + }) + if err := w.Work(context.Background(), job("msg_legacy_paused", 1)); !isSnooze(err) { + t.Fatalf("paused legacy: err = %v, want snooze", err) + } + st = &fakeStore{job: acceptedJob("msg_legacy_orphan")} + w = outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(allowAll()).WithOperationResolver(func(context.Context, string) (sendingpolicy.AcceptanceDecision, sendingpolicy.OperationRef, error) { + return "", sendingpolicy.OperationRef{}, sendingpolicy.ErrSourceUnavailable + }) + if err := w.Work(context.Background(), job("msg_legacy_orphan", 1)); !isCancel(err) || len(st.failed) != 1 { + t.Fatalf("orphan legacy: err=%v failed=%d, want cancel with one local failure", err, len(st.failed)) + } + st = &fakeStore{job: acceptedJob("msg_legacy_unwired")} + dl = &fakeDeliverer{} + if err := outboundsend.NewSendWorker(st, dl).WithGate(allowAll()).Work(context.Background(), job("msg_legacy_unwired", 1)); !isCancel(err) || dl.calls != 0 { + t.Fatalf("unwired resolver: err=%v delivers=%d, want cancel with no I/O", err, dl.calls) + } +} + +func TestGatedWorker_TenantReadinessMovesSetupHoldToRateClassOnce(t *testing.T) { + anchor := time.Now().Add(-70 * time.Hour) + ready := anchor.Add(60 * time.Hour) // inside the 72h setup deadline + j := acceptedJob("msg_ready") + j.LocalHoldClass, j.LocalHoldAnchor, j.TenantReadyAt = outboundsend.HoldTenantSetup, anchor, ready + st := &fakeStore{job: j} + dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{ProviderMessageID: "ses-ready"}} + if err := outboundsend.NewSendWorker(st, dl).WithGate(allowAll()).Work(context.Background(), gatedJob("msg_ready", 1)); err != nil { + t.Fatalf("Work: %v", err) + } + if len(st.holds) != 1 || st.holds[0].class != outboundsend.HoldRateRampOrProvider || !st.holds[0].anchor.Equal(ready) { + t.Fatalf("holds = %+v, want the one-way move to rate_ramp_or_provider anchored at readiness", st.holds) + } + + // Readiness that landed AFTER the setup deadline does not rescue the + // message: it expires as setup on its next hold. + late := acceptedJob("msg_late_ready") + late.LocalHoldClass, late.LocalHoldAnchor, late.TenantReadyAt = outboundsend.HoldTenantSetup, time.Now().Add(-80*time.Hour), time.Now().Add(-time.Hour) + st = &fakeStore{job: late} + g := &fakeGate{reserve: sendingpolicy.Decision{Allow: false, Reason: sendingpolicy.ReasonRampCapacity, RetryAt: time.Now().Add(time.Hour)}} + err := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(g).Work(context.Background(), gatedJob("msg_late_ready", 1)) + if !isCancel(err) || len(st.failed) != 1 || st.failed[0].reason != messagelifecycle.ReasonSubmissionSendingSetupExpired { + t.Fatalf("late readiness: err=%v failed=%+v, want setup expiry", err, st.failed) + } +} + +func TestGatedWorker_FirstHoldAnchorsAtTheLatestOfAcceptScheduleReviewResume(t *testing.T) { + base := time.Now().Add(-10 * 24 * time.Hour) + j := acceptedJob("msg_anchor") + j.AcceptedAt = base + j.ScheduledAt = base.Add(24 * time.Hour) + j.ReviewedAt = base.Add(48 * time.Hour) + j.LastResumedAt = base.Add(9*24*time.Hour + 23*time.Hour) // an hour ago: the latest + st := &fakeStore{job: j} + g := &fakeGate{reserve: sendingpolicy.Decision{Allow: false, Reason: sendingpolicy.ReasonRampCapacity, RetryAt: time.Now().Add(time.Hour)}} + if err := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(g).Work(context.Background(), gatedJob("msg_anchor", 1)); !isSnooze(err) { + t.Fatalf("err = %v, want snooze — a ten-day-old accept is not the clock, the resume an hour ago is", err) + } + if len(st.holds) != 1 || !st.holds[0].anchor.Equal(j.LastResumedAt) { + t.Fatalf("holds = %+v, want anchored at the last resume", st.holds) + } +} + +func TestGatedWorker_AcceptanceUnknownIsRetriedAsANewOrdinalNotSettled(t *testing.T) { + st := &fakeStore{job: acceptedJob("msg_unknown")} + dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{Err: errors.New("data final: acceptance unknown"), AcceptanceUnknown: true}} + g := allowAll() + err := outboundsend.NewSendWorker(st, dl).WithGate(g).Work(context.Background(), gatedJob("msg_unknown", 1)) + if err == nil || isSnooze(err) || isCancel(err) { + t.Fatalf("err = %v, want a plain retryable error (River's next attempt returns to Reserve)", err) + } + if len(st.temporary) != 1 || len(st.failed) != 0 || len(g.settled) != 0 { + t.Fatalf("temporary=%d failed=%d settled=%v, want a temporary record and nothing settled", len(st.temporary), len(st.failed), g.settled) + } +} + +func TestGatedWorker_HoldConstantsMatchThePolicyDefault(t *testing.T) { + if got := time.Duration(sendingpolicy.DisabledPolicy().BudgetHoldMaxDays) * 24 * time.Hour; got != outboundsend.PolicyBudgetHoldHorizon { + t.Fatalf("PolicyBudgetHoldHorizon = %s, policy budget_hold_max_days default = %s", outboundsend.PolicyBudgetHoldHorizon, got) + } +} diff --git a/internal/outboundsend/jobs.go b/internal/outboundsend/jobs.go index 8636ba6d9..e9255f141 100644 --- a/internal/outboundsend/jobs.go +++ b/internal/outboundsend/jobs.go @@ -2,6 +2,7 @@ package outboundsend import ( "context" + "fmt" "time" "github.com/jackc/pgx/v5" @@ -9,6 +10,7 @@ import ( "github.com/riverqueue/river" "github.com/tokencanopy/e2a/internal/jobs" + "github.com/tokencanopy/e2a/internal/sendingpolicy" ) // Jobs is the outbound-send integration on the shared River client: a @@ -19,7 +21,7 @@ import ( type Jobs struct { store Store deliverer Deliverer - ramp RampGate + gate sendingpolicy.Gate rate RateGate pool *pgxpool.Pool enq jobs.Enqueuer @@ -27,15 +29,30 @@ type Jobs struct { } // NewJobs builds the integration with its dependencies (no client yet). pool -// backs the periodic terminal-state reconciler's scan. -func NewJobs(store Store, deliverer Deliverer, pool *pgxpool.Pool, ramp ...RampGate) *Jobs { - j := &Jobs{store: store, deliverer: deliverer, pool: pool, metrics: noopMetrics{}} - if len(ramp) > 0 { - j.ramp = ramp[0] +// backs the periodic terminal-state reconciler's scan and the legacy-argument +// resolver's transaction. +func NewJobs(store Store, deliverer Deliverer, pool *pgxpool.Pool) *Jobs { + return &Jobs{store: store, deliverer: deliverer, pool: pool, metrics: noopMetrics{}} +} + +// WithGate injects the sending-protection gate. Every enqueue then prepares a +// durable operation in the accept transaction, and every worker execution +// authorizes through it. Chainable; nil keeps the gateless default (unit +// tests only — see NewSendWorker). +func (j *Jobs) WithGate(g sendingpolicy.Gate) *Jobs { + if g != nil { + j.gate = g } return j } +// Gate exposes the wired sending-protection gate, for the composition root's +// wiring test. nil when none is wired. +func (j *Jobs) Gate() sendingpolicy.Gate { return j.gate } + +// Deliverer exposes the wired provider deliverer, for the same test. +func (j *Jobs) Deliverer() Deliverer { return j.deliverer } + // SetEnqueuer injects the shared client so EnqueueSendTx can insert jobs. func (j *Jobs) SetEnqueuer(e jobs.Enqueuer) { j.enq = e } @@ -62,8 +79,8 @@ func (j *Jobs) WithRateGate(g RateGate) *Jobs { // RegisterJobs adds the SendWorker and terminal-state safety net to the shared // client's bundle. Implements jobs.Registrar. func (j *Jobs) RegisterJobs(w *river.Workers) []*river.PeriodicJob { - river.AddWorker(w, NewSendWorker(j.store, j.deliverer, j.ramp).WithMetrics(j.metrics).WithRateGate(j.rate)) - river.AddWorker(w, NewTerminalReconcileWorker(j.pool, j.store, j.ramp).WithMetrics(j.metrics)) + river.AddWorker(w, NewSendWorker(j.store, j.deliverer).WithMetrics(j.metrics).WithRateGate(j.rate).WithGate(j.gate).WithOperationResolver(j.ResolveLegacyOperation)) + river.AddWorker(w, NewTerminalReconcileWorker(j.pool, j.store).WithMetrics(j.metrics).WithGate(j.gate)) return []*river.PeriodicJob{ river.NewPeriodicJob( river.PeriodicInterval(terminalReconcileInterval), @@ -119,7 +136,26 @@ func (j *Jobs) EnqueueScheduledSendTx(ctx context.Context, tx pgx.Tx, messageID // enqueueSendTx is the shared outbox insert behind the immediate and scheduled // entry points. A non-zero `at` sets InsertOpts.ScheduledAt; a zero value omits // it (River defaults ScheduledAt to now, i.e. immediately available). +// +// With a gate wired, the durable provider operation is prepared HERE, after +// the message insert and before the River insert, in the caller's transaction: +// a paused account is refused at the door (ErrSendingPaused) rather than +// queueing mail that can never leave, and the job carries the operation +// reference so the worker never derives purpose or attribution on its own. func (j *Jobs) enqueueSendTx(ctx context.Context, tx pgx.Tx, messageID string, at time.Time) (int64, error) { + args := OutboundSendArgs{MessageID: messageID} + if j.gate != nil { + decision, ref, err := j.gate.PrepareExternalTx(ctx, tx, messageID) + if err != nil { + return 0, fmt.Errorf("prepare sending operation: %w", err) + } + if decision == sendingpolicy.AcceptanceSendingPaused { + return 0, ErrSendingPaused + } + if !ref.IsZero() { + args.OperationRef = &ref + } + } opts := &river.InsertOpts{ Queue: jobs.QueueOutbound, MaxAttempts: MaxSendAttempts, @@ -127,9 +163,34 @@ func (j *Jobs) enqueueSendTx(ctx context.Context, tx pgx.Tx, messageID string, a if !at.IsZero() { opts.ScheduledAt = at } - res, err := j.enq.InsertTx(ctx, tx, OutboundSendArgs{MessageID: messageID}, opts) + res, err := j.enq.InsertTx(ctx, tx, args, opts) if err != nil { return 0, err } return res.Job.ID, nil } + +// ResolveLegacyOperation is the compatibility resolver for a job enqueued by +// a pre-floor slot with no operation reference. It runs the same +// PrepareExternalTx an accept transaction runs — idempotent on the durable +// operation row — in its own committed transaction, so an old job and a new +// one authorize identically. There is deliberately no other way to obtain an +// operation from a bare message id. +func (j *Jobs) ResolveLegacyOperation(ctx context.Context, messageID string) (sendingpolicy.AcceptanceDecision, sendingpolicy.OperationRef, error) { + if j.gate == nil || j.pool == nil { + return "", sendingpolicy.OperationRef{}, fmt.Errorf("legacy operation resolver is not wired") + } + tx, err := j.pool.Begin(ctx) + if err != nil { + return "", sendingpolicy.OperationRef{}, fmt.Errorf("begin legacy resolve: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + decision, ref, err := j.gate.PrepareExternalTx(ctx, tx, messageID) + if err != nil { + return "", sendingpolicy.OperationRef{}, err + } + if err := tx.Commit(ctx); err != nil { + return "", sendingpolicy.OperationRef{}, fmt.Errorf("commit legacy resolve: %w", err) + } + return decision, ref, nil +} diff --git a/internal/outboundsend/jobs_gate_test.go b/internal/outboundsend/jobs_gate_test.go new file mode 100644 index 000000000..d9f4046b7 --- /dev/null +++ b/internal/outboundsend/jobs_gate_test.go @@ -0,0 +1,286 @@ +package outboundsend_test + +import ( + "context" + "errors" + "testing" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/riverqueue/river" + "github.com/riverqueue/river/rivertype" + + "github.com/tokencanopy/e2a/internal/agent" + "github.com/tokencanopy/e2a/internal/identity" + "github.com/tokencanopy/e2a/internal/jobs" + "github.com/tokencanopy/e2a/internal/outboundsend" + "github.com/tokencanopy/e2a/internal/sendingpolicy" + "github.com/tokencanopy/e2a/internal/testutil" + "github.com/tokencanopy/e2a/internal/usage" + "github.com/tokencanopy/e2a/internal/webhookpub" +) + +// These tests drive the real gate against real Postgres through the jobs +// bundle: the accept transaction prepares the operation, a paused account is +// refused at the door, and a legacy job with no reference authorizes through +// the same path as a new one. + +type gateFixture struct { + t *testing.T + ctx context.Context + pool *pgxpool.Pool + store *identity.Store + adapter outboundsend.Store + gate sendingpolicy.Gate + userID string + agentID string + client jobs.Enqueuer + gated *outboundsend.Jobs + legacy *outboundsend.Jobs +} + +func newGateFixture(t *testing.T) *gateFixture { + t.Helper() + ctx := context.Background() + pool := testutil.TestDB(t) + store := identity.NewStore(pool) + if err := jobs.Migrate(ctx, pool); err != nil { + t.Fatalf("jobs.Migrate: %v", err) + } + user, err := store.CreateOrGetUser(ctx, "owner-gate@example.test", "Owner", "google-gate") + if err != nil { + t.Fatalf("CreateOrGetUser: %v", err) + } + domain := "gate.example.test" + if _, err := store.ClaimOrCreateDomain(ctx, domain, user.ID); err != nil { + t.Fatalf("ClaimOrCreateDomain: %v", err) + } + if err := store.VerifyDomain(ctx, domain, user.ID); err != nil { + t.Fatalf("VerifyDomain: %v", err) + } + ag, err := store.CreateAgent(ctx, "bot@"+domain, domain, "", "", "local", user.ID) + if err != nil { + t.Fatalf("CreateAgent: %v", err) + } + adapter := agent.NewOutboundSendStore(store, webhookpub.NewOutbox(pool, webhookpub.StaticFlag(true)), usage.NewNoopUsageTracker()) + gate := sendingpolicy.NewGate(pool, sendingpolicy.Secrets{}, sendingpolicy.PolicySourceConfig, sendingpolicy.DisabledPolicy()) + gated := outboundsend.NewJobs(adapter, &fakeDeliverer{}, pool).WithGate(gate) + legacy := outboundsend.NewJobs(adapter, &fakeDeliverer{}, pool) + client, err := jobs.New(pool, jobs.Config{}, gated) + if err != nil { + t.Fatalf("jobs.New: %v", err) + } + gated.SetEnqueuer(client) + legacy.SetEnqueuer(client) + return &gateFixture{t: t, ctx: ctx, pool: pool, store: store, adapter: adapter, gate: gate, userID: user.ID, agentID: ag.ID, client: client, gated: gated, legacy: legacy} +} + +// accept runs the accept transaction the API runs, through the given bundle. +func (f *gateFixture) accept(bundle *outboundsend.Jobs, label string) (messageID string, jobID int64, err error) { + f.t.Helper() + err = f.store.WithTx(f.ctx, func(tx pgx.Tx) error { + m, err := f.store.CreateOutboundMessageTx(f.ctx, tx, f.agentID, + []string{label + "@example.test"}, nil, nil, label, "send", "smtp", "", "conv-"+label, + []byte("From: bot\r\n\r\nbody"), "accepted", "bot@gate.example.test", "relay") + if err != nil { + return err + } + messageID = m.ID + jobID, err = bundle.EnqueueSendTx(f.ctx, tx, messageID) + if err != nil { + return err + } + return f.store.StampSendJobIDTx(f.ctx, tx, messageID, jobID) + }) + return messageID, jobID, err +} + +func (f *gateFixture) operationExists(messageID string) bool { + f.t.Helper() + var n int + if err := f.pool.QueryRow(f.ctx, `SELECT count(*) FROM sending_provider_operations WHERE operation_id = $1 AND purpose = 'customer_message'`, messageID).Scan(&n); err != nil { + f.t.Fatal(err) + } + return n == 1 +} + +func TestJobs_EnqueuePreparesTheOperationInTheAcceptTransaction(t *testing.T) { + f := newGateFixture(t) + messageID, jobID, err := f.accept(f.gated, "prepared") + if err != nil { + t.Fatalf("accept: %v", err) + } + var refID string + if err := f.pool.QueryRow(f.ctx, `SELECT args->'operation_ref'->>'id' FROM river_job WHERE id = $1`, jobID).Scan(&refID); err != nil { + t.Fatal(err) + } + if refID != messageID { + t.Fatalf("job carries operation_ref id %q, want the message id %q", refID, messageID) + } + if !f.operationExists(messageID) { + t.Fatal("no customer_message operation was prepared in the accept transaction") + } +} + +func TestJobs_EnqueueRefusesAPausedAccountAndRollsBack(t *testing.T) { + f := newGateFixture(t) + if _, err := f.pool.Exec(f.ctx, ` + INSERT INTO account_sending_controls (user_id, state, reason, actor) VALUES ($1, 'paused', 'test', 'test') + ON CONFLICT (user_id) DO UPDATE SET state = 'paused'`, f.userID); err != nil { + t.Fatal(err) + } + messageID, _, err := f.accept(f.gated, "paused") + if !errors.Is(err, outboundsend.ErrSendingPaused) { + t.Fatalf("accept on a paused account err = %v, want ErrSendingPaused", err) + } + var rows int + if err := f.pool.QueryRow(f.ctx, `SELECT count(*) FROM messages WHERE id = $1`, messageID).Scan(&rows); err != nil { + t.Fatal(err) + } + if rows != 0 { + t.Fatalf("message row survived the refused accept; the transaction must roll back") + } +} + +func TestJobs_LegacyJobResolvesAndAuthorizesThroughTheGate(t *testing.T) { + f := newGateFixture(t) + // A pre-floor slot enqueued this job: no operation reference in its args. + messageID, jobID, err := f.accept(f.legacy, "legacy") + if err != nil { + t.Fatalf("legacy accept: %v", err) + } + var hasRef bool + if err := f.pool.QueryRow(f.ctx, `SELECT args ? 'operation_ref' FROM river_job WHERE id = $1`, jobID).Scan(&hasRef); err != nil { + t.Fatal(err) + } + if hasRef || f.operationExists(messageID) { + t.Fatal("the legacy enqueue must carry no reference and prepare nothing") + } + + dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{ProviderMessageID: ""}} + w := outboundsend.NewSendWorker(f.adapter, dl).WithGate(f.gate).WithOperationResolver(f.gated.ResolveLegacyOperation) + rj := &river.Job[outboundsend.OutboundSendArgs]{ + JobRow: &rivertype.JobRow{ID: jobID, Attempt: 1, MaxAttempts: outboundsend.MaxSendAttempts, Kind: outboundsend.OutboundSendArgs{}.Kind()}, + Args: outboundsend.OutboundSendArgs{MessageID: messageID}, + } + if err := w.Work(f.ctx, rj); err != nil { + t.Fatalf("Work: %v", err) + } + if dl.calls != 1 { + t.Fatalf("provider calls = %d, want exactly one", dl.calls) + } + if !f.operationExists(messageID) { + t.Fatal("the resolver did not prepare the operation") + } + var state, callState string + if err := f.pool.QueryRow(f.ctx, ` + SELECT state, call_state FROM sending_budget_reservations + WHERE operation_id = $1 AND submission_attempt = 1`, messageID).Scan(&state, &callState); err != nil { + t.Fatalf("read reservation: %v", err) + } + if state != "confirmed" { + t.Fatalf("attempt state = %s, want confirmed (final authorization ran)", state) + } + var status string + if err := f.pool.QueryRow(f.ctx, `SELECT delivery_status FROM messages WHERE id = $1`, messageID).Scan(&status); err != nil { + t.Fatal(err) + } + if status != "sent" { + t.Fatalf("delivery_status = %s, want sent", status) + } +} + +func TestJobs_GatedWorkerAuthorizesANewJob(t *testing.T) { + f := newGateFixture(t) + messageID, jobID, err := f.accept(f.gated, "gated") + if err != nil { + t.Fatalf("accept: %v", err) + } + dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{ProviderMessageID: ""}} + w := outboundsend.NewSendWorker(f.adapter, dl).WithGate(f.gate) + ref := refFor(messageID) + rj := &river.Job[outboundsend.OutboundSendArgs]{ + JobRow: &rivertype.JobRow{ID: jobID, Attempt: 1, MaxAttempts: outboundsend.MaxSendAttempts, Kind: outboundsend.OutboundSendArgs{}.Kind()}, + Args: outboundsend.OutboundSendArgs{MessageID: messageID, OperationRef: &ref}, + } + if err := w.Work(f.ctx, rj); err != nil { + t.Fatalf("Work: %v", err) + } + if dl.calls != 1 || len(dl.auths) != 1 || dl.auths[0].IsZero() { + t.Fatalf("calls=%d auths=%d, want one provider call carrying a real authorization", dl.calls, len(dl.auths)) + } + // A re-drive of the sent row is a no-op: no new ordinal, no new call. + if err := w.Work(f.ctx, rj); err != nil { + t.Fatalf("re-drive: %v", err) + } + var attempts int + if err := f.pool.QueryRow(f.ctx, `SELECT current_attempt FROM sending_provider_operations WHERE operation_id = $1`, messageID).Scan(&attempts); err != nil { + t.Fatal(err) + } + if dl.calls != 1 || attempts != 1 { + t.Fatalf("after re-drive calls=%d current_attempt=%d, want 1/1", dl.calls, attempts) + } +} + +// TestJobs_ReconcilerSettlesTheDialedAttemptFromEvidence: the worker dialed +// (the token was redeemed) but lost the 250; SES's feedback later proved +// acceptance; the job is terminal. The reconciler settles the row as sent and, +// through the gate, settles the attempt that dialed — binding the provider id +// to its correlation — without resubmitting or reserving anything. +func TestJobs_ReconcilerSettlesTheDialedAttemptFromEvidence(t *testing.T) { + f := newGateFixture(t) + messageID, jobID, err := f.accept(f.gated, "evidence") + if err != nil { + t.Fatalf("accept: %v", err) + } + dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{Err: errors.New("data final: lost"), AcceptanceUnknown: true}} + w := outboundsend.NewSendWorker(f.adapter, dl).WithGate(f.gate) + ref := refFor(messageID) + rj := &river.Job[outboundsend.OutboundSendArgs]{ + JobRow: &rivertype.JobRow{ID: jobID, Attempt: 1, MaxAttempts: outboundsend.MaxSendAttempts, Kind: outboundsend.OutboundSendArgs{}.Kind()}, + Args: outboundsend.OutboundSendArgs{MessageID: messageID, OperationRef: &ref}, + } + if err := w.Work(f.ctx, rj); err == nil { + t.Fatal("an acceptance-unknown failure must return a retryable error") + } + // The production submitter redeems before it dials; the fake did not, so + // redeem the token it was handed to reproduce "dialed, answer lost". + if len(dl.auths) != 1 { + t.Fatalf("auths = %d, want the one the worker handed over", len(dl.auths)) + } + if err := f.gate.RedeemProviderCall(f.ctx, dl.auths[0]); err != nil { + t.Fatalf("redeem: %v", err) + } + // SES feedback proved acceptance; River gave up on the job. + if _, err := f.pool.Exec(f.ctx, ` + UPDATE messages SET provider_accepted_at = now(), provider_message_id = '' + WHERE id = $1`, messageID); err != nil { + t.Fatal(err) + } + if _, err := f.pool.Exec(f.ctx, `UPDATE river_job SET state = 'discarded', finalized_at = now() - interval '16 minutes' WHERE id = $1`, jobID); err != nil { + t.Fatal(err) + } + + if err := outboundsend.NewTerminalReconcileWorker(f.pool, f.adapter).WithGate(f.gate).Work(f.ctx, &river.Job[outboundsend.TerminalReconcileArgs]{}); err != nil { + t.Fatalf("reconcile: %v", err) + } + var status string + if err := f.pool.QueryRow(f.ctx, `SELECT delivery_status FROM messages WHERE id = $1`, messageID).Scan(&status); err != nil { + t.Fatal(err) + } + if status != "sent" { + t.Fatalf("delivery_status = %s, want sent from evidence", status) + } + var bound *string + if err := f.pool.QueryRow(f.ctx, ` + SELECT provider_message_id FROM sending_feedback_correlations + WHERE operation_id = $1 AND submission_attempt = 1`, messageID).Scan(&bound); err != nil { + t.Fatalf("read correlation: %v", err) + } + if bound == nil || *bound != "ses-evidence-000000" { + t.Fatalf("correlation provider id = %v, want the bare evidence id bound to the dialed attempt", bound) + } + if dl.calls != 1 { + t.Fatalf("provider calls = %d, want the original one only", dl.calls) + } +} diff --git a/internal/outboundsend/rate_test.go b/internal/outboundsend/rate_test.go index cb71b5294..f05f0c602 100644 --- a/internal/outboundsend/rate_test.go +++ b/internal/outboundsend/rate_test.go @@ -222,13 +222,12 @@ func TestSendWorker_RateLimitedPastRetryHorizonFailsTerminally(t *testing.T) { j.Domain, j.MessageType, j.SentAs = "new.example.com", "send", "own_address" st := &fakeStore{job: j} dl := &fakeDeliverer{} - ramp := &fakeRampGate{decision: outboundsend.RampDecision{Allowed: true}} gate := &fakeRateGate{decision: outboundsend.RateDecision{ Allowed: false, RetryAt: time.Now().Add(30 * time.Second), }} rec := &recordingMetrics{} - w := outboundsend.NewSendWorker(st, dl, ramp).WithRateGate(gate).WithMetrics(rec) + w := outboundsend.NewSendWorker(st, dl).WithRateGate(gate).WithMetrics(rec) err := w.Work(context.Background(), job("msg_1", 4)) if err == nil { @@ -248,9 +247,6 @@ func TestSendWorker_RateLimitedPastRetryHorizonFailsTerminally(t *testing.T) { t.Errorf("terminal = {detail %q, source %v}, want {send_rate_timeout, local}", got.detail, got.source) } - if len(ramp.released) != 1 || ramp.released[0] != "msg_1" { - t.Errorf("ramp releases = %v, want [msg_1] (timeout releases the reservation)", ramp.released) - } if !stringsEqual(rec.terminals, []string{"failed_local_retries"}) { t.Errorf("terminals = %v, want [failed_local_retries]", rec.terminals) } @@ -266,10 +262,9 @@ func TestSendWorker_RateGateErrorPastRetryHorizonFailsTerminally(t *testing.T) { j.AcceptedAt = time.Now().Add(-73 * time.Hour) j.Domain, j.MessageType, j.SentAs = "new.example.com", "send", "own_address" st := &fakeStore{job: j} - ramp := &fakeRampGate{decision: outboundsend.RampDecision{Allowed: true}} gate := &fakeRateGate{err: errors.New("rate store down")} rec := &recordingMetrics{} - w := outboundsend.NewSendWorker(st, &fakeDeliverer{}, ramp).WithRateGate(gate).WithMetrics(rec) + w := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithRateGate(gate).WithMetrics(rec) err := w.Work(context.Background(), job("msg_1", 4)) if err == nil { @@ -286,36 +281,6 @@ func TestSendWorker_RateGateErrorPastRetryHorizonFailsTerminally(t *testing.T) { t.Errorf("terminal = {detail %q, source %v}, want {send_rate_timeout: rate store down, local}", got.detail, got.source) } - if len(ramp.released) != 1 || ramp.released[0] != "msg_1" { - t.Errorf("ramp releases = %v, want [msg_1] (timeout releases the reservation)", ramp.released) - } -} - -// TestSendWorker_RateLimitedDeferralKeepsRampReservation pins the complement -// of the horizon path: an ordinary deferral releases the SEND CLAIM but keeps -// the ramp reservation — same-message Reserve is idempotent, while a released -// reservation is terminal and cannot be re-reserved. -func TestSendWorker_RateLimitedDeferralKeepsRampReservation(t *testing.T) { - j := acceptedJob("msg_1") - j.Domain, j.MessageType, j.SentAs = "new.example.com", "send", "own_address" - st := &fakeStore{job: j} - ramp := &fakeRampGate{decision: outboundsend.RampDecision{Allowed: true}} - gate := &fakeRateGate{decision: outboundsend.RateDecision{ - Allowed: false, - RetryAt: time.Now().Add(30 * time.Second), - }} - w := outboundsend.NewSendWorker(st, &fakeDeliverer{}, ramp).WithRateGate(gate) - - requireSnooze(t, w.Work(context.Background(), job("msg_1", 1))) - if len(ramp.calls) != 1 { - t.Errorf("ramp reserves = %d, want 1 (taken before the rate gate)", len(ramp.calls)) - } - if len(ramp.released) != 0 { - t.Errorf("ramp releases = %v, want none — a deferral keeps the reservation", ramp.released) - } - if len(st.released) != 1 || st.released[0] != "msg_1" { - t.Errorf("send-claim releases = %v, want [msg_1]", st.released) - } } // TestSendWorker_RateGateAllowsSubmission: an allowed reservation falls diff --git a/internal/outboundsend/reconcile_test.go b/internal/outboundsend/reconcile_test.go index c81325f89..8657318aa 100644 --- a/internal/outboundsend/reconcile_test.go +++ b/internal/outboundsend/reconcile_test.go @@ -20,6 +20,7 @@ import ( "github.com/tokencanopy/e2a/internal/jobs" "github.com/tokencanopy/e2a/internal/messagelifecycle" "github.com/tokencanopy/e2a/internal/outboundsend" + "github.com/tokencanopy/e2a/internal/sendingpolicy" "github.com/tokencanopy/e2a/internal/testutil" "github.com/tokencanopy/e2a/internal/usage" "github.com/tokencanopy/e2a/internal/webhookpub" @@ -481,9 +482,8 @@ func TestTerminalReconcileWorker_ReconcilesOnlyTerminalJobs(t *testing.T) { sentID := f.seed(t, "sent", "sent", "completed", false) missingID := f.seed(t, "missing", "accepted", "", true) - gate := &fakeRampGate{} rec := &recordingMetrics{} - worker := outboundsend.NewTerminalReconcileWorker(pool, adapter, gate).WithMetrics(rec) + worker := outboundsend.NewTerminalReconcileWorker(pool, adapter).WithMetrics(rec) if err := worker.Work(context.Background(), &river.Job[outboundsend.TerminalReconcileArgs]{}); err != nil { t.Fatalf("Work: %v", err) } @@ -541,9 +541,6 @@ func TestTerminalReconcileWorker_ReconcilesOnlyTerminalJobs(t *testing.T) { } f.assertEventCarriesOnly(t, tc.id, webhookpub.EventEmailFailed, tr) } - if len(gate.resolved) != 4 { - t.Errorf("ramp resolutions = %v, want four terminal outcomes", gate.resolved) - } // One terminal metric per settled row; all four sweeps here wrote a // locally inferred failure (no provider provenance, no suppression list). // One terminal per settled row, labeled by provenance: the cancelled-state @@ -572,82 +569,6 @@ func TestTerminalReconcileWorker_ReconcilesOnlyTerminalJobs(t *testing.T) { } } -func TestTerminalReconcileWorker_ResolvesReservedRampForTerminalMessage(t *testing.T) { - pool := testutil.TestDB(t) - store := identity.NewStore(pool) - adapter := agent.NewOutboundSendStore(store, - webhookpub.NewOutbox(pool, webhookpub.StaticFlag(true)), usage.NewNoopUsageTracker()) - f := newTerminalFixture(t, pool, store, adapter) - messageID := f.seed(t, "terminal-ramp-cleanup", "accepted", "cancelled", false) - - ctx := context.Background() - var userID string - if err := pool.QueryRow(ctx, `SELECT user_id FROM agent_identities WHERE id=$1`, f.agentID).Scan(&userID); err != nil { - t.Fatalf("read agent owner: %v", err) - } - if _, err := pool.Exec(ctx, - `UPDATE messages SET delivery_status='failed' WHERE id=$1`, messageID); err != nil { - t.Fatalf("make message terminal: %v", err) - } - if _, err := pool.Exec(ctx, - `INSERT INTO domain_send_counters (user_id, domain, day, reserved_count, confirmed_count, daily_limit) - VALUES ($1, 'example.com', current_date, 1, 0, 50)`, userID); err != nil { - t.Fatalf("seed ramp counter: %v", err) - } - if _, err := pool.Exec(ctx, - `INSERT INTO sending_ramp_reservations (message_id, day, user_id, domain, units) - VALUES ($1, current_date, $2, 'example.com', 1)`, messageID, userID); err != nil { - t.Fatalf("seed reserved ramp: %v", err) - } - - gate := &fakeRampGate{} - worker := outboundsend.NewTerminalReconcileWorker(pool, adapter, gate) - if err := worker.Work(ctx, &river.Job[outboundsend.TerminalReconcileArgs]{}); err != nil { - t.Fatalf("Work: %v", err) - } - if len(gate.resolved) != 1 || gate.resolved[0] != messageID { - t.Fatalf("ramp resolutions = %v, want [%s]", gate.resolved, messageID) - } -} - -func TestTerminalReconcileWorker_ResolvesReleasedRampAfterProviderCorrection(t *testing.T) { - pool := testutil.TestDB(t) - store := identity.NewStore(pool) - adapter := agent.NewOutboundSendStore(store, - webhookpub.NewOutbox(pool, webhookpub.StaticFlag(true)), usage.NewNoopUsageTracker()) - f := newTerminalFixture(t, pool, store, adapter) - messageID := f.seed(t, "released-ramp-provider-correction", "accepted", "cancelled", false) - - ctx := context.Background() - var userID string - if err := pool.QueryRow(ctx, `SELECT user_id FROM agent_identities WHERE id=$1`, f.agentID).Scan(&userID); err != nil { - t.Fatalf("read agent owner: %v", err) - } - if _, err := pool.Exec(ctx, - `UPDATE messages SET delivery_status='delivered' WHERE id=$1`, messageID); err != nil { - t.Fatalf("apply provider correction: %v", err) - } - if _, err := pool.Exec(ctx, - `INSERT INTO domain_send_counters (user_id, domain, day, reserved_count, confirmed_count, daily_limit) - VALUES ($1, 'example.com', current_date, 0, 0, 50)`, userID); err != nil { - t.Fatalf("seed ramp counter: %v", err) - } - if _, err := pool.Exec(ctx, - `INSERT INTO sending_ramp_reservations (message_id, day, user_id, domain, units, state) - VALUES ($1, current_date, $2, 'example.com', 1, 'released')`, messageID, userID); err != nil { - t.Fatalf("seed released ramp: %v", err) - } - - gate := &fakeRampGate{} - worker := outboundsend.NewTerminalReconcileWorker(pool, adapter, gate) - if err := worker.Work(ctx, &river.Job[outboundsend.TerminalReconcileArgs]{}); err != nil { - t.Fatalf("Work: %v", err) - } - if len(gate.resolved) != 1 || gate.resolved[0] != messageID { - t.Fatalf("ramp resolutions = %v, want [%s]", gate.resolved, messageID) - } -} - // TestTerminalReconcileWorker_GraceWindowHoldsFreshTerminalJobs pins the §3.1 // grace behavior: a row whose job just reached a terminal state is NOT failed // while provider evidence may still be arriving; it is failed once the job has @@ -662,8 +583,7 @@ func TestTerminalReconcileWorker_GraceWindowHoldsFreshTerminalJobs(t *testing.T) freshID := f.seed(t, "fresh-discard", "accepted", "discarded", false) f.freshenJob(t, freshID) // terminal seconds ago — inside the grace window - gate := &fakeRampGate{} - worker := outboundsend.NewTerminalReconcileWorker(pool, adapter, gate) + worker := outboundsend.NewTerminalReconcileWorker(pool, adapter) if err := worker.Work(context.Background(), &river.Job[outboundsend.TerminalReconcileArgs]{}); err != nil { t.Fatalf("Work: %v", err) } @@ -721,8 +641,7 @@ func TestTerminalReconcileWorker_ProviderEvidenceSettlesAsSent(t *testing.T) { t.Fatal(err) } - gate := &fakeRampGate{} - worker := outboundsend.NewTerminalReconcileWorker(pool, adapter, gate) + worker := outboundsend.NewTerminalReconcileWorker(pool, adapter) if err := worker.Work(context.Background(), &river.Job[outboundsend.TerminalReconcileArgs]{}); err != nil { t.Fatalf("Work: %v", err) } @@ -784,9 +703,6 @@ func TestTerminalReconcileWorker_ProviderEvidenceSettlesAsSent(t *testing.T) { if got := f.failedEventCount(t, evidenceID); got != 0 { t.Errorf("email.failed count = %d, want 0 — evidence must suppress the false failure", got) } - if len(gate.resolved) != 1 || gate.resolved[0] != evidenceID { - t.Errorf("ramp resolutions = %v, want evidence message", gate.resolved) - } // Idempotent: a second pass no-ops (the row left accepted/sending). if err := worker.Work(context.Background(), &river.Job[outboundsend.TerminalReconcileArgs]{}); err != nil { @@ -938,14 +854,19 @@ func testLocalFallbackReason(t *testing.T, label string, want messagelifecycle.R if _, err := pool.Exec(context.Background(), `UPDATE messages SET sent_as='own_address' WHERE id=$1`, messageID); err != nil { t.Fatal(err) } - worker = outboundsend.NewSendWorker(adapter, &fakeDeliverer{}, &fakeRampGate{err: permanentRampError{msg: "invalid ramp"}}) + // A terminal gate hold (the account is gone) is the local cancellation + // this reason describes. + worker = outboundsend.NewSendWorker(adapter, &fakeDeliverer{}).WithGate(&fakeGate{ + reserve: sendingpolicy.Decision{Allow: false, Reason: sendingpolicy.ReasonAccountDeleted, Terminal: true}, + }) } else { if _, err := pool.Exec(context.Background(), `UPDATE messages SET created_at=now()-interval '73 hours' WHERE id=$1`, messageID); err != nil { t.Fatal(err) } worker = outboundsend.NewSendWorker(adapter, &fakeDeliverer{out: outboundsend.DeliverOutcome{Err: errors.New("provider unavailable"), Outage: true}}) } - rj := &river.Job[outboundsend.OutboundSendArgs]{JobRow: &rivertype.JobRow{ID: jobID, Attempt: 3, CreatedAt: time.Now().UTC()}, Args: outboundsend.OutboundSendArgs{MessageID: messageID}} + ref := refFor(messageID) + rj := &river.Job[outboundsend.OutboundSendArgs]{JobRow: &rivertype.JobRow{ID: jobID, Attempt: 3, CreatedAt: time.Now().UTC()}, Args: outboundsend.OutboundSendArgs{MessageID: messageID, OperationRef: &ref}} if err := worker.Work(context.Background(), rj); err == nil { t.Fatal("terminal branch must return cancellation/error") } @@ -988,8 +909,7 @@ func testProviderRejectionAtomicFailure(t *testing.T, label, install, uninstall } t.Cleanup(func() { _, _ = pool.Exec(context.Background(), uninstall) }) deliverer := &fakeDeliverer{out: outboundsend.DeliverOutcome{Err: errors.New("550 explicit rejection"), Permanent: true}} - ramp := &fakeRampGate{decision: outboundsend.RampDecision{Allowed: true}} - w := outboundsend.NewSendWorker(adapter, deliverer, ramp) + w := outboundsend.NewSendWorker(adapter, deliverer) rj := &river.Job[outboundsend.OutboundSendArgs]{JobRow: &rivertype.JobRow{ID: jobID, Attempt: 2, CreatedAt: time.Now().UTC()}, Args: outboundsend.OutboundSendArgs{MessageID: messageID}} if err := w.Work(context.Background(), rj); err == nil { t.Fatal("provider rejection must cancel") @@ -1018,9 +938,6 @@ func testProviderRejectionAtomicFailure(t *testing.T, label, install, uninstall if deliverer.calls != 1 { t.Fatalf("fallback re-drive provider calls=%d, want exactly the original call", deliverer.calls) } - if len(ramp.calls) != 1 || len(ramp.released) != 1 || len(ramp.resolved) != 1 { - t.Fatalf("fallback ramp reserve=%d release=%v resolve=%v, want one of each without re-reserve", len(ramp.calls), ramp.released, ramp.resolved) - } if _, err := pool.Exec(context.Background(), uninstall); err != nil { t.Fatal(err) } @@ -1132,6 +1049,9 @@ func (s failingTerminalStore) ClaimSend(context.Context, string, int64) (*outbou return nil, nil } func (s failingTerminalStore) ReleaseSend(context.Context, string, int64) error { return nil } +func (s failingTerminalStore) RecordHold(context.Context, string, outboundsend.HoldClass, time.Time) error { + return nil +} func (s failingTerminalStore) MarkSent(context.Context, string, int64, int, time.Time, string, string) error { return nil } diff --git a/internal/outboundsend/suppression_test.go b/internal/outboundsend/suppression_test.go index cb535827e..80bd3803e 100644 --- a/internal/outboundsend/suppression_test.go +++ b/internal/outboundsend/suppression_test.go @@ -17,12 +17,13 @@ import ( "testing" "github.com/tokencanopy/e2a/internal/outboundsend" + "github.com/tokencanopy/e2a/internal/sendingpolicy" ) // trippingDeliverer fails the test if any provider I/O is attempted. type trippingDeliverer struct{ t *testing.T } -func (d trippingDeliverer) Deliver(_ context.Context, j *outboundsend.SendJob) outboundsend.DeliverOutcome { +func (d trippingDeliverer) Deliver(_ context.Context, j *outboundsend.SendJob, _ sendingpolicy.ProviderAuthorization) outboundsend.DeliverOutcome { d.t.Errorf("provider Deliver called for %s despite suppression guard", j.MessageID) return outboundsend.DeliverOutcome{} } @@ -31,8 +32,7 @@ func TestSendWorker_SuppressedRecipientFailsTerminallyWithoutProviderIO(t *testi j := acceptedJob("msg_1") j.Domain, j.MessageType, j.SentAs = "new.example.com", "send", "own_address" st := &fakeStore{job: j, suppressed: []string{"b@y.com"}} - gate := &fakeRampGate{decision: outboundsend.RampDecision{Allowed: true}} - w := outboundsend.NewSendWorker(st, trippingDeliverer{t}, gate) + w := outboundsend.NewSendWorker(st, trippingDeliverer{t}) err := w.Work(context.Background(), job("msg_1", 1)) if err == nil { @@ -57,9 +57,6 @@ func TestSendWorker_SuppressedRecipientFailsTerminallyWithoutProviderIO(t *testi if st.suppressionAgentID != st.job.AgentID { t.Errorf("suppression check agent = %q, want %q", st.suppressionAgentID, st.job.AgentID) } - if len(gate.released) != 1 || gate.released[0] != "msg_1" { - t.Errorf("ramp releases = %v, want [msg_1]", gate.released) - } } // A store error on the guard is conservative: no provider I/O, no terminal @@ -83,45 +80,6 @@ func TestSendWorker_SuppressionCheckErrorFailsClosed(t *testing.T) { } } -func TestSendWorker_SuppressionCheckErrorAfterRampPreservesReservation(t *testing.T) { - j := acceptedJob("msg_1") - j.Domain, j.MessageType, j.SentAs = "new.example.com", "send", "own_address" - st := &fakeStore{job: j, suppressedErr: errors.New("suppression store down")} - gate := &fakeRampGate{decision: outboundsend.RampDecision{Allowed: true}} - w := outboundsend.NewSendWorker(st, trippingDeliverer{t}, gate) - - if err := w.Work(context.Background(), job("msg_1", 1)); err == nil { - t.Fatal("suppression-store error must retry") - } - if len(gate.released) != 0 { - t.Fatalf("ramp releases = %v, want none so same-day retry stays idempotent", gate.released) - } - if len(st.released) != 1 || st.released[0] != "msg_1" { - t.Fatalf("claim releases = %v, want [msg_1]", st.released) - } -} - -func TestSendWorker_SuppressionCheckErrorKeepsRampReservationWhenClaimReleaseFails(t *testing.T) { - lookupErr := errors.New("suppression store down") - claimErr := errors.New("claim release down") - j := acceptedJob("msg_1") - j.Domain, j.MessageType, j.SentAs = "new.example.com", "send", "own_address" - st := &fakeStore{job: j, suppressedErr: lookupErr, releaseErr: claimErr} - gate := &fakeRampGate{decision: outboundsend.RampDecision{Allowed: true}} - w := outboundsend.NewSendWorker(st, trippingDeliverer{t}, gate) - - err := w.Work(context.Background(), job("msg_1", 1)) - if !errors.Is(err, lookupErr) || !errors.Is(err, claimErr) { - t.Fatalf("error = %v, want joined lookup and claim-release causes", err) - } - if len(st.released) != 1 { - t.Fatalf("claim release calls = %v, want one attempt", st.released) - } - if len(gate.released) != 0 { - t.Fatalf("ramp releases = %v, want none while claim remains held", gate.released) - } -} - func TestSendWorker_UnsuppressedRecipientStillSends(t *testing.T) { st := &fakeStore{job: acceptedJob("msg_1")} // no suppressions dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{ProviderMessageID: "ses-ok", SentAs: "relay"}} diff --git a/internal/outboundsend/terminal_reconcile.go b/internal/outboundsend/terminal_reconcile.go index 584361204..c4f7a10fd 100644 --- a/internal/outboundsend/terminal_reconcile.go +++ b/internal/outboundsend/terminal_reconcile.go @@ -2,6 +2,7 @@ package outboundsend import ( "context" + "errors" "fmt" "log" "time" @@ -12,6 +13,7 @@ import ( "github.com/tokencanopy/e2a/internal/delivery" "github.com/tokencanopy/e2a/internal/jobs" "github.com/tokencanopy/e2a/internal/messagelifecycle" + "github.com/tokencanopy/e2a/internal/sendingpolicy" ) const terminalReconcileInterval = time.Minute @@ -49,15 +51,21 @@ type TerminalReconcileWorker struct { river.WorkerDefaults[TerminalReconcileArgs] pool *pgxpool.Pool store Store - ramp RampGate + gate sendingpolicy.Gate metrics Metrics } // NewTerminalReconcileWorker builds the periodic safety-net worker. -func NewTerminalReconcileWorker(pool *pgxpool.Pool, store Store, ramps ...RampGate) *TerminalReconcileWorker { - w := &TerminalReconcileWorker{pool: pool, store: store, metrics: noopMetrics{}} - if len(ramps) > 0 { - w.ramp = ramps[0] +func NewTerminalReconcileWorker(pool *pgxpool.Pool, store Store) *TerminalReconcileWorker { + return &TerminalReconcileWorker{pool: pool, store: store, metrics: noopMetrics{}} +} + +// WithGate injects the sending-protection gate so an evidence-settled row can +// also settle its provider attempt (ramp progress, provider-id binding). +// Reconciliation is settlement-only: it never resubmits and never reserves. +func (w *TerminalReconcileWorker) WithGate(g sendingpolicy.Gate) *TerminalReconcileWorker { + if g != nil { + w.gate = g } return w } @@ -86,6 +94,7 @@ type terminalCandidate struct { failureOccurredAt *time.Time failureAttempt *int failureBlockedRecipients []string + providerMessageID string } // submissionAnchor is this candidate's acceptance→terminal SLI baseline — the @@ -116,7 +125,8 @@ func (w *TerminalReconcileWorker) Work(ctx context.Context, _ *river.Job[Termina r.finalized_at, m.created_at, m.scheduled_at, m.reviewed_at, COALESCE(m.delivery_failure_source,''),COALESCE(m.delivery_detail,''),COALESCE(m.delivery_failure_reason_code,''), - m.delivery_failure_occurred_at,m.delivery_failure_attempt,m.delivery_failure_blocked_recipients + m.delivery_failure_occurred_at,m.delivery_failure_attempt,m.delivery_failure_blocked_recipients, + COALESCE(m.provider_message_id,'') FROM messages m LEFT JOIN river_job r ON r.id = m.send_job_id WHERE m.direction = 'outbound' @@ -138,7 +148,7 @@ func (w *TerminalReconcileWorker) Work(ctx context.Context, _ *river.Job[Termina candidates := make([]terminalCandidate, 0) for rows.Next() { var candidate terminalCandidate - if err := rows.Scan(&candidate.messageID, &candidate.jobID, &candidate.attempt, &candidate.state, &candidate.finalizedAt, &candidate.acceptedAt, &candidate.scheduledAt, &candidate.reviewedAt, &candidate.failureSource, &candidate.detail, &candidate.failureReason, &candidate.failureOccurredAt, &candidate.failureAttempt, &candidate.failureBlockedRecipients); err != nil { + if err := rows.Scan(&candidate.messageID, &candidate.jobID, &candidate.attempt, &candidate.state, &candidate.finalizedAt, &candidate.acceptedAt, &candidate.scheduledAt, &candidate.reviewedAt, &candidate.failureSource, &candidate.detail, &candidate.failureReason, &candidate.failureOccurredAt, &candidate.failureAttempt, &candidate.failureBlockedRecipients, &candidate.providerMessageID); err != nil { return err } candidates = append(candidates, candidate) @@ -204,69 +214,34 @@ func (w *TerminalReconcileWorker) Work(ctx context.Context, _ *river.Job[Termina emitTerminal(w.metrics, terminalOutcome(source, reason, candidate.failureBlockedRecipients), candidate.submissionAnchor(), settledAt) case delivery.StatusSent: emitTerminal(w.metrics, terminalSent, candidate.submissionAnchor(), settledAt) - } - if w.ramp != nil { - if err := w.ramp.Resolve(ctx, candidate.messageID); err != nil { - return fmt.Errorf("resolve sending ramp for %s: %w", candidate.messageID, err) - } + // Provider evidence settled the row; settle the attempt that + // dialed, so ramp progress and the provider-id binding catch up. + // Best effort and idempotent — an attempt that predates the gate + // has nothing to settle. + w.settleFromEvidence(ctx, candidate.messageID, candidate.providerMessageID) } processed++ } if processed > 0 { log.Printf("[outbound-terminal-reconcile] processed %d candidates", processed) } - return w.resolveTerminalRampReservations(ctx) + return nil } -// resolveTerminalRampReservations is the durable safety net for the narrow -// window where a worker commits a terminal message outcome, then cannot settle -// its sending-ramp reservation. That worker returns an error and normally fixes -// the reservation on its next (unclaimable-message) retry, but its last River -// attempt can be discarded before another retry. The sweep also revisits a -// released reservation when authoritative provider feedback later corrects a -// locally inferred failure. The reservation table's state/updated_at index -// makes this bounded sweep cheap; Resolve derives confirm versus release from -// the message's durable delivery status. -func (w *TerminalReconcileWorker) resolveTerminalRampReservations(ctx context.Context) error { - if w.ramp == nil { - return nil +func (w *TerminalReconcileWorker) settleFromEvidence(ctx context.Context, messageID, providerMessageID string) { + if w.gate == nil { + return } - rows, err := w.pool.Query(ctx, - `SELECT r.message_id - FROM sending_ramp_reservations r - JOIN messages m ON m.id = r.message_id - WHERE (r.state = 'reserved' - AND m.delivery_status IN ('sent', 'failed', 'deferred', 'delivered', 'bounced', 'complained')) - OR (r.state = 'released' - AND m.delivery_status IN ('sent', 'deferred', 'delivered', 'bounced', 'complained')) - ORDER BY r.updated_at ASC, r.message_id ASC - LIMIT $1`, - jobs.DefaultReconcileBatch, - ) + ref, err := w.gate.LookupOperation(ctx, messageID) if err != nil { - return err - } - messageIDs := make([]string, 0) - for rows.Next() { - var messageID string - if err := rows.Scan(&messageID); err != nil { - rows.Close() - return err + if !errors.Is(err, sendingpolicy.ErrSourceUnavailable) { + log.Printf("[outbound-terminal-reconcile] lookup operation for %s: %v", messageID, err) } - messageIDs = append(messageIDs, messageID) + return } - if err := rows.Err(); err != nil { - rows.Close() - return err + if err := w.gate.SettleOperation(ctx, ref, sendingpolicy.SettlementProviderAccepted, providerMessageID); err != nil && !errors.Is(err, sendingpolicy.ErrAttemptStale) { + log.Printf("[outbound-terminal-reconcile] settle %s from provider evidence: %v", messageID, err) } - rows.Close() - - for _, messageID := range messageIDs { - if err := w.ramp.Resolve(ctx, messageID); err != nil { - return fmt.Errorf("resolve terminal sending ramp for %s: %w", messageID, err) - } - } - return nil } func terminalReconcilePeriodicConstructor() (river.JobArgs, *river.InsertOpts) { diff --git a/internal/outboundsend/worker.go b/internal/outboundsend/worker.go index a1bd86181..166a9956b 100644 --- a/internal/outboundsend/worker.go +++ b/internal/outboundsend/worker.go @@ -15,6 +15,15 @@ // ambiguously defers its terminal write to the reconciler's provider-evidence // grace window rather than firing an immediate — possibly false — email.failed. // +// Every provider call passes through the sending-protection Gate +// (internal/sendingpolicy). The worker order is fixed: Reserve the durable +// attempt; on a hold, snooze without provider I/O; on a rate deferral, +// DeferAttempt; on a final suppression match, CancelAttempt; ConsumeAttempt is +// the last serialized decision; the authorized submitter redeems the token +// immediately before the socket opens and settles the provider's answer. A +// later execution after a confirmed attempt returns to Reserve, which +// allocates the next ordinal — the worker never chooses one. +// // One SMTP attempt per job attempt — River owns the multi-attempt envelope via // NextRetry, so Work() stays short (the deliverer does a single submit, not an // internal retry loop). See the design's "claim + rescue, not a lease" note. @@ -35,6 +44,7 @@ import ( "github.com/tokencanopy/e2a/internal/delivery" "github.com/tokencanopy/e2a/internal/messagelifecycle" "github.com/tokencanopy/e2a/internal/outbound" + "github.com/tokencanopy/e2a/internal/sendingpolicy" "github.com/tokencanopy/e2a/internal/sendrate" ) @@ -57,34 +67,92 @@ const MaxSendAttempts = 6 // MaxSendAttempts (design §8 circuit breaker). const outageSnoozeInterval = 5 * time.Minute -// rampErrorSnoozeInterval keeps a durable message queued when the ramp store is -// temporarily unavailable. JobSnooze does not consume a River attempt. -const rampErrorSnoozeInterval = time.Minute +// gateErrorSnoozeInterval keeps a durable message queued when the sending +// protection gate is temporarily unavailable. JobSnooze does not consume a +// River attempt: fail toward retry, never toward an unauthorized submit. +const gateErrorSnoozeInterval = time.Minute // rateErrorSnoozeInterval keeps a durable message queued when the fire-time -// rate store is temporarily unavailable — mirroring rampErrorSnoozeInterval: -// fail toward retry, never toward an unthrottled submit. +// rate store is temporarily unavailable — fail toward retry, never toward an +// unthrottled submit. const rateErrorSnoozeInterval = time.Minute // rateMinSnooze floors a rate deferral so a RetryAt at (or just past) now — // the window-boundary race — cannot hot-loop the queue. const rateMinSnooze = 250 * time.Millisecond -// SendRetryHorizon bounds the outage-tolerant tail: past this age (from accept) an -// outage-snoozing job stops deferring and is declared terminally failed. 72h matches -// the industry MTA retry horizon (and the webhook deliverer's envelope) — long enough -// to ride out a multi-hour regional SES incident, not forever. +// indefiniteHoldSnooze paces a hold that has no clock of its own — an account +// pause waits for an operator, not for midnight. +const indefiniteHoldSnooze = time.Hour + +// SendRetryHorizon bounds the outage-tolerant tail: past this age a message in a +// rate/ramp/provider or tenant-setup hold is declared terminally failed. 72h +// matches the industry MTA retry horizon (and the webhook deliverer's envelope) +// — long enough to ride out a multi-hour regional SES incident, not forever. const SendRetryHorizon = 72 * time.Hour -// OutboundSendArgs drives one outbound send. Args carry only the message id; the -// worker re-reads the messages row (the source of truth) each attempt. +// PolicyBudgetHoldHorizon bounds a sending-budget hold: a message may wait +// through several UTC days for capacity, but not forever. Seven days is the +// policy's budget_hold_max_days default; the worker holds it as a constant +// because the deadline is derived, never stored, and every execution must +// derive the same one. +const PolicyBudgetHoldHorizon = 7 * 24 * time.Hour + +// HoldClass is the durable finite-hold classification persisted on the message +// the first time it waits for something with a clock. +type HoldClass string + +const ( + // HoldRateRampOrProvider: per-agent rate, custom-domain ramp, or provider + // outage. 72-hour deadline; expiry reason submission.local_retries_exhausted. + HoldRateRampOrProvider HoldClass = "rate_ramp_or_provider" + // HoldTenantSetup: the account's SES tenant is not ready. 72-hour deadline; + // expiry reason submission.sending_setup_expired. Transitions exactly once + // to HoldRateRampOrProvider when readiness lands before the setup deadline. + HoldTenantSetup HoldClass = "tenant_setup" + // HoldPolicyBudget: a sending-budget pool is exhausted. Seven-day deadline + // from the existing anchor; every finite class promotes to it and nothing + // moves it afterwards. Expiry reason submission.policy_budget_expired. + HoldPolicyBudget HoldClass = "policy_budget" +) + +// horizon is the class's absolute deadline measured from its anchor. +func (c HoldClass) horizon() time.Duration { + if c == HoldPolicyBudget { + return PolicyBudgetHoldHorizon + } + return SendRetryHorizon +} + +// expiryReason is the lifecycle reason a class emits when its deadline passes. +func (c HoldClass) expiryReason() messagelifecycle.ReasonCode { + switch c { + case HoldPolicyBudget: + return messagelifecycle.ReasonSubmissionPolicyBudgetExpired + case HoldTenantSetup: + return messagelifecycle.ReasonSubmissionSendingSetupExpired + } + return messagelifecycle.ReasonSubmissionLocalRetriesExhausted +} + +// ErrSendingPaused is returned by the enqueue entry points when the owning +// account is paused: the acceptance surface must reject the request rather +// than queue mail that can never leave. +var ErrSendingPaused = errors.New("outboundsend: account sending is paused") + +// OutboundSendArgs drives one outbound send. Args carry the message id and the +// durable operation reference the accept transaction prepared; the worker +// re-reads the messages row (the source of truth) each attempt. A job enqueued +// before the reference existed (a pre-floor slot) carries none and is resolved +// at fire time through the same Prepare path. type OutboundSendArgs struct { - MessageID string `json:"message_id"` + MessageID string `json:"message_id"` + OperationRef *sendingpolicy.OperationRef `json:"operation_ref,omitempty"` } func (OutboundSendArgs) Kind() string { return "outbound_send" } -// SendJob is the send payload the worker loads from the messages row (Store.LoadForSend). +// SendJob is the send payload the worker loads from the messages row (Store.ClaimSend). type SendJob struct { MessageID string // UserID is the owning account — the tenant scope for the pre-provider @@ -98,46 +166,34 @@ type SendJob struct { Recipients []string RawMessage []byte // composed MIME SentAs string // From identity decided at accept ("own_address"|"relay") - // AcceptedAt is messages.created_at — the outage tail's clock, so a job that has - // been snoozing through an outage past SendRetryHorizon can be terminated. + // AcceptedAt is messages.created_at. AcceptedAt time.Time // ScheduledAt is messages.scheduled_at for a scheduled send (zero for an - // immediate one). The retry horizon is measured from max(AcceptedAt, - // ScheduledAt): a send scheduled far past accept still gets the full - // outage-tolerant tail from its fire time, instead of a horizon already blown - // the moment it first runs. + // immediate one). ScheduledAt time.Time // ReviewedAt is messages.reviewed_at — when a HITL hold was resolved into the - // send pipeline (human approve or TTL auto-approve), zero for a message that - // was never held. Consumed ONLY by submissionAnchor for the latency SLI; the - // retry horizon deliberately still measures from AcceptedAt, so the F2 - // limitation in docs/design/hitl-ttl-async-send.md is unchanged by this field. + // send pipeline, zero for a message that was never held. ReviewedAt time.Time // ProviderAccepted is set when authoritatively correlated provider-accept - // evidence (an SNS-verified, header- or provider-id-matched SES - // notification) has been recorded for this message: the provider already - // has it — an earlier attempt's submit landed in the SMTP-accept↔mark-sent - // crash window — so the worker settles the row as sent instead of - // re-submitting a duplicate. + // evidence has been recorded for this message: the provider already has it, + // so the worker settles the row as sent instead of re-submitting a duplicate. ProviderAccepted bool ProviderAcceptedAt *time.Time // ProviderMessageID is the evidence-repaired provider id accompanying // ProviderAccepted ('' when no evidence). ProviderMessageID string -} - -// pastRetryHorizon reports whether the accept is older than the outage-tolerant -// retry horizon. Zero AcceptedAt (unknown) is treated as not-past so an outage keeps -// deferring rather than being falsely terminated on a missing timestamp. -func (j *SendJob) pastRetryHorizon() bool { - // Measure from max(accept, scheduled): a scheduled send's outage tail starts - // when it fires, not when it was accepted, so a >72h-out schedule isn't - // terminally failed on its very first attempt. - start := j.AcceptedAt - if j.ScheduledAt.After(start) { - start = j.ScheduledAt - } - return !start.IsZero() && time.Since(start) > SendRetryHorizon + // LocalHoldClass / LocalHoldAnchor are the durable finite-hold pair a + // previous execution persisted (empty/zero when never held). The deadline + // is derived from them on every execution and never stored. + LocalHoldClass HoldClass + LocalHoldAnchor time.Time + // LastResumedAt is the owning account's last pause→active transition; a + // first finite hold anchors no earlier than it, so a pause that preceded + // the hold does not consume its horizon. Zero when unknown. + LastResumedAt time.Time + // TenantReadyAt is when the account's SES tenant became ready (zero until + // it is). Drives the one-way tenant_setup → rate_ramp_or_provider move. + TenantReadyAt time.Time } // submissionAnchor is this job's acceptance→terminal SLI baseline — see the @@ -160,7 +216,20 @@ func (j *SendJob) alreadyDone() bool { return s != delivery.StatusAccepted && s != delivery.StatusSending } -// DeliverOutcome is the result of one SMTP submit attempt. +// initialHoldAnchor is where a message's first finite hold starts its clock: +// the latest of accept, schedule, review, and the account's last resume, so +// time spent in review or under an earlier pause is not charged to the hold. +func (j *SendJob) initialHoldAnchor() time.Time { + anchor := j.AcceptedAt + for _, t := range []time.Time{j.ScheduledAt, j.ReviewedAt, j.LastResumedAt} { + if t.After(anchor) { + anchor = t + } + } + return anchor +} + +// DeliverOutcome is the result of one authorized provider submission. type DeliverOutcome struct { ProviderMessageID string SentAs string @@ -172,33 +241,23 @@ type DeliverOutcome struct { // the worker snoozes without burning an attempt (design §8), up to the retry // horizon. Mutually exclusive with Permanent in practice. Outage bool + // AcceptanceUnknown marks a failure AFTER the whole body was handed to the + // provider (the 250 never came): the provider may hold the message. Never + // permanent; the next attempt is a new ordinal, and provider feedback + // carrying the attempt header is the only authoritative answer. + AcceptanceUnknown bool + // SettlementErr reports that the provider ACCEPTED the message but the + // local settlement did not commit. The send happened; the caller must not + // resubmit. + SettlementErr error } -// Deliverer performs a SINGLE SMTP submit — River owns re-attempts. Implemented in -// the binary over internal/outbound's single-attempt path. +// Deliverer performs a SINGLE authorized SMTP submit — River owns re-attempts. +// The token is the authorization for exactly this call; the production +// implementation (the outbound.ProviderSubmitter) redeems it immediately before +// the socket opens and refuses to dial without it. type Deliverer interface { - Deliver(ctx context.Context, j *SendJob) DeliverOutcome -} - -type RampRequest struct { - MessageID string - UserID string - Domain string - Units int -} - -type RampDecision struct { - Allowed bool - RetryAt time.Time -} - -// RampGate reserves recipient capacity for an eligible custom-domain send. -// Implementations must make a same-message/day call idempotent. -type RampGate interface { - Reserve(ctx context.Context, req RampRequest) (RampDecision, error) - Confirm(ctx context.Context, messageID string) error - Release(ctx context.Context, messageID string) error - Resolve(ctx context.Context, messageID string) error + Deliver(ctx context.Context, j *SendJob, auth sendingpolicy.ProviderAuthorization) DeliverOutcome } // RateDecision is the fire-time rate gate's answer for one submission slot: @@ -210,21 +269,20 @@ type RateDecision = sendrate.Decision // RateGate reserves one slot in the per-agent fire-time submission budget // (internal/sendrate) — the durable counterpart to the acceptance-time // in-memory send limit, enforced immediately before provider submission so -// scheduled-send bursts and multi-replica deployments cannot exceed it. -// Unlike RampGate there is no Confirm/Release: the slot is consumed at -// Reserve and ages out of the sliding window on its own (see the sendrate -// package doc for the crash semantics). A nil gate allows everything. -// Window exposes the gate's sliding window so the deferral snooze clamp -// cannot diverge from the limiter's real window. +// scheduled-send bursts and multi-replica deployments cannot exceed it. It +// stays separate from the sending-protection gate because it controls provider +// throughput, not reputation admission. A nil gate allows everything. type RateGate interface { Reserve(ctx context.Context, agentID string) (RateDecision, error) Window() time.Duration } -// Store is the messages-store surface the worker needs. Implemented over -// internal/identity in the binary. ClaimSend atomically checks that the message -// and agent are live and persists delivery_status='sending' for the stamped River -// job before provider I/O begins. +// OperationResolver recovers the durable operation for a job that carries no +// reference — a legacy argument shape from a pre-floor slot. It runs the same +// Prepare path an accept transaction runs, idempotently, so an old job and a +// new one authorize identically. +type OperationResolver func(ctx context.Context, messageID string) (sendingpolicy.AcceptanceDecision, sendingpolicy.OperationRef, error) + // DailyQuotaDeferredError is returned by Store.ClaimSend when the owning // account's per-day send cap is exhausted at fire time. The store has already // released the send claim; the worker snoozes the job until RetryAt (the next @@ -237,6 +295,10 @@ func (e *DailyQuotaDeferredError) Error() string { return fmt.Sprintf("daily send cap exhausted; deferred until %s", e.RetryAt.Format(time.RFC3339)) } +// Store is the messages-store surface the worker needs. Implemented over +// internal/identity in the binary. ClaimSend atomically checks that the message +// and agent are live and persists delivery_status='sending' for the stamped River +// job before provider I/O begins. type Store interface { // ClaimSend returns nil when the message is gone, trashed, terminal, or owned // by a different River job. It returns *DailyQuotaDeferredError (claim @@ -245,6 +307,9 @@ type Store interface { ClaimSend(ctx context.Context, messageID string, jobID int64) (*SendJob, error) // ReleaseSend clears a side-effect-free attempt before River backoff. ReleaseSend(ctx context.Context, messageID string, jobID int64) error + // RecordHold persists the message's finite-hold class and anchor. Terminal + // writes clear the pair. + RecordHold(ctx context.Context, messageID string, class HoldClass, anchor time.Time) error // MarkSent records the provider outcome monotonically from a pre-terminal // state, including when trash won after ClaimSend. MarkSent(ctx context.Context, messageID string, jobID int64, attempt int, occurredAt time.Time, providerMessageID, sentAs string) error @@ -256,10 +321,7 @@ type Store interface { // state", not to unconditionally fail. // The returned status reports what the guarded write actually did: // StatusFailed, StatusSent (evidence settle), or "" (no-op). The returned - // time is the occurred_at the write actually used — the provider-accept - // evidence time on an evidence settle, the passed occurredAt on a - // failure, zero on a no-op — so observability reports what the write - // did, not what the caller asked for. + // time is the occurred_at the write actually used. MarkFailed(ctx context.Context, messageID string, jobID int64, attempt int, occurredAt time.Time, detail string, source delivery.FailureSource, reason messagelifecycle.ReasonCode, blockedRecipients []string) (delivery.Status, time.Time, error) PreserveTerminalFailure(ctx context.Context, messageID string, jobID int64, attempt int, occurredAt time.Time, detail string, source delivery.FailureSource, reason messagelifecycle.ReasonCode, blockedRecipients []string) error // DeferTerminalFailure records a final attempt's diagnostic + releases the @@ -281,17 +343,19 @@ type SendWorker struct { river.WorkerDefaults[OutboundSendArgs] store Store deliverer Deliverer - ramp RampGate + gate sendingpolicy.Gate + resolve OperationResolver rate RateGate metrics Metrics + now func() time.Time } -func NewSendWorker(store Store, deliverer Deliverer, ramp ...RampGate) *SendWorker { - w := &SendWorker{store: store, deliverer: deliverer, metrics: noopMetrics{}} - if len(ramp) > 0 { - w.ramp = ramp[0] - } - return w +// NewSendWorker builds a worker with no sending-protection gate. Without a +// gate every provider call is made with an empty authorization, which the +// production submitter refuses before it dials; the composition root always +// installs one via WithGate, and its wiring test proves it. +func NewSendWorker(store Store, deliverer Deliverer) *SendWorker { + return &SendWorker{store: store, deliverer: deliverer, metrics: noopMetrics{}, now: time.Now} } // WithMetrics injects the SLI recorder. Chainable; nil keeps the no-op @@ -312,6 +376,32 @@ func (w *SendWorker) WithRateGate(g RateGate) *SendWorker { return w } +// WithGate injects the sending-protection gate every provider call must pass. +// Chainable; nil keeps the gateless default described on NewSendWorker. +func (w *SendWorker) WithGate(g sendingpolicy.Gate) *SendWorker { + if g != nil { + w.gate = g + } + return w +} + +// WithOperationResolver injects the legacy-argument resolver. Chainable; nil +// leaves a legacy job failing closed. +func (w *SendWorker) WithOperationResolver(r OperationResolver) *SendWorker { + if r != nil { + w.resolve = r + } + return w +} + +// WithClock overrides the worker's clock for deadline tests. Chainable. +func (w *SendWorker) WithClock(now func() time.Time) *SendWorker { + if now != nil { + w.now = now + } + return w +} + // NextRetry overrides River's default backoff with the decided send envelope. func (w *SendWorker) NextRetry(job *river.Job[OutboundSendArgs]) time.Time { i := job.Attempt @@ -325,12 +415,10 @@ func (w *SendWorker) NextRetry(job *river.Job[OutboundSendArgs]) time.Time { // River's 60s default JobTimeout. (Contrast the maintenance/sweep workers, which // override it because they can run for minutes.) func (w *SendWorker) Work(ctx context.Context, job *river.Job[OutboundSendArgs]) error { - // Queue-wait SLI: due→pickup latency for THIS attempt (River stamps - // scheduled_at at enqueue, at each retry's backoff target, and on snooze; - // attempted_at at claim). scheduled_at — NOT created_at — is the baseline: - // a retried/snoozed/ramp-deferred message would otherwise record its entire - // cumulative age as "queue wait" on every pass, poisoning the p95. Guarded - // against zero/negative deltas (clock skew, hand-built rows). + // Queue-wait SLI: due→pickup latency for THIS attempt. scheduled_at — NOT + // created_at — is the baseline: a retried/snoozed/deferred message would + // otherwise record its entire cumulative age as "queue wait" on every + // pass, poisoning the p95. Guarded against zero/negative deltas. if job.AttemptedAt != nil && !job.ScheduledAt.IsZero() { if wait := job.AttemptedAt.Sub(job.ScheduledAt); wait > 0 { w.metrics.OutboundQueueWait(wait.Seconds()) @@ -353,23 +441,9 @@ func (w *SendWorker) Work(ctx context.Context, job *river.Job[OutboundSendArgs]) return err // DB error — retryable } if j == nil { - // A previous terminal attempt may have committed the durable message - // outcome before ramp cleanup failed. Terminal rows cannot be claimed on - // retry, so resolve any reservation from that durable outcome here. Resolve - // is also safe for deleted, non-ramped, and missing messages. - if w.ramp != nil { - if err := w.ramp.Resolve(ctx, job.Args.MessageID); err != nil { - return fmt.Errorf("resolve sending ramp for unclaimable message: %w", err) - } - } return nil // message gone or already terminal — nothing to provider-submit } if j.alreadyDone() { - if w.ramp != nil && j.rampEligible() { - if err := w.ramp.Resolve(ctx, j.MessageID); err != nil { - return fmt.Errorf("resolve sending ramp for completed message: %w", err) - } - } return nil // already submitted (sent+) — idempotent re-drive } if j.ProviderAccepted { @@ -384,138 +458,80 @@ func (w *SendWorker) Work(ctx context.Context, job *river.Job[OutboundSendArgs]) return err } // Terminal 'sent', but NOT an attempt — the submit happened on an - // earlier attempt; only the settle lands here. occurredAt is the - // provider-accept evidence time, so the latency measures - // acceptance→provider-accept, not acceptance→settle. + // earlier attempt; only the settle lands here. emitTerminal(w.metrics, terminalSent, j.submissionAnchor(), observedAt) - if w.ramp != nil && j.rampEligible() { - return w.ramp.Confirm(ctx, j.MessageID) - } + w.settleFromEvidence(ctx, job, j.ProviderMessageID) return nil } - // Ramp only mail that uses a verified customer identity. Platform-originated - // test mail uses the relay identity and remains exempt; loopback never enters - // this worker. Reserve after the provider-evidence guard. The final suppression - // check deliberately follows an allowed reservation, closing the policy window - // while Reserve waits on shared capacity. Retryable work after Reserve keeps - // that reservation: same-message/day Reserve is idempotent, while a released - // reservation is terminal and cannot be re-reserved. - if w.ramp != nil && j.rampEligible() { - decision, rerr := w.ramp.Reserve(ctx, RampRequest{ - MessageID: j.MessageID, - UserID: j.UserID, - Domain: j.Domain, - Units: uniqueRecipientCount(j.Recipients), - }) - observedAt = time.Now().UTC() - if rerr != nil { - if isPermanentRampError(rerr) { - if err := w.markFailed(ctx, j.MessageID, job.ID, job.Attempt, j.submissionAnchor(), observedAt, "sending_ramp_invalid: "+rerr.Error(), delivery.FailureSourceLocal, messagelifecycle.ReasonSubmissionCancelled, nil); err != nil { - return err - } - return river.JobCancel(rerr) - } - if j.pastRetryHorizon() { - if err := w.markFailed(ctx, j.MessageID, job.ID, job.Attempt, j.submissionAnchor(), observedAt, "ramp_capacity_timeout: "+rerr.Error(), delivery.FailureSourceLocal, messagelifecycle.ReasonSubmissionLocalRetriesExhausted, nil); err != nil { - return err - } - _ = w.ramp.Release(ctx, j.MessageID) - return river.JobCancel(fmt.Errorf("sending ramp unavailable past %s horizon: %w", SendRetryHorizon, rerr)) - } - if err := w.store.ReleaseSend(ctx, j.MessageID, job.ID); err != nil { - return fmt.Errorf("release outbound send claim after ramp-check failure: %w", err) - } - log.Printf("[outbound-send] ramp reservation failed for %s (snoozing): %v", j.MessageID, rerr) - return river.JobSnooze(rampErrorSnoozeInterval) + // A message whose SES tenant became ready in time leaves the setup class + // before any later gate is consulted, so the setup deadline it has already + // escaped cannot fail it and the new 72-hour horizon starts at readiness. + if err := w.applyTenantReadiness(ctx, j); err != nil { + return err + } + + // Without a gate (unit tests only) the gate steps are skipped and every + // other guard still runs; the production deliverer refuses the empty + // authorization that results, so a deployment that reaches the provider + // this way sends nothing. The composition root's wiring test proves + // production never builds this shape. + var attempt sendingpolicy.AttemptRef + if w.gate != nil { + ref, holdErr := w.operationFor(ctx, job, j) + if holdErr != nil { + return holdErr } - if !decision.Allowed { - if j.pastRetryHorizon() { - if err := w.markFailed(ctx, j.MessageID, job.ID, job.Attempt, j.submissionAnchor(), observedAt, "ramp_capacity_timeout", delivery.FailureSourceLocal, messagelifecycle.ReasonSubmissionLocalRetriesExhausted, nil); err != nil { - return err - } - if err := w.ramp.Release(ctx, j.MessageID); err != nil { - return fmt.Errorf("release ramp reservation after timeout: %w", err) - } - return river.JobCancel(fmt.Errorf("sending ramp deferred past %s horizon", SendRetryHorizon)) - } - if err := w.store.ReleaseSend(ctx, j.MessageID, job.ID); err != nil { - return fmt.Errorf("release outbound send claim after ramp deferral: %w", err) - } - delay := time.Until(decision.RetryAt) - if delay < time.Minute { - delay = time.Minute + // 1. Reserve the durable attempt. Reserve is idempotent per ordinal, + // so a re-driven execution that never reached ConsumeAttempt finds + // its own reservation, and a confirmed one is followed by a fresh + // ordinal. + early, reserved, err := w.gate.Reserve(ctx, ref) + observedAt = w.now().UTC() + if err != nil { + if errors.Is(err, sendingpolicy.ErrSourceUnavailable) { + return w.cancelTerminally(ctx, job, j, reserved, observedAt, "sending_policy: operation unavailable: "+err.Error()) } - return river.JobSnooze(delay) + return w.snoozeOnGateError(ctx, job, j, "reserve", err) + } + if !early.Allow { + return w.hold(ctx, job, j, reserved, early, observedAt) } + attempt = reserved } - // Fire-time per-agent rate gate (internal/sendrate): the durable, - // cross-replica counterpart of the acceptance-time in-memory send limit — - // scheduled sends accumulate as River jobs and would otherwise burst past - // the advertised 60/min/agent at the provider when they fire. Grouped with - // the other wait-gates: after the ramp reservation, before the final - // suppression check. A deferral RELEASES the send claim but KEEPS the ramp - // reservation (same invariant as the outage snooze above — same-message - // Reserve is idempotent, a released reservation is terminal), and snoozes - // WITHOUT burning an attempt, metering, or emitting lifecycle/terminal - // events: the message simply fires when the window frees capacity. + // 2. Fire-time per-agent rate gate: a deferral DeferAttempts (the budget + // is given back; the ramp reservation is kept) and snoozes WITHOUT + // burning an attempt, metering, or emitting lifecycle/terminal events. if w.rate != nil { decision, rerr := w.rate.Reserve(ctx, j.AgentID) - observedAt = time.Now().UTC() - if rerr != nil { - // Fail toward retry, never toward an unthrottled submit: the - // provider is never exposed because the limiter is down. - if j.pastRetryHorizon() { - if err := w.markFailed(ctx, j.MessageID, job.ID, job.Attempt, j.submissionAnchor(), observedAt, "send_rate_timeout: "+rerr.Error(), delivery.FailureSourceLocal, messagelifecycle.ReasonSubmissionLocalRetriesExhausted, nil); err != nil { - return err - } - if w.ramp != nil && j.rampEligible() { - _ = w.ramp.Release(ctx, j.MessageID) - } - return river.JobCancel(fmt.Errorf("send rate gate unavailable past %s horizon: %w", SendRetryHorizon, rerr)) - } - if err := w.store.ReleaseSend(ctx, j.MessageID, job.ID); err != nil { - return fmt.Errorf("release outbound send claim after rate-gate failure: %w", err) - } - log.Printf("[outbound-send] rate gate unavailable for %s (snoozing): %v", j.MessageID, rerr) - return river.JobSnooze(rateErrorSnoozeInterval) - } - if !decision.Allowed { - if j.pastRetryHorizon() { - if err := w.markFailed(ctx, j.MessageID, job.ID, job.Attempt, j.submissionAnchor(), observedAt, "send_rate_timeout", delivery.FailureSourceLocal, messagelifecycle.ReasonSubmissionLocalRetriesExhausted, nil); err != nil { - return err - } - if w.ramp != nil && j.rampEligible() { - if err := w.ramp.Release(ctx, j.MessageID); err != nil { - return fmt.Errorf("release ramp reservation after send-rate timeout: %w", err) - } - } - return river.JobCancel(fmt.Errorf("send rate deferred past %s horizon", SendRetryHorizon)) - } - if err := w.store.ReleaseSend(ctx, j.MessageID, job.ID); err != nil { - return fmt.Errorf("release outbound send claim after rate deferral: %w", err) + observedAt = w.now().UTC() + if rerr != nil || !decision.Allowed { + w.deferAttempt(ctx, attempt, "rate") + if rerr != nil { + log.Printf("[outbound-send] rate gate unavailable for %s (snoozing): %v", j.MessageID, rerr) + return w.holdFinite(ctx, job, j, attempt, HoldRateRampOrProvider, "send_rate_timeout: "+rerr.Error(), rateErrorSnoozeInterval, observedAt) } delay := clampRateSnooze(time.Until(decision.RetryAt), w.rate.Window()) + rateJitter(j.MessageID, w.rate.Window()) - w.metrics.OutboundRateDeferred() - // IDs only — never recipient data. - log.Printf("[outbound-send] rate_limited agent=%s msg=%s retry_in=%s", j.AgentID, j.MessageID, delay) - return river.JobSnooze(delay) + if !w.holdExpired(j, HoldRateRampOrProvider, observedAt) { + // A deferral is counted only when it defers; an expiry is a + // terminal outcome and is counted as one by markFailed. + w.metrics.OutboundRateDeferred() + // IDs only — never recipient data. + log.Printf("[outbound-send] rate_limited agent=%s msg=%s retry_in=%s", j.AgentID, j.MessageID, delay) + } + return w.holdFinite(ctx, job, j, attempt, HoldRateRampOrProvider, "send_rate_timeout", delay, observedAt) } } - // Final suppression guard immediately before provider I/O: a suppression - // added after acceptance or while an allowed ramp reservation was in flight - // must still prevent delivery. A match is terminal; a store error fails - // closed, releasing the side-effect-free claim while preserving an allowed - // ramp reservation for the idempotent River retry. + // 3. Final suppression guard immediately before authorization: a + // suppression added after acceptance must still prevent delivery. A + // match is terminal and cancels the attempt (both ledgers); a store + // error fails closed, releasing the side-effect-free claim. suppressed, serr := w.store.SuppressedRecipients(ctx, j.UserID, j.AgentID, j.Recipients) - observedAt = time.Now().UTC() + observedAt = w.now().UTC() if serr != nil { if err := w.store.ReleaseSend(ctx, j.MessageID, job.ID); err != nil { - // Keep the idempotent ramp reservation while the message claim remains - // held. Releasing capacity first would let another message consume it, - // then a retry could reserve the same message a second time. return fmt.Errorf("suppression check and claim cleanup before outbound send: %w", errors.Join(serr, fmt.Errorf("release outbound send claim: %w", err))) } @@ -526,17 +542,40 @@ func (w *SendWorker) Work(ctx context.Context, job *river.Job[OutboundSendArgs]) if err := w.markFailed(ctx, j.MessageID, job.ID, job.Attempt, j.submissionAnchor(), observedAt, supErr.Error(), delivery.FailureSourceLocal, messagelifecycle.ReasonSubmissionCancelled, suppressed); err != nil { return err } - if w.ramp != nil && j.rampEligible() { - if err := w.ramp.Release(ctx, j.MessageID); err != nil { - return fmt.Errorf("release ramp reservation after suppression: %w", err) - } - } + w.cancelAttempt(ctx, attempt, "suppression") return river.JobCancel(supErr) } + if w.gate == nil { + return w.submit(ctx, job, j, sendingpolicy.ProviderAuthorization{}, observedAt) + } + + // 4. Final authorization. ConsumeAttempt re-checks account state, tenant + // readiness, both ledgers, and the post-lock UTC day under lock; a hold + // here is handled exactly like an early one, and an error leaves the + // reservation standing for the idempotent retry. + decision, auth, err := w.gate.ConsumeAttempt(ctx, attempt) + observedAt = w.now().UTC() + if err != nil { + if errors.Is(err, sendingpolicy.ErrSourceUnavailable) { + return w.cancelTerminally(ctx, job, j, attempt, observedAt, "sending_policy: operation unavailable: "+err.Error()) + } + return w.snoozeOnGateError(ctx, job, j, "authorize", err) + } + if !decision.Allow || auth == nil { + return w.hold(ctx, job, j, attempt, decision, observedAt) + } + + // 5-6. The authorized submitter redeems the token immediately before the + // socket opens and settles the provider's answer. + return w.submit(ctx, job, j, *auth, observedAt) +} + +// submit makes the single authorized provider call and records its outcome. +func (w *SendWorker) submit(ctx context.Context, job *river.Job[OutboundSendArgs], j *SendJob, auth sendingpolicy.ProviderAuthorization, observedAt time.Time) error { deliverStart := time.Now() - out := w.deliverer.Deliver(ctx, j) - observedAt = time.Now().UTC() + out := w.deliverer.Deliver(ctx, j, auth) + observedAt = w.now().UTC() // Every Deliver call is exactly one submission attempt; classify it here // so no downstream branch (outage, horizon, deferral) can drop the sample. deliverSeconds := time.Since(deliverStart).Seconds() @@ -556,49 +595,42 @@ func (w *SendWorker) Work(ctx context.Context, job *river.Job[OutboundSendArgs]) // Emitted even when MarkSent was a no-op (the row was already // finalized sent by a racing SNS delivery notification): that path is // NOT instrumented, so this is still the message's ONLY sent count. - // If FinalizeProviderAcceptedTx is ever given its own emission, this - // site must become status-aware (like MarkFailed) or the race - // double-counts. The latency observation shares this exactly-once - // contract — emitTerminal emits count and latency together, here and - // everywhere else, and the SNS-feedback path stays uninstrumented - // for both. emitTerminal(w.metrics, terminalSent, j.submissionAnchor(), observedAt) - if w.ramp != nil && j.rampEligible() { - if err := w.ramp.Confirm(ctx, j.MessageID); err != nil { - return fmt.Errorf("confirm sending ramp: %w", err) - } + if out.SettlementErr != nil { + // The provider has the message; only the local settlement (ramp + // progress, provider-id binding) is behind. Never a resend. The + // delayed feedback path settles the same attempt idempotently. + log.Printf("[outbound-send] WARNING: %s accepted by provider but not settled: %v", j.MessageID, out.SettlementErr) } return nil } // Permanent failure (validation / permanent 5xx) — terminal now, no retries. // Provenance 'provider': SES itself refused this submission, so the §3.1 - // correction never revives it. + // correction never revives it. The submitter has already settled it. if out.Permanent { if err := w.markFailed(ctx, j.MessageID, job.ID, job.Attempt, j.submissionAnchor(), observedAt, out.Err.Error(), delivery.FailureSourceProvider, messagelifecycle.ReasonSubmissionProviderRejected, nil); err != nil { return err } - if w.ramp != nil && j.rampEligible() { - if err := w.ramp.Release(ctx, j.MessageID); err != nil { - return fmt.Errorf("release ramp reservation after provider rejection: %w", err) - } - } return river.JobCancel(out.Err) } // Provider outage (relay unreachable) — snooze WITHOUT burning an attempt so a // multi-hour SES incident defers instead of exhausting MaxSendAttempts and - // mass-firing false email.failed (§8 circuit breaker). Bounded by the retry - // horizon: once the accept is older than SendRetryHorizon, give up terminally - // (provenance 'local': the provider never confirmed a rejection). + // mass-firing false email.failed (§8 circuit breaker). Bounded by the hold + // deadline: a message under a policy_budget hold keeps its seven-day + // clock; any other message gets the 72-hour provider horizon. if out.Outage { - if j.pastRetryHorizon() { - if err := w.markFailed(ctx, j.MessageID, job.ID, job.Attempt, j.submissionAnchor(), observedAt, out.Err.Error(), delivery.FailureSourceLocal, messagelifecycle.ReasonSubmissionLocalRetriesExhausted, nil); err != nil { - return err + class, anchor, changed := w.nextHoldState(j, HoldRateRampOrProvider, observedAt) + if changed { + if err := w.store.RecordHold(ctx, j.MessageID, class, anchor); err != nil { + return fmt.Errorf("record outbound hold: %w", err) } - if w.ramp != nil && j.rampEligible() { - _ = w.ramp.Release(ctx, j.MessageID) + } + if !observedAt.Before(anchor.Add(class.horizon())) { + if err := w.markFailed(ctx, j.MessageID, job.ID, job.Attempt, j.submissionAnchor(), observedAt, out.Err.Error(), delivery.FailureSourceLocal, class.expiryReason(), nil); err != nil { + return err } - return fmt.Errorf("outbound send failed (provider outage past %s horizon): %w", SendRetryHorizon, out.Err) + return fmt.Errorf("outbound send failed (provider outage past %s horizon): %w", class.horizon(), out.Err) } if err := w.store.RecordTemporaryFailure(ctx, j.MessageID, job.ID, job.Attempt, observedAt, out.Err.Error()); err != nil { return fmt.Errorf("record outbound provider outage and release claim: %w", err) @@ -615,21 +647,234 @@ func (w *SendWorker) Work(ctx context.Context, job *river.Job[OutboundSendArgs]) if err := w.store.DeferTerminalFailure(ctx, j.MessageID, job.ID, job.Attempt, observedAt, out.Err.Error()); err != nil { log.Printf("[outbound-send] defer terminal failure for %s: %v", j.MessageID, err) } - // Not counted as terminal: the reconciler declares the real outcome - // (sent on evidence, failed otherwise) after the grace window and - // emits it then — counting the deferral too would double-count the - // message in e2a_outbound_terminal_total. return fmt.Errorf("outbound send failed (final attempt %d; outcome deferred to terminal reconciler): %w", job.Attempt, out.Err) } - // Retryable — River reschedules per NextRetry. + // Retryable — River reschedules per NextRetry. The next execution returns + // to Reserve, which allocates the next ordinal; an acceptance-unknown + // failure takes the same path because only provider feedback can say + // whether the body was kept. if err := w.store.RecordTemporaryFailure(ctx, j.MessageID, job.ID, job.Attempt, observedAt, out.Err.Error()); err != nil { return fmt.Errorf("record outbound temporary failure and release claim: %w", err) } return fmt.Errorf("outbound send attempt %d failed: %w", job.Attempt, out.Err) } -func (j *SendJob) rampEligible() bool { - return j.SentAs == "own_address" && j.MessageType != "test" +// operationFor returns the job's durable operation, resolving a legacy job +// through the accept path. It returns a River verdict (snooze/cancel) as its +// error when the message cannot proceed. +func (w *SendWorker) operationFor(ctx context.Context, job *river.Job[OutboundSendArgs], j *SendJob) (sendingpolicy.OperationRef, error) { + if job.Args.OperationRef != nil && !job.Args.OperationRef.IsZero() { + return *job.Args.OperationRef, nil + } + observedAt := w.now().UTC() + if w.resolve == nil { + return sendingpolicy.OperationRef{}, w.cancelTerminally(ctx, job, j, sendingpolicy.AttemptRef{}, observedAt, "sending_policy: legacy job carries no operation and no resolver is wired") + } + decision, ref, err := w.resolve(ctx, j.MessageID) + if err != nil { + if errors.Is(err, sendingpolicy.ErrSourceUnavailable) { + return sendingpolicy.OperationRef{}, w.cancelTerminally(ctx, job, j, sendingpolicy.AttemptRef{}, observedAt, "sending_policy: legacy source unavailable: "+err.Error()) + } + return sendingpolicy.OperationRef{}, w.snoozeOnGateError(ctx, job, j, "resolve", err) + } + if decision == sendingpolicy.AcceptanceSendingPaused { + return sendingpolicy.OperationRef{}, w.hold(ctx, job, j, sendingpolicy.AttemptRef{}, sendingpolicy.Decision{Reason: sendingpolicy.ReasonAccountPaused}, observedAt) + } + if ref.IsZero() { + // The only accepted shape with no operation is an exact self-send, + // which never enqueues. A queued message that resolves to nothing is + // not something this worker can authorize. + return sendingpolicy.OperationRef{}, w.cancelTerminally(ctx, job, j, sendingpolicy.AttemptRef{}, observedAt, "sending_policy: message has no provider operation") + } + return ref, nil +} + +// hold handles a gate hold: a terminal one fails the message now; a pause +// waits for an operator; every other one is a finite hold with a clock. +func (w *SendWorker) hold(ctx context.Context, job *river.Job[OutboundSendArgs], j *SendJob, attempt sendingpolicy.AttemptRef, d sendingpolicy.Decision, observedAt time.Time) error { + if d.Terminal { + return w.cancelTerminally(ctx, job, j, attempt, observedAt, "sending_policy: "+d.Reason) + } + class := holdClassFor(d.Reason) + delay := indefiniteHoldSnooze + if !d.RetryAt.IsZero() { + delay = time.Until(d.RetryAt) + if delay < time.Minute { + delay = time.Minute + } + } + if class == "" { + // An account pause has no clock of its own. It does not start a finite + // hold, but a deadline already running keeps running. + if j.LocalHoldClass == "" { + if err := w.store.ReleaseSend(ctx, j.MessageID, job.ID); err != nil { + return fmt.Errorf("release outbound send claim during account pause: %w", err) + } + return river.JobSnooze(delay) + } + class = j.LocalHoldClass + } + return w.holdFinite(ctx, job, j, attempt, class, "sending_policy_hold: "+d.Reason, delay, observedAt) +} + +// holdFinite persists the hold state, expires the message when its derived +// deadline has passed, and otherwise releases the claim and snoozes. +func (w *SendWorker) holdFinite(ctx context.Context, job *river.Job[OutboundSendArgs], j *SendJob, attempt sendingpolicy.AttemptRef, requested HoldClass, detail string, delay time.Duration, observedAt time.Time) error { + class, anchor, changed := w.nextHoldState(j, requested, observedAt) + if changed { + if err := w.store.RecordHold(ctx, j.MessageID, class, anchor); err != nil { + return fmt.Errorf("record outbound hold: %w", err) + } + j.LocalHoldClass, j.LocalHoldAnchor = class, anchor + } + if !observedAt.Before(anchor.Add(class.horizon())) { + if err := w.markFailed(ctx, j.MessageID, job.ID, job.Attempt, j.submissionAnchor(), observedAt, detail, delivery.FailureSourceLocal, class.expiryReason(), nil); err != nil { + return err + } + w.cancelAttempt(ctx, attempt, "hold expiry") + return river.JobCancel(fmt.Errorf("%s: %s hold expired after %s", detail, class, class.horizon())) + } + if err := w.store.ReleaseSend(ctx, j.MessageID, job.ID); err != nil { + return fmt.Errorf("release outbound send claim during hold: %w", err) + } + return river.JobSnooze(delay) +} + +// holdExpired reports whether encountering `requested` now would find the +// message past its derived deadline, without persisting anything. +func (w *SendWorker) holdExpired(j *SendJob, requested HoldClass, observedAt time.Time) bool { + class, anchor, _ := w.nextHoldState(j, requested, observedAt) + return !observedAt.Before(anchor.Add(class.horizon())) +} + +// nextHoldState applies the durable hold rules to the message's persisted pair +// and the class it is now encountering, reporting whether anything changed. +// +// - First finite hold: the requested class, anchored at the latest of accept, +// schedule, review, and last resume — or at the observation time for a +// tenant-setup hold observed later than that. +// - A budget hold promotes any class to policy_budget, keeping the anchor. +// - policy_budget never changes again. +// - Otherwise the persisted class stands: a later readiness loss does not +// replace a rate class, and a rate hold does not replace a setup class. +func (w *SendWorker) nextHoldState(j *SendJob, requested HoldClass, observedAt time.Time) (HoldClass, time.Time, bool) { + if j.LocalHoldClass == "" { + anchor := j.initialHoldAnchor() + // A tenant-setup hold observed later than the anchor starts its + // clock at the observation; so does a message whose timestamps are + // unknown, which must never be treated as already expired. + if (requested == HoldTenantSetup && observedAt.After(anchor)) || anchor.IsZero() { + anchor = observedAt + } + return requested, anchor, true + } + if j.LocalHoldClass == HoldPolicyBudget { + return HoldPolicyBudget, j.LocalHoldAnchor, false + } + if requested == HoldPolicyBudget { + return HoldPolicyBudget, j.LocalHoldAnchor, true + } + return j.LocalHoldClass, j.LocalHoldAnchor, false +} + +// applyTenantReadiness performs the one-way setup→rate transition when the +// tenant became ready on or before the setup deadline. The comparison uses the +// stored readiness time, so a worker waking after the old deadline still +// honors readiness that committed in time. +func (w *SendWorker) applyTenantReadiness(ctx context.Context, j *SendJob) error { + if j.LocalHoldClass != HoldTenantSetup || j.TenantReadyAt.IsZero() { + return nil + } + if j.TenantReadyAt.After(j.LocalHoldAnchor.Add(HoldTenantSetup.horizon())) { + return nil + } + if err := w.store.RecordHold(ctx, j.MessageID, HoldRateRampOrProvider, j.TenantReadyAt); err != nil { + return fmt.Errorf("record tenant readiness transition: %w", err) + } + j.LocalHoldClass, j.LocalHoldAnchor = HoldRateRampOrProvider, j.TenantReadyAt + return nil +} + +// holdClassFor maps a gate hold reason to its finite-hold class; "" means the +// hold has no clock (an account pause). +func holdClassFor(reason string) HoldClass { + switch { + case reason == sendingpolicy.ReasonAccountPaused: + return "" + case strings.HasSuffix(reason, "_budget_exhausted"): + return HoldPolicyBudget + case reason == sendingpolicy.ReasonTenantNotReady, reason == sendingpolicy.ReasonTenantUnnamed: + return HoldTenantSetup + } + return HoldRateRampOrProvider +} + +// cancelTerminally fails the message for a reason no retry can change and +// gives its attempt back where the gate still allows it. +func (w *SendWorker) cancelTerminally(ctx context.Context, job *river.Job[OutboundSendArgs], j *SendJob, attempt sendingpolicy.AttemptRef, observedAt time.Time, detail string) error { + if err := w.markFailed(ctx, j.MessageID, job.ID, job.Attempt, j.submissionAnchor(), observedAt, detail, delivery.FailureSourceLocal, messagelifecycle.ReasonSubmissionCancelled, nil); err != nil { + return err + } + w.cancelAttempt(ctx, attempt, "terminal") + return river.JobCancel(errors.New(detail)) +} + +// snoozeOnGateError releases the claim and snoozes when the gate itself is +// unavailable: fail toward retry, never toward an unauthorized submit, and +// never burn a River attempt on infrastructure. +func (w *SendWorker) snoozeOnGateError(ctx context.Context, job *river.Job[OutboundSendArgs], j *SendJob, step string, gerr error) error { + if err := w.store.ReleaseSend(ctx, j.MessageID, job.ID); err != nil { + return fmt.Errorf("release outbound send claim after gate %s failure: %w", step, errors.Join(gerr, err)) + } + log.Printf("[outbound-send] sending policy %s failed for %s (snoozing): %v", step, j.MessageID, gerr) + return river.JobSnooze(gateErrorSnoozeInterval) +} + +// deferAttempt gives the budget back for a rate deferral; a stale or already +// released attempt is not an error here — the next Reserve is idempotent. +func (w *SendWorker) deferAttempt(ctx context.Context, attempt sendingpolicy.AttemptRef, why string) { + if w.gate == nil { + return + } + if err := w.gate.DeferAttempt(ctx, attempt); err != nil && + !errors.Is(err, sendingpolicy.ErrAttemptStale) && !errors.Is(err, sendingpolicy.ErrSourceUnavailable) { + log.Printf("[outbound-send] defer attempt (%s) for %s: %v", why, attempt.OperationID(), err) + } +} + +// cancelAttempt gives both ledgers back for a terminal local outcome. A +// started attempt cannot be refunded and says so; that is expected on a +// terminal path reached after a socket opened. +func (w *SendWorker) cancelAttempt(ctx context.Context, attempt sendingpolicy.AttemptRef, why string) { + if w.gate == nil { + return + } + // A zero attempt (no reservation was ever made) has nothing to give back; + // the gate says so with ErrSourceUnavailable and that is not worth a log. + if err := w.gate.CancelAttempt(ctx, attempt); err != nil && + !errors.Is(err, sendingpolicy.ErrAttemptStale) && !errors.Is(err, sendingpolicy.ErrProviderCallStarted) && + !errors.Is(err, sendingpolicy.ErrSourceUnavailable) { + log.Printf("[outbound-send] cancel attempt (%s) for %s: %v", why, attempt.OperationID(), err) + } +} + +// settleFromEvidence applies provider-accept evidence to the operation's +// latest dialed attempt. Best effort: the row is already settled as sent, and +// an attempt that predates the gate has nothing to settle. +func (w *SendWorker) settleFromEvidence(ctx context.Context, job *river.Job[OutboundSendArgs], providerMessageID string) { + if w.gate == nil { + return + } + ref, err := w.gate.LookupOperation(ctx, job.Args.MessageID) + if err != nil { + if !errors.Is(err, sendingpolicy.ErrSourceUnavailable) { + log.Printf("[outbound-send] lookup operation for evidence settle of %s: %v", job.Args.MessageID, err) + } + return + } + if err := w.gate.SettleOperation(ctx, ref, sendingpolicy.SettlementProviderAccepted, providerMessageID); err != nil && !errors.Is(err, sendingpolicy.ErrAttemptStale) { + log.Printf("[outbound-send] settle %s from provider evidence: %v", job.Args.MessageID, err) + } } // clampRateSnooze bounds a rate deferral to [rateMinSnooze, window]: the floor @@ -672,11 +917,6 @@ func rateJitter(messageID string, window time.Duration) time.Duration { return time.Duration(h.Sum32()%uint32(ms)) * time.Millisecond } -func isPermanentRampError(err error) bool { - var permanent interface{ Permanent() bool } - return errors.As(err, &permanent) && permanent.Permanent() -} - func uniqueRecipientCount(recipients []string) int { seen := make(map[string]struct{}, len(recipients)) for _, recipient := range recipients { diff --git a/internal/outboundsend/worker_test.go b/internal/outboundsend/worker_test.go index 4ca28b6ac..581177d9c 100644 --- a/internal/outboundsend/worker_test.go +++ b/internal/outboundsend/worker_test.go @@ -2,16 +2,19 @@ package outboundsend_test import ( "context" + "encoding/json" "errors" "testing" "time" + "github.com/jackc/pgx/v5" "github.com/riverqueue/river" "github.com/riverqueue/river/rivertype" "github.com/tokencanopy/e2a/internal/delivery" "github.com/tokencanopy/e2a/internal/messagelifecycle" "github.com/tokencanopy/e2a/internal/outboundsend" + "github.com/tokencanopy/e2a/internal/sendingpolicy" ) type fakeStore struct { @@ -32,6 +35,7 @@ type fakeStore struct { suppressedErr error sent []sentCall + holds []holdCall failed []failedCall deferred []failedCall temporary []failedCall @@ -43,12 +47,18 @@ type fakeStore struct { } type sentCall struct{ id, provider, sentAs string } +type holdCall struct { + id string + class outboundsend.HoldClass + anchor time.Time +} type failedCall struct { id string attempt int occurredAt time.Time detail string source delivery.FailureSource + reason messagelifecycle.ReasonCode blockedRecipients []string } @@ -62,8 +72,8 @@ func (f *fakeStore) MarkSent(_ context.Context, id string, _ int64, _ int, _ tim f.sent = append(f.sent, sentCall{id, provider, sentAs}) return f.markSentErr } -func (f *fakeStore) MarkFailed(_ context.Context, id string, _ int64, attempt int, occurredAt time.Time, detail string, source delivery.FailureSource, _ messagelifecycle.ReasonCode, blockedRecipients []string) (delivery.Status, time.Time, error) { - f.failed = append(f.failed, failedCall{id: id, attempt: attempt, occurredAt: occurredAt, detail: detail, source: source, blockedRecipients: blockedRecipients}) +func (f *fakeStore) MarkFailed(_ context.Context, id string, _ int64, attempt int, occurredAt time.Time, detail string, source delivery.FailureSource, reason messagelifecycle.ReasonCode, blockedRecipients []string) (delivery.Status, time.Time, error) { + f.failed = append(f.failed, failedCall{id: id, attempt: attempt, occurredAt: occurredAt, detail: detail, source: source, reason: reason, blockedRecipients: blockedRecipients}) status := f.settleStatus if status == "" { status = delivery.StatusFailed @@ -86,6 +96,13 @@ func (f *fakeStore) RecordTemporaryFailure(_ context.Context, id string, _ int64 f.temporary = append(f.temporary, failedCall{id: id}) return f.releaseErr } +func (f *fakeStore) RecordHold(_ context.Context, id string, class outboundsend.HoldClass, anchor time.Time) error { + f.holds = append(f.holds, holdCall{id: id, class: class, anchor: anchor}) + if f.job != nil && f.job.MessageID == id { + f.job.LocalHoldClass, f.job.LocalHoldAnchor = class, anchor + } + return nil +} func (f *fakeStore) ReleaseSend(_ context.Context, id string, _ int64) error { f.released = append(f.released, id) return f.releaseErr @@ -101,45 +118,11 @@ type fakeDeliverer struct { out outboundsend.DeliverOutcome calls int returnedAt time.Time + auths []sendingpolicy.ProviderAuthorization } -type fakeRampGate struct { - decision outboundsend.RampDecision - err error - calls []outboundsend.RampRequest - confirmed []string - released []string - resolved []string - confirmErr error - releaseErr error -} - -func (f *fakeRampGate) Reserve(_ context.Context, req outboundsend.RampRequest) (outboundsend.RampDecision, error) { - f.calls = append(f.calls, req) - return f.decision, f.err -} - -func (f *fakeRampGate) Confirm(_ context.Context, messageID string) error { - f.confirmed = append(f.confirmed, messageID) - return f.confirmErr -} - -func (f *fakeRampGate) Release(_ context.Context, messageID string) error { - f.released = append(f.released, messageID) - return f.releaseErr -} - -func (f *fakeRampGate) Resolve(_ context.Context, messageID string) error { - f.resolved = append(f.resolved, messageID) - return nil -} - -type permanentRampError struct{ msg string } - -func (e permanentRampError) Error() string { return e.msg } -func (e permanentRampError) Permanent() bool { return true } - -func (f *fakeDeliverer) Deliver(_ context.Context, _ *outboundsend.SendJob) outboundsend.DeliverOutcome { +func (f *fakeDeliverer) Deliver(_ context.Context, _ *outboundsend.SendJob, auth sendingpolicy.ProviderAuthorization) outboundsend.DeliverOutcome { + f.auths = append(f.auths, auth) f.calls++ f.returnedAt = time.Now().UTC() return f.out @@ -286,172 +269,6 @@ func TestSendWorker_SuppressionObservationTimeFollowsDecision(t *testing.T) { } } -func TestSendWorker_RampLimitedReleasesAndSnoozesWithoutProviderIO(t *testing.T) { - j := acceptedJob("msg_1") - j.Domain = "new.example.com" - j.MessageType = "send" - j.SentAs = "own_address" - j.Recipients = []string{"One@example.net", "one@example.net", "two@example.net"} - st := &fakeStore{job: j} - dl := &fakeDeliverer{} - gate := &fakeRampGate{decision: outboundsend.RampDecision{ - Allowed: false, - RetryAt: time.Now().Add(6 * time.Hour), - }} - - err := outboundsend.NewSendWorker(st, dl, gate).Work(context.Background(), job("msg_1", 5)) - if err == nil { - t.Fatal("limited send should snooze") - } - if dl.calls != 0 { - t.Fatalf("provider calls = %d, want 0", dl.calls) - } - if len(st.released) != 1 || st.released[0] != "msg_1" { - t.Fatalf("released = %v, want msg_1", st.released) - } - if len(gate.calls) != 1 || gate.calls[0].Units != 2 || gate.calls[0].Domain != "new.example.com" { - t.Fatalf("gate calls = %+v, want two deduplicated recipients", gate.calls) - } -} - -func TestSendWorker_RampErrorFailsClosedAndSnoozes(t *testing.T) { - j := acceptedJob("msg_1") - j.Domain, j.MessageType, j.SentAs = "new.example.com", "send", "own_address" - st := &fakeStore{job: j} - dl := &fakeDeliverer{} - gate := &fakeRampGate{err: errors.New("database unavailable")} - - if err := outboundsend.NewSendWorker(st, dl, gate).Work(context.Background(), job("msg_1", 1)); err == nil { - t.Fatal("ramp storage error should snooze") - } - if dl.calls != 0 || len(st.released) != 1 { - t.Fatalf("gate error must release without provider I/O: calls=%d released=%v", dl.calls, st.released) - } -} - -func TestSendWorker_RampExemptsPlatformTest(t *testing.T) { - j := acceptedJob("msg_test") - j.Domain, j.MessageType, j.SentAs = "new.example.com", "test", "relay" - st := &fakeStore{job: j} - dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{ProviderMessageID: "ses-test"}} - gate := &fakeRampGate{decision: outboundsend.RampDecision{Allowed: false}} - - if err := outboundsend.NewSendWorker(st, dl, gate).Work(context.Background(), job("msg_test", 1)); err != nil { - t.Fatalf("Work: %v", err) - } - if len(gate.calls) != 0 || dl.calls != 1 { - t.Fatalf("platform test should bypass ramp: gate=%d provider=%d", len(gate.calls), dl.calls) - } -} - -func TestSendWorker_ProviderEvidencePrecedesRamp(t *testing.T) { - j := acceptedJob("msg_1") - j.Domain, j.MessageType, j.SentAs = "new.example.com", "send", "own_address" - j.ProviderAccepted, j.ProviderMessageID = true, "ses-evidence" - st := &fakeStore{job: j} - gate := &fakeRampGate{decision: outboundsend.RampDecision{Allowed: false}} - - if err := outboundsend.NewSendWorker(st, &fakeDeliverer{}, gate).Work(context.Background(), job("msg_1", 2)); err != nil { - t.Fatalf("Work: %v", err) - } - if len(gate.calls) != 0 { - t.Fatalf("provider evidence must settle before ramp reservation, got %+v", gate.calls) - } -} - -func TestSendWorker_ConfirmsRampAfterMarkSent(t *testing.T) { - j := acceptedJob("msg_confirm") - j.Domain, j.MessageType, j.SentAs = "new.example.com", "send", "own_address" - st := &fakeStore{job: j} - gate := &fakeRampGate{decision: outboundsend.RampDecision{Allowed: true}} - dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{ProviderMessageID: "ses-confirm", SentAs: "own_address"}} - if err := outboundsend.NewSendWorker(st, dl, gate).Work(context.Background(), job(j.MessageID, 1)); err != nil { - t.Fatalf("Work: %v", err) - } - if len(st.sent) != 1 || len(gate.confirmed) != 1 || gate.confirmed[0] != j.MessageID { - t.Fatalf("sent=%v confirmed=%v", st.sent, gate.confirmed) - } -} - -func TestSendWorker_RepairsRampConfirmationForAlreadySentMessage(t *testing.T) { - j := acceptedJob("msg_repair") - j.Domain, j.MessageType, j.SentAs, j.Status = "new.example.com", "send", "own_address", "sent" - gate := &fakeRampGate{} - dl := &fakeDeliverer{} - if err := outboundsend.NewSendWorker(&fakeStore{job: j}, dl, gate).Work(context.Background(), job(j.MessageID, 2)); err != nil { - t.Fatalf("Work: %v", err) - } - if dl.calls != 0 || len(gate.resolved) != 1 { - t.Fatalf("deliver=%d resolved=%v", dl.calls, gate.resolved) - } -} - -func TestSendWorker_ReleasesRampOnPermanentProviderFailure(t *testing.T) { - j := acceptedJob("msg_release") - j.Domain, j.MessageType, j.SentAs = "new.example.com", "send", "own_address" - gate := &fakeRampGate{decision: outboundsend.RampDecision{Allowed: true}} - dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{Err: errors.New("rejected"), Permanent: true}} - _ = outboundsend.NewSendWorker(&fakeStore{job: j}, dl, gate).Work(context.Background(), job(j.MessageID, 1)) - if len(gate.released) != 1 || gate.released[0] != j.MessageID { - t.Fatalf("released=%v", gate.released) - } -} - -func TestSendWorker_RetainsRampOnAmbiguousFailure(t *testing.T) { - j := acceptedJob("msg_ambiguous") - j.Domain, j.MessageType, j.SentAs = "new.example.com", "send", "own_address" - gate := &fakeRampGate{decision: outboundsend.RampDecision{Allowed: true}} - dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{Err: errors.New("connection reset")}} - _ = outboundsend.NewSendWorker(&fakeStore{job: j}, dl, gate).Work(context.Background(), job(j.MessageID, 1)) - if len(gate.released) != 0 { - t.Fatalf("ambiguous failure released ramp: %v", gate.released) - } -} - -func TestSendWorker_FailsPermanentRampInvariant(t *testing.T) { - j := acceptedJob("msg_bad_ramp") - j.Domain, j.MessageType, j.SentAs = "new.example.com", "send", "own_address" - st := &fakeStore{job: j} - gate := &fakeRampGate{err: permanentRampError{"domain missing"}} - if err := outboundsend.NewSendWorker(st, &fakeDeliverer{}, gate).Work(context.Background(), job(j.MessageID, 1)); err == nil { - t.Fatal("permanent ramp invariant should terminate") - } - if len(st.failed) != 1 { - t.Fatalf("failed=%v", st.failed) - } -} - -func TestSendWorker_FailsRampDeferredMessagePastHorizon(t *testing.T) { - j := acceptedJob("msg_ramp_timeout") - j.Domain, j.MessageType, j.SentAs = "new.example.com", "send", "own_address" - j.AcceptedAt = time.Now().Add(-73 * time.Hour) - st := &fakeStore{job: j} - gate := &fakeRampGate{decision: outboundsend.RampDecision{Allowed: false, RetryAt: time.Now().Add(time.Hour)}} - if err := outboundsend.NewSendWorker(st, &fakeDeliverer{}, gate).Work(context.Background(), job(j.MessageID, 1)); err == nil { - t.Fatal("past-horizon ramp deferral should terminate") - } - if len(st.failed) != 1 || len(gate.released) != 1 { - t.Fatalf("failed=%v released=%v", st.failed, gate.released) - } -} - -// A scheduled send measures its retry horizon from scheduled_at, not accept: -// accepted 10 days ago but firing ~now, a ramp deferral must snooze/retry — NOT -// terminally fail as the immediate-send case above does at the same accept age. -// Guards the fix for the long-scheduled-send false-failure blocker. -func TestSendWorker_ScheduledSendHorizonMeasuredFromScheduledAt(t *testing.T) { - j := acceptedJob("msg_sched_horizon") - j.Domain, j.MessageType, j.SentAs = "new.example.com", "send", "own_address" - j.AcceptedAt = time.Now().Add(-10 * 24 * time.Hour) // long before fire - j.ScheduledAt = time.Now() // just fired — inside the horizon - st := &fakeStore{job: j} - gate := &fakeRampGate{decision: outboundsend.RampDecision{Allowed: false, RetryAt: time.Now().Add(time.Hour)}} - err := outboundsend.NewSendWorker(st, &fakeDeliverer{}, gate).Work(context.Background(), job(j.MessageID, 1)) - if len(st.failed) != 0 { - t.Fatalf("a just-fired long-scheduled send must not be terminated on a ramp deferral; failed=%v err=%v", st.failed, err) - } -} - func TestSendWorker_RetryableFailureDoesNotMarkFailed(t *testing.T) { st := &fakeStore{job: acceptedJob("msg_1")} dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{Err: errors.New("transient 421")}} @@ -483,34 +300,6 @@ func TestSendWorker_RetryableFailureReleaseErrorRetries(t *testing.T) { } } -func TestSendWorker_TerminalRampReleaseFailureResolvesOnRetry(t *testing.T) { - j := acceptedJob("msg_1") - j.Domain, j.MessageType, j.SentAs = "new.example.com", "send", "own_address" - st := &fakeStore{job: j, terminalAfterFailure: true} - dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{Err: errors.New("provider rejected message"), Permanent: true}} - gate := &fakeRampGate{ - decision: outboundsend.RampDecision{Allowed: true}, - releaseErr: errors.New("ramp database unavailable"), - } - w := outboundsend.NewSendWorker(st, dl, gate) - - if err := w.Work(context.Background(), job(j.MessageID, 1)); err == nil || !errors.Is(err, gate.releaseErr) { - t.Fatalf("first Work error = %v, want ramp release failure", err) - } - if len(st.failed) != 1 || len(gate.released) != 1 { - t.Fatalf("first Work failed/released = %v/%v, want one each", st.failed, gate.released) - } - - // MarkFailed made the message terminal, so the retry cannot claim it. The - // worker must still settle the orphaned reservation from the durable outcome. - if err := w.Work(context.Background(), job(j.MessageID, 2)); err != nil { - t.Fatalf("retry Work: %v", err) - } - if len(gate.resolved) != 1 || gate.resolved[0] != j.MessageID { - t.Fatalf("resolved reservations = %v, want [%s]", gate.resolved, j.MessageID) - } -} - func TestSendWorker_OutageSnoozesWithoutBurningAttempt(t *testing.T) { j := acceptedJob("msg_1") j.AcceptedAt = time.Now() // fresh accept — within the retry horizon @@ -561,3 +350,92 @@ func TestSendWorker_NextRetryMatchesEnvelope(t *testing.T) { } } } + +// fakeGate is a scriptable sendingpolicy.Gate. Its references and tokens are +// zero values — the worker never inspects them beyond nil/zero checks — and it +// records every ledger call so tests can assert the fixed worker order. +type fakeGate struct { + reserve sendingpolicy.Decision + reserveErr error + consume sendingpolicy.Decision + consumeErr error + deferred []string + cancelled []string + settled []sendingpolicy.SettlementOutcome + reserves int + consumes int + lookupErr error + lookupCalls int +} + +func allowAll() *fakeGate { + return &fakeGate{reserve: sendingpolicy.Decision{Allow: true}, consume: sendingpolicy.Decision{Allow: true}} +} + +func (g *fakeGate) PrepareExternalTx(context.Context, pgx.Tx, string) (sendingpolicy.AcceptanceDecision, sendingpolicy.OperationRef, error) { + return sendingpolicy.AcceptanceAccept, refFor("msg_prepared"), nil +} +func (g *fakeGate) PrepareNotificationTx(context.Context, pgx.Tx, sendingpolicy.NotificationRef) (sendingpolicy.OperationRef, error) { + return sendingpolicy.OperationRef{}, nil +} +func (g *fakeGate) PrepareProtectionNoticeTx(context.Context, pgx.Tx, sendingpolicy.ProtectionNoticeRef) (sendingpolicy.OperationRef, error) { + return sendingpolicy.OperationRef{}, nil +} +func (g *fakeGate) PreparePublicFeedback(context.Context, sendingpolicy.PublicFeedbackRef) (sendingpolicy.OperationRef, error) { + return sendingpolicy.OperationRef{}, nil +} +func (g *fakeGate) Reserve(context.Context, sendingpolicy.OperationRef) (sendingpolicy.Decision, sendingpolicy.AttemptRef, error) { + g.reserves++ + return g.reserve, sendingpolicy.AttemptRef{}, g.reserveErr +} +func (g *fakeGate) ConsumeAttempt(context.Context, sendingpolicy.AttemptRef) (sendingpolicy.Decision, *sendingpolicy.ProviderAuthorization, error) { + g.consumes++ + if g.consumeErr != nil || !g.consume.Allow { + return g.consume, nil, g.consumeErr + } + return g.consume, &sendingpolicy.ProviderAuthorization{}, nil +} +func (g *fakeGate) RedeemProviderCall(context.Context, sendingpolicy.ProviderAuthorization) error { + return nil +} +func (g *fakeGate) DeferAttempt(_ context.Context, a sendingpolicy.AttemptRef) error { + g.deferred = append(g.deferred, a.OperationID()) + return nil +} +func (g *fakeGate) CancelAttempt(_ context.Context, a sendingpolicy.AttemptRef) error { + g.cancelled = append(g.cancelled, a.OperationID()) + return nil +} +func (g *fakeGate) SettleProvider(_ context.Context, s sendingpolicy.ProviderSettlement) error { + g.settled = append(g.settled, s.Outcome) + return nil +} +func (g *fakeGate) SettleOperation(_ context.Context, _ sendingpolicy.OperationRef, o sendingpolicy.SettlementOutcome, _ string) error { + g.settled = append(g.settled, o) + return nil +} +func (g *fakeGate) LookupOperation(_ context.Context, id string) (sendingpolicy.OperationRef, error) { + g.lookupCalls++ + if g.lookupErr != nil { + return sendingpolicy.OperationRef{}, g.lookupErr + } + return refFor(id), nil +} + +// refFor builds an operation reference the way a River job carries one: the +// versioned wire form holding only the id. +func refFor(id string) sendingpolicy.OperationRef { + var ref sendingpolicy.OperationRef + if err := json.Unmarshal([]byte(`{"v":1,"id":"`+id+`"}`), &ref); err != nil { + panic(err) + } + return ref +} + +// gatedJob is job() with the operation reference the accept path would stamp. +func gatedJob(id string, attempt int) *river.Job[outboundsend.OutboundSendArgs] { + j := job(id, attempt) + ref := refFor(id) + j.Args.OperationRef = &ref + return j +} diff --git a/internal/testutil/contract_server.go b/internal/testutil/contract_server.go index 86a3e3835..63ae44ad5 100644 --- a/internal/testutil/contract_server.go +++ b/internal/testutil/contract_server.go @@ -18,6 +18,7 @@ import ( "github.com/tokencanopy/e2a/internal/outbound" "github.com/tokencanopy/e2a/internal/outboundsend" "github.com/tokencanopy/e2a/internal/relay" + "github.com/tokencanopy/e2a/internal/sendingpolicy" "github.com/tokencanopy/e2a/internal/testutil/testdb" "github.com/tokencanopy/e2a/internal/unsubscribe" "github.com/tokencanopy/e2a/internal/usage" @@ -117,11 +118,15 @@ func StartContractServer(ctx context.Context, dbURL string) (*ContractServer, er // River enqueue semantics without submitting external email. outboundSendStore := agent.NewOutboundSendStore(store, outbox, noopUsage) store.SetScheduledSendFinalizer(outboundSendStore) + // The same composition production uses: a config-source gate running the + // disabled policy (pass-through admission, every attempt still durable) + // and the authorized submitter that refuses to dial without its token. + sendingGate := sendingpolicy.NewGate(pool, sendingpolicy.Secrets{}, sendingpolicy.PolicySourceConfig, sendingpolicy.DisabledPolicy()) outboundJobs := outboundsend.NewJobs( outboundSendStore, - agent.NewOutboundDeliverer(sender), + agent.NewOutboundDeliverer(outbound.NewProviderSubmitter(smtpRelay, sendingGate)), pool, - ) + ).WithGate(sendingGate) jobsClient, err := jobs.New(pool, jobs.Config{OutboundWorkers: 1}, outboundJobs) if err != nil { pool.Close() diff --git a/internal/testutil/server.go b/internal/testutil/server.go index c96a3d686..22d37e500 100644 --- a/internal/testutil/server.go +++ b/internal/testutil/server.go @@ -27,6 +27,7 @@ import ( "github.com/tokencanopy/e2a/internal/outbound" "github.com/tokencanopy/e2a/internal/outboundsend" "github.com/tokencanopy/e2a/internal/relay" + "github.com/tokencanopy/e2a/internal/sendingpolicy" "github.com/tokencanopy/e2a/internal/usage" "github.com/tokencanopy/e2a/internal/webhook" "github.com/tokencanopy/e2a/internal/webhookdelivery" @@ -217,11 +218,15 @@ func TestServer(t *testing.T, pool *pgxpool.Pool, opts ...TestServerOption) *E2A } outboundSendStore := agent.NewOutboundSendStore(store, outbox, noopUsage) store.SetScheduledSendFinalizer(outboundSendStore) + // The same composition production uses: a config-source gate running the + // disabled policy (pass-through admission, every attempt still durable) + // and the authorized submitter that refuses to dial without its token. + sendingGate := sendingpolicy.NewGate(pool, sendingpolicy.Secrets{}, sendingpolicy.PolicySourceConfig, sendingpolicy.DisabledPolicy()) outboundJobs := outboundsend.NewJobs( outboundSendStore, - agent.NewOutboundDeliverer(sender), + agent.NewOutboundDeliverer(outbound.NewProviderSubmitter(smtpRelay, sendingGate)), pool, - ) + ).WithGate(sendingGate) jobsClient, err := jobs.New(pool, jobs.Config{OutboundWorkers: 2}, outboundJobs) if err != nil { t.Fatalf("build River client: %v", err) From 7dc40400334d20ea83d32c3f8a887b40a71c28ab Mon Sep 17 00:00:00 2001 From: Josh Zhang <39790535+jiashuoz@users.noreply.github.com> Date: Fri, 4 Sep 2026 22:04:29 -0700 Subject: [PATCH 05/12] style(messagelifecycle): gofmt the reason catalog Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01AjfGxvXW6fNKWGFHuo68yX --- internal/messagelifecycle/catalog.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/messagelifecycle/catalog.go b/internal/messagelifecycle/catalog.go index a85c270ea..17ccd5b68 100644 --- a/internal/messagelifecycle/catalog.go +++ b/internal/messagelifecycle/catalog.go @@ -73,13 +73,13 @@ const ( // ReasonSubmissionSendingSetupExpired means the account's provider-side // sending setup (SES tenant readiness) did not complete within the // 72-hour setup deadline. - ReasonSubmissionSendingSetupExpired ReasonCode = "submission.sending_setup_expired" - ReasonDeliveryRecipientServerAccepted ReasonCode = "delivery.recipient_server_accepted" - ReasonDeliveryTemporaryDelay ReasonCode = "delivery.temporary_delay" - ReasonDeliveryPermanentBounce ReasonCode = "delivery.permanent_bounce" - ReasonDeliveryTransientBounce ReasonCode = "delivery.transient_bounce" - ReasonDeliveryUndeterminedBounce ReasonCode = "delivery.undetermined_bounce" - ReasonComplaintRecipientReported ReasonCode = "complaint.recipient_reported" + ReasonSubmissionSendingSetupExpired ReasonCode = "submission.sending_setup_expired" + ReasonDeliveryRecipientServerAccepted ReasonCode = "delivery.recipient_server_accepted" + ReasonDeliveryTemporaryDelay ReasonCode = "delivery.temporary_delay" + ReasonDeliveryPermanentBounce ReasonCode = "delivery.permanent_bounce" + ReasonDeliveryTransientBounce ReasonCode = "delivery.transient_bounce" + ReasonDeliveryUndeterminedBounce ReasonCode = "delivery.undetermined_bounce" + ReasonComplaintRecipientReported ReasonCode = "complaint.recipient_reported" ) // Definition is the fixed meaning of a reason code. From 1c67415a587931efc5ae8fdecfdc50f96da8d865 Mon Sep 17 00:00:00 2001 From: Josh Zhang <39790535+jiashuoz@users.noreply.github.com> Date: Fri, 4 Sep 2026 22:19:50 -0700 Subject: [PATCH 06/12] feat(api): publish sending_paused and the two hold-expiry reasons The machine-checked contracts caught three vocabularies the worker cutover widened without saying so: the error-code catalog and the ErrorBody.Code documentation (sending_paused, 403, auth family), the lifecycle reason table in docs/api.md (submission.policy_budget_expired, submission.sending_setup_expired), and the OpenAPI description the two generated SDK models embed. Both SDK error maps classify sending_paused as a non-retryable permission error, with tests. The email-eval integration runner's job-args parser insisted on exactly one key; the accept transaction now stamps operation_ref beside message_id, so the parser admits that key and still rejects any other. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01AjfGxvXW6fNKWGFHuo68yX --- api/openapi.yaml | 7 ++++++- docs/api.md | 3 +++ docs/events.md | 2 +- internal/e2e/email_eval_runner_e2e_test.go | 11 ++++++++++- internal/httpapi/error_catalog.go | 1 + internal/httpapi/errors.go | 2 +- sdks/python/src/e2a/v1/errors.py | 1 + sdks/python/src/e2a/v1/generated/models/error_body.py | 2 +- sdks/python/tests/test_v1_errors.py | 5 +++++ sdks/typescript/src/v1/errors.ts | 1 + sdks/typescript/src/v1/generated/models/ErrorBody.ts | 2 +- sdks/typescript/test/v1/errors.test.ts | 4 ++++ 12 files changed, 35 insertions(+), 6 deletions(-) diff --git a/api/openapi.yaml b/api/openapi.yaml index 3d49b67b7..34f436afb 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -1984,7 +1984,7 @@ components: additionalProperties: true properties: code: - description: "Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status." + description: "Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, sending_paused, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental), sending_paused (403, outbound sending is paused for the account by the platform's abuse controls; queued mail is held, new sends are refused until an operator resumes). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status." type: string x-e2a-error-contracts: address_in_trash: @@ -2259,6 +2259,11 @@ components: retryable: false statuses: - 409 + sending_paused: + family: auth + retryable: false + statuses: + - 403 starter_template_not_found: family: not_found retryable: false diff --git a/docs/api.md b/docs/api.md index c4aad7739..bfa9b737e 100644 --- a/docs/api.md +++ b/docs/api.md @@ -313,6 +313,7 @@ retryable ones (the per-row retry notes in the table below are authoritative). | `unauthorized` | 401 | Missing or invalid credentials (REST and the WebSocket handshake). | | `forbidden` | 403 | Authenticated but not allowed (key scope, cross-tenant access). | | `blocked_by_policy` | 403 | **Experimental.** The outbound message was blocked by the agent's outbound policy gate. | +| `sending_paused` | 403 | Outbound sending is paused for the account by the platform's abuse controls. Nothing was queued; queued mail is held until an operator resumes. | | **Validation** | | | | `invalid_request` | 400 / 422 | The canonical input-validation code — malformed (400) or semantically invalid (422). `error.details` carries the per-field list. | | `invalid_cursor` | 400 | Bad pagination cursor — drop it and re-fetch from the start. | @@ -859,6 +860,8 @@ retryability; clients must not reinterpret those fields independently: | `submission.provider_rejected` | `submission` | `failed` | false | | `submission.local_retries_exhausted` | `submission` | `failed` | true | | `submission.cancelled` | `submission` | `failed` | false | +| `submission.policy_budget_expired` | `submission` | `failed` | true | +| `submission.sending_setup_expired` | `submission` | `failed` | true | | `delivery.recipient_server_accepted` | `delivery` | `delivered` | false | | `delivery.temporary_delay` | `delivery` | `deferred` | true | | `delivery.permanent_bounce` | `delivery` | `bounced` | false | diff --git a/docs/events.md b/docs/events.md index 4ec7e0105..40ba564d4 100644 --- a/docs/events.md +++ b/docs/events.md @@ -106,7 +106,7 @@ The event-to-reason mapping is: |---|---| | `email.received` | `acceptance.inbound_smtp` (or `acceptance.local_loopback`); DMARC `pass` → `authentication.dmarc_pass`, DMARC `fail` → `authentication.dmarc_fail`, DMARC `none` → `authentication.dmarc_none`, DMARC `temperror` → `authentication.dmarc_temporary_error`, and DMARC `permerror` → `authentication.dmarc_permanent_error`; plus `queue.inbound_processing` when async intake was durably queued. | | `email.sent` | `submission.upstream_accepted` or `submission.local_loopback_accepted`. | -| `email.failed` | `submission.provider_rejected`, `submission.local_retries_exhausted`, or `submission.cancelled`, matching the terminal cause. Temporary attempts use `submission.temporary_failure` in the ledger but do not emit a terminal `email.failed` event. | +| `email.failed` | `submission.provider_rejected`, `submission.local_retries_exhausted`, `submission.cancelled`, `submission.policy_budget_expired`, or `submission.sending_setup_expired`, matching the terminal cause. Temporary attempts use `submission.temporary_failure` in the ledger but do not emit a terminal `email.failed` event. | | `email.delivered` | `delivery.recipient_server_accepted` for `delivered_to`. | | `email.bounced` | `delivery.permanent_bounce`, `delivery.transient_bounce`, or `delivery.undetermined_bounce` for `delivered_to`. | | `email.complained` | `complaint.recipient_reported` for `delivered_to`. | diff --git a/internal/e2e/email_eval_runner_e2e_test.go b/internal/e2e/email_eval_runner_e2e_test.go index eaf48b538..559fc6527 100644 --- a/internal/e2e/email_eval_runner_e2e_test.go +++ b/internal/e2e/email_eval_runner_e2e_test.go @@ -1490,9 +1490,18 @@ func waitForOutboundJobsTerminal( func outboundJobMessageID(job outboundJobRecord) (string, error) { var args map[string]json.RawMessage - if json.Unmarshal([]byte(job.Args), &args) != nil || len(args) != 1 { + if json.Unmarshal([]byte(job.Args), &args) != nil { return "", errors.New("invalid outbound job args") } + // The accept transaction stamps the durable sending operation reference + // beside the message id (sending abuse prevention, slice B6). Nothing + // else may appear: the eval's safety claim is that the queue holds only + // the jobs it knows the shape of. + for key := range args { + if key != "message_id" && key != "operation_ref" { + return "", errors.New("invalid outbound job args") + } + } var messageID string if json.Unmarshal(args["message_id"], &messageID) != nil || messageID == "" { return "", errors.New("invalid outbound job message identity") diff --git a/internal/httpapi/error_catalog.go b/internal/httpapi/error_catalog.go index 558bab232..e71b00587 100644 --- a/internal/httpapi/error_catalog.go +++ b/internal/httpapi/error_catalog.go @@ -21,6 +21,7 @@ var errorCodeCatalog = []errorCodeContract{ {Code: "unauthorized", Status: "401", Family: "auth"}, {Code: "forbidden", Status: "403", Family: "auth"}, {Code: "blocked_by_policy", Status: "403", Family: "auth"}, + {Code: "sending_paused", Status: "403", Family: "auth"}, {Code: "invalid_request", Status: "400 / 422", Family: "validation", DetailsSchema: "ValidationErrorDetails"}, {Code: "invalid_cursor", Status: "400", Family: "validation"}, {Code: "invalid_filter", Status: "400", Family: "validation"}, diff --git a/internal/httpapi/errors.go b/internal/httpapi/errors.go index 50b3c6470..47b396bd2 100644 --- a/internal/httpapi/errors.go +++ b/internal/httpapi/errors.go @@ -54,7 +54,7 @@ type ErrorEnvelope struct { // ErrorBody is the inner object of the envelope. type ErrorBody struct { - Code string `json:"code" doc:"Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status."` + Code string `json:"code" doc:"Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, sending_paused, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental), sending_paused (403, outbound sending is paused for the account by the platform's abuse controls; queued mail is held, new sends are refused until an operator resumes). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status."` Message string `json:"message" doc:"Human-readable explanation. Not for branching — use code."` Details any `json:"details,omitempty" doc:"Optional structured context, polymorphic by code. Treat it as an open object keyed off code; unknown codes and fields must be preserved."` RequestID string `json:"request_id" doc:"Echoes the X-Request-Id response header so a failing call is greppable in logs."` diff --git a/sdks/python/src/e2a/v1/errors.py b/sdks/python/src/e2a/v1/errors.py index 700fa6658..f7abe5e1d 100644 --- a/sdks/python/src/e2a/v1/errors.py +++ b/sdks/python/src/e2a/v1/errors.py @@ -193,6 +193,7 @@ def is_retryable_status(status: int) -> bool: # 403 family "forbidden": (E2APermissionError, False), "blocked_by_policy": (E2APermissionError, False), + "sending_paused": (E2APermissionError, False), # 404/410 family — also covers *_not_found via the suffix check in _resolve. "not_found": (E2ANotFoundError, False), "gone": (E2ANotFoundError, False), diff --git a/sdks/python/src/e2a/v1/generated/models/error_body.py b/sdks/python/src/e2a/v1/generated/models/error_body.py index fc57b1a03..e6bad6318 100644 --- a/sdks/python/src/e2a/v1/generated/models/error_body.py +++ b/sdks/python/src/e2a/v1/generated/models/error_body.py @@ -26,7 +26,7 @@ class ErrorBody(BaseModel): """ ErrorBody """ # noqa: E501 - code: StrictStr = Field(description="Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status.") + code: StrictStr = Field(description="Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, sending_paused, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental), sending_paused (403, outbound sending is paused for the account by the platform's abuse controls; queued mail is held, new sends are refused until an operator resumes). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status.") details: Optional[Dict[str, Any]] = Field(default=None, description="Optional structured context, polymorphic by code. Treat it as an open object keyed off code; unknown codes and fields must be preserved.") message: StrictStr = Field(description="Human-readable explanation. Not for branching — use code.") request_id: StrictStr = Field(description="Echoes the X-Request-Id response header so a failing call is greppable in logs.") diff --git a/sdks/python/tests/test_v1_errors.py b/sdks/python/tests/test_v1_errors.py index c7be267c9..a1c12658d 100644 --- a/sdks/python/tests/test_v1_errors.py +++ b/sdks/python/tests/test_v1_errors.py @@ -231,6 +231,11 @@ def test_catalog_family_overrides(): ), E2APermissionError, ) + paused = from_api_exception( + _exc(403, body='{"error":{"code":"sending_paused","message":"x"}}') + ) + assert isinstance(paused, E2APermissionError) + assert paused.retryable is False assert isinstance( from_api_exception( _exc(409, body='{"error":{"code":"message_not_pending","message":"x"}}') diff --git a/sdks/typescript/src/v1/errors.ts b/sdks/typescript/src/v1/errors.ts index bed04c315..f56e9c0c5 100644 --- a/sdks/typescript/src/v1/errors.ts +++ b/sdks/typescript/src/v1/errors.ts @@ -108,6 +108,7 @@ const CODE_TABLE: Record = { // 403 forbidden: { make: mkPermission, retryable: false }, blocked_by_policy: { make: mkPermission, retryable: false }, + sending_paused: { make: mkPermission, retryable: false }, // 404 / 410 — the *_not_found suffix family resolves in resolve() below. not_found: { make: mkNotFound, retryable: false }, gone: { make: mkNotFound, retryable: false }, diff --git a/sdks/typescript/src/v1/generated/models/ErrorBody.ts b/sdks/typescript/src/v1/generated/models/ErrorBody.ts index efb982e43..47d599481 100644 --- a/sdks/typescript/src/v1/generated/models/ErrorBody.ts +++ b/sdks/typescript/src/v1/generated/models/ErrorBody.ts @@ -14,7 +14,7 @@ import { HttpFile } from '../http/http.js'; export class ErrorBody { /** - * Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status. + * Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, sending_paused, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental), sending_paused (403, outbound sending is paused for the account by the platform's abuse controls; queued mail is held, new sends are refused until an operator resumes). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status. */ 'code': string; /** diff --git a/sdks/typescript/test/v1/errors.test.ts b/sdks/typescript/test/v1/errors.test.ts index cb1dfdf72..d1a249da8 100644 --- a/sdks/typescript/test/v1/errors.test.ts +++ b/sdks/typescript/test/v1/errors.test.ts @@ -168,6 +168,10 @@ describe("code-first class selection (F2)", () => { expect(toE2AError({ status: 403, code: "blocked_by_policy", message: "x" })).toBeInstanceOf( E2APermissionError, ); + expect(toE2AError({ status: 403, code: "sending_paused", message: "x" })).toBeInstanceOf( + E2APermissionError, + ); + expect(toE2AError({ status: 403, code: "sending_paused", message: "x" }).retryable).toBe(false); expect(toE2AError({ status: 409, code: "message_not_pending", message: "x" })).toBeInstanceOf( E2AConflictError, ); From 1374857d3330c3f190d2e452360bedd5c6b9604d Mon Sep 17 00:00:00 2001 From: Josh Zhang <39790535+jiashuoz@users.noreply.github.com> Date: Fri, 4 Sep 2026 22:39:53 -0700 Subject: [PATCH 07/12] fix(outbound): harden the worker cutover after review Two parallel reviews (correctness + adversarial) over the first cut. Every item has a named test. Contract surfaces (the blocker in both): the two new lifecycle reasons join the hand-maintained reason_code enum tag, both closed-vocabulary tests, the regenerated spec, both generated SDK models (mirrored by hand; the generator needs Docker, and the description now carries no apostrophe so the two generators agree), the web lifecycle parser and timeline, docs/api.md and docs/events.md. sending_paused is registered in the error catalog, the ErrorBody.Code doc, docs/api.md, and both SDK error maps. Worker: - A gate outage is a bounded rate/ramp/provider hold, not an unbounded snooze. - A paused job evaluates no deadline; a persisted deadline is not extended and the first hold after resume applies it. - A provider outage never emits the setup reason (expiryReasonFor). - markFailed's evidence-settle branch settles the dialed attempt, as the reconciler already did; a failed post-acceptance settlement is retried before it is logged as critical; a provider-id conflict is surfaced as an invariant alarm. - The job's operation reference must name its own message; a mismatch cancels before any ledger call. Enqueue refuses a zero reference. - HoldClassFor maps reasons by name; the armed worker RegisterJobs builds is exposed (Jobs.SendWorker) so the wiring test can prove it carries the gate and the legacy resolver and the submitter carries the configuration set. Gate: SettleOperation prefers the oldest dialed attempt with no provider id yet, so evidence arriving in send order binds each attempt when several dialed. Paused accounts on every enqueue path: 403 sending_paused on the direct, platform-test, and HITL-approve paths; the TTL auto-approve sweep defers a paused account's expired review by an hour (DeferReviewExpiry) instead of re-picking it first every cycle and starving the batch. The email-eval integration runner admits the operation_ref args key. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01AjfGxvXW6fNKWGFHuo68yX --- api/openapi.yaml | 4 +- cmd/e2a/sending_policy_wiring_test.go | 16 ++ docs/api.md | 2 +- internal/agent/api.go | 3 + internal/agent/hitl_api.go | 5 + internal/hitlworker/async_approve_test.go | 48 ++++++ internal/hitlworker/worker.go | 17 ++ internal/httpapi/errors.go | 2 +- internal/httpapi/spec_review_test.go | 2 + internal/identity/review.go | 17 ++ internal/messagelifecycle/model.go | 2 +- internal/messagelifecycle/model_test.go | 2 +- internal/outbound/provider_submit.go | 4 + internal/outboundsend/gate_worker_test.go | 147 +++++++++++++++++- internal/outboundsend/jobs.go | 25 ++- internal/outboundsend/jobs_gate_test.go | 52 +++++++ internal/outboundsend/terminal_reconcile.go | 4 + internal/outboundsend/worker.go | 140 +++++++++++++---- internal/sendingpolicy/gate.go | 20 ++- internal/sendingpolicy/provider_token_test.go | 44 ++++++ .../src/e2a/v1/generated/models/error_body.py | 2 +- .../models/message_lifecycle_transition.py | 4 +- .../src/v1/generated/models/ErrorBody.ts | 2 +- .../models/MessageLifecycleTransition.ts | 2 + .../messages/MessageLifecycleTimeline.tsx | 4 + web/src/lib/messageLifecycle.ts | 1 + 26 files changed, 519 insertions(+), 52 deletions(-) diff --git a/api/openapi.yaml b/api/openapi.yaml index 34f436afb..b5436deed 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -1984,7 +1984,7 @@ components: additionalProperties: true properties: code: - description: "Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, sending_paused, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental), sending_paused (403, outbound sending is paused for the account by the platform's abuse controls; queued mail is held, new sends are refused until an operator resumes). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status." + description: "Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, sending_paused, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental), sending_paused (403, outbound sending is paused for the account by the platform abuse controls; queued mail is held, new sends are refused until an operator resumes). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status." type: string x-e2a-error-contracts: address_in_trash: @@ -2896,6 +2896,8 @@ components: - submission.provider_rejected - submission.local_retries_exhausted - submission.cancelled + - submission.policy_budget_expired + - submission.sending_setup_expired - delivery.recipient_server_accepted - delivery.temporary_delay - delivery.permanent_bounce diff --git a/cmd/e2a/sending_policy_wiring_test.go b/cmd/e2a/sending_policy_wiring_test.go index 45da21822..16b7f985b 100644 --- a/cmd/e2a/sending_policy_wiring_test.go +++ b/cmd/e2a/sending_policy_wiring_test.go @@ -39,9 +39,25 @@ func TestSendingPolicyWiring(t *testing.T) { if composed.submitter == nil { t.Fatal("no authorized submitter composed") } + if got := composed.submitter.SESConfigurationSet(); got != "e2a-delivery-test" { + t.Fatalf("submitter configuration set = %q, want the deployment's — delivery feedback must stay on", got) + } if composed.jobs.Gate() != composed.gate { t.Fatal("the jobs bundle does not hold the composed gate") } + // The worker RegisterJobs registers is what runs in production; it, not + // the bundle, must carry the gate and the legacy resolver. Without the + // resolver every job in flight at cutover would fail closed. + worker := composed.jobs.SendWorker() + if worker.Gate() != composed.gate { + t.Fatal("the registered send worker does not hold the composed gate") + } + if !worker.HasOperationResolver() { + t.Fatal("the registered send worker has no legacy operation resolver") + } + if composed.jobs.TerminalReconcileWorker() == nil { + t.Fatal("no terminal reconciler composed") + } if got := fmt.Sprintf("%T", composed.jobs.Deliverer()); !strings.HasSuffix(got, "agent.outboundDeliverer") { t.Fatalf("worker deliverer is %s, want the ProviderSubmitter-backed agent.outboundDeliverer", got) } diff --git a/docs/api.md b/docs/api.md index bfa9b737e..b9ea43c1c 100644 --- a/docs/api.md +++ b/docs/api.md @@ -313,7 +313,7 @@ retryable ones (the per-row retry notes in the table below are authoritative). | `unauthorized` | 401 | Missing or invalid credentials (REST and the WebSocket handshake). | | `forbidden` | 403 | Authenticated but not allowed (key scope, cross-tenant access). | | `blocked_by_policy` | 403 | **Experimental.** The outbound message was blocked by the agent's outbound policy gate. | -| `sending_paused` | 403 | Outbound sending is paused for the account by the platform's abuse controls. Nothing was queued; queued mail is held until an operator resumes. | +| `sending_paused` | 403 | Outbound sending is paused for the account by the platform abuse controls. Nothing was queued; queued mail is held until an operator resumes. | | **Validation** | | | | `invalid_request` | 400 / 422 | The canonical input-validation code — malformed (400) or semantically invalid (422). `error.details` carries the per-field list. | | `invalid_cursor` | 400 | Bad pagination cursor — drop it and re-fetch from the start. | diff --git a/internal/agent/api.go b/internal/agent/api.go index c34180b7b..fe16be741 100644 --- a/internal/agent/api.go +++ b/internal/agent/api.go @@ -1735,6 +1735,9 @@ func (a *API) acceptPlatformSend(ctx context.Context, agent *identity.AgentIdent accepted = msg return nil }); txErr != nil { + if errors.Is(txErr, outboundsend.ErrSendingPaused) { + return nil, &OutboundError{Status: http.StatusForbidden, Code: "sending_paused", Msg: "sending is paused for this account"} + } log.Printf("[api] platform accept tx failed: agent=%s to_count=%d to_domains=%v error=%v", agent.Domain, len(req.To), logredact.AddressDomains(req.To), txErr) return nil, &OutboundError{Status: http.StatusInternalServerError, Code: "internal_error", Msg: "failed to accept message for send"} } diff --git a/internal/agent/hitl_api.go b/internal/agent/hitl_api.go index fe36bf636..38a7c7cd8 100644 --- a/internal/agent/hitl_api.go +++ b/internal/agent/hitl_api.go @@ -15,6 +15,7 @@ import ( "github.com/tokencanopy/e2a/internal/limits" "github.com/tokencanopy/e2a/internal/logredact" "github.com/tokencanopy/e2a/internal/outbound" + "github.com/tokencanopy/e2a/internal/outboundsend" ) // approveRequest is the JSON body accepted by the approve endpoint. Every @@ -280,6 +281,10 @@ func approveAsyncError(agentID, messageID string, err error) *OutboundError { return &OutboundError{Status: http.StatusConflict, Code: "message_not_pending", Msg: "message is not pending approval"} case errors.Is(err, identity.ErrMessageNotFound): return &OutboundError{Status: http.StatusNotFound, Code: "not_found", Msg: "message not found"} + case errors.Is(err, outboundsend.ErrSendingPaused): + // The draft stays pending_review (the approval transaction rolled + // back); the reviewer learns why rather than seeing a 500. + return &OutboundError{Status: http.StatusForbidden, Code: "sending_paused", Msg: "sending is paused for this account; the draft remains pending"} default: var ve *outbound.ValidationError if errors.As(err, &ve) { diff --git a/internal/hitlworker/async_approve_test.go b/internal/hitlworker/async_approve_test.go index 65ddf1d0f..962308d72 100644 --- a/internal/hitlworker/async_approve_test.go +++ b/internal/hitlworker/async_approve_test.go @@ -8,6 +8,7 @@ import ( "github.com/jackc/pgx/v5" "github.com/tokencanopy/e2a/internal/identity" + "github.com/tokencanopy/e2a/internal/outboundsend" ) // fakeEnq records EnqueueSendTx / EnqueueScheduledSendTx calls (the outbound_send @@ -16,10 +17,14 @@ import ( type fakeEnq struct { calls []string scheduledCalls map[string]time.Time + err error } func (f *fakeEnq) EnqueueSendTx(_ context.Context, _ pgx.Tx, messageID string) (int64, error) { f.calls = append(f.calls, messageID) + if f.err != nil { + return 0, f.err + } return 7777, nil } @@ -155,3 +160,46 @@ func TestWorkerAutoApproveAsync_SelfSendStaysLoopback(t *testing.T) { t.Errorf("self-send status = %q, want %q (resolved via loopback)", status, identity.MessageStatusReviewExpiredApproved) } } + +// TestWorkerAutoApprovePausedAccountDefersWithoutBlocking: a TTL-expired hold on +// an account paused for sending stays pending — held, as the pause promises — +// but its TTL is pushed forward so it does not sit at the head of the sweep and +// starve every other expired review, and it is not retried every cycle. +func TestWorkerAutoApprovePausedAccountDefersWithoutBlocking(t *testing.T) { + w, store, pool, smtpDone := setupWorker(t) + ctx := context.Background() + agent := prepareAgent(t, store, "approve-paused", identity.HITLExpirationApprove) + enq := &fakeEnq{err: outboundsend.ErrSendingPaused} + w.SetOutboundEnqueuer(enq) + msg, err := store.CreatePendingOutboundMessage(ctx, agent.ID, + []string{"alice@external.test"}, nil, nil, + "Held", "body", "

html

", nil, "send", "", "", "", 60) + if err != nil { + t.Fatal(err) + } + backdateExpiry(t, pool, msg.ID) + + w.RunOnce(ctx) + if msgs := smtpDone(); len(msgs) != 0 { + t.Fatalf("paused account must not send inline, got %d SMTP messages", len(msgs)) + } + if len(enq.calls) != 1 { + t.Fatalf("enqueue attempts = %v, want exactly one", enq.calls) + } + var status string + var expiresAt time.Time + if err := pool.QueryRow(ctx, `SELECT status, approval_expires_at FROM messages WHERE id=$1`, msg.ID).Scan(&status, &expiresAt); err != nil { + t.Fatal(err) + } + if status != identity.MessageStatusPendingReview { + t.Fatalf("status = %q, want pending_review (held, not rejected)", status) + } + if expiresAt.Before(time.Now().Add(50 * time.Minute)) { + t.Fatalf("approval_expires_at = %v, want deferred about an hour ahead", expiresAt) + } + // Deferred out of the window: the next sweep leaves it alone. + w.RunOnce(ctx) + if len(enq.calls) != 1 { + t.Fatalf("enqueue attempts after deferral = %v, want still one", enq.calls) + } +} diff --git a/internal/hitlworker/worker.go b/internal/hitlworker/worker.go index ca4fbe5ec..00a1467e2 100644 --- a/internal/hitlworker/worker.go +++ b/internal/hitlworker/worker.go @@ -27,6 +27,7 @@ import ( "github.com/tokencanopy/e2a/internal/loopback" "github.com/tokencanopy/e2a/internal/messagelifecycle" "github.com/tokencanopy/e2a/internal/outbound" + "github.com/tokencanopy/e2a/internal/outboundsend" "github.com/tokencanopy/e2a/internal/piguard" "github.com/tokencanopy/e2a/internal/usage" "github.com/tokencanopy/e2a/internal/webhookpub" @@ -61,6 +62,11 @@ const DefaultBatchSize = 100 // Worker runs the TTL sweep. Construct with New; its RunOnce is driven on a // schedule by the River maintenance periodic (see maintenance.go). +// pausedReviewRetry is how far a TTL-expired review on a paused account is +// deferred before the sweep looks at it again. Long enough not to churn, short +// enough that a resume is picked up within the hour. +const pausedReviewRetry = time.Hour + type Worker struct { store *identity.Store sender *outbound.Sender @@ -406,6 +412,17 @@ func (w *Worker) autoApproveAsync(ctx context.Context, agent *identity.AgentIden if errors.Is(err, identity.ErrNotPendingApproval) { return true // resolved between load and transition } + if errors.Is(err, outboundsend.ErrSendingPaused) { + // The account is paused for sending. The draft stays pending_review + // — that is the held queue the pause promises — but it must not + // stay the sweep's oldest candidate, or it is re-picked first every + // cycle and starves every other expired review. Defer its TTL; the + // sweep after resume resolves it. + if derr := w.store.DeferReviewExpiry(ctx, c.MessageID, time.Now().Add(pausedReviewRetry)); derr != nil { + log.Printf("[hitl-worker] auto-approve %s: defer while account is paused: %v", c.MessageID, derr) + } + return true + } // Transient tx/enqueue failure: leave the row pending_review for the next // cycle. Do NOT autoReject — no send happened, so this is not a "stuck" send. log.Printf("[hitl-worker] auto-approve %s: accept+enqueue: %v", c.MessageID, err) diff --git a/internal/httpapi/errors.go b/internal/httpapi/errors.go index 47b396bd2..ac2162b52 100644 --- a/internal/httpapi/errors.go +++ b/internal/httpapi/errors.go @@ -54,7 +54,7 @@ type ErrorEnvelope struct { // ErrorBody is the inner object of the envelope. type ErrorBody struct { - Code string `json:"code" doc:"Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, sending_paused, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental), sending_paused (403, outbound sending is paused for the account by the platform's abuse controls; queued mail is held, new sends are refused until an operator resumes). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status."` + Code string `json:"code" doc:"Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, sending_paused, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental), sending_paused (403, outbound sending is paused for the account by the platform abuse controls; queued mail is held, new sends are refused until an operator resumes). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status."` Message string `json:"message" doc:"Human-readable explanation. Not for branching — use code."` Details any `json:"details,omitempty" doc:"Optional structured context, polymorphic by code. Treat it as an open object keyed off code; unknown codes and fields must be preserved."` RequestID string `json:"request_id" doc:"Echoes the X-Request-Id response header so a failing call is greppable in logs."` diff --git a/internal/httpapi/spec_review_test.go b/internal/httpapi/spec_review_test.go index abf340c3f..13a9c62f7 100644 --- a/internal/httpapi/spec_review_test.go +++ b/internal/httpapi/spec_review_test.go @@ -263,6 +263,8 @@ func assertMessageLifecycleContractSchema(t *testing.T, doc map[string]any) { "suppression.recipient_blocked", "suppression.hard_bounce_applied", "suppression.complaint_applied", "queue.inbound_processing", "queue.outbound_submission", "submission.upstream_accepted", "submission.local_loopback_accepted", "submission.temporary_failure", "submission.provider_rejected", "submission.local_retries_exhausted", "submission.cancelled", + "submission.policy_budget_expired", + "submission.sending_setup_expired", "delivery.recipient_server_accepted", "delivery.temporary_delay", "delivery.permanent_bounce", "delivery.transient_bounce", "delivery.undetermined_bounce", "complaint.recipient_reported", }, diff --git a/internal/identity/review.go b/internal/identity/review.go index dc627786f..9fd35c958 100644 --- a/internal/identity/review.go +++ b/internal/identity/review.go @@ -349,6 +349,23 @@ func (s *Store) ExpireApproveReviewWithTransition(ctx context.Context, messageID return s.transitionReview(ctx, messageID, "", MessageStatusReviewExpiredApproved, nil, "") } +// DeferReviewExpiry pushes a pending review's TTL forward without resolving +// it. The expiration sweep orders candidates by approval_expires_at, so a +// hold that cannot resolve yet — its account is paused for sending — would +// otherwise stay the oldest candidate and be re-picked first every cycle, +// starving every other expired review once enough of them accumulate. +// Deferring it yields the slot; when the account resumes, the next sweep +// after the deferred instant resolves it normally. +func (s *Store) DeferReviewExpiry(ctx context.Context, messageID string, until time.Time) error { + _, err := s.pool.Exec(ctx, + `UPDATE messages SET approval_expires_at = $2 WHERE id = $1 AND status = 'pending_review'`, + messageID, until.UTC()) + if err != nil { + return fmt.Errorf("defer review expiry: %w", err) + } + return nil +} + // ExpireRejectReview is the worker-side TTL auto-reject: drops the message // (status review_expired_rejected) with no human reviewer. System-scoped. func (s *Store) ExpireRejectReview(ctx context.Context, messageID, reason string) error { diff --git a/internal/messagelifecycle/model.go b/internal/messagelifecycle/model.go index 783187f82..9c2c2b0ab 100644 --- a/internal/messagelifecycle/model.go +++ b/internal/messagelifecycle/model.go @@ -69,7 +69,7 @@ type MessageLifecycleTransition struct { Recipient string `json:"recipient,omitempty" nullable:"true"` Stage Stage `json:"stage" enum:"accepted,authentication,review,suppression,queued,submission,delivery,complaint"` Outcome Outcome `json:"outcome" enum:"accepted,passed,failed,indeterminate,pending,approved,rejected,blocked,applied,enqueued,deferred,delivered,bounced,reported"` - ReasonCode ReasonCode `json:"reason_code" enum:"acceptance.inbound_smtp,acceptance.outbound_api,acceptance.local_loopback,authentication.dmarc_pass,authentication.dmarc_fail,authentication.dmarc_none,authentication.dmarc_temporary_error,authentication.dmarc_permanent_error,review.hold_created,review.approved,review.rejected,review.expired_approved,review.expired_rejected,suppression.recipient_blocked,suppression.hard_bounce_applied,suppression.complaint_applied,queue.inbound_processing,queue.outbound_submission,submission.upstream_accepted,submission.local_loopback_accepted,submission.temporary_failure,submission.provider_rejected,submission.local_retries_exhausted,submission.cancelled,delivery.recipient_server_accepted,delivery.temporary_delay,delivery.permanent_bounce,delivery.transient_bounce,delivery.undetermined_bounce,complaint.recipient_reported"` + ReasonCode ReasonCode `json:"reason_code" enum:"acceptance.inbound_smtp,acceptance.outbound_api,acceptance.local_loopback,authentication.dmarc_pass,authentication.dmarc_fail,authentication.dmarc_none,authentication.dmarc_temporary_error,authentication.dmarc_permanent_error,review.hold_created,review.approved,review.rejected,review.expired_approved,review.expired_rejected,suppression.recipient_blocked,suppression.hard_bounce_applied,suppression.complaint_applied,queue.inbound_processing,queue.outbound_submission,submission.upstream_accepted,submission.local_loopback_accepted,submission.temporary_failure,submission.provider_rejected,submission.local_retries_exhausted,submission.cancelled,submission.policy_budget_expired,submission.sending_setup_expired,delivery.recipient_server_accepted,delivery.temporary_delay,delivery.permanent_bounce,delivery.transient_bounce,delivery.undetermined_bounce,complaint.recipient_reported"` Retryable bool `json:"retryable"` Evidence map[string]any `json:"evidence"` CorrelationIDs map[string]string `json:"correlation_ids"` diff --git a/internal/messagelifecycle/model_test.go b/internal/messagelifecycle/model_test.go index 7e741d490..21295bc6e 100644 --- a/internal/messagelifecycle/model_test.go +++ b/internal/messagelifecycle/model_test.go @@ -502,7 +502,7 @@ func TestNewTransitionSchemaEnumTags(t *testing.T) { assertTag("Direction", "enum", "inbound,outbound") assertTag("Stage", "enum", "accepted,authentication,review,suppression,queued,submission,delivery,complaint") assertTag("Outcome", "enum", "accepted,passed,failed,indeterminate,pending,approved,rejected,blocked,applied,enqueued,deferred,delivered,bounced,reported") - assertTag("ReasonCode", "enum", "acceptance.inbound_smtp,acceptance.outbound_api,acceptance.local_loopback,authentication.dmarc_pass,authentication.dmarc_fail,authentication.dmarc_none,authentication.dmarc_temporary_error,authentication.dmarc_permanent_error,review.hold_created,review.approved,review.rejected,review.expired_approved,review.expired_rejected,suppression.recipient_blocked,suppression.hard_bounce_applied,suppression.complaint_applied,queue.inbound_processing,queue.outbound_submission,submission.upstream_accepted,submission.local_loopback_accepted,submission.temporary_failure,submission.provider_rejected,submission.local_retries_exhausted,submission.cancelled,delivery.recipient_server_accepted,delivery.temporary_delay,delivery.permanent_bounce,delivery.transient_bounce,delivery.undetermined_bounce,complaint.recipient_reported") + assertTag("ReasonCode", "enum", "acceptance.inbound_smtp,acceptance.outbound_api,acceptance.local_loopback,authentication.dmarc_pass,authentication.dmarc_fail,authentication.dmarc_none,authentication.dmarc_temporary_error,authentication.dmarc_permanent_error,review.hold_created,review.approved,review.rejected,review.expired_approved,review.expired_rejected,suppression.recipient_blocked,suppression.hard_bounce_applied,suppression.complaint_applied,queue.inbound_processing,queue.outbound_submission,submission.upstream_accepted,submission.local_loopback_accepted,submission.temporary_failure,submission.provider_rejected,submission.local_retries_exhausted,submission.cancelled,submission.policy_budget_expired,submission.sending_setup_expired,delivery.recipient_server_accepted,delivery.temporary_delay,delivery.permanent_bounce,delivery.transient_bounce,delivery.undetermined_bounce,complaint.recipient_reported") } func validAppendInput() AppendInput { diff --git a/internal/outbound/provider_submit.go b/internal/outbound/provider_submit.go index 79668a82b..fa01de163 100644 --- a/internal/outbound/provider_submit.go +++ b/internal/outbound/provider_submit.go @@ -139,6 +139,10 @@ func NewProviderSubmitter(relay *SMTPRelay, gate sendingpolicy.Gate) *ProviderSu // tagged with. Empty means no header (dev/self-host without SES). func (s *ProviderSubmitter) SetSESConfigurationSet(name string) { s.sesConfigSet = name } +// SESConfigurationSet reports the configured configuration set, for wiring +// tests that must prove delivery feedback stayed switched on. +func (s *ProviderSubmitter) SESConfigurationSet() string { return s.sesConfigSet } + // SubmitOnce makes exactly one provider call for one authorized attempt. // // The sequence is fixed and every early exit is I/O-free: prove the envelope is diff --git a/internal/outboundsend/gate_worker_test.go b/internal/outboundsend/gate_worker_test.go index 536c80410..f3943c39c 100644 --- a/internal/outboundsend/gate_worker_test.go +++ b/internal/outboundsend/gate_worker_test.go @@ -99,17 +99,24 @@ func TestGatedWorker_PauseHoldIsIndefiniteAndPersistsNothing(t *testing.T) { } } -func TestGatedWorker_PauseDoesNotExtendARunningBudgetDeadline(t *testing.T) { +func TestGatedWorker_PauseNeverEvaluatesADeadlineButNeverExtendsIt(t *testing.T) { + // Paused with a budget deadline already eight days gone: the paused job + // only waits. Nothing is failed, nothing rewritten. j := acceptedJob("msg_paused_budget") j.LocalHoldClass, j.LocalHoldAnchor = outboundsend.HoldPolicyBudget, time.Now().Add(-8*24*time.Hour) st := &fakeStore{job: j} g := &fakeGate{reserve: sendingpolicy.Decision{Allow: false, Reason: sendingpolicy.ReasonAccountPaused}} - err := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(g).Work(context.Background(), gatedJob("msg_paused_budget", 1)) - if !isCancel(err) { - t.Fatalf("err = %v, want cancel — the seven-day budget deadline governs every later wait", err) + if err := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(g).Work(context.Background(), gatedJob("msg_paused_budget", 1)); !isSnooze(err) { + t.Fatalf("paused err = %v, want snooze — a paused job evaluates no deadline", err) } - if len(st.failed) != 1 || st.failed[0].source != delivery.FailureSourceLocal { - t.Fatalf("failed = %+v, want one local failure", st.failed) + if len(st.failed) != 0 || len(st.holds) != 0 { + t.Fatalf("failed=%+v holds=%+v, want nothing touched while paused", st.failed, st.holds) + } + // After resume the first hold it meets applies the unextended deadline. + g = &fakeGate{reserve: sendingpolicy.Decision{Allow: false, Reason: sendingpolicy.ReasonGlobalAllBudget, RetryAt: time.Now().Add(time.Hour)}} + err := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(g).Work(context.Background(), gatedJob("msg_paused_budget", 2)) + if !isCancel(err) || len(st.failed) != 1 || st.failed[0].reason != messagelifecycle.ReasonSubmissionPolicyBudgetExpired { + t.Fatalf("after resume err=%v failed=%+v, want the budget deadline to fire with its own reason", err, st.failed) } } @@ -368,3 +375,131 @@ func TestGatedWorker_HoldConstantsMatchThePolicyDefault(t *testing.T) { t.Fatalf("PolicyBudgetHoldHorizon = %s, policy budget_hold_max_days default = %s", outboundsend.PolicyBudgetHoldHorizon, got) } } + +func TestGatedWorker_GateOutageIsBoundedByTheHoldDeadline(t *testing.T) { + j := acceptedJob("msg_gate_down_long") + j.AcceptedAt = time.Now().Add(-73 * time.Hour) + st := &fakeStore{job: j} + dl := &fakeDeliverer{} + g := &fakeGate{reserveErr: errors.New("policy db down")} + err := outboundsend.NewSendWorker(st, dl).WithGate(g).Work(context.Background(), gatedJob("msg_gate_down_long", 1)) + if !isCancel(err) || dl.calls != 0 { + t.Fatalf("err=%v delivers=%d, want the 72-hour expiry with no I/O", err, dl.calls) + } + if len(st.failed) != 1 || st.failed[0].reason != messagelifecycle.ReasonSubmissionLocalRetriesExhausted { + t.Fatalf("failed = %+v, want local_retries_exhausted", st.failed) + } + // Inside the horizon it holds as rate/ramp/provider and snoozes. + j = acceptedJob("msg_gate_down_short") + j.AcceptedAt = time.Now().Add(-time.Hour) + st = &fakeStore{job: j} + if err := outboundsend.NewSendWorker(st, dl).WithGate(g).Work(context.Background(), gatedJob("msg_gate_down_short", 1)); !isSnooze(err) { + t.Fatalf("err = %v, want snooze", err) + } + if len(st.holds) != 1 || st.holds[0].class != outboundsend.HoldRateRampOrProvider { + t.Fatalf("holds = %+v, want a rate/ramp/provider hold", st.holds) + } +} + +func TestGatedWorker_ReadinessLossDoesNotReplaceARateClass(t *testing.T) { + anchor := time.Now().Add(-time.Hour) + j := acceptedJob("msg_keep_rate") + j.LocalHoldClass, j.LocalHoldAnchor = outboundsend.HoldRateRampOrProvider, anchor + st := &fakeStore{job: j} + g := &fakeGate{reserve: sendingpolicy.Decision{Allow: false, Reason: sendingpolicy.ReasonTenantNotReady}} + if err := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(g).Work(context.Background(), gatedJob("msg_keep_rate", 1)); !isSnooze(err) { + t.Fatalf("err = %v, want snooze", err) + } + if len(st.holds) != 0 { + t.Fatalf("holds = %+v, want the persisted rate class left alone", st.holds) + } +} + +func TestGatedWorker_ProviderOutagePersistsTheHoldAndHonorsTheBudgetClock(t *testing.T) { + // First outage: enters the rate/ramp/provider class anchored at accept. + j := acceptedJob("msg_outage") + j.AcceptedAt = time.Now().Add(-time.Hour) + st := &fakeStore{job: j} + dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{Err: errors.New("connection refused"), Outage: true}} + if err := outboundsend.NewSendWorker(st, dl).WithGate(allowAll()).Work(context.Background(), gatedJob("msg_outage", 1)); !isSnooze(err) { + t.Fatalf("err = %v, want snooze", err) + } + if len(st.holds) != 1 || st.holds[0].class != outboundsend.HoldRateRampOrProvider || !st.holds[0].anchor.Equal(j.AcceptedAt) { + t.Fatalf("holds = %+v, want rate/ramp/provider anchored at accept", st.holds) + } + // Under a policy_budget hold four days old, an outage keeps waiting on + // the seven-day clock instead of the 72-hour one. + j = acceptedJob("msg_outage_budget") + j.AcceptedAt = time.Now().Add(-5 * 24 * time.Hour) + j.LocalHoldClass, j.LocalHoldAnchor = outboundsend.HoldPolicyBudget, time.Now().Add(-4*24*time.Hour) + st = &fakeStore{job: j} + if err := outboundsend.NewSendWorker(st, dl).WithGate(allowAll()).Work(context.Background(), gatedJob("msg_outage_budget", 1)); !isSnooze(err) { + t.Fatalf("budget-held outage err = %v, want snooze on the seven-day clock", err) + } + if len(st.holds) != 0 || len(st.failed) != 0 { + t.Fatalf("holds=%+v failed=%+v, want the budget class untouched", st.holds, st.failed) + } + // An outage that expires a tenant_setup class never emits the setup + // reason: setup was not what blocked the send at the end. + j = acceptedJob("msg_outage_setup") + j.LocalHoldClass, j.LocalHoldAnchor = outboundsend.HoldTenantSetup, time.Now().Add(-73*time.Hour) + st = &fakeStore{job: j} + err := outboundsend.NewSendWorker(st, dl).WithGate(allowAll()).Work(context.Background(), gatedJob("msg_outage_setup", 1)) + if err == nil || len(st.failed) != 1 || st.failed[0].reason != messagelifecycle.ReasonSubmissionLocalRetriesExhausted { + t.Fatalf("err=%v failed=%+v, want local_retries_exhausted, never sending_setup_expired", err, st.failed) + } +} + +func TestGatedWorker_EvidenceSettleUnderATerminalWriteSettlesTheOperation(t *testing.T) { + // A suppression arrives for a message whose earlier attempt dialed and + // whose provider evidence has since landed: the guarded terminal write + // settles the row as SENT, and the dialed attempt must be settled too. + st := &fakeStore{job: acceptedJob("msg_late_evidence"), suppressed: []string{"b@y.com"}, settleStatus: delivery.StatusSent} + g := allowAll() + if err := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(g).Work(context.Background(), gatedJob("msg_late_evidence", 2)); !isCancel(err) { + t.Fatalf("err = %v, want cancel", err) + } + if g.lookupCalls != 1 || len(g.settled) != 1 || g.settled[0] != sendingpolicy.SettlementProviderAccepted { + t.Fatalf("lookups=%d settled=%v, want the operation settled as accepted from the evidence", g.lookupCalls, g.settled) + } +} + +func TestHoldClassForNamesEveryReasonExplicitly(t *testing.T) { + // Every hold reason the gate can emit decides a horizon; the mapping is + // by name, and an unknown name takes the shorter clock. + cases := map[string]outboundsend.HoldClass{ + sendingpolicy.ReasonAccountPaused: "", + sendingpolicy.ReasonAccountDailyBudget: outboundsend.HoldPolicyBudget, + sendingpolicy.ReasonAccountSharedBudget: outboundsend.HoldPolicyBudget, + sendingpolicy.ReasonGlobalAllBudget: outboundsend.HoldPolicyBudget, + sendingpolicy.ReasonGlobalProbation: outboundsend.HoldPolicyBudget, + sendingpolicy.ReasonGlobalCritical: outboundsend.HoldPolicyBudget, + sendingpolicy.ReasonGlobalViolation: outboundsend.HoldPolicyBudget, + sendingpolicy.ReasonTenantNotReady: outboundsend.HoldTenantSetup, + sendingpolicy.ReasonTenantUnnamed: outboundsend.HoldTenantSetup, + sendingpolicy.ReasonRampCapacity: outboundsend.HoldRateRampOrProvider, + sendingpolicy.ReasonSendingIdentityUnverified: outboundsend.HoldRateRampOrProvider, + "some_future_budget_exhausted": outboundsend.HoldRateRampOrProvider, + } + for reason, want := range cases { + if got := outboundsend.HoldClassFor(reason); got != want { + t.Errorf("HoldClassFor(%q) = %q, want %q", reason, got, want) + } + } +} + +func TestGatedWorker_OperationReferenceMustNameThisMessage(t *testing.T) { + st := &fakeStore{job: acceptedJob("msg_a")} + dl := &fakeDeliverer{} + g := allowAll() + rj := job("msg_a", 1) + other := refFor("msg_b") + rj.Args.OperationRef = &other + err := outboundsend.NewSendWorker(st, dl).WithGate(g).Work(context.Background(), rj) + if !isCancel(err) || dl.calls != 0 || g.reserves != 0 { + t.Fatalf("err=%v delivers=%d reserves=%d, want cancel before any ledger call", err, dl.calls, g.reserves) + } + if len(st.failed) != 1 || st.failed[0].reason != messagelifecycle.ReasonSubmissionCancelled { + t.Fatalf("failed = %+v, want one local cancellation", st.failed) + } +} diff --git a/internal/outboundsend/jobs.go b/internal/outboundsend/jobs.go index e9255f141..d782f1983 100644 --- a/internal/outboundsend/jobs.go +++ b/internal/outboundsend/jobs.go @@ -46,6 +46,18 @@ func (j *Jobs) WithGate(g sendingpolicy.Gate) *Jobs { return j } +// SendWorker builds the fully armed send worker RegisterJobs registers: the +// gate, the legacy resolver, the rate gate, and metrics. It is the one place +// those are wired, and the composition root's test inspects its result. +func (j *Jobs) SendWorker() *SendWorker { + return NewSendWorker(j.store, j.deliverer).WithMetrics(j.metrics).WithRateGate(j.rate).WithGate(j.gate).WithOperationResolver(j.ResolveLegacyOperation) +} + +// TerminalReconcileWorker builds the reconciler RegisterJobs registers. +func (j *Jobs) TerminalReconcileWorker() *TerminalReconcileWorker { + return NewTerminalReconcileWorker(j.pool, j.store).WithMetrics(j.metrics).WithGate(j.gate) +} + // Gate exposes the wired sending-protection gate, for the composition root's // wiring test. nil when none is wired. func (j *Jobs) Gate() sendingpolicy.Gate { return j.gate } @@ -79,8 +91,8 @@ func (j *Jobs) WithRateGate(g RateGate) *Jobs { // RegisterJobs adds the SendWorker and terminal-state safety net to the shared // client's bundle. Implements jobs.Registrar. func (j *Jobs) RegisterJobs(w *river.Workers) []*river.PeriodicJob { - river.AddWorker(w, NewSendWorker(j.store, j.deliverer).WithMetrics(j.metrics).WithRateGate(j.rate).WithGate(j.gate).WithOperationResolver(j.ResolveLegacyOperation)) - river.AddWorker(w, NewTerminalReconcileWorker(j.pool, j.store).WithMetrics(j.metrics).WithGate(j.gate)) + river.AddWorker(w, j.SendWorker()) + river.AddWorker(w, j.TerminalReconcileWorker()) return []*river.PeriodicJob{ river.NewPeriodicJob( river.PeriodicInterval(terminalReconcileInterval), @@ -152,9 +164,14 @@ func (j *Jobs) enqueueSendTx(ctx context.Context, tx pgx.Tx, messageID string, a if decision == sendingpolicy.AcceptanceSendingPaused { return 0, ErrSendingPaused } - if !ref.IsZero() { - args.OperationRef = &ref + if ref.IsZero() { + // The only accepted shape without an operation is an exact + // self-send, and those never enqueue. Refusing here keeps a + // prepared-but-operationless job from masquerading as a legacy + // one that the worker would then kill. + return 0, fmt.Errorf("prepare sending operation: message %s has no provider operation", messageID) } + args.OperationRef = &ref } opts := &river.InsertOpts{ Queue: jobs.QueueOutbound, diff --git a/internal/outboundsend/jobs_gate_test.go b/internal/outboundsend/jobs_gate_test.go index d9f4046b7..0c73f4add 100644 --- a/internal/outboundsend/jobs_gate_test.go +++ b/internal/outboundsend/jobs_gate_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "testing" + "time" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" @@ -284,3 +285,54 @@ func TestJobs_ReconcilerSettlesTheDialedAttemptFromEvidence(t *testing.T) { t.Fatalf("provider calls = %d, want the original one only", dl.calls) } } + +func TestJobs_RateDeferralReleasesTheRealReservation(t *testing.T) { + f := newGateFixture(t) + messageID, jobID, err := f.accept(f.gated, "rate") + if err != nil { + t.Fatalf("accept: %v", err) + } + gate := &fakeRateGate{decision: outboundsend.RateDecision{Allowed: false, RetryAt: time.Now().Add(30 * time.Second)}, window: time.Minute} + dl := &fakeDeliverer{} + w := outboundsend.NewSendWorker(f.adapter, dl).WithGate(f.gate).WithRateGate(gate) + ref := refFor(messageID) + rj := &river.Job[outboundsend.OutboundSendArgs]{ + JobRow: &rivertype.JobRow{ID: jobID, Attempt: 1, MaxAttempts: outboundsend.MaxSendAttempts, Kind: outboundsend.OutboundSendArgs{}.Kind()}, + Args: outboundsend.OutboundSendArgs{MessageID: messageID, OperationRef: &ref}, + } + if err := w.Work(f.ctx, rj); !isSnooze(err) || dl.calls != 0 { + t.Fatalf("err=%v delivers=%d, want snooze with no I/O", err, dl.calls) + } + var state string + if err := f.pool.QueryRow(f.ctx, `SELECT state FROM sending_budget_reservations WHERE operation_id = $1 AND submission_attempt = 1`, messageID).Scan(&state); err != nil { + t.Fatalf("read reservation: %v", err) + } + if state != "released" { + t.Fatalf("reservation state = %s, want released — the deferral must give the budget back", state) + } +} + +// zeroRefGate accepts but prepares nothing — the shape only an exact +// self-send produces, which never enqueues. +type zeroRefGate struct{ *fakeGate } + +func (zeroRefGate) PrepareExternalTx(context.Context, pgx.Tx, string) (sendingpolicy.AcceptanceDecision, sendingpolicy.OperationRef, error) { + return sendingpolicy.AcceptanceAccept, sendingpolicy.OperationRef{}, nil +} + +func TestJobs_EnqueueRefusesAnAcceptWithoutAnOperation(t *testing.T) { + f := newGateFixture(t) + bundle := outboundsend.NewJobs(f.adapter, &fakeDeliverer{}, f.pool).WithGate(zeroRefGate{allowAll()}) + bundle.SetEnqueuer(f.client) + messageID, _, err := f.accept(bundle, "zero-ref") + if err == nil { + t.Fatal("an accept that prepared no operation was enqueued as a legacy-looking job") + } + var rows int + if err := f.pool.QueryRow(f.ctx, `SELECT count(*) FROM messages WHERE id = $1`, messageID).Scan(&rows); err != nil { + t.Fatal(err) + } + if rows != 0 { + t.Fatal("the refused accept left a message row behind") + } +} diff --git a/internal/outboundsend/terminal_reconcile.go b/internal/outboundsend/terminal_reconcile.go index c4f7a10fd..f0dbc5b16 100644 --- a/internal/outboundsend/terminal_reconcile.go +++ b/internal/outboundsend/terminal_reconcile.go @@ -240,6 +240,10 @@ func (w *TerminalReconcileWorker) settleFromEvidence(ctx context.Context, messag return } if err := w.gate.SettleOperation(ctx, ref, sendingpolicy.SettlementProviderAccepted, providerMessageID); err != nil && !errors.Is(err, sendingpolicy.ErrAttemptStale) { + if errors.Is(err, sendingpolicy.ErrProviderMessageIDConflict) { + log.Printf("[outbound-terminal-reconcile] CRITICAL: provider id conflict settling %s from evidence: %v", messageID, err) + return + } log.Printf("[outbound-terminal-reconcile] settle %s from provider evidence: %v", messageID, err) } } diff --git a/internal/outboundsend/worker.go b/internal/outboundsend/worker.go index 166a9956b..2190c5e55 100644 --- a/internal/outboundsend/worker.go +++ b/internal/outboundsend/worker.go @@ -402,6 +402,15 @@ func (w *SendWorker) WithClock(now func() time.Time) *SendWorker { return w } +// Gate exposes the wired sending-protection gate (nil when none), for the +// composition root's wiring test. +func (w *SendWorker) Gate() sendingpolicy.Gate { return w.gate } + +// HasOperationResolver reports whether a legacy-argument resolver is wired. +// Without one every job from a pre-floor slot fails closed, so the wiring +// test insists on it. +func (w *SendWorker) HasOperationResolver() bool { return w.resolve != nil } + // NextRetry overrides River's default backoff with the decided send envelope. func (w *SendWorker) NextRetry(job *river.Job[OutboundSendArgs]) time.Time { i := job.Attempt @@ -460,7 +469,7 @@ func (w *SendWorker) Work(ctx context.Context, job *river.Job[OutboundSendArgs]) // Terminal 'sent', but NOT an attempt — the submit happened on an // earlier attempt; only the settle lands here. emitTerminal(w.metrics, terminalSent, j.submissionAnchor(), observedAt) - w.settleFromEvidence(ctx, job, j.ProviderMessageID) + w.settleFromEvidence(ctx, j.MessageID, j.ProviderMessageID) return nil } @@ -598,9 +607,10 @@ func (w *SendWorker) submit(ctx context.Context, job *river.Job[OutboundSendArgs emitTerminal(w.metrics, terminalSent, j.submissionAnchor(), observedAt) if out.SettlementErr != nil { // The provider has the message; only the local settlement (ramp - // progress, provider-id binding) is behind. Never a resend. The - // delayed feedback path settles the same attempt idempotently. - log.Printf("[outbound-send] WARNING: %s accepted by provider but not settled: %v", j.MessageID, out.SettlementErr) + // progress, provider-id binding) is behind. Never a resend: retry + // the settlement itself, idempotently, and leave the delayed + // feedback path to finish it if that fails too. + w.resettle(ctx, j.MessageID, out.ProviderMessageID, out.SettlementErr) } return nil } @@ -625,9 +635,10 @@ func (w *SendWorker) submit(ctx context.Context, job *river.Job[OutboundSendArgs if err := w.store.RecordHold(ctx, j.MessageID, class, anchor); err != nil { return fmt.Errorf("record outbound hold: %w", err) } + j.LocalHoldClass, j.LocalHoldAnchor = class, anchor } if !observedAt.Before(anchor.Add(class.horizon())) { - if err := w.markFailed(ctx, j.MessageID, job.ID, job.Attempt, j.submissionAnchor(), observedAt, out.Err.Error(), delivery.FailureSourceLocal, class.expiryReason(), nil); err != nil { + if err := w.markFailed(ctx, j.MessageID, job.ID, job.Attempt, j.submissionAnchor(), observedAt, out.Err.Error(), delivery.FailureSourceLocal, expiryReasonFor(class, true), nil); err != nil { return err } return fmt.Errorf("outbound send failed (provider outage past %s horizon): %w", class.horizon(), out.Err) @@ -663,10 +674,18 @@ func (w *SendWorker) submit(ctx context.Context, job *river.Job[OutboundSendArgs // through the accept path. It returns a River verdict (snooze/cancel) as its // error when the message cannot proceed. func (w *SendWorker) operationFor(ctx context.Context, job *river.Job[OutboundSendArgs], j *SendJob) (sendingpolicy.OperationRef, error) { + observedAt := w.now().UTC() if job.Args.OperationRef != nil && !job.Args.OperationRef.IsZero() { + // A customer message's operation IS its message id. A job whose + // reference names another operation would charge that operation's + // account and route this message's feedback to that message; the + // gate cannot tell, because every reference reloads its row. This is + // the one place the two ids meet, so this is where they must agree. + if job.Args.OperationRef.ID() != j.MessageID { + return sendingpolicy.OperationRef{}, w.cancelTerminally(ctx, job, j, sendingpolicy.AttemptRef{}, observedAt, "sending_policy: job operation reference does not name this message") + } return *job.Args.OperationRef, nil } - observedAt := w.now().UTC() if w.resolve == nil { return sendingpolicy.OperationRef{}, w.cancelTerminally(ctx, job, j, sendingpolicy.AttemptRef{}, observedAt, "sending_policy: legacy job carries no operation and no resolver is wired") } @@ -695,7 +714,7 @@ func (w *SendWorker) hold(ctx context.Context, job *river.Job[OutboundSendArgs], if d.Terminal { return w.cancelTerminally(ctx, job, j, attempt, observedAt, "sending_policy: "+d.Reason) } - class := holdClassFor(d.Reason) + class := HoldClassFor(d.Reason) delay := indefiniteHoldSnooze if !d.RetryAt.IsZero() { delay = time.Until(d.RetryAt) @@ -704,15 +723,15 @@ func (w *SendWorker) hold(ctx context.Context, job *river.Job[OutboundSendArgs], } } if class == "" { - // An account pause has no clock of its own. It does not start a finite - // hold, but a deadline already running keeps running. - if j.LocalHoldClass == "" { - if err := w.store.ReleaseSend(ctx, j.MessageID, job.ID); err != nil { - return fmt.Errorf("release outbound send claim during account pause: %w", err) - } - return river.JobSnooze(delay) + // An account pause has no clock of its own. It starts no finite hold + // and evaluates none: a paused job only waits. A deadline persisted + // before the pause is not extended either — after resume the job + // either continues within its remaining time or expires with its + // class's own reason on the next hold it meets. + if err := w.store.ReleaseSend(ctx, j.MessageID, job.ID); err != nil { + return fmt.Errorf("release outbound send claim during account pause: %w", err) } - class = j.LocalHoldClass + return river.JobSnooze(delay) } return w.holdFinite(ctx, job, j, attempt, class, "sending_policy_hold: "+d.Reason, delay, observedAt) } @@ -728,7 +747,7 @@ func (w *SendWorker) holdFinite(ctx context.Context, job *river.Job[OutboundSend j.LocalHoldClass, j.LocalHoldAnchor = class, anchor } if !observedAt.Before(anchor.Add(class.horizon())) { - if err := w.markFailed(ctx, j.MessageID, job.ID, job.Attempt, j.submissionAnchor(), observedAt, detail, delivery.FailureSourceLocal, class.expiryReason(), nil); err != nil { + if err := w.markFailed(ctx, j.MessageID, job.ID, job.Attempt, j.submissionAnchor(), observedAt, detail, delivery.FailureSourceLocal, expiryReasonFor(class, false), nil); err != nil { return err } w.cancelAttempt(ctx, attempt, "hold expiry") @@ -795,17 +814,35 @@ func (w *SendWorker) applyTenantReadiness(ctx context.Context, j *SendJob) error return nil } -// holdClassFor maps a gate hold reason to its finite-hold class; "" means the +// expiryReasonFor picks the lifecycle reason for a hold that expired. The +// persisted class decides, with the one exception the design names: a later +// provider outage cannot emit the setup reason, because the provider — not +// setup — is what blocked the send at the end. A rate or ramp wait met by a +// setup-class message still expires as setup: missing or late readiness is +// the story of that message. +func expiryReasonFor(class HoldClass, providerOutage bool) messagelifecycle.ReasonCode { + if class == HoldTenantSetup && providerOutage { + return HoldRateRampOrProvider.expiryReason() + } + return class.expiryReason() +} + +// HoldClassFor maps a gate hold reason to its finite-hold class; "" means the // hold has no clock (an account pause). -func holdClassFor(reason string) HoldClass { - switch { - case reason == sendingpolicy.ReasonAccountPaused: +func HoldClassFor(reason string) HoldClass { + switch reason { + case sendingpolicy.ReasonAccountPaused: return "" - case strings.HasSuffix(reason, "_budget_exhausted"): + case sendingpolicy.ReasonAccountDailyBudget, sendingpolicy.ReasonAccountSharedBudget, + sendingpolicy.ReasonGlobalAllBudget, sendingpolicy.ReasonGlobalProbation, + sendingpolicy.ReasonGlobalCritical, sendingpolicy.ReasonGlobalViolation: return HoldPolicyBudget - case reason == sendingpolicy.ReasonTenantNotReady, reason == sendingpolicy.ReasonTenantUnnamed: + case sendingpolicy.ReasonTenantNotReady, sendingpolicy.ReasonTenantUnnamed: return HoldTenantSetup } + // Ramp capacity, an unverified sending identity, and any hold reason this + // worker does not know by name all wait on the 72-hour clock: unknown is + // the shorter horizon, never the longer one. return HoldRateRampOrProvider } @@ -822,12 +859,13 @@ func (w *SendWorker) cancelTerminally(ctx context.Context, job *river.Job[Outbou // snoozeOnGateError releases the claim and snoozes when the gate itself is // unavailable: fail toward retry, never toward an unauthorized submit, and // never burn a River attempt on infrastructure. +// +// It is a bounded wait like every other one: the message enters (or stays +// in) the rate/ramp/provider class and expires at that class's deadline, so a +// gate that is down for days does not park mail forever. func (w *SendWorker) snoozeOnGateError(ctx context.Context, job *river.Job[OutboundSendArgs], j *SendJob, step string, gerr error) error { - if err := w.store.ReleaseSend(ctx, j.MessageID, job.ID); err != nil { - return fmt.Errorf("release outbound send claim after gate %s failure: %w", step, errors.Join(gerr, err)) - } log.Printf("[outbound-send] sending policy %s failed for %s (snoozing): %v", step, j.MessageID, gerr) - return river.JobSnooze(gateErrorSnoozeInterval) + return w.holdFinite(ctx, job, j, sendingpolicy.AttemptRef{}, HoldRateRampOrProvider, "sending_policy_unavailable: "+step+": "+gerr.Error(), gateErrorSnoozeInterval, w.now().UTC()) } // deferAttempt gives the budget back for a rate deferral; a stale or already @@ -858,22 +896,59 @@ func (w *SendWorker) cancelAttempt(ctx context.Context, attempt sendingpolicy.At } } +// resettle retries a settlement that failed after the provider accepted the +// message. It mirrors markFailed's bounded retry: a transient database error +// should not cost a domain its ramp progress for the day. +func (w *SendWorker) resettle(ctx context.Context, messageID, providerMessageID string, first error) { + if w.gate == nil { + return + } + err := first + for i := 0; i < terminalWriteRetries; i++ { + select { + case <-ctx.Done(): + return + case <-time.After(time.Duration(i+1) * terminalWriteBackoff): + } + ref, lerr := w.gate.LookupOperation(ctx, messageID) + if lerr != nil { + err = lerr + continue + } + err = w.gate.SettleOperation(ctx, ref, sendingpolicy.SettlementProviderAccepted, providerMessageID) + if err == nil || errors.Is(err, sendingpolicy.ErrAttemptStale) { + return + } + if errors.Is(err, sendingpolicy.ErrProviderMessageIDConflict) { + break + } + } + log.Printf("[outbound-send] CRITICAL: %s accepted by provider but not settled after retries: %v", messageID, err) +} + // settleFromEvidence applies provider-accept evidence to the operation's // latest dialed attempt. Best effort: the row is already settled as sent, and // an attempt that predates the gate has nothing to settle. -func (w *SendWorker) settleFromEvidence(ctx context.Context, job *river.Job[OutboundSendArgs], providerMessageID string) { +func (w *SendWorker) settleFromEvidence(ctx context.Context, messageID, providerMessageID string) { if w.gate == nil { return } - ref, err := w.gate.LookupOperation(ctx, job.Args.MessageID) + ref, err := w.gate.LookupOperation(ctx, messageID) if err != nil { if !errors.Is(err, sendingpolicy.ErrSourceUnavailable) { - log.Printf("[outbound-send] lookup operation for evidence settle of %s: %v", job.Args.MessageID, err) + log.Printf("[outbound-send] lookup operation for evidence settle of %s: %v", messageID, err) } return } if err := w.gate.SettleOperation(ctx, ref, sendingpolicy.SettlementProviderAccepted, providerMessageID); err != nil && !errors.Is(err, sendingpolicy.ErrAttemptStale) { - log.Printf("[outbound-send] settle %s from provider evidence: %v", job.Args.MessageID, err) + if errors.Is(err, sendingpolicy.ErrProviderMessageIDConflict) { + // Two physical sends for one charged attempt, or evidence + // attributed to the wrong attempt: an invariant violation, never + // a transient. Surface it as such. + log.Printf("[outbound-send] CRITICAL: provider id conflict settling %s from evidence: %v", messageID, err) + return + } + log.Printf("[outbound-send] settle %s from provider evidence: %v", messageID, err) } } @@ -959,6 +1034,11 @@ func (w *SendWorker) markFailed(ctx context.Context, messageID string, jobID int emitTerminal(w.metrics, terminalOutcome(source, reason, blockedRecipients), anchorAt, settledAt) case delivery.StatusSent: emitTerminal(w.metrics, terminalSent, anchorAt, settledAt) + // Provider evidence settled the row under a terminal write that + // expected to fail it. The attempt that dialed still needs + // settling — ramp progress and the correlation binding — and + // only the operation, not this call's attempt, names it. + w.settleFromEvidence(ctx, messageID, "") } return nil } diff --git a/internal/sendingpolicy/gate.go b/internal/sendingpolicy/gate.go index 2a47826fa..abe543796 100644 --- a/internal/sendingpolicy/gate.go +++ b/internal/sendingpolicy/gate.go @@ -1875,10 +1875,24 @@ func (m *Module) settle(ctx context.Context, operationID string, attempt int, se return err } if attempt == 0 { + // Evidence without a token names an operation, not an ordinal. When + // several attempts dialed, the oldest one that has no provider id yet + // is the best owner: feedback arrives in send order far more often + // than not, and each binding retires its attempt from this choice. + // With every dialed attempt already bound, the latest one takes the + // replay, and the bind refuses a different id rather than absorb it. if err := tx.QueryRow(ctx, ` - SELECT COALESCE(MAX(submission_attempt), 0) - FROM sending_budget_reservations - WHERE operation_id = $1 AND call_state = 'started'`, operationID, + SELECT COALESCE( + (SELECT MIN(r.submission_attempt) + FROM sending_budget_reservations r + LEFT JOIN sending_feedback_correlations c + ON c.operation_id = r.operation_id AND c.submission_attempt = r.submission_attempt + WHERE r.operation_id = $1 AND r.call_state = 'started' + AND c.provider_message_id IS NULL), + (SELECT MAX(submission_attempt) + FROM sending_budget_reservations + WHERE operation_id = $1 AND call_state = 'started'), + 0)`, operationID, ).Scan(&attempt); err != nil { return fmt.Errorf("sendingpolicy: find started attempt: %w", err) } diff --git a/internal/sendingpolicy/provider_token_test.go b/internal/sendingpolicy/provider_token_test.go index 9c0a3085e..e98b6e6cd 100644 --- a/internal/sendingpolicy/provider_token_test.go +++ b/internal/sendingpolicy/provider_token_test.go @@ -338,3 +338,47 @@ func TestProviderTokenLookupOperationResolvesOnlyDurableOperations(t *testing.T) t.Fatalf("lookup of an empty id err = %v, want ErrSourceUnavailable", err) } } + +// TestProviderTokenSettleOperationPrefersTheOldestUnboundDialedAttempt: two +// attempts dialed and both lost their 250. Evidence arriving in send order +// binds attempt one first, then attempt two — neither steals the other's id. +func TestProviderTokenSettleOperationPrefersTheOldestUnboundDialedAttempt(t *testing.T) { + f := newFixture(t) + g := f.gate(enforcingPolicy(nil)) + agent := f.agent(f.user("standard")) + ref, attempt := f.prepareAndReserve(g, agent, 1) + for i := 1; i <= 2; i++ { + if i == 2 { + var err error + if _, attempt, err = g.Reserve(f.ctx, ref); err != nil || attempt.Attempt() != 2 { + t.Fatalf("reserve ordinal two: attempt=%v err=%v", attempt, err) + } + } + _, auth, err := g.ConsumeAttempt(f.ctx, attempt) + if err != nil || auth == nil { + t.Fatalf("authorize %d: auth=%v err=%v", i, auth, err) + } + if err := g.RedeemProviderCall(f.ctx, *auth); err != nil { + t.Fatalf("redeem %d: %v", i, err) + } + } + if err := g.SettleOperation(f.ctx, ref, sendingpolicy.SettlementProviderAccepted, "ses-first"); err != nil { + t.Fatalf("settle first evidence: %v", err) + } + if err := g.SettleOperation(f.ctx, ref, sendingpolicy.SettlementProviderAccepted, "ses-second"); err != nil { + t.Fatalf("settle second evidence: %v", err) + } + if got := f.providerMessageID(ref.ID(), 1); got == nil || *got != "ses-first" { + t.Fatalf("attempt one bound = %v, want ses-first", got) + } + if got := f.providerMessageID(ref.ID(), 2); got == nil || *got != "ses-second" { + t.Fatalf("attempt two bound = %v, want ses-second", got) + } + // Everything bound: a replay of either id is idempotent, a third id is a conflict. + if err := g.SettleOperation(f.ctx, ref, sendingpolicy.SettlementProviderAccepted, "ses-second"); err != nil { + t.Fatalf("replay: %v", err) + } + if err := g.SettleOperation(f.ctx, ref, sendingpolicy.SettlementProviderAccepted, "ses-third"); !errors.Is(err, sendingpolicy.ErrProviderMessageIDConflict) { + t.Fatalf("third id err = %v, want ErrProviderMessageIDConflict", err) + } +} diff --git a/sdks/python/src/e2a/v1/generated/models/error_body.py b/sdks/python/src/e2a/v1/generated/models/error_body.py index e6bad6318..7c57352d5 100644 --- a/sdks/python/src/e2a/v1/generated/models/error_body.py +++ b/sdks/python/src/e2a/v1/generated/models/error_body.py @@ -26,7 +26,7 @@ class ErrorBody(BaseModel): """ ErrorBody """ # noqa: E501 - code: StrictStr = Field(description="Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, sending_paused, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental), sending_paused (403, outbound sending is paused for the account by the platform's abuse controls; queued mail is held, new sends are refused until an operator resumes). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status.") + code: StrictStr = Field(description="Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, sending_paused, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental), sending_paused (403, outbound sending is paused for the account by the platform abuse controls; queued mail is held, new sends are refused until an operator resumes). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status.") details: Optional[Dict[str, Any]] = Field(default=None, description="Optional structured context, polymorphic by code. Treat it as an open object keyed off code; unknown codes and fields must be preserved.") message: StrictStr = Field(description="Human-readable explanation. Not for branching — use code.") request_id: StrictStr = Field(description="Echoes the X-Request-Id response header so a failing call is greppable in logs.") diff --git a/sdks/python/src/e2a/v1/generated/models/message_lifecycle_transition.py b/sdks/python/src/e2a/v1/generated/models/message_lifecycle_transition.py index edc2fac22..094aa1b2d 100644 --- a/sdks/python/src/e2a/v1/generated/models/message_lifecycle_transition.py +++ b/sdks/python/src/e2a/v1/generated/models/message_lifecycle_transition.py @@ -59,8 +59,8 @@ def outcome_validate_enum(cls, value): @field_validator('reason_code') def reason_code_validate_enum(cls, value): """Validates the enum""" - if value not in set(['acceptance.inbound_smtp', 'acceptance.outbound_api', 'acceptance.local_loopback', 'authentication.dmarc_pass', 'authentication.dmarc_fail', 'authentication.dmarc_none', 'authentication.dmarc_temporary_error', 'authentication.dmarc_permanent_error', 'review.hold_created', 'review.approved', 'review.rejected', 'review.expired_approved', 'review.expired_rejected', 'suppression.recipient_blocked', 'suppression.hard_bounce_applied', 'suppression.complaint_applied', 'queue.inbound_processing', 'queue.outbound_submission', 'submission.upstream_accepted', 'submission.local_loopback_accepted', 'submission.temporary_failure', 'submission.provider_rejected', 'submission.local_retries_exhausted', 'submission.cancelled', 'delivery.recipient_server_accepted', 'delivery.temporary_delay', 'delivery.permanent_bounce', 'delivery.transient_bounce', 'delivery.undetermined_bounce', 'complaint.recipient_reported']): - raise ValueError("must be one of enum values ('acceptance.inbound_smtp', 'acceptance.outbound_api', 'acceptance.local_loopback', 'authentication.dmarc_pass', 'authentication.dmarc_fail', 'authentication.dmarc_none', 'authentication.dmarc_temporary_error', 'authentication.dmarc_permanent_error', 'review.hold_created', 'review.approved', 'review.rejected', 'review.expired_approved', 'review.expired_rejected', 'suppression.recipient_blocked', 'suppression.hard_bounce_applied', 'suppression.complaint_applied', 'queue.inbound_processing', 'queue.outbound_submission', 'submission.upstream_accepted', 'submission.local_loopback_accepted', 'submission.temporary_failure', 'submission.provider_rejected', 'submission.local_retries_exhausted', 'submission.cancelled', 'delivery.recipient_server_accepted', 'delivery.temporary_delay', 'delivery.permanent_bounce', 'delivery.transient_bounce', 'delivery.undetermined_bounce', 'complaint.recipient_reported')") + if value not in set(['acceptance.inbound_smtp', 'acceptance.outbound_api', 'acceptance.local_loopback', 'authentication.dmarc_pass', 'authentication.dmarc_fail', 'authentication.dmarc_none', 'authentication.dmarc_temporary_error', 'authentication.dmarc_permanent_error', 'review.hold_created', 'review.approved', 'review.rejected', 'review.expired_approved', 'review.expired_rejected', 'suppression.recipient_blocked', 'suppression.hard_bounce_applied', 'suppression.complaint_applied', 'queue.inbound_processing', 'queue.outbound_submission', 'submission.upstream_accepted', 'submission.local_loopback_accepted', 'submission.temporary_failure', 'submission.provider_rejected', 'submission.local_retries_exhausted', 'submission.cancelled', 'submission.policy_budget_expired', 'submission.sending_setup_expired', 'delivery.recipient_server_accepted', 'delivery.temporary_delay', 'delivery.permanent_bounce', 'delivery.transient_bounce', 'delivery.undetermined_bounce', 'complaint.recipient_reported']): + raise ValueError("must be one of enum values ('acceptance.inbound_smtp', 'acceptance.outbound_api', 'acceptance.local_loopback', 'authentication.dmarc_pass', 'authentication.dmarc_fail', 'authentication.dmarc_none', 'authentication.dmarc_temporary_error', 'authentication.dmarc_permanent_error', 'review.hold_created', 'review.approved', 'review.rejected', 'review.expired_approved', 'review.expired_rejected', 'suppression.recipient_blocked', 'suppression.hard_bounce_applied', 'suppression.complaint_applied', 'queue.inbound_processing', 'queue.outbound_submission', 'submission.upstream_accepted', 'submission.local_loopback_accepted', 'submission.temporary_failure', 'submission.provider_rejected', 'submission.local_retries_exhausted', 'submission.cancelled', 'submission.policy_budget_expired', 'submission.sending_setup_expired', 'delivery.recipient_server_accepted', 'delivery.temporary_delay', 'delivery.permanent_bounce', 'delivery.transient_bounce', 'delivery.undetermined_bounce', 'complaint.recipient_reported')") return value @field_validator('stage') diff --git a/sdks/typescript/src/v1/generated/models/ErrorBody.ts b/sdks/typescript/src/v1/generated/models/ErrorBody.ts index 47d599481..0f6d56fb3 100644 --- a/sdks/typescript/src/v1/generated/models/ErrorBody.ts +++ b/sdks/typescript/src/v1/generated/models/ErrorBody.ts @@ -14,7 +14,7 @@ import { HttpFile } from '../http/http.js'; export class ErrorBody { /** - * Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, sending_paused, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental), sending_paused (403, outbound sending is paused for the account by the platform's abuse controls; queued mail is held, new sends are refused until an operator resumes). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status. + * Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, sending_paused, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental), sending_paused (403, outbound sending is paused for the account by the platform abuse controls; queued mail is held, new sends are refused until an operator resumes). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status. */ 'code': string; /** diff --git a/sdks/typescript/src/v1/generated/models/MessageLifecycleTransition.ts b/sdks/typescript/src/v1/generated/models/MessageLifecycleTransition.ts index cfd726c5e..4c59b8575 100644 --- a/sdks/typescript/src/v1/generated/models/MessageLifecycleTransition.ts +++ b/sdks/typescript/src/v1/generated/models/MessageLifecycleTransition.ts @@ -157,6 +157,8 @@ export enum MessageLifecycleTransitionReasonCodeEnum { SubmissionProviderRejected = 'submission.provider_rejected', SubmissionLocalRetriesExhausted = 'submission.local_retries_exhausted', SubmissionCancelled = 'submission.cancelled', + SubmissionPolicyBudgetExpired = 'submission.policy_budget_expired', + SubmissionSendingSetupExpired = 'submission.sending_setup_expired', DeliveryRecipientServerAccepted = 'delivery.recipient_server_accepted', DeliveryTemporaryDelay = 'delivery.temporary_delay', DeliveryPermanentBounce = 'delivery.permanent_bounce', diff --git a/web/src/app/components/messages/MessageLifecycleTimeline.tsx b/web/src/app/components/messages/MessageLifecycleTimeline.tsx index 2b599ed6c..4aa74f51f 100644 --- a/web/src/app/components/messages/MessageLifecycleTimeline.tsx +++ b/web/src/app/components/messages/MessageLifecycleTimeline.tsx @@ -40,6 +40,8 @@ export const LIFECYCLE_PRESENTATION: Record = "submission.provider_rejected": { title: "Delivery provider rejected message", description: "The delivery provider refused the message, so it was not handed off." }, "submission.local_retries_exhausted": { title: "Delivery failed", description: "e2a could not hand off the message after repeated attempts." }, "submission.cancelled": { title: "Delivery cancelled", description: "Delivery was stopped before the message was handed off." }, + "submission.policy_budget_expired": { title: "Delivery failed", description: "The message waited for sending capacity for seven days and was not handed off." }, + "submission.sending_setup_expired": { title: "Delivery failed", description: "Sending setup for this account did not complete in time, so the message was not handed off." }, "delivery.recipient_server_accepted": { title: "Accepted by recipient server", description: "The recipient's mail server accepted the message. This does not confirm inbox placement." }, "delivery.temporary_delay": { title: "Delivery delayed", description: "The delivery provider reported a temporary delay." }, "delivery.permanent_bounce": { title: "Delivery failed permanently", description: "The recipient's mail server permanently rejected the message." }, @@ -79,6 +81,8 @@ function lifecycleSummary(last: MessageLifecycleTransitionWire): string { case "submission.provider_rejected": case "submission.local_retries_exhausted": case "submission.cancelled": + case "submission.policy_budget_expired": + case "submission.sending_setup_expired": case "suppression.recipient_blocked": return "Failed"; default: diff --git a/web/src/lib/messageLifecycle.ts b/web/src/lib/messageLifecycle.ts index 8b6fae737..96d361e43 100644 --- a/web/src/lib/messageLifecycle.ts +++ b/web/src/lib/messageLifecycle.ts @@ -19,6 +19,7 @@ export const MESSAGE_LIFECYCLE_REASON_CODES = [ "submission.upstream_accepted", "submission.local_loopback_accepted", "submission.temporary_failure", "submission.provider_rejected", "submission.local_retries_exhausted", "submission.cancelled", + "submission.policy_budget_expired", "submission.sending_setup_expired", "delivery.recipient_server_accepted", "delivery.temporary_delay", "delivery.permanent_bounce", "delivery.transient_bounce", "delivery.undetermined_bounce", "complaint.recipient_reported", From bc8289bf0aee25c84efd89e3f17e5714db77f7d3 Mon Sep 17 00:00:00 2001 From: Josh Zhang <39790535+jiashuoz@users.noreply.github.com> Date: Fri, 4 Sep 2026 23:13:29 -0700 Subject: [PATCH 08/12] fix(outbound): settle, wire, and mark precisely after re-review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mutation-tested re-review of the previous fix round: no blockers, six should-fixes. - SettleOperation without a token resolves the attempt as: one already bound to this exact provider id (a replay stays home), else the oldest dialed attempt with no id, else the latest dialed. The earlier oldest-unbound-first rule let a replay for attempt one bind attempt two; the test now covers that shape. - snoozeOnGateError threads the live reservation into the bounded hold, so an expiry at final authorization gives the attempt back instead of stranding it under an enforcing policy. - MarkFailed returns the evidence's provider id, and the worker's evidence settle under a terminal write carries it — the reconciler already did. The two evidence paths now agree. - resettle logs at critical level when the context ends mid-retry; a dedicated test covers the retry itself. - The wiring test registers workers exactly as main does and inspects the worker River received (Jobs.RegisteredSendWorker), so a RegisterJobs that bypassed the armed constructor fails it. - sending_paused is marked experimental beside blocked_by_policy in the stability extension, the docs, and the description, since the pause control ships disabled and pre-GA. The Python forward-compat table gains the two lifecycle reasons. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01AjfGxvXW6fNKWGFHuo68yX --- api/openapi.yaml | 3 +- cmd/e2a/sending_policy_wiring_test.go | 10 ++++++- docs/api.md | 8 +++-- internal/agent/outbound_async.go | 10 +++---- internal/agent/outbound_async_test.go | 2 +- internal/httpapi/errors.go | 2 +- internal/httpapi/stability.go | 7 +++-- internal/httpapi/stability_test.go | 9 +++--- internal/outboundsend/gate_worker_test.go | 23 ++++++++++++++- internal/outboundsend/jobs.go | 13 ++++++++- internal/outboundsend/reconcile_test.go | 6 ++-- internal/outboundsend/terminal_reconcile.go | 7 +++-- internal/outboundsend/worker.go | 25 ++++++++++------ internal/outboundsend/worker_test.go | 10 +++++-- internal/sendingpolicy/gate.go | 22 +++++++++----- internal/sendingpolicy/provider_token_test.go | 29 +++++++++++++++++-- .../src/e2a/v1/generated/models/error_body.py | 2 +- sdks/python/tests/test_enum_forward_compat.py | 2 ++ .../src/v1/generated/models/ErrorBody.ts | 2 +- 19 files changed, 142 insertions(+), 50 deletions(-) diff --git a/api/openapi.yaml b/api/openapi.yaml index b5436deed..f4e240e0c 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -1984,7 +1984,7 @@ components: additionalProperties: true properties: code: - description: "Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, sending_paused, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental), sending_paused (403, outbound sending is paused for the account by the platform abuse controls; queued mail is held, new sends are refused until an operator resumes). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status." + description: "Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, sending_paused, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental), sending_paused (403, outbound sending is paused for the account by the platform abuse controls; queued mail is held, new sends are refused until an operator resumes; experimental). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status." type: string x-e2a-error-contracts: address_in_trash: @@ -2322,6 +2322,7 @@ components: - 400 x-experimental-values: - blocked_by_policy + - sending_paused details: additionalProperties: true description: Optional structured context, polymorphic by code. Treat it as an open object keyed off code; unknown codes and fields must be preserved. diff --git a/cmd/e2a/sending_policy_wiring_test.go b/cmd/e2a/sending_policy_wiring_test.go index 16b7f985b..a7eebc1a4 100644 --- a/cmd/e2a/sending_policy_wiring_test.go +++ b/cmd/e2a/sending_policy_wiring_test.go @@ -6,6 +6,8 @@ import ( "strings" "testing" + "github.com/riverqueue/river" + "github.com/tokencanopy/e2a/internal/config" "github.com/tokencanopy/e2a/internal/outbound" "github.com/tokencanopy/e2a/internal/sendingpolicy" @@ -48,7 +50,13 @@ func TestSendingPolicyWiring(t *testing.T) { // The worker RegisterJobs registers is what runs in production; it, not // the bundle, must carry the gate and the legacy resolver. Without the // resolver every job in flight at cutover would fail closed. - worker := composed.jobs.SendWorker() + // Register exactly as main does and inspect what River received — the + // constructor alone would not catch a RegisterJobs that bypassed it. + composed.jobs.RegisterJobs(river.NewWorkers()) + worker := composed.jobs.RegisteredSendWorker() + if worker == nil { + t.Fatal("RegisterJobs registered no send worker") + } if worker.Gate() != composed.gate { t.Fatal("the registered send worker does not hold the composed gate") } diff --git a/docs/api.md b/docs/api.md index b9ea43c1c..db8b9b960 100644 --- a/docs/api.md +++ b/docs/api.md @@ -85,7 +85,8 @@ stable field are beta, `x-experimental-values` on that field): the screening + review-hold event types (`email.flagged`, `email.blocked`, `email.review_requested`, `email.review_approved`, `email.review_rejected` — marked via `x-experimental-values` on the stable `type` field). The stable -`error.code` vocabulary likewise marks only `blocked_by_policy` experimental. +`error.code` vocabulary likewise marks only `blocked_by_policy` and +`sending_paused` experimental. See [events.md](events.md). The exact operation-level list is repeated with methods and paths in @@ -313,7 +314,7 @@ retryable ones (the per-row retry notes in the table below are authoritative). | `unauthorized` | 401 | Missing or invalid credentials (REST and the WebSocket handshake). | | `forbidden` | 403 | Authenticated but not allowed (key scope, cross-tenant access). | | `blocked_by_policy` | 403 | **Experimental.** The outbound message was blocked by the agent's outbound policy gate. | -| `sending_paused` | 403 | Outbound sending is paused for the account by the platform abuse controls. Nothing was queued; queued mail is held until an operator resumes. | +| `sending_paused` | 403 | **Experimental.** Outbound sending is paused for the account by the platform abuse controls. Nothing was queued; queued mail is held until an operator resumes. | | **Validation** | | | | `invalid_request` | 400 / 422 | The canonical input-validation code — malformed (400) or semantically invalid (422). `error.details` carries the per-field list. | | `invalid_cursor` | 400 | Bad pagination cursor — drop it and re-fetch from the start. | @@ -466,7 +467,8 @@ every `/v1` operation not listed here is covered by the GA freeze. `x-experimental-values` listing exactly those values — the field itself stays stable, the listed values (and their payloads) may still change, and every unlisted value is stable. The stable `ErrorBody.code` discriminator - similarly marks only `blocked_by_policy` experimental. Anything not marked + similarly marks only `blocked_by_policy` and `sending_paused` experimental. + Anything not marked beta or experimental is stable surface. One deliberate schema-level use of the beta marker under a **stable** operation: the account export's interior record schemas (`GET /v1/account/export`) are beta-marked because they are diff --git a/internal/agent/outbound_async.go b/internal/agent/outbound_async.go index 2fa40cb85..55b46411d 100644 --- a/internal/agent/outbound_async.go +++ b/internal/agent/outbound_async.go @@ -146,7 +146,7 @@ func (a *outboundSendStore) ClaimSend(ctx context.Context, messageID string, job anchor = *p.ScheduledAt } if !anchor.IsZero() && time.Since(anchor) > outboundsend.SendRetryHorizon { - if _, _, failErr := a.MarkFailed(ctx, p.ID, jobID, 0, time.Now().UTC(), + if _, _, _, failErr := a.MarkFailed(ctx, p.ID, jobID, 0, time.Now().UTC(), "daily_send_cap_timeout: daily send limit still exceeded past the retry horizon", delivery.FailureSourceLocal, messagelifecycle.ReasonSubmissionLocalRetriesExhausted, nil); failErr != nil { return nil, failErr @@ -164,7 +164,7 @@ func (a *outboundSendStore) ClaimSend(ctx context.Context, messageID string, job log.Printf("[outbound-send:%s] daily send cap exhausted at fire time, deferring to %s", p.ID, retryAt.Format(time.RFC3339)) return nil, &outboundsend.DailyQuotaDeferredError{RetryAt: retryAt} } - if _, _, failErr := a.MarkFailed(ctx, p.ID, jobID, 0, time.Now().UTC(), + if _, _, _, failErr := a.MarkFailed(ctx, p.ID, jobID, 0, time.Now().UTC(), "send canceled: monthly send limit exceeded at send time", delivery.FailureSourceLocal, messagelifecycle.ReasonSubmissionCancelled, nil); failErr != nil { return nil, failErr @@ -367,7 +367,7 @@ func (a *outboundSendStore) FinalizeScheduledCancellationTx( // time is the occurred_at the write actually used: the provider-accept // evidence time on an evidence settle, the caller's occurredAt on a failure, // zero on a no-op. -func (a *outboundSendStore) MarkFailed(ctx context.Context, messageID string, jobID int64, attempt int, occurredAt time.Time, detail string, source delivery.FailureSource, reason messagelifecycle.ReasonCode, blockedRecipients []string) (delivery.Status, time.Time, error) { +func (a *outboundSendStore) MarkFailed(ctx context.Context, messageID string, jobID int64, attempt int, occurredAt time.Time, detail string, source delivery.FailureSource, reason messagelifecycle.ReasonCode, blockedRecipients []string) (delivery.Status, time.Time, string, error) { detail = messagelifecycle.SafeDiagnostic(detail) blockedRecipients = normalizeBlockedRecipients(blockedRecipients) var settled delivery.Status @@ -415,12 +415,12 @@ func (a *outboundSendStore) MarkFailed(ctx context.Context, messageID string, jo e.ID = webhookpub.DeterministicEventID(messageID, webhookpub.EventEmailFailed) return a.outbox.PublishTx(ctx, tx, e) }); err != nil { - return "", time.Time{}, err + return "", time.Time{}, "", err } if resolved != nil { log.Printf("[outbound-send] %s: terminal-failure guard settled as sent on provider evidence (provider id %q)", messageID, resolvedProviderID) } - return settled, settledAt, nil + return settled, settledAt, resolvedProviderID, nil } func (a *outboundSendStore) PreserveTerminalFailure(ctx context.Context, messageID string, jobID int64, attempt int, occurredAt time.Time, detail string, source delivery.FailureSource, reason messagelifecycle.ReasonCode, blockedRecipients []string) error { diff --git a/internal/agent/outbound_async_test.go b/internal/agent/outbound_async_test.go index fc4396c43..574fd3409 100644 --- a/internal/agent/outbound_async_test.go +++ b/internal/agent/outbound_async_test.go @@ -1271,7 +1271,7 @@ func TestOutboundSendStore_MarkFailed(t *testing.T) { adapter := agent.NewOutboundSendStore(store, outbox, usage.NewNoopUsageTracker()) occurredAt := time.Now().UTC() - settled, settledAt, err := adapter.MarkFailed(ctx, res.MessageID, 999, 6, occurredAt, "550 mailbox unavailable", delivery.FailureSourceProvider, messagelifecycle.ReasonSubmissionProviderRejected, nil) + settled, settledAt, _, err := adapter.MarkFailed(ctx, res.MessageID, 999, 6, occurredAt, "550 mailbox unavailable", delivery.FailureSourceProvider, messagelifecycle.ReasonSubmissionProviderRejected, nil) if err != nil { t.Fatalf("MarkFailed: %v", err) } diff --git a/internal/httpapi/errors.go b/internal/httpapi/errors.go index ac2162b52..049c48b54 100644 --- a/internal/httpapi/errors.go +++ b/internal/httpapi/errors.go @@ -54,7 +54,7 @@ type ErrorEnvelope struct { // ErrorBody is the inner object of the envelope. type ErrorBody struct { - Code string `json:"code" doc:"Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, sending_paused, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental), sending_paused (403, outbound sending is paused for the account by the platform abuse controls; queued mail is held, new sends are refused until an operator resumes). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status."` + Code string `json:"code" doc:"Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, sending_paused, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental), sending_paused (403, outbound sending is paused for the account by the platform abuse controls; queued mail is held, new sends are refused until an operator resumes; experimental). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status."` Message string `json:"message" doc:"Human-readable explanation. Not for branching — use code."` Details any `json:"details,omitempty" doc:"Optional structured context, polymorphic by code. Treat it as an open object keyed off code; unknown codes and fields must be preserved."` RequestID string `json:"request_id" doc:"Echoes the X-Request-Id response header so a failing call is greppable in logs."` diff --git a/internal/httpapi/stability.go b/internal/httpapi/stability.go index 7eea7ad91..7d30d3b26 100644 --- a/internal/httpapi/stability.go +++ b/internal/httpapi/stability.go @@ -244,9 +244,10 @@ func (s *Server) applyEvolutionStance() { for _, schema := range []string{"HoldReasonView", "ProtectionFindingView", "ThreatCategoryView"} { markSchema(schemas, schema, extStabilityLevel, stabilityBeta) } - // ErrorBody.code is a stable open discriminator; only the outbound - // gate-policy value remains experimental. - markProperty(schemas, "ErrorBody", "code", extExperimentalValues, []string{"blocked_by_policy"}) + // ErrorBody.code is a stable open discriminator; the outbound gate-policy + // value and the sending-abuse pause value remain experimental — both are + // produced by controls that ship disabled. + markProperty(schemas, "ErrorBody", "code", extExperimentalValues, []string{"blocked_by_policy", "sending_paused"}) // // The template hooks on send are beta (templates are beta) even though // sendMessage itself is stable. diff --git a/internal/httpapi/stability_test.go b/internal/httpapi/stability_test.go index dfdb3dd0f..1f37e285d 100644 --- a/internal/httpapi/stability_test.go +++ b/internal/httpapi/stability_test.go @@ -436,12 +436,13 @@ func TestSpecBetaMarkers(t *testing.T) { } } - // The error discriminator remains stable; only the gate-policy value is - // experimental. + // The error discriminator remains stable; only the two values produced by + // controls that ship disabled — the outbound gate policy and the sending + // abuse pause — are experimental. errorCode, _ := schemaProps(t, doc, "ErrorBody")["code"].(map[string]any) rawErrorValues, _ := errorCode["x-experimental-values"].([]any) - if len(rawErrorValues) != 1 || rawErrorValues[0] != "blocked_by_policy" { - t.Errorf("ErrorBody.code x-experimental-values = %v, want [blocked_by_policy]", rawErrorValues) + if len(rawErrorValues) != 2 || rawErrorValues[0] != "blocked_by_policy" || rawErrorValues[1] != "sending_paused" { + t.Errorf("ErrorBody.code x-experimental-values = %v, want [blocked_by_policy sending_paused]", rawErrorValues) } // Managed unsubscribe is a beta opt-in nested inside otherwise-stable diff --git a/internal/outboundsend/gate_worker_test.go b/internal/outboundsend/gate_worker_test.go index f3943c39c..78d9ec641 100644 --- a/internal/outboundsend/gate_worker_test.go +++ b/internal/outboundsend/gate_worker_test.go @@ -273,6 +273,9 @@ func TestGatedWorker_ProviderEvidenceSettlesTheOperation(t *testing.T) { if g.lookupCalls != 1 || len(g.settled) != 1 || g.settled[0] != sendingpolicy.SettlementProviderAccepted { t.Fatalf("lookups=%d settled=%v, want the operation settled as accepted", g.lookupCalls, g.settled) } + if len(g.settledIDs) != 1 || g.settledIDs[0] != "ses-evidence" { + t.Fatalf("settled ids = %v, want the evidence's provider id carried into the settlement", g.settledIDs) + } } func TestGatedWorker_LegacyJobResolvesThroughTheAcceptPath(t *testing.T) { @@ -454,7 +457,7 @@ func TestGatedWorker_EvidenceSettleUnderATerminalWriteSettlesTheOperation(t *tes // A suppression arrives for a message whose earlier attempt dialed and // whose provider evidence has since landed: the guarded terminal write // settles the row as SENT, and the dialed attempt must be settled too. - st := &fakeStore{job: acceptedJob("msg_late_evidence"), suppressed: []string{"b@y.com"}, settleStatus: delivery.StatusSent} + st := &fakeStore{job: acceptedJob("msg_late_evidence"), suppressed: []string{"b@y.com"}, settleStatus: delivery.StatusSent, settleProviderID: "ses-under-terminal"} g := allowAll() if err := outboundsend.NewSendWorker(st, &fakeDeliverer{}).WithGate(g).Work(context.Background(), gatedJob("msg_late_evidence", 2)); !isCancel(err) { t.Fatalf("err = %v, want cancel", err) @@ -462,6 +465,9 @@ func TestGatedWorker_EvidenceSettleUnderATerminalWriteSettlesTheOperation(t *tes if g.lookupCalls != 1 || len(g.settled) != 1 || g.settled[0] != sendingpolicy.SettlementProviderAccepted { t.Fatalf("lookups=%d settled=%v, want the operation settled as accepted from the evidence", g.lookupCalls, g.settled) } + if len(g.settledIDs) != 1 || g.settledIDs[0] != "ses-under-terminal" { + t.Fatalf("settled ids = %v, want the store's resolved provider id carried into the settlement", g.settledIDs) + } } func TestHoldClassForNamesEveryReasonExplicitly(t *testing.T) { @@ -503,3 +509,18 @@ func TestGatedWorker_OperationReferenceMustNameThisMessage(t *testing.T) { t.Fatalf("failed = %+v, want one local cancellation", st.failed) } } + +func TestGatedWorker_FailedSettlementAfterAcceptanceIsRetriedNotResent(t *testing.T) { + st := &fakeStore{job: acceptedJob("msg_resettle")} + dl := &fakeDeliverer{out: outboundsend.DeliverOutcome{ProviderMessageID: "ses-resettle", SettlementErr: errors.New("settle: db blip")}} + g := allowAll() + if err := outboundsend.NewSendWorker(st, dl).WithGate(g).Work(context.Background(), gatedJob("msg_resettle", 1)); err != nil { + t.Fatalf("Work: %v — an accepted send must never surface a settlement failure as a send error", err) + } + if dl.calls != 1 || len(st.sent) != 1 { + t.Fatalf("delivers=%d sent=%d, want exactly one of each", dl.calls, len(st.sent)) + } + if len(g.settled) != 1 || g.settled[0] != sendingpolicy.SettlementProviderAccepted || g.settledIDs[0] != "ses-resettle" { + t.Fatalf("settlements = %v / %v, want one retried acceptance carrying the provider id", g.settled, g.settledIDs) + } +} diff --git a/internal/outboundsend/jobs.go b/internal/outboundsend/jobs.go index d782f1983..2819eb3f1 100644 --- a/internal/outboundsend/jobs.go +++ b/internal/outboundsend/jobs.go @@ -26,6 +26,9 @@ type Jobs struct { pool *pgxpool.Pool enq jobs.Enqueuer metrics Metrics + + // registered is the send worker the last RegisterJobs call handed to River. + registered *SendWorker } // NewJobs builds the integration with its dependencies (no client yet). pool @@ -58,6 +61,10 @@ func (j *Jobs) TerminalReconcileWorker() *TerminalReconcileWorker { return NewTerminalReconcileWorker(j.pool, j.store).WithMetrics(j.metrics).WithGate(j.gate) } +// RegisteredSendWorker returns the send worker the last RegisterJobs call +// registered with River, or nil before any registration. +func (j *Jobs) RegisteredSendWorker() *SendWorker { return j.registered } + // Gate exposes the wired sending-protection gate, for the composition root's // wiring test. nil when none is wired. func (j *Jobs) Gate() sendingpolicy.Gate { return j.gate } @@ -91,7 +98,11 @@ func (j *Jobs) WithRateGate(g RateGate) *Jobs { // RegisterJobs adds the SendWorker and terminal-state safety net to the shared // client's bundle. Implements jobs.Registrar. func (j *Jobs) RegisterJobs(w *river.Workers) []*river.PeriodicJob { - river.AddWorker(w, j.SendWorker()) + // The worker registered here is recorded so the composition root's + // wiring test can inspect the exact object River will run, not merely + // what a constructor would produce. + j.registered = j.SendWorker() + river.AddWorker(w, j.registered) river.AddWorker(w, j.TerminalReconcileWorker()) return []*river.PeriodicJob{ river.NewPeriodicJob( diff --git a/internal/outboundsend/reconcile_test.go b/internal/outboundsend/reconcile_test.go index 8657318aa..c048843bc 100644 --- a/internal/outboundsend/reconcile_test.go +++ b/internal/outboundsend/reconcile_test.go @@ -1055,11 +1055,11 @@ func (s failingTerminalStore) RecordHold(context.Context, string, outboundsend.H func (s failingTerminalStore) MarkSent(context.Context, string, int64, int, time.Time, string, string) error { return nil } -func (s failingTerminalStore) MarkFailed(_ context.Context, _ string, _ int64, _ int, occurredAt time.Time, _ string, _ delivery.FailureSource, _ messagelifecycle.ReasonCode, _ []string) (delivery.Status, time.Time, error) { +func (s failingTerminalStore) MarkFailed(_ context.Context, _ string, _ int64, _ int, occurredAt time.Time, _ string, _ delivery.FailureSource, _ messagelifecycle.ReasonCode, _ []string) (delivery.Status, time.Time, string, error) { if s.err != nil { - return "", time.Time{}, s.err + return "", time.Time{}, "", s.err } - return delivery.StatusFailed, occurredAt, nil + return delivery.StatusFailed, occurredAt, "", nil } func (s failingTerminalStore) PreserveTerminalFailure(context.Context, string, int64, int, time.Time, string, delivery.FailureSource, messagelifecycle.ReasonCode, []string) error { return nil diff --git a/internal/outboundsend/terminal_reconcile.go b/internal/outboundsend/terminal_reconcile.go index f0dbc5b16..e212c7477 100644 --- a/internal/outboundsend/terminal_reconcile.go +++ b/internal/outboundsend/terminal_reconcile.go @@ -194,7 +194,7 @@ func (w *TerminalReconcileWorker) Work(ctx context.Context, _ *river.Job[Termina // fails it with provenance 'local' so later authoritative evidence can // still correct it. The stored detail of a deferred final attempt is // preferred over this generic sweep detail. - settled, settledAt, err := w.store.MarkFailed(ctx, candidate.messageID, candidate.jobID, attempt, occurredAt, detail, source, reason, candidate.failureBlockedRecipients) + settled, settledAt, providerID, err := w.store.MarkFailed(ctx, candidate.messageID, candidate.jobID, attempt, occurredAt, detail, source, reason, candidate.failureBlockedRecipients) if err != nil { if processed > 0 { log.Printf("[outbound-terminal-reconcile] processed %d candidates", processed) @@ -218,7 +218,10 @@ func (w *TerminalReconcileWorker) Work(ctx context.Context, _ *river.Job[Termina // dialed, so ramp progress and the provider-id binding catch up. // Best effort and idempotent — an attempt that predates the gate // has nothing to settle. - w.settleFromEvidence(ctx, candidate.messageID, candidate.providerMessageID) + if providerID == "" { + providerID = candidate.providerMessageID + } + w.settleFromEvidence(ctx, candidate.messageID, providerID) } processed++ } diff --git a/internal/outboundsend/worker.go b/internal/outboundsend/worker.go index 2190c5e55..c43a612ee 100644 --- a/internal/outboundsend/worker.go +++ b/internal/outboundsend/worker.go @@ -321,8 +321,11 @@ type Store interface { // state", not to unconditionally fail. // The returned status reports what the guarded write actually did: // StatusFailed, StatusSent (evidence settle), or "" (no-op). The returned - // time is the occurred_at the write actually used. - MarkFailed(ctx context.Context, messageID string, jobID int64, attempt int, occurredAt time.Time, detail string, source delivery.FailureSource, reason messagelifecycle.ReasonCode, blockedRecipients []string) (delivery.Status, time.Time, error) + // time is the occurred_at the write actually used, and the returned + // provider id is the evidence's provider message id on an evidence + // settle ('' otherwise), so the attempt that dialed can be settled with + // it. + MarkFailed(ctx context.Context, messageID string, jobID int64, attempt int, occurredAt time.Time, detail string, source delivery.FailureSource, reason messagelifecycle.ReasonCode, blockedRecipients []string) (delivery.Status, time.Time, string, error) PreserveTerminalFailure(ctx context.Context, messageID string, jobID int64, attempt int, occurredAt time.Time, detail string, source delivery.FailureSource, reason messagelifecycle.ReasonCode, blockedRecipients []string) error // DeferTerminalFailure records a final attempt's diagnostic + releases the // I/O claim WITHOUT declaring failed: the terminal reconciler declares the @@ -501,7 +504,7 @@ func (w *SendWorker) Work(ctx context.Context, job *river.Job[OutboundSendArgs]) if errors.Is(err, sendingpolicy.ErrSourceUnavailable) { return w.cancelTerminally(ctx, job, j, reserved, observedAt, "sending_policy: operation unavailable: "+err.Error()) } - return w.snoozeOnGateError(ctx, job, j, "reserve", err) + return w.snoozeOnGateError(ctx, job, j, reserved, "reserve", err) } if !early.Allow { return w.hold(ctx, job, j, reserved, early, observedAt) @@ -569,7 +572,7 @@ func (w *SendWorker) Work(ctx context.Context, job *river.Job[OutboundSendArgs]) if errors.Is(err, sendingpolicy.ErrSourceUnavailable) { return w.cancelTerminally(ctx, job, j, attempt, observedAt, "sending_policy: operation unavailable: "+err.Error()) } - return w.snoozeOnGateError(ctx, job, j, "authorize", err) + return w.snoozeOnGateError(ctx, job, j, attempt, "authorize", err) } if !decision.Allow || auth == nil { return w.hold(ctx, job, j, attempt, decision, observedAt) @@ -694,7 +697,7 @@ func (w *SendWorker) operationFor(ctx context.Context, job *river.Job[OutboundSe if errors.Is(err, sendingpolicy.ErrSourceUnavailable) { return sendingpolicy.OperationRef{}, w.cancelTerminally(ctx, job, j, sendingpolicy.AttemptRef{}, observedAt, "sending_policy: legacy source unavailable: "+err.Error()) } - return sendingpolicy.OperationRef{}, w.snoozeOnGateError(ctx, job, j, "resolve", err) + return sendingpolicy.OperationRef{}, w.snoozeOnGateError(ctx, job, j, sendingpolicy.AttemptRef{}, "resolve", err) } if decision == sendingpolicy.AcceptanceSendingPaused { return sendingpolicy.OperationRef{}, w.hold(ctx, job, j, sendingpolicy.AttemptRef{}, sendingpolicy.Decision{Reason: sendingpolicy.ReasonAccountPaused}, observedAt) @@ -863,9 +866,9 @@ func (w *SendWorker) cancelTerminally(ctx context.Context, job *river.Job[Outbou // It is a bounded wait like every other one: the message enters (or stays // in) the rate/ramp/provider class and expires at that class's deadline, so a // gate that is down for days does not park mail forever. -func (w *SendWorker) snoozeOnGateError(ctx context.Context, job *river.Job[OutboundSendArgs], j *SendJob, step string, gerr error) error { +func (w *SendWorker) snoozeOnGateError(ctx context.Context, job *river.Job[OutboundSendArgs], j *SendJob, attempt sendingpolicy.AttemptRef, step string, gerr error) error { log.Printf("[outbound-send] sending policy %s failed for %s (snoozing): %v", step, j.MessageID, gerr) - return w.holdFinite(ctx, job, j, sendingpolicy.AttemptRef{}, HoldRateRampOrProvider, "sending_policy_unavailable: "+step+": "+gerr.Error(), gateErrorSnoozeInterval, w.now().UTC()) + return w.holdFinite(ctx, job, j, attempt, HoldRateRampOrProvider, "sending_policy_unavailable: "+step+": "+gerr.Error(), gateErrorSnoozeInterval, w.now().UTC()) } // deferAttempt gives the budget back for a rate deferral; a stale or already @@ -907,6 +910,9 @@ func (w *SendWorker) resettle(ctx context.Context, messageID, providerMessageID for i := 0; i < terminalWriteRetries; i++ { select { case <-ctx.Done(): + // Shutdown or the job timeout: the one moment a lost settlement + // is likeliest, so it must not also be the one that goes unlogged. + log.Printf("[outbound-send] CRITICAL: %s accepted by provider but not settled (context ended before retry): %v", messageID, err) return case <-time.After(time.Duration(i+1) * terminalWriteBackoff): } @@ -1019,7 +1025,8 @@ func (w *SendWorker) markFailed(ctx context.Context, messageID string, jobID int for i := 0; i < terminalWriteRetries; i++ { var settled delivery.Status var settledAt time.Time - if settled, settledAt, err = w.store.MarkFailed(ctx, messageID, jobID, attempt, occurredAt, detail, source, reason, blockedRecipients); err == nil { + var providerID string + if settled, settledAt, providerID, err = w.store.MarkFailed(ctx, messageID, jobID, attempt, occurredAt, detail, source, reason, blockedRecipients); err == nil { // Emit what the guarded write actually did, exactly once, only // after the durable write: a failure with the caller's provenance, // or "sent" when provider evidence settled the row. A no-op write @@ -1038,7 +1045,7 @@ func (w *SendWorker) markFailed(ctx context.Context, messageID string, jobID int // expected to fail it. The attempt that dialed still needs // settling — ramp progress and the correlation binding — and // only the operation, not this call's attempt, names it. - w.settleFromEvidence(ctx, messageID, "") + w.settleFromEvidence(ctx, messageID, providerID) } return nil } diff --git a/internal/outboundsend/worker_test.go b/internal/outboundsend/worker_test.go index 581177d9c..1bcd55426 100644 --- a/internal/outboundsend/worker_test.go +++ b/internal/outboundsend/worker_test.go @@ -27,6 +27,8 @@ type fakeStore struct { // occurred_at to the provider-accept evidence time for the durable write. settleStatus delivery.Status settleAt time.Time + // settleProviderID is the evidence's provider id an evidence settle reports. + settleProviderID string // terminalAfterFailure mirrors the production store: once MarkFailed commits, // a retry can no longer claim the terminal message and ClaimSend returns nil. terminalAfterFailure bool @@ -72,7 +74,7 @@ func (f *fakeStore) MarkSent(_ context.Context, id string, _ int64, _ int, _ tim f.sent = append(f.sent, sentCall{id, provider, sentAs}) return f.markSentErr } -func (f *fakeStore) MarkFailed(_ context.Context, id string, _ int64, attempt int, occurredAt time.Time, detail string, source delivery.FailureSource, reason messagelifecycle.ReasonCode, blockedRecipients []string) (delivery.Status, time.Time, error) { +func (f *fakeStore) MarkFailed(_ context.Context, id string, _ int64, attempt int, occurredAt time.Time, detail string, source delivery.FailureSource, reason messagelifecycle.ReasonCode, blockedRecipients []string) (delivery.Status, time.Time, string, error) { f.failed = append(f.failed, failedCall{id: id, attempt: attempt, occurredAt: occurredAt, detail: detail, source: source, reason: reason, blockedRecipients: blockedRecipients}) status := f.settleStatus if status == "" { @@ -82,7 +84,7 @@ func (f *fakeStore) MarkFailed(_ context.Context, id string, _ int64, attempt in if at.IsZero() { at = occurredAt } - return status, at, nil + return status, at, f.settleProviderID, nil } func (f *fakeStore) PreserveTerminalFailure(context.Context, string, int64, int, time.Time, string, delivery.FailureSource, messagelifecycle.ReasonCode, []string) error { return nil @@ -362,6 +364,7 @@ type fakeGate struct { deferred []string cancelled []string settled []sendingpolicy.SettlementOutcome + settledIDs []string reserves int consumes int lookupErr error @@ -410,8 +413,9 @@ func (g *fakeGate) SettleProvider(_ context.Context, s sendingpolicy.ProviderSet g.settled = append(g.settled, s.Outcome) return nil } -func (g *fakeGate) SettleOperation(_ context.Context, _ sendingpolicy.OperationRef, o sendingpolicy.SettlementOutcome, _ string) error { +func (g *fakeGate) SettleOperation(_ context.Context, _ sendingpolicy.OperationRef, o sendingpolicy.SettlementOutcome, id string) error { g.settled = append(g.settled, o) + g.settledIDs = append(g.settledIDs, id) return nil } func (g *fakeGate) LookupOperation(_ context.Context, id string) (sendingpolicy.OperationRef, error) { diff --git a/internal/sendingpolicy/gate.go b/internal/sendingpolicy/gate.go index abe543796..15534515c 100644 --- a/internal/sendingpolicy/gate.go +++ b/internal/sendingpolicy/gate.go @@ -1875,14 +1875,22 @@ func (m *Module) settle(ctx context.Context, operationID string, attempt int, se return err } if attempt == 0 { - // Evidence without a token names an operation, not an ordinal. When - // several attempts dialed, the oldest one that has no provider id yet - // is the best owner: feedback arrives in send order far more often - // than not, and each binding retires its attempt from this choice. - // With every dialed attempt already bound, the latest one takes the - // replay, and the bind refuses a different id rather than absorb it. + // Evidence without a token names an operation, not an ordinal. The + // attempt is chosen in this order: one already bound to this exact + // provider id (a replay, which must be idempotent and must not spill + // onto a later attempt); else the oldest dialed attempt with no + // provider id yet, because feedback arrives in send order far more + // often than not and each binding retires its attempt from this + // choice; else the latest dialed attempt, whose bind refuses a + // different id rather than absorb it. if err := tx.QueryRow(ctx, ` SELECT COALESCE( + (SELECT MIN(r.submission_attempt) + FROM sending_budget_reservations r + JOIN sending_feedback_correlations c + ON c.operation_id = r.operation_id AND c.submission_attempt = r.submission_attempt + WHERE r.operation_id = $1 AND r.call_state = 'started' + AND $2 <> '' AND c.provider_message_id = $2), (SELECT MIN(r.submission_attempt) FROM sending_budget_reservations r LEFT JOIN sending_feedback_correlations c @@ -1892,7 +1900,7 @@ func (m *Module) settle(ctx context.Context, operationID string, attempt int, se (SELECT MAX(submission_attempt) FROM sending_budget_reservations WHERE operation_id = $1 AND call_state = 'started'), - 0)`, operationID, + 0)`, operationID, NormalizeProviderMessageID(settlement.ProviderMessageID), ).Scan(&attempt); err != nil { return fmt.Errorf("sendingpolicy: find started attempt: %w", err) } diff --git a/internal/sendingpolicy/provider_token_test.go b/internal/sendingpolicy/provider_token_test.go index e98b6e6cd..721eb4d5b 100644 --- a/internal/sendingpolicy/provider_token_test.go +++ b/internal/sendingpolicy/provider_token_test.go @@ -374,11 +374,34 @@ func TestProviderTokenSettleOperationPrefersTheOldestUnboundDialedAttempt(t *tes if got := f.providerMessageID(ref.ID(), 2); got == nil || *got != "ses-second" { t.Fatalf("attempt two bound = %v, want ses-second", got) } - // Everything bound: a replay of either id is idempotent, a third id is a conflict. + // A replay for attempt one arriving while a LATER attempt is still + // unbound must return to attempt one, never spill onto the unbound one. + // Set that shape up on ordinal three. + if _, third, err := g.Reserve(f.ctx, ref); err != nil || third.Attempt() != 3 { + t.Fatalf("reserve ordinal three: attempt=%v err=%v", third, err) + } else { + _, auth, err := g.ConsumeAttempt(f.ctx, third) + if err != nil || auth == nil { + t.Fatalf("authorize 3: auth=%v err=%v", auth, err) + } + if err := g.RedeemProviderCall(f.ctx, *auth); err != nil { + t.Fatalf("redeem 3: %v", err) + } + } + if err := g.SettleOperation(f.ctx, ref, sendingpolicy.SettlementProviderAccepted, "ses-first"); err != nil { + t.Fatalf("replay of attempt one with attempt three unbound: %v", err) + } + if got := f.providerMessageID(ref.ID(), 3); got != nil { + t.Fatalf("attempt three bound = %q by a replay of attempt one's id", *got) + } + if err := g.SettleOperation(f.ctx, ref, sendingpolicy.SettlementProviderAccepted, "ses-third"); err != nil { + t.Fatalf("attempt three's own evidence: %v", err) + } + // Everything bound: a replay of any id is idempotent, a fourth id is a conflict. if err := g.SettleOperation(f.ctx, ref, sendingpolicy.SettlementProviderAccepted, "ses-second"); err != nil { t.Fatalf("replay: %v", err) } - if err := g.SettleOperation(f.ctx, ref, sendingpolicy.SettlementProviderAccepted, "ses-third"); !errors.Is(err, sendingpolicy.ErrProviderMessageIDConflict) { - t.Fatalf("third id err = %v, want ErrProviderMessageIDConflict", err) + if err := g.SettleOperation(f.ctx, ref, sendingpolicy.SettlementProviderAccepted, "ses-fourth"); !errors.Is(err, sendingpolicy.ErrProviderMessageIDConflict) { + t.Fatalf("fourth id err = %v, want ErrProviderMessageIDConflict", err) } } diff --git a/sdks/python/src/e2a/v1/generated/models/error_body.py b/sdks/python/src/e2a/v1/generated/models/error_body.py index 7c57352d5..8e60232ef 100644 --- a/sdks/python/src/e2a/v1/generated/models/error_body.py +++ b/sdks/python/src/e2a/v1/generated/models/error_body.py @@ -26,7 +26,7 @@ class ErrorBody(BaseModel): """ ErrorBody """ # noqa: E501 - code: StrictStr = Field(description="Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, sending_paused, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental), sending_paused (403, outbound sending is paused for the account by the platform abuse controls; queued mail is held, new sends are refused until an operator resumes). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status.") + code: StrictStr = Field(description="Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, sending_paused, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental), sending_paused (403, outbound sending is paused for the account by the platform abuse controls; queued mail is held, new sends are refused until an operator resumes; experimental). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status.") details: Optional[Dict[str, Any]] = Field(default=None, description="Optional structured context, polymorphic by code. Treat it as an open object keyed off code; unknown codes and fields must be preserved.") message: StrictStr = Field(description="Human-readable explanation. Not for branching — use code.") request_id: StrictStr = Field(description="Echoes the X-Request-Id response header so a failing call is greppable in logs.") diff --git a/sdks/python/tests/test_enum_forward_compat.py b/sdks/python/tests/test_enum_forward_compat.py index 1275ef8f0..3c47279dc 100644 --- a/sdks/python/tests/test_enum_forward_compat.py +++ b/sdks/python/tests/test_enum_forward_compat.py @@ -76,6 +76,8 @@ "submission.provider_rejected", "submission.local_retries_exhausted", "submission.cancelled", + "submission.policy_budget_expired", + "submission.sending_setup_expired", "delivery.recipient_server_accepted", "delivery.temporary_delay", "delivery.permanent_bounce", diff --git a/sdks/typescript/src/v1/generated/models/ErrorBody.ts b/sdks/typescript/src/v1/generated/models/ErrorBody.ts index 0f6d56fb3..ba6f627b2 100644 --- a/sdks/typescript/src/v1/generated/models/ErrorBody.ts +++ b/sdks/typescript/src/v1/generated/models/ErrorBody.ts @@ -14,7 +14,7 @@ import { HttpFile } from '../http/http.js'; export class ErrorBody { /** - * Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, sending_paused, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental), sending_paused (403, outbound sending is paused for the account by the platform abuse controls; queued mail is held, new sends are refused until an operator resumes). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status. + * Machine-branchable error code — the stable discriminator clients switch on. Open set: treat it as a string and tolerate unknown values, since new codes may be added over time (branch on the ones you handle, fall back to the HTTP status otherwise). Exact current vocabulary (machine-checked): unauthorized, forbidden, blocked_by_policy, sending_paused, invalid_request, invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope, reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty, recipient_suppressed, not_found, attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found, gone, conflict, precondition_failed, agent_taken, domain_taken, alias_taken, address_in_trash, message_held, message_not_pending, message_not_yet_delivered, not_in_trash, purge_in_progress, send_in_progress, webhook_disabled, webhook_cooldown, domain_not_registered, domain_has_agents, domain_not_verified, inbound_mx_missing, limit_exceeded, rate_limited, contact_limit_reached, template_limit_reached, webhook_limit_reached, idempotency_in_flight, idempotency_key_reuse, payload_too_large, attachment_too_large, not_implemented, events_log_disabled, limits_unavailable, inbound_mx_check_failed, auth_unavailable, internal_error, method_not_allowed, unsupported_media_type, error. Grouped semantics: auth: unauthorized (401), forbidden (403), blocked_by_policy (403, outbound policy gate; experimental), sending_paused (403, outbound sending is paused for the account by the platform abuse controls; queued mail is held, new sends are refused until an operator resumes; experimental). Validation: invalid_request is the single canonical code for input-validation failures whether they arrive as 400 (malformed) or 422 (semantically invalid); field/resource-specific invalid_* refinements (invalid_cursor, invalid_filter, invalid_domain, invalid_slug, invalid_recipient, invalid_attachment, invalid_template, invalid_event_type, invalid_webhook_url, invalid_expires_at, invalid_scope), reserved_domain, too_many_recipients, template_render_failed, template_rendered_empty (all 400); recipient_suppressed (422). Not found: not_found (404) plus the *_not_found family (attachment_not_found, contact_not_found, engagement_not_found, import_batch_not_found, template_not_found, starter_template_not_found); gone (410, past retention). Conflict/state: conflict (409, generic), precondition_failed (412, optimistic-concurrency validator is stale), the *_taken family — the requested identifier is already claimed — (agent_taken, domain_taken, alias_taken, all 409), address_in_trash (409), message_held (409), message_not_pending (409), message_not_yet_delivered (409, retry after the source outbound message is sent), not_in_trash (409), purge_in_progress (409, permanent delete already claimed), send_in_progress (409), webhook_disabled (409), webhook_cooldown (409), domain_not_registered (400), domain_has_agents (400), domain_not_verified (400 on create-agent, 403 on send), inbound_mx_missing (400). Capacity: limit_exceeded (402, plan quota — see LimitExceededDetails), rate_limited (429, request rate — see RateLimitedDetails), contact_limit_reached, template_limit_reached and webhook_limit_reached (400, fixed per-account caps). Idempotency: idempotency_in_flight (409, wait then retry the byte-identical request), idempotency_key_reuse (422, caller bug — do not retry as-is). Size: payload_too_large (413, request body), attachment_too_large (413, inline fetch over the cap — use download_url). Availability: not_implemented (501, feature not available on this deployment), events_log_disabled (501), limits_unavailable (503), inbound_mx_check_failed (503), auth_unavailable (503, an auth backend — e.g. a delegated-token verifier or the identity store — could not judge the credential; retry). Server/fallback: internal_error (5xx), method_not_allowed (405), unsupported_media_type (415), and the generic code error for any otherwise-unmapped status. */ 'code': string; /** From 128e8e8720334bee53635e31a8d35f0aaf24b437 Mon Sep 17 00:00:00 2001 From: jiashuoz <39790535+jiashuoz@users.noreply.github.com> Date: Fri, 4 Sep 2026 23:31:49 -0700 Subject: [PATCH 09/12] feat(outbound): close the provider seam for every sender Slice B7 of the sending abuse prevention plan. The relay no longer exports a send method; ProviderSubmitter.SubmitOnce with a gate token is the only way to reach the provider, and a tracked-closure test parses every production file to keep it that way. - hitlnotify + webhooknotify: enqueue prepares a customer_notification operation in the source transaction and stamps it on the job; workers run Reserve -> early hold -> ConsumeAttempt -> authorized submit and snooze on a hold without provider I/O; pre-floor jobs resolve at fire time and are stamped once (jobs.StampJobArg). - public feedback: server-keyed public_feedback_notification operation with a bounded per-attempt Reserve/Consume/Submit loop; a definite rejection or a lost acceptance stops it. - e2a -reconcile-legacy-sending-jobs: stamps pending outbound_send / hitl_notify / webhook_notify jobs that carry no operation, cancels orphans, exits nonzero unless every scanned job was decided. - main, TestServer and the contract server build the notifiers over the shared submitter and hand the API the submitter + gate. - design addendum in docs/design/async-message-pipeline.md. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01AjfGxvXW6fNKWGFHuo68yX --- cmd/e2a/main.go | 13 +- cmd/e2a/sending_policy.go | 7 +- cmd/e2a/sending_policy_test.go | 11 + cmd/e2a/sending_reconcile.go | 221 ++++++++++++++++ cmd/e2a/sending_reconcile_test.go | 183 +++++++++++++ docs/design/async-message-pipeline.md | 34 +++ internal/agent/api.go | 87 +++++- internal/agent/api_test.go | 15 +- internal/agent/feedback_github_test.go | 14 + internal/agent/feedback_seam_test.go | 248 ++++++++++++++++++ internal/hitlnotify/e2e_test.go | 6 +- internal/hitlnotify/jobs.go | 71 ++++- internal/hitlnotify/notifier.go | 31 ++- internal/hitlnotify/notifier_test.go | 110 ++++++-- internal/hitlnotify/worker.go | 126 ++++++++- internal/hitlnotify/worker_test.go | 144 +++++++++- internal/jobs/argstamp.go | 40 +++ .../provider_authorization_guard_test.go | 140 ++++++++++ internal/outbound/provider_submit.go | 2 +- internal/outbound/sender.go | 77 ------ internal/outbound/smtp_relay.go | 89 +------ internal/outbound/smtp_relay_test.go | 14 +- internal/testutil/contract_server.go | 4 +- internal/testutil/server.go | 4 +- internal/webhooknotify/e2e_test.go | 6 +- internal/webhooknotify/jobs.go | 70 ++++- internal/webhooknotify/notifier.go | 25 +- internal/webhooknotify/notifier_test.go | 33 ++- internal/webhooknotify/worker.go | 124 ++++++++- internal/webhooknotify/worker_test.go | 120 ++++++++- 30 files changed, 1812 insertions(+), 257 deletions(-) create mode 100644 cmd/e2a/sending_reconcile.go create mode 100644 cmd/e2a/sending_reconcile_test.go create mode 100644 internal/agent/feedback_seam_test.go create mode 100644 internal/jobs/argstamp.go create mode 100644 internal/outbound/provider_authorization_guard_test.go diff --git a/cmd/e2a/main.go b/cmd/e2a/main.go index 7e13c1726..e3d730456 100644 --- a/cmd/e2a/main.go +++ b/cmd/e2a/main.go @@ -121,6 +121,7 @@ func main() { flag.IntVar(&spFlags.activeBillingContract, "active-billing-contract", -1, "verified active billing contract level") flag.StringVar(&spFlags.rollbackBillingDigest, "rollback-billing-digest", "", "verified rollback billing image digest") flag.IntVar(&spFlags.rollbackBillingContract, "rollback-billing-contract", -1, "verified rollback billing contract level") + flag.BoolVar(&spFlags.reconcile, "reconcile-legacy-sending-jobs", false, "stamp a sending operation reference onto every pending provider-submitting job enqueued without one (cancelling orphans whose source row is gone), print counts, then exit; nonzero unless every job was decided") flag.BoolVar(&spFlags.capabilities, "print-capabilities", false, "print the machine-readable capability marker (contract level, policy source, operator commitments), then exit") flag.StringVar(&spFlags.reason, "reason", "", "nonblank reason recorded in the audit row of a sending-protection mutation") flag.Parse() @@ -365,6 +366,9 @@ func main() { }) outboundJobs := outboundSending.jobs registrars = append(registrars, outboundJobs) + // Platform mail the API sends itself (public feedback) crosses the same + // seam with tokens from the same gate. + sendingGate, providerSubmitter := outboundSending.gate, outboundSending.submitter registrars = append(registrars, sendramp.NewMaintenanceJobs(rampStore)) // Queue depth/age gauges: a 30s maintenance periodic sampling river_job // per queue+state (docs/observability.md). @@ -392,7 +396,7 @@ func main() { var notifyJobs *hitlnotify.Jobs notifierEnabled := cfg.OutboundSMTP.FromDomain != "" && cfg.HTTP.PublicURL != "" if notifierEnabled { - notifyJobs = hitlnotify.NewJobs(store) + notifyJobs = hitlnotify.NewJobs(store).WithGate(sendingGate, pool) registrars = append(registrars, notifyJobs) } @@ -407,7 +411,7 @@ func main() { // (pre-feature behavior). var webhookNotifyJobs *webhooknotify.Jobs if cfg.OutboundSMTP.FromDomain != "" { - webhookNotifyJobs = webhooknotify.NewJobs(store).WithMetrics(metrics) + webhookNotifyJobs = webhooknotify.NewJobs(store).WithMetrics(metrics).WithGate(sendingGate, pool) registrars = append(registrars, webhookNotifyJobs) } @@ -697,7 +701,7 @@ func main() { // unreachable in practice — kept as a defensive guard against future drift. log.Printf("[hitl] notifier disabled: notification job pipeline not registered") } else { - notifier := hitlnotify.New(store, smtpRelay, approvalSigner, cfg.OutboundSMTP.FromDomain, cfg.Notifications.FromAddress, cfg.Notifications.ReplyTo, cfg.HTTP.PublicURL).WithDKIM(store) + notifier := hitlnotify.New(store, providerSubmitter, approvalSigner, cfg.OutboundSMTP.FromDomain, cfg.Notifications.FromAddress, cfg.Notifications.ReplyTo, cfg.HTTP.PublicURL).WithDKIM(store) // Late-bind the concrete Deliverer onto the registered NotifyWorker (which // has been running since jobsClient.Start; jobs enqueued before this bind // simply retry) and give the hold path its accept-tx enqueuer. The HTTP @@ -718,7 +722,7 @@ func main() { // a BYODKIM custom from-address domain is signed here or not at all. // Fail-open — no stored key (self-host default) sends unsigned. if webhookNotifyJobs != nil { - whNotifier := webhooknotify.New(store, smtpRelay, cfg.OutboundSMTP.FromDomain, cfg.Notifications.FromAddress, cfg.Notifications.ReplyTo, cfg.HTTP.PublicURL).WithDKIM(store) + whNotifier := webhooknotify.New(store, providerSubmitter, cfg.OutboundSMTP.FromDomain, cfg.Notifications.FromAddress, cfg.Notifications.ReplyTo, cfg.HTTP.PublicURL).WithDKIM(store) webhookNotifyJobs.SetDeliverer(whNotifier) log.Printf("[webhook-notify] enabled (from=%s)", whNotifier.FromAddress()) } else { @@ -833,6 +837,7 @@ func main() { // The outbound accept-tx enqueuer is mandatory: DeliverOutbound always // persists+enqueues and returns accepted before provider submission. api.SetOutboundEnqueuer(outboundJobs) + api.SetProviderSubmitter(providerSubmitter, sendingGate) // Slices 6 + 7: customer-facing events API needs the raw pool to // query webhook_events and write webhook_subscriber_deliveries on // replay. Kept as a separate setter so a future refactor can route diff --git a/cmd/e2a/sending_policy.go b/cmd/e2a/sending_policy.go index 02c2efc41..a9d8d733c 100644 --- a/cmd/e2a/sending_policy.go +++ b/cmd/e2a/sending_policy.go @@ -24,6 +24,7 @@ type sendingProtectionFlags struct { register bool attest bool capabilities bool + reconcile bool expectedGeneration int64 expectedPolicySHA string @@ -40,12 +41,12 @@ type sendingProtectionFlags struct { } func (f *sendingProtectionFlags) commandRequested() bool { - return f.inspect || f.activate || f.register || f.attest || f.capabilities + return f.inspect || f.activate || f.register || f.attest || f.capabilities || f.reconcile } func (f *sendingProtectionFlags) selectedCount() int { n := 0 - for _, set := range []bool{f.inspect, f.activate, f.register, f.attest, f.capabilities} { + for _, set := range []bool{f.inspect, f.activate, f.register, f.attest, f.capabilities, f.reconcile} { if set { n++ } @@ -105,6 +106,8 @@ func runSendingProtectionCommand(ctx context.Context, cfg *config.Config, pool * return runRuntimeAttest(ctx, module, f, stdout) case f.capabilities: return runPrintCapabilities(source, secrets, stdout) + case f.reconcile: + return runReconcileLegacySendingJobs(ctx, pool, sendingpolicy.NewGate(pool, secrets, source, policy), stdout) } return errors.New("no sending-protection command selected") } diff --git a/cmd/e2a/sending_policy_test.go b/cmd/e2a/sending_policy_test.go index aa30648eb..9a8beb0d7 100644 --- a/cmd/e2a/sending_policy_test.go +++ b/cmd/e2a/sending_policy_test.go @@ -312,6 +312,17 @@ func TestSendingProtectionCommands(t *testing.T) { } }) + t.Run("reconcile-legacy-sending-jobs dispatches", func(t *testing.T) { + resetRiverJobs(t, pool) + out, err := run(&sendingProtectionFlags{reconcile: true}) + if err != nil { + t.Fatalf("reconcile: %v", err) + } + if !strings.Contains(out, "scanned: 0") || !strings.Contains(out, "remaining: 0") { + t.Errorf("reconcile output = %q", out) + } + }) + t.Run("print-capabilities", func(t *testing.T) { clearEnvForTest(t) out, err := run(&sendingProtectionFlags{capabilities: true}) diff --git a/cmd/e2a/sending_reconcile.go b/cmd/e2a/sending_reconcile.go new file mode 100644 index 000000000..f4c6b8e7c --- /dev/null +++ b/cmd/e2a/sending_reconcile.go @@ -0,0 +1,221 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/riverqueue/river/rivertype" + + "github.com/tokencanopy/e2a/internal/hitlnotify" + "github.com/tokencanopy/e2a/internal/jobs" + "github.com/tokencanopy/e2a/internal/outboundsend" + "github.com/tokencanopy/e2a/internal/sendingpolicy" + "github.com/tokencanopy/e2a/internal/webhooknotify" +) + +// legacySendingJobKinds are the River job kinds that submit mail to the +// provider and therefore must carry a sending operation reference. A job of +// one of these kinds without an operation_ref was enqueued by a pre-floor +// slot: the worker resolves it at fire time, but an operator can also settle +// the backlog up front with -reconcile-legacy-sending-jobs so the cutover +// leaves no job whose attribution is decided later than its enqueue. +var legacySendingJobKinds = []string{ + outboundsend.OutboundSendArgs{}.Kind(), + hitlnotify.HITLNotifyArgs{}.Kind(), + webhooknotify.WebhookNotifyArgs{}.Kind(), +} + +// legacyReconcileStates are the job states a reconcile touches: those River +// may still pick up. A running job is left to its worker, and a finalized job +// (completed, cancelled, discarded) has nothing left to authorize. +var legacyReconcileStates = []string{ + string(rivertype.JobStateAvailable), + string(rivertype.JobStatePending), + string(rivertype.JobStateRetryable), + string(rivertype.JobStateScheduled), +} + +// legacyReconcileCounts is the operator-facing summary of one reconcile pass. +type legacyReconcileCounts struct { + Scanned int + Stamped int + Cancelled int + Paused int + Failed int +} + +// remaining is the number of scanned jobs that still carry no operation +// reference after the pass: those the resolver could not decide. A job whose +// account is paused is deliberately left for the worker's hold path, so it is +// not counted as remaining. +func (c legacyReconcileCounts) remaining() int { return c.Failed } + +// runReconcileLegacySendingJobs stamps an operation reference onto every +// pending provider-submitting job that has none, cancelling the ones whose +// source row no longer exists. Each job is handled in its own transaction, +// through exactly the Prepare path its enqueue would have used, so a stamped +// job and a natively enqueued job authorize identically. Exit status is +// nonzero unless every scanned job was decided. +func runReconcileLegacySendingJobs(ctx context.Context, pool *pgxpool.Pool, gate sendingpolicy.Gate, stdout io.Writer) error { + client, err := jobs.New(pool, jobs.Config{}) + if err != nil { + return fmt.Errorf("river client: %w", err) + } + rows, err := pool.Query(ctx, ` + SELECT id, kind, args + FROM river_job + WHERE kind = ANY($1) + AND state = ANY($2) + AND NOT (args ? 'operation_ref') + ORDER BY id`, legacySendingJobKinds, legacyReconcileStates) + if err != nil { + return fmt.Errorf("scan legacy sending jobs: %w", err) + } + type legacyJob struct { + id int64 + kind string + args []byte + } + var pending []legacyJob + for rows.Next() { + var j legacyJob + if err := rows.Scan(&j.id, &j.kind, &j.args); err != nil { + rows.Close() + return fmt.Errorf("scan legacy sending job: %w", err) + } + pending = append(pending, j) + } + rows.Close() + if err := rows.Err(); err != nil { + return fmt.Errorf("scan legacy sending jobs: %w", err) + } + + var counts legacyReconcileCounts + for _, j := range pending { + counts.Scanned++ + outcome, err := reconcileLegacySendingJob(ctx, pool, client, gate, j.id, j.kind, j.args) + if err != nil { + counts.Failed++ + fmt.Fprintf(stdout, "job %d (%s): %v\n", j.id, j.kind, err) + continue + } + switch outcome { + case legacyOutcomeStamped: + counts.Stamped++ + case legacyOutcomeCancelled: + counts.Cancelled++ + case legacyOutcomePaused: + counts.Paused++ + } + } + + fmt.Fprintf(stdout, "scanned: %d\n", counts.Scanned) + fmt.Fprintf(stdout, "stamped: %d\n", counts.Stamped) + fmt.Fprintf(stdout, "cancelled: %d\n", counts.Cancelled) + fmt.Fprintf(stdout, "paused: %d\n", counts.Paused) + fmt.Fprintf(stdout, "failed: %d\n", counts.Failed) + fmt.Fprintf(stdout, "remaining: %d\n", counts.remaining()) + if counts.remaining() != 0 { + return fmt.Errorf("%d legacy sending job(s) could not be reconciled", counts.remaining()) + } + return nil +} + +type legacyOutcome int + +const ( + legacyOutcomeStamped legacyOutcome = iota + 1 + legacyOutcomeCancelled + legacyOutcomePaused +) + +// reconcileLegacySendingJob decides one job inside one transaction: the +// source row is locked by the Prepare call, the reference is stamped (or the +// orphan cancelled) in the same transaction, and a failure rolls both back so +// a rerun sees the job untouched. +func reconcileLegacySendingJob(ctx context.Context, pool *pgxpool.Pool, client *jobs.Client, gate sendingpolicy.Gate, jobID int64, kind string, rawArgs []byte) (legacyOutcome, error) { + tx, err := pool.Begin(ctx) + if err != nil { + return 0, fmt.Errorf("begin: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + + var ref sendingpolicy.OperationRef + var cancelReason string + switch kind { + case outboundsend.OutboundSendArgs{}.Kind(): + var args outboundsend.OutboundSendArgs + if err := json.Unmarshal(rawArgs, &args); err != nil { + return 0, fmt.Errorf("decode args: %w", err) + } + decision, prepared, err := gate.PrepareExternalTx(ctx, tx, args.MessageID) + switch { + case errors.Is(err, sendingpolicy.ErrSourceUnavailable): + cancelReason = "legacy source unavailable" + case err != nil: + return 0, err + case decision == sendingpolicy.AcceptanceSendingPaused: + // The worker's hold path owns a paused account: it records the + // hold on the message and waits for the operator. Nothing to + // stamp yet; the rerun after the resume picks it up. + return legacyOutcomePaused, nil + case prepared.IsZero(): + // The only accepted shape with no operation is an exact + // self-send, which never enqueues; a queued job that resolves to + // nothing cannot be authorized by any worker. + cancelReason = "message has no provider operation" + default: + ref = prepared + } + case hitlnotify.HITLNotifyArgs{}.Kind(): + var args hitlnotify.HITLNotifyArgs + if err := json.Unmarshal(rawArgs, &args); err != nil { + return 0, fmt.Errorf("decode args: %w", err) + } + ref, cancelReason, err = prepareLegacyNotification(ctx, tx, gate, sendingpolicy.NewHITLNotificationRef(args.MessageID)) + if err != nil { + return 0, err + } + case webhooknotify.WebhookNotifyArgs{}.Kind(): + var args webhooknotify.WebhookNotifyArgs + if err := json.Unmarshal(rawArgs, &args); err != nil { + return 0, fmt.Errorf("decode args: %w", err) + } + ref, cancelReason, err = prepareLegacyNotification(ctx, tx, gate, sendingpolicy.NewWebhookHealthNotificationRef(args.WebhookID)) + if err != nil { + return 0, err + } + default: + return 0, fmt.Errorf("unexpected job kind %q", kind) + } + + outcome := legacyOutcomeStamped + if cancelReason != "" { + if err := client.CancelTx(ctx, tx, jobID); err != nil { + return 0, fmt.Errorf("cancel (%s): %w", cancelReason, err) + } + outcome = legacyOutcomeCancelled + } else if err := jobs.StampJobArg(ctx, tx, jobID, "operation_ref", ref); err != nil { + return 0, err + } + if err := tx.Commit(ctx); err != nil { + return 0, fmt.Errorf("commit: %w", err) + } + return outcome, nil +} + +func prepareLegacyNotification(ctx context.Context, tx pgx.Tx, gate sendingpolicy.Gate, nref sendingpolicy.NotificationRef) (sendingpolicy.OperationRef, string, error) { + ref, err := gate.PrepareNotificationTx(ctx, tx, nref) + if errors.Is(err, sendingpolicy.ErrSourceUnavailable) { + return sendingpolicy.OperationRef{}, "legacy source unavailable", nil + } + if err != nil { + return sendingpolicy.OperationRef{}, "", err + } + return ref, "", nil +} diff --git a/cmd/e2a/sending_reconcile_test.go b/cmd/e2a/sending_reconcile_test.go new file mode 100644 index 000000000..bfe4fc847 --- /dev/null +++ b/cmd/e2a/sending_reconcile_test.go @@ -0,0 +1,183 @@ +package main + +import ( + "bytes" + "context" + "strings" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/tokencanopy/e2a/internal/identity" + "github.com/tokencanopy/e2a/internal/jobs" + "github.com/tokencanopy/e2a/internal/sendingpolicy" + "github.com/tokencanopy/e2a/internal/testutil" +) + +// insertLegacyJob enqueues a River job the way a pre-floor slot did: the +// args carry no operation_ref. Raw SQL on purpose — the typed enqueuers +// always prepare a reference now, so the only way to produce a legacy job in +// a test is to write one the old way. +func insertLegacyJob(t *testing.T, pool *pgxpool.Pool, kind, args string) int64 { + t.Helper() + var id int64 + if err := pool.QueryRow(context.Background(), + `INSERT INTO river_job (args, kind, max_attempts) VALUES ($1::jsonb, $2, 3) RETURNING id`, + args, kind).Scan(&id); err != nil { + t.Fatalf("insert legacy %s job: %v", kind, err) + } + return id +} + +// resetRiverJobs empties the shared per-package river_job table: the test DB +// helper leaves River's tables alone, so legacy rows one test writes would +// otherwise be scanned by the next. +func resetRiverJobs(t *testing.T, pool *pgxpool.Pool) { + t.Helper() + if err := jobs.Migrate(context.Background(), pool); err != nil { + t.Fatalf("jobs.Migrate: %v", err) + } + if _, err := pool.Exec(context.Background(), `TRUNCATE river_job RESTART IDENTITY`); err != nil { + t.Fatalf("reset river_job: %v", err) + } +} + +func legacyJobState(t *testing.T, pool *pgxpool.Pool, id int64) (state, opID string) { + t.Helper() + if err := pool.QueryRow(context.Background(), + `SELECT state, COALESCE(args->'operation_ref'->>'id', '') FROM river_job WHERE id = $1`, id, + ).Scan(&state, &opID); err != nil { + t.Fatalf("read job %d: %v", id, err) + } + return state, opID +} + +func seedReconcileSource(t *testing.T, store *identity.Store, slug string) (*identity.Message, *identity.Webhook) { + t.Helper() + ctx := context.Background() + user, err := store.CreateOrGetUser(ctx, "owner-"+slug+"@reviewer.test", "Owner", "google-reconcile-"+slug) + if err != nil { + t.Fatal(err) + } + if _, err := store.ClaimOrCreateDomain(ctx, slug+".bot.test", user.ID); err != nil { + t.Fatal(err) + } + if err := store.VerifyDomain(ctx, slug+".bot.test", user.ID); err != nil { + t.Fatal(err) + } + a, err := store.CreateAgent(ctx, "bot@"+slug+".bot.test", slug+".bot.test", "", "https://example.com/webhook", "", user.ID) + if err != nil { + t.Fatal(err) + } + msg, err := store.CreatePendingOutboundMessage(ctx, a.ID, + []string{"alice@example.com"}, nil, nil, + "Held draft", "body", "", nil, "send", "conv_"+slug, "", "", 3600) + if err != nil { + t.Fatal(err) + } + wh, err := store.CreateWebhook(ctx, user.ID, "https://hooks.example.com/e2a", "", + []string{"email.received"}, identity.WebhookFilters{}) + if err != nil { + t.Fatal(err) + } + return msg, wh +} + +// TestReconcileLegacySendingJobs: every pending provider-submitting job +// without an operation reference is decided in one pass — stamped when its +// source row exists, cancelled when it does not — and a second pass finds +// nothing left. A job River already finalized is out of scope. +func TestReconcileLegacySendingJobs(t *testing.T) { + ctx := context.Background() + pool := testutil.TestDB(t) + resetRiverJobs(t, pool) + store := identity.NewStore(pool) + gate := sendingpolicy.NewGate(pool, sendingpolicy.Secrets{}, sendingpolicy.PolicySourceConfig, sendingpolicy.DisabledPolicy()) + msg, wh := seedReconcileSource(t, store, "reconcile") + + sendLive := insertLegacyJob(t, pool, "outbound_send", `{"message_id":"`+msg.ID+`"}`) + sendGone := insertLegacyJob(t, pool, "outbound_send", `{"message_id":"msg_does_not_exist"}`) + hitlLive := insertLegacyJob(t, pool, "hitl_notify", `{"message_id":"`+msg.ID+`"}`) + whLive := insertLegacyJob(t, pool, "webhook_notify", `{"webhook_id":"`+wh.ID+`","kind":"warning"}`) + whGone := insertLegacyJob(t, pool, "webhook_notify", `{"webhook_id":"wh_does_not_exist","kind":"disabled"}`) + finalized := insertLegacyJob(t, pool, "outbound_send", `{"message_id":"msg_finalized"}`) + if _, err := pool.Exec(ctx, `UPDATE river_job SET state = 'completed', finalized_at = now() WHERE id = $1`, finalized); err != nil { + t.Fatal(err) + } + other := insertLegacyJob(t, pool, "outbound_terminal_reconcile", `{"message_id":"`+msg.ID+`"}`) + + var out bytes.Buffer + if err := runReconcileLegacySendingJobs(ctx, pool, gate, &out); err != nil { + t.Fatalf("reconcile: %v\nOUTPUT:\n%s", err, out.String()) + } + for _, want := range []string{"scanned: 5", "stamped: 3", "cancelled: 2", "failed: 0", "remaining: 0"} { + if !strings.Contains(out.String(), want) { + t.Errorf("output missing %q:\n%s", want, out.String()) + } + } + + if state, op := legacyJobState(t, pool, sendLive); state != "available" || op != msg.ID { + t.Errorf("live send job: state=%s op=%q, want available with the message id", state, op) + } + if state, op := legacyJobState(t, pool, hitlLive); state != "available" || !strings.HasPrefix(op, "op_") { + t.Errorf("live hitl job: state=%s op=%q, want available with a notification operation", state, op) + } + if state, op := legacyJobState(t, pool, whLive); state != "available" || !strings.HasPrefix(op, "op_") { + t.Errorf("live webhook job: state=%s op=%q, want available with a notification operation", state, op) + } + for name, id := range map[string]int64{"send": sendGone, "webhook": whGone} { + if state, op := legacyJobState(t, pool, id); state != "cancelled" || op != "" { + t.Errorf("orphan %s job: state=%s op=%q, want cancelled and unstamped", name, state, op) + } + } + if state, op := legacyJobState(t, pool, finalized); state != "completed" || op != "" { + t.Errorf("finalized job touched: state=%s op=%q", state, op) + } + if state, op := legacyJobState(t, pool, other); state != "available" || op != "" { + t.Errorf("non-submitting kind touched: state=%s op=%q", state, op) + } + + // The stamped reference must round-trip: the same bytes a native enqueue + // would have written, so a worker reading it authorizes identically. + var raw []byte + if err := pool.QueryRow(ctx, `SELECT args->'operation_ref' FROM river_job WHERE id = $1`, sendLive).Scan(&raw); err != nil { + t.Fatal(err) + } + var ref sendingpolicy.OperationRef + if err := ref.UnmarshalJSON(raw); err != nil || ref.ID() != msg.ID { + t.Fatalf("stamped reference does not decode to the message operation: err=%v id=%q", err, ref.ID()) + } + + out.Reset() + if err := runReconcileLegacySendingJobs(ctx, pool, gate, &out); err != nil { + t.Fatalf("second pass: %v", err) + } + if !strings.Contains(out.String(), "scanned: 0") { + t.Errorf("second pass should find nothing:\n%s", out.String()) + } +} + +// TestReconcileLegacySendingJobsReportsUndecided: a job the resolver cannot +// decide is reported, left untouched, and makes the command exit nonzero so a +// cutover script cannot mistake a partial pass for a clean one. +func TestReconcileLegacySendingJobsReportsUndecided(t *testing.T) { + ctx := context.Background() + pool := testutil.TestDB(t) + resetRiverJobs(t, pool) + gate := sendingpolicy.NewGate(pool, sendingpolicy.Secrets{}, sendingpolicy.PolicySourceConfig, sendingpolicy.DisabledPolicy()) + broken := insertLegacyJob(t, pool, "outbound_send", `{"message_id":123}`) + + var out bytes.Buffer + err := runReconcileLegacySendingJobs(ctx, pool, gate, &out) + if err == nil || !strings.Contains(err.Error(), "1 legacy sending job(s) could not be reconciled") { + t.Fatalf("err = %v, want the undecided count", err) + } + for _, want := range []string{"failed: 1", "remaining: 1", "decode args"} { + if !strings.Contains(out.String(), want) { + t.Errorf("output missing %q:\n%s", want, out.String()) + } + } + if state, op := legacyJobState(t, pool, broken); state != "available" || op != "" { + t.Errorf("undecided job touched: state=%s op=%q", state, op) + } +} diff --git a/docs/design/async-message-pipeline.md b/docs/design/async-message-pipeline.md index 7aff7c3de..e6ba9649b 100644 --- a/docs/design/async-message-pipeline.md +++ b/docs/design/async-message-pipeline.md @@ -292,3 +292,37 @@ respectively. An account pause has no clock and starts no hold, but a deadline already running keeps running. Terminal reconciliation is settlement-only: an evidence-settled row also settles the attempt that dialed (`Gate.SettleOperation`). + +## Addendum (2026-09-05): every provider call is an authorized attempt (B7) + +Slice B7 closed the seam B5 opened. `outbound.SMTPRelay` no longer exports a +send method: the only way to open a socket to the provider is +`ProviderSubmitter.SubmitOnce` with a `sendingpolicy.ProviderAuthorization`, +and `internal/outbound`'s tracked-closure test parses every production file +to keep it that way (no `net/smtp` import and no call to the relay's socket +core outside the named exceptions). The paths that used to bypass the gate now +cross it: + +- **HITL approval notifications** (`internal/hitlnotify`) and **webhook health + notices** (`internal/webhooknotify`): the enqueue prepares a + `customer_notification` operation in the same transaction as the source + row (`PrepareNotificationTx`, charged to the triggering account, shared + reputation class) and stamps it on the job; the worker runs the same + Reserve → early hold → ConsumeAttempt → authorized submit order as the + message worker, snoozing on a hold without provider I/O. A job from a + pre-floor slot resolves its operation at fire time and stamps it once + (`jobs.StampJobArg`), so the derivation never repeats. +- **Public feedback mail** (`POST /api/feedback`): the operation is keyed by + a server-minted submission id and its envelope is the configured notify + set, never the request, so the form cannot become a relay. No queue owns + this path, so its bounded in-request retry loop is the whole envelope and + every physical attempt is its own charged ordinal; a definite rejection and + a lost acceptance both stop the loop. + +Operators cutting over a slot with a queued backlog run +`e2a -reconcile-legacy-sending-jobs`: it stamps an operation onto every +pending `outbound_send` / `hitl_notify` / `webhook_notify` job that has none, +through exactly the Prepare path its enqueue would have used, cancels the +ones whose source row is gone, and exits nonzero unless every scanned job was +decided. The workers resolve legacy jobs themselves, so the command is a +convenience for a clean cutover, not a prerequisite. diff --git a/internal/agent/api.go b/internal/agent/api.go index fe16be741..570b4dcdd 100644 --- a/internal/agent/api.go +++ b/internal/agent/api.go @@ -2,6 +2,8 @@ package agent import ( "context" + "crypto/rand" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -37,6 +39,7 @@ import ( "github.com/tokencanopy/e2a/internal/outboundsend" "github.com/tokencanopy/e2a/internal/piguard" "github.com/tokencanopy/e2a/internal/ratelimit" + "github.com/tokencanopy/e2a/internal/sendingpolicy" "github.com/tokencanopy/e2a/internal/telemetry" "github.com/tokencanopy/e2a/internal/usage" "github.com/tokencanopy/e2a/internal/webhook" @@ -178,7 +181,12 @@ type API struct { // identically to a wire roundtrip of the same message. inboundScreen *piguard.Engine smtpRelay *outbound.SMTPRelay - userAuth *auth.UserAuth + // submitter and gate are the authorized provider seam for platform mail + // this API sends itself (public feedback). Wired via SetProviderSubmitter; + // unset means the platform cannot send feedback mail. + submitter *outbound.ProviderSubmitter + gate sendingpolicy.Gate + userAuth *auth.UserAuth // oidcAuth wires optional, generic OpenID Connect browser login. Nil means // both OIDC routes are absent; it is independent of legacy Google login. oidcAuth *auth.OIDCAuth @@ -1629,6 +1637,13 @@ func (a *API) DeliverOutbound(ctx context.Context, user *identity.User, agent *i return &OutboundResult{MessageID: accepted.ID, Status: acceptStatus, ScheduledAt: scheduledAt, SentAs: comp.SentAs, Method: comp.Method}, nil } +// SetProviderSubmitter wires the authorized provider seam and the gate that +// issues its tokens, for the platform mail this API sends on its own behalf. +func (a *API) SetProviderSubmitter(submitter *outbound.ProviderSubmitter, gate sendingpolicy.Gate) { + a.submitter = submitter + a.gate = gate +} + // SendTestCore accepts (or HITL-holds) a platform test email to the agent's // own address. HTTP-free; shared by the legacy handler and the v1 layer. The // caller has already authed, resolved + owned the agent, domain-verified, @@ -1925,7 +1940,7 @@ func (a *API) handleFeedback(w http.ResponseWriter, r *http.Request) { // notification reaches them directly; compose-layer header sanitization // neutralizes any CR/LF in that user-controlled value. func (a *API) sendFeedbackEmail(ctx context.Context, title, category, message, submitterEmail, ghNote string, to, cc []string) error { - if a.smtpRelay == nil || !a.smtpRelay.Configured() || a.fromDomain == "" { + if a.submitter == nil || a.gate == nil || a.smtpRelay == nil || !a.smtpRelay.Configured() || a.fromDomain == "" { return fmt.Errorf("outbound SMTP relay not configured") } @@ -1951,12 +1966,70 @@ func (a *API) sendFeedbackEmail(ctx context.Context, title, category, message, s rcpts = append(rcpts, to...) rcpts = append(rcpts, cc...) - // Send (not SendOnce) — no job queue owns retries for this path, so the - // relay's own transient-4xx backoff is the only retry envelope. - if _, err := a.smtpRelay.SendWithContext(ctx, from, rcpts, raw); err != nil { - return fmt.Errorf("smtp send: %w", err) + // No job queue owns retries for this path, so the request's bounded + // retry loop is the whole envelope — and every physical attempt is its + // own charged ordinal: Reserve, ConsumeAttempt, one authorized submit. + // The operation is keyed by a server-minted submission id and its + // envelope is configuration, never the request, so the form cannot + // become an open relay however it is retried. + submissionID := feedbackSubmissionID() + ref, err := a.gate.PreparePublicFeedback(ctx, sendingpolicy.NewPublicFeedbackRef(submissionID, rcpts)) + if err != nil { + return fmt.Errorf("prepare feedback operation: %w", err) + } + var last error + for attempt := 0; attempt < feedbackSendAttempts; attempt++ { + if attempt > 0 { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(feedbackRetryBackoff[attempt-1]): + } + } + early, attemptRef, err := a.gate.Reserve(ctx, ref) + if err != nil { + return fmt.Errorf("reserve feedback attempt: %w", err) + } + if !early.Allow { + return fmt.Errorf("feedback send held by sending policy: %s", early.Reason) + } + decision, auth, err := a.gate.ConsumeAttempt(ctx, attemptRef) + if err != nil { + return fmt.Errorf("authorize feedback attempt: %w", err) + } + if !decision.Allow || auth == nil { + return fmt.Errorf("feedback send held by sending policy: %s", decision.Reason) + } + _, err = a.submitter.SubmitOnce(ctx, *auth, outbound.Envelope{From: from, Recipients: rcpts, Message: raw}) + if err == nil { + return nil + } + last = err + if outbound.IsPermanentSMTPError(err) || errors.Is(err, outbound.ErrProviderAcceptanceUnknown) { + // Definite rejection: retrying resends nothing. Acceptance unknown: + // the provider may hold the message, and a retry would be a + // duplicate copy of platform mail nobody asked for twice. + break + } + } + return fmt.Errorf("smtp send: %w", last) +} + +// feedbackSendAttempts bounds the physical submissions one feedback request +// may make; feedbackRetryBackoff paces them. Each is a distinct charged +// attempt on the feedback operation. +const feedbackSendAttempts = 4 + +var feedbackRetryBackoff = []time.Duration{time.Second, 5 * time.Second, 15 * time.Second} + +// feedbackSubmissionID mints the server-side identity one feedback request's +// operation is keyed by. +func feedbackSubmissionID() string { + var b [12]byte + if _, err := rand.Read(b[:]); err != nil { + panic(fmt.Sprintf("feedback submission id: %v", err)) } - return nil + return hex.EncodeToString(b[:]) } // splitFeedbackAddrs parses a comma-separated address list from env config, diff --git a/internal/agent/api_test.go b/internal/agent/api_test.go index 702208f9e..2235cf563 100644 --- a/internal/agent/api_test.go +++ b/internal/agent/api_test.go @@ -8,6 +8,7 @@ import ( "mime" "net/http" "net/http/httptest" + "sort" "strings" "testing" @@ -19,6 +20,7 @@ import ( "github.com/tokencanopy/e2a/internal/idempotency" "github.com/tokencanopy/e2a/internal/identity" "github.com/tokencanopy/e2a/internal/outbound" + "github.com/tokencanopy/e2a/internal/sendingpolicy" "github.com/tokencanopy/e2a/internal/testutil" "github.com/tokencanopy/e2a/internal/usage" ) @@ -139,6 +141,9 @@ func setupAPIWithSMTP(t *testing.T) (*httptest.Server, *identity.Store, *pgxpool sender := outbound.NewSender(smtpRelay, "test.e2a.dev") noopUsage := usage.NewNoopUsageTracker() api := agent.NewAPI(store, sender, smtpRelay, nil, noopUsage, "e2a.dev", "test.e2a.dev", "agents.e2a.dev", "", false) + // Platform mail (public feedback) crosses the authorized provider seam. + gate := sendingpolicy.NewGate(pool, sendingpolicy.Secrets{}, sendingpolicy.PolicySourceConfig, sendingpolicy.DisabledPolicy()) + api.SetProviderSubmitter(outbound.NewProviderSubmitter(smtpRelay, gate), gate) api.SetIdempotencyStore(idempotency.NewStore(pool)) router := mux.NewRouter() api.RegisterRoutes(router) @@ -364,8 +369,12 @@ func TestFeedback_EmailNotification(t *testing.T) { if m.From != "noreply@test.e2a.dev" { t.Errorf("envelope from = %q, want noreply@test.e2a.dev", m.From) } - wantRcpts := []string{"feedback-to@example.com", "feedback-cc@example.com"} - if strings.Join(m.Recipients, ",") != strings.Join(wantRcpts, ",") { + // RCPT TO is issued from the token's canonical (sorted) recipient set, + // so compare as a set: the wire order is the seam's, not the form's. + gotRcpts := append([]string(nil), m.Recipients...) + sort.Strings(gotRcpts) + wantRcpts := []string{"feedback-cc@example.com", "feedback-to@example.com"} + if strings.Join(gotRcpts, ",") != strings.Join(wantRcpts, ",") { t.Errorf("recipients = %v, want %v", m.Recipients, wantRcpts) } for _, want := range []string{ @@ -413,6 +422,8 @@ func TestFeedback_AllChannelsFail_500(t *testing.T) { deadRelay := outbound.NewSMTPRelay(&config.OutboundSMTPConfig{Host: "127.0.0.1", Port: 1}) sender := outbound.NewSender(deadRelay, "test.e2a.dev") api := agent.NewAPI(store, sender, deadRelay, nil, usage.NewNoopUsageTracker(), "e2a.dev", "test.e2a.dev", "agents.e2a.dev", "", false) + deadGate := sendingpolicy.NewGate(pool, sendingpolicy.Secrets{}, sendingpolicy.PolicySourceConfig, sendingpolicy.DisabledPolicy()) + api.SetProviderSubmitter(outbound.NewProviderSubmitter(deadRelay, deadGate), deadGate) router := mux.NewRouter() api.RegisterRoutes(router) server := httptest.NewServer(router) diff --git a/internal/agent/feedback_github_test.go b/internal/agent/feedback_github_test.go index 3afd92def..fe6d0f8fa 100644 --- a/internal/agent/feedback_github_test.go +++ b/internal/agent/feedback_github_test.go @@ -22,6 +22,8 @@ import ( "github.com/gorilla/mux" "github.com/tokencanopy/e2a/internal/config" "github.com/tokencanopy/e2a/internal/outbound" + "github.com/tokencanopy/e2a/internal/sendingpolicy" + "github.com/tokencanopy/e2a/internal/testutil/testdb" "github.com/tokencanopy/e2a/internal/usage" ) @@ -132,6 +134,7 @@ func TestFeedbackGitHubTimeoutStillDeliversEmail(t *testing.T) { relay := outbound.NewSMTPRelay(&config.OutboundSMTPConfig{Host: smtpHost, Port: smtpPort}) sender := outbound.NewSender(relay, "test.e2a.dev") api := NewAPI(nil, sender, relay, nil, usage.NewNoopUsageTracker(), "e2a.dev", "test.e2a.dev", "agents.e2a.dev", "", false) + wireFeedbackSubmitter(t, api, relay) router := mux.NewRouter() api.RegisterRoutes(router) server := httptest.NewServer(router) @@ -188,6 +191,7 @@ func TestFeedbackEmailTimeoutReturnsAfterGitHubDelivery(t *testing.T) { relay := outbound.NewSMTPRelay(&config.OutboundSMTPConfig{Host: smtpHost, Port: smtpPort}) sender := outbound.NewSender(relay, "test.e2a.dev") api := NewAPI(nil, sender, relay, nil, usage.NewNoopUsageTracker(), "e2a.dev", "test.e2a.dev", "agents.e2a.dev", "", false) + wireFeedbackSubmitter(t, api, relay) router := mux.NewRouter() api.RegisterRoutes(router) server := httptest.NewServer(router) @@ -239,6 +243,7 @@ func TestFeedbackNoRepoConfigured_RefusesToFileRatherThanDefaultingToOperatorRep relay := outbound.NewSMTPRelay(&config.OutboundSMTPConfig{Host: smtpHost, Port: smtpPort}) sender := outbound.NewSender(relay, "test.e2a.dev") api := NewAPI(nil, sender, relay, nil, usage.NewNoopUsageTracker(), "e2a.dev", "test.e2a.dev", "agents.e2a.dev", "", false) + wireFeedbackSubmitter(t, api, relay) router := mux.NewRouter() api.RegisterRoutes(router) server := httptest.NewServer(router) @@ -402,3 +407,12 @@ func TestFeedbackGitHubClient_Precedence(t *testing.T) { t.Errorf("bad app key: got client=%v err=%v, want nil,error", c, err) } } + +// wireFeedbackSubmitter gives an API the authorized provider seam the feedback +// path submits through, backed by a disabled-policy gate on the test DB. +func wireFeedbackSubmitter(t *testing.T, api *API, relay *outbound.SMTPRelay) { + t.Helper() + pool := testdb.TestDB(t) + gate := sendingpolicy.NewGate(pool, sendingpolicy.Secrets{}, sendingpolicy.PolicySourceConfig, sendingpolicy.DisabledPolicy()) + api.SetProviderSubmitter(outbound.NewProviderSubmitter(relay, gate), gate) +} diff --git a/internal/agent/feedback_seam_test.go b/internal/agent/feedback_seam_test.go new file mode 100644 index 000000000..f2c4ac340 --- /dev/null +++ b/internal/agent/feedback_seam_test.go @@ -0,0 +1,248 @@ +package agent + +import ( + "bufio" + "context" + "errors" + "fmt" + "net" + "strings" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/tokencanopy/e2a/internal/config" + "github.com/tokencanopy/e2a/internal/outbound" + "github.com/tokencanopy/e2a/internal/sendingpolicy" + "github.com/tokencanopy/e2a/internal/testutil/testdb" + "github.com/tokencanopy/e2a/internal/usage" +) + +// scriptedSMTP answers one connection per script entry. The entry is the +// reply the server gives after the message body: an SMTP code ("250", "451", +// "554") or "drop", which closes the socket without any reply — the lost-250 +// shape the relay reports as ErrProviderAcceptanceUnknown. +type scriptedSMTP struct { + host string + port int + + mu sync.Mutex + messages []string + conns int +} + +func startScriptedSMTP(t *testing.T, script ...string) *scriptedSMTP { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = listener.Close() }) + addr := listener.Addr().(*net.TCPAddr) + s := &scriptedSMTP{host: addr.IP.String(), port: addr.Port} + + go func() { + for _, reply := range script { + conn, acceptErr := listener.Accept() + if acceptErr != nil { + return + } + s.mu.Lock() + s.conns++ + s.mu.Unlock() + s.serve(conn, reply) + } + }() + return s +} + +func (s *scriptedSMTP) serve(conn net.Conn, reply string) { + defer conn.Close() + reader := bufio.NewReader(conn) + fmt.Fprint(conn, "220 scripted ready\r\n") + var data []string + inData := false + for { + line, err := reader.ReadString('\n') + if err != nil { + return + } + line = strings.TrimRight(line, "\r\n") + if inData { + if line != "." { + data = append(data, line) + continue + } + s.mu.Lock() + s.messages = append(s.messages, strings.Join(data, "\n")) + s.mu.Unlock() + if reply == "drop" { + return + } + fmt.Fprintf(conn, "%s scripted reply\r\n", reply) + inData = false + continue + } + switch { + case strings.EqualFold(line, "DATA"): + inData = true + fmt.Fprint(conn, "354 Go ahead\r\n") + case strings.EqualFold(line, "QUIT"): + fmt.Fprint(conn, "221 Bye\r\n") + return + default: + fmt.Fprint(conn, "250 OK\r\n") + } + } +} + +func (s *scriptedSMTP) received() ([]string, int) { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.messages...), s.conns +} + +func attemptHeader(wire string) string { + for _, line := range strings.Split(wire, "\n") { + if strings.HasPrefix(line, outbound.ProviderAttemptHeader+": ") { + return strings.TrimPrefix(line, outbound.ProviderAttemptHeader+": ") + } + } + return "" +} + +func countFeedbackAttempts(t *testing.T, pool *pgxpool.Pool) int { + t.Helper() + var n int + if err := pool.QueryRow(context.Background(), + `SELECT count(*) FROM sending_budget_reservations WHERE purpose = 'public_feedback_notification' AND call_state = 'started'`, + ).Scan(&n); err != nil { + t.Fatal(err) + } + return n +} + +func newFeedbackSeamAPI(t *testing.T, s *scriptedSMTP) (*API, *pgxpool.Pool) { + t.Helper() + pool := testdb.TestDB(t) + relay := outbound.NewSMTPRelay(&config.OutboundSMTPConfig{Host: s.host, Port: s.port}) + api := NewAPI(nil, outbound.NewSender(relay, "test.e2a.dev"), relay, nil, usage.NewNoopUsageTracker(), "e2a.dev", "test.e2a.dev", "agents.e2a.dev", "", false) + gate := sendingpolicy.NewGate(pool, sendingpolicy.Secrets{}, sendingpolicy.PolicySourceConfig, sendingpolicy.DisabledPolicy()) + api.SetProviderSubmitter(outbound.NewProviderSubmitter(relay, gate), gate) + return api, pool +} + +func fastFeedbackBackoff(t *testing.T) { + t.Helper() + old := feedbackRetryBackoff + feedbackRetryBackoff = []time.Duration{time.Millisecond, time.Millisecond, time.Millisecond} + t.Cleanup(func() { feedbackRetryBackoff = old }) +} + +// TestFeedbackSeam_EachPhysicalAttemptIsItsOwnOrdinal: a transient provider +// reply is retried, and the retry is a NEW authorized attempt — a distinct +// ordinal in the ledger and a distinct attempt id on the wire — not a replay +// of the first token. +func TestFeedbackSeam_EachPhysicalAttemptIsItsOwnOrdinal(t *testing.T) { + fastFeedbackBackoff(t) + s := startScriptedSMTP(t, "451", "250") + api, pool := newFeedbackSeamAPI(t, s) + before := countFeedbackAttempts(t, pool) + + err := api.sendFeedbackEmail(context.Background(), "t", "bug", "m", "", "", []string{"feedback@example.test"}, nil) + if err != nil { + t.Fatalf("sendFeedbackEmail: %v", err) + } + msgs, conns := s.received() + if conns != 2 || len(msgs) != 2 { + t.Fatalf("conns=%d messages=%d, want 2/2 (one retry)", conns, len(msgs)) + } + a1, a2 := attemptHeader(msgs[0]), attemptHeader(msgs[1]) + if a1 == "" || a2 == "" || a1 == a2 { + t.Fatalf("attempt ids on the wire = %q / %q, want two distinct non-empty ids", a1, a2) + } + if got := countFeedbackAttempts(t, pool) - before; got != 2 { + t.Fatalf("started feedback attempts = %d, want 2", got) + } +} + +// TestFeedbackSeam_DefiniteRejectionIsNotRetried: a 5xx is the provider's +// answer to the message; retrying it resends nothing. +func TestFeedbackSeam_DefiniteRejectionIsNotRetried(t *testing.T) { + fastFeedbackBackoff(t) + s := startScriptedSMTP(t, "554", "250") + api, pool := newFeedbackSeamAPI(t, s) + before := countFeedbackAttempts(t, pool) + + err := api.sendFeedbackEmail(context.Background(), "t", "bug", "m", "", "", []string{"feedback@example.test"}, nil) + if err == nil || !outbound.IsPermanentSMTPError(err) { + t.Fatalf("err = %v, want the permanent SMTP rejection", err) + } + if _, conns := s.received(); conns != 1 { + t.Fatalf("conns = %d, want 1 (no retry after a definite rejection)", conns) + } + if got := countFeedbackAttempts(t, pool) - before; got != 1 { + t.Fatalf("started feedback attempts = %d, want 1", got) + } +} + +// TestFeedbackSeam_LostAcceptanceIsNotRetried: a body the provider took but +// never answered may already be queued; a retry would be a second copy. +func TestFeedbackSeam_LostAcceptanceIsNotRetried(t *testing.T) { + fastFeedbackBackoff(t) + s := startScriptedSMTP(t, "drop", "250") + api, _ := newFeedbackSeamAPI(t, s) + + err := api.sendFeedbackEmail(context.Background(), "t", "bug", "m", "", "", []string{"feedback@example.test"}, nil) + if !errors.Is(err, outbound.ErrProviderAcceptanceUnknown) { + t.Fatalf("err = %v, want ErrProviderAcceptanceUnknown", err) + } + if _, conns := s.received(); conns != 1 { + t.Fatalf("conns = %d, want 1 (no retry after a lost acceptance)", conns) + } +} + +// TestFeedbackSeam_RetriesAreBounded: transient failures stop at the attempt +// cap, each one charged. +func TestFeedbackSeam_RetriesAreBounded(t *testing.T) { + fastFeedbackBackoff(t) + s := startScriptedSMTP(t, "451", "451", "451", "451", "250") + api, pool := newFeedbackSeamAPI(t, s) + before := countFeedbackAttempts(t, pool) + + err := api.sendFeedbackEmail(context.Background(), "t", "bug", "m", "", "", []string{"feedback@example.test"}, nil) + if err == nil { + t.Fatal("expected the exhausted retry loop to fail") + } + if _, conns := s.received(); conns != feedbackSendAttempts { + t.Fatalf("conns = %d, want %d", conns, feedbackSendAttempts) + } + if got := countFeedbackAttempts(t, pool) - before; got != feedbackSendAttempts { + t.Fatalf("started feedback attempts = %d, want %d", got, feedbackSendAttempts) + } +} + +// TestFeedbackSeam_EnvelopeIsConfigurationNotRequest: the recipients on the +// wire are exactly the configured notify set the operation was prepared with; +// the form's own address only ever appears as Reply-To. +func TestFeedbackSeam_EnvelopeIsConfigurationNotRequest(t *testing.T) { + s := startScriptedSMTP(t, "250") + api, _ := newFeedbackSeamAPI(t, s) + + err := api.sendFeedbackEmail(context.Background(), "t", "bug", "m", "someone@attacker.test", "", []string{"feedback@example.test"}, []string{"ops@example.test"}) + if err != nil { + t.Fatalf("sendFeedbackEmail: %v", err) + } + msgs, _ := s.received() + if len(msgs) != 1 { + t.Fatalf("messages = %d, want 1", len(msgs)) + } + if !strings.Contains(msgs[0], "Reply-To: someone@attacker.test") { + t.Errorf("submitter address should be the Reply-To only") + } + if attemptHeader(msgs[0]) == "" { + t.Errorf("feedback mail left without the provider attempt header: it did not cross the authorized seam") + } +} diff --git a/internal/hitlnotify/e2e_test.go b/internal/hitlnotify/e2e_test.go index ac7691941..54e42b0d2 100644 --- a/internal/hitlnotify/e2e_test.go +++ b/internal/hitlnotify/e2e_test.go @@ -13,6 +13,7 @@ import ( "github.com/tokencanopy/e2a/internal/identity" "github.com/tokencanopy/e2a/internal/jobs" "github.com/tokencanopy/e2a/internal/outbound" + "github.com/tokencanopy/e2a/internal/sendingpolicy" "github.com/tokencanopy/e2a/internal/testutil" ) @@ -34,7 +35,8 @@ func TestEndToEnd_AcceptTxThroughRiverToSMTP(t *testing.T) { Host: smtpAddr.Host, Port: smtpAddr.Port, FromDomain: "notify.test", }) signer := approvaltoken.NewSigner("hitl-notify-e2e-secret") - notifier := hitlnotify.New(store, relay, signer, "notify.test", "", "", "https://app.example.test") + gate := sendingpolicy.NewGate(pool, sendingpolicy.Secrets{}, sendingpolicy.PolicySourceConfig, sendingpolicy.DisabledPolicy()) + notifier := hitlnotify.New(store, outbound.NewProviderSubmitter(relay, gate), signer, "notify.test", "", "", "https://app.example.test") // Seed a verified HITL agent + owner. user, err := store.CreateOrGetUser(ctx, "owner-e2e@reviewer.test", "Owner", "google-notify-e2e") @@ -54,7 +56,7 @@ func TestEndToEnd_AcceptTxThroughRiverToSMTP(t *testing.T) { } // Build the integration on a real client and bind the concrete Notifier. - j := hitlnotify.NewJobs(store) + j := hitlnotify.NewJobs(store).WithGate(gate, pool) client, err := jobs.New(pool, jobs.Config{}, j) if err != nil { t.Fatalf("jobs.New: %v", err) diff --git a/internal/hitlnotify/jobs.go b/internal/hitlnotify/jobs.go index c8c510236..7702d14d6 100644 --- a/internal/hitlnotify/jobs.go +++ b/internal/hitlnotify/jobs.go @@ -3,6 +3,7 @@ package hitlnotify import ( "context" "errors" + "fmt" "sync" "github.com/jackc/pgx/v5" @@ -11,6 +12,7 @@ import ( "github.com/tokencanopy/e2a/internal/identity" "github.com/tokencanopy/e2a/internal/jobs" + "github.com/tokencanopy/e2a/internal/sendingpolicy" ) // Jobs is the HITL-notification integration on the shared River client: a @@ -23,6 +25,8 @@ import ( type Jobs struct { store Store enq jobs.Enqueuer + gate sendingpolicy.Gate + pool *pgxpool.Pool mu sync.RWMutex deliverer Deliverer @@ -31,6 +35,20 @@ type Jobs struct { // NewJobs builds the integration with just its store (no client, no deliverer yet). func NewJobs(store Store) *Jobs { return &Jobs{store: store} } +// WithGate injects the sending-protection gate and the pool its legacy +// resolver and arg stamp use. Every enqueue then prepares a notification +// operation in the hold's transaction and every worker execution authorizes +// through the gate. Chainable; nil keeps the gateless default (tests only). +func (j *Jobs) WithGate(g sendingpolicy.Gate, pool *pgxpool.Pool) *Jobs { + if g != nil { + j.gate = g + } + if pool != nil { + j.pool = pool + } + return j +} + // SetEnqueuer injects the shared client so EnqueueNotifyTx can insert jobs. func (j *Jobs) SetEnqueuer(e jobs.Enqueuer) { j.enq = e } @@ -47,29 +65,74 @@ func (j *Jobs) SetDeliverer(d Deliverer) { // set via SetDeliverer. Until that is wired (the brief startup window before the // notifier is built) it returns a retryable outcome, so a pending job simply // retries rather than dropping on a nil deliverer. -func (j *Jobs) Deliver(ctx context.Context, pn *identity.PendingNotify) DeliverOutcome { +func (j *Jobs) Deliver(ctx context.Context, pn *identity.PendingNotify, auth sendingpolicy.ProviderAuthorization) DeliverOutcome { j.mu.RLock() d := j.deliverer j.mu.RUnlock() if d == nil { return DeliverOutcome{Err: errors.New("hitl notifier not wired yet — retrying")} } - return d.Deliver(ctx, pn) + return d.Deliver(ctx, pn, auth) } // RegisterJobs adds the NotifyWorker (with Jobs as the late-binding Deliverer). // No periodics — the reconciler is a one-shot startup cutover. Implements // jobs.Registrar. func (j *Jobs) RegisterJobs(w *river.Workers) []*river.PeriodicJob { - river.AddWorker(w, NewNotifyWorker(j.store, j)) + river.AddWorker(w, j.NotifyWorker()) return nil } +// NotifyWorker builds the fully armed worker RegisterJobs registers. +func (j *Jobs) NotifyWorker() *NotifyWorker { + w := NewNotifyWorker(j.store, j).WithGate(j.gate).WithOperationResolver(j.ResolveLegacyOperation) + if j.pool != nil { + w = w.WithArgStamper(func(ctx context.Context, jobID int64, ref sendingpolicy.OperationRef) error { + return jobs.StampJobArg(ctx, j.pool, jobID, "operation_ref", ref) + }) + } + return w +} + +// ResolveLegacyOperation prepares the notification operation for a job that +// carries no reference, in its own committed transaction, through the same +// PrepareNotificationTx an enqueue runs. +func (j *Jobs) ResolveLegacyOperation(ctx context.Context, messageID string) (sendingpolicy.OperationRef, error) { + if j.gate == nil || j.pool == nil { + return sendingpolicy.OperationRef{}, fmt.Errorf("hitl notify: legacy operation resolver is not wired") + } + tx, err := j.pool.Begin(ctx) + if err != nil { + return sendingpolicy.OperationRef{}, fmt.Errorf("begin legacy resolve: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + ref, err := j.gate.PrepareNotificationTx(ctx, tx, sendingpolicy.NewHITLNotificationRef(messageID)) + if err != nil { + return sendingpolicy.OperationRef{}, err + } + if err := tx.Commit(ctx); err != nil { + return sendingpolicy.OperationRef{}, fmt.Errorf("commit legacy resolve: %w", err) + } + return ref, nil +} + // EnqueueNotifyTx inserts the hitl_notify job in the caller's hold accept-tx (the // same tx as the pending_review insert), returning the River job id to stamp on the // message so a committed pending_review row always has its notification job. +// +// With a gate wired the notification's operation is prepared here, in the +// same transaction, against the locked source row: the triggering account is +// charged, never the platform, and the worker never derives attribution. func (j *Jobs) EnqueueNotifyTx(ctx context.Context, tx pgx.Tx, messageID string) (int64, error) { - res, err := j.enq.InsertTx(ctx, tx, HITLNotifyArgs{MessageID: messageID}, &river.InsertOpts{ + args := HITLNotifyArgs{MessageID: messageID} + if j.gate != nil { + ref, err := j.gate.PrepareNotificationTx(ctx, tx, sendingpolicy.NewHITLNotificationRef(messageID)) + if err != nil { + return 0, fmt.Errorf("prepare notification operation: %w", err) + } + args.OperationRef = &ref + } + res, err := j.enq.InsertTx(ctx, tx, args, &river.InsertOpts{ Queue: jobs.QueueNotify, MaxAttempts: MaxNotifyAttempts, }) diff --git a/internal/hitlnotify/notifier.go b/internal/hitlnotify/notifier.go index 835739cfb..ac89c19bd 100644 --- a/internal/hitlnotify/notifier.go +++ b/internal/hitlnotify/notifier.go @@ -26,6 +26,7 @@ import ( "github.com/tokencanopy/e2a/internal/approvaltoken" "github.com/tokencanopy/e2a/internal/identity" "github.com/tokencanopy/e2a/internal/outbound" + "github.com/tokencanopy/e2a/internal/sendingpolicy" ) // notifyLocalPart is the default local-part of the notification sender @@ -50,9 +51,9 @@ const tokenGraceAfterTTL = 10 * time.Minute // call NotifyPendingApproval from the HITL gate right after the pending // row is written. Errors are logged, never returned upstream. type Notifier struct { - store *identity.Store - relay *outbound.SMTPRelay - signer *approvaltoken.Signer + store *identity.Store + submitter *outbound.ProviderSubmitter + signer *approvaltoken.Signer // fromAddress is the resolved sender: notifications.from_address when // set, else notifyLocalPart on fromDomain. fromAddress string @@ -80,7 +81,7 @@ type Notifier struct { // distinct and separately filterable. Resolution deliberately mirrors // webhooknotify.New line for line; it is a copy rather than a shared helper, // so changing one means changing the other. -func New(store *identity.Store, relay *outbound.SMTPRelay, signer *approvaltoken.Signer, fromDomain, fromAddress, replyTo, publicURL string) *Notifier { +func New(store *identity.Store, submitter *outbound.ProviderSubmitter, signer *approvaltoken.Signer, fromDomain, fromAddress, replyTo, publicURL string) *Notifier { addr := strings.TrimSpace(fromAddress) if addr == "" { addr = fmt.Sprintf("%s@%s", notifyLocalPart, fromDomain) @@ -91,7 +92,7 @@ func New(store *identity.Store, relay *outbound.SMTPRelay, signer *approvaltoken } return &Notifier{ store: store, - relay: relay, + submitter: submitter, signer: signer, fromAddress: addr, fromDomain: msgIDDomain, @@ -113,7 +114,7 @@ func (n *Notifier) WithDKIM(lookup outbound.DKIMKeyLookup) *Notifier { // message, submitting once (SendOnce). It is the compose+send core the River // NotifyWorker drives via Deliver; the returned error is classified there into // retry/permanent/outage. -func (n *Notifier) NotifyPendingApproval(ctx context.Context, msg *identity.Message, agent *identity.AgentIdentity) error { +func (n *Notifier) NotifyPendingApproval(ctx context.Context, msg *identity.Message, agent *identity.AgentIdentity, auth sendingpolicy.ProviderAuthorization) error { if n == nil { return nil } @@ -225,10 +226,16 @@ func (n *Notifier) NotifyPendingApproval(ctx context.Context, msg *identity.Mess message = signed } - // SendOnce, not Send: this runs inside a River job, so River (not the relay's - // in-process loop) owns retries. The %w keeps the SMTP error classifiable by - // Deliver via internal/outbound's IsPermanentSMTPError / IsConnectionError. - if _, err := n.relay.SendOnce(fromAddr, []string{owner.Email}, message); err != nil { + // One authorized submission: the submitter redeems the token immediately + // before the socket opens and settles the provider's answer; River (not + // the relay's in-process loop) owns retries, each as a fresh attempt. The + // %w keeps the SMTP error classifiable by Deliver via internal/outbound's + // IsPermanentSMTPError / IsConnectionError. + if _, err := n.submitter.SubmitOnce(ctx, auth, outbound.Envelope{ + From: fromAddr, + Recipients: []string{owner.Email}, + Message: message, + }); err != nil { return fmt.Errorf("notify: smtp send: %w", err) } @@ -242,8 +249,8 @@ func (n *Notifier) NotifyPendingApproval(ctx context.Context, msg *identity.Mess // (no retry), an unreachable relay is an Outage (snooze), everything else retries. // Implements hitlnotify.Deliverer. The classifiers key on the SMTP code / net // error preserved through NotifyPendingApproval's %w wrapping. -func (n *Notifier) Deliver(ctx context.Context, pn *identity.PendingNotify) DeliverOutcome { - if err := n.NotifyPendingApproval(ctx, pn.Message, pn.Agent); err != nil { +func (n *Notifier) Deliver(ctx context.Context, pn *identity.PendingNotify, auth sendingpolicy.ProviderAuthorization) DeliverOutcome { + if err := n.NotifyPendingApproval(ctx, pn.Message, pn.Agent, auth); err != nil { return DeliverOutcome{ Err: err, Permanent: outbound.IsPermanentSMTPError(err), diff --git a/internal/hitlnotify/notifier_test.go b/internal/hitlnotify/notifier_test.go index 17e0b2878..906fba2a1 100644 --- a/internal/hitlnotify/notifier_test.go +++ b/internal/hitlnotify/notifier_test.go @@ -5,12 +5,14 @@ import ( "strings" "testing" + "github.com/jackc/pgx/v5/pgxpool" "github.com/tokencanopy/e2a/internal/approvaltoken" "github.com/tokencanopy/e2a/internal/config" "github.com/tokencanopy/e2a/internal/dkim" "github.com/tokencanopy/e2a/internal/hitlnotify" "github.com/tokencanopy/e2a/internal/identity" "github.com/tokencanopy/e2a/internal/outbound" + "github.com/tokencanopy/e2a/internal/sendingpolicy" "github.com/tokencanopy/e2a/internal/testutil" ) @@ -38,10 +40,54 @@ func newNotifier(t *testing.T) ( FromDomain: notifyFromDomain, }) signer := approvaltoken.NewSigner(notifySecret) - n := hitlnotify.New(store, relay, signer, notifyFromDomain, "", "", publicURL) + gate := sendingpolicy.NewGate(pool, sendingpolicy.Secrets{}, sendingpolicy.PolicySourceConfig, sendingpolicy.DisabledPolicy()) + notifierGates[store] = gatePool{gate: gate, pool: pool} + n := hitlnotify.New(store, outbound.NewProviderSubmitter(relay, gate), signer, notifyFromDomain, "", "", publicURL) return n, store, signer, smtpDone } +type gatePool struct { + gate sendingpolicy.Gate + pool *pgxpool.Pool +} + +// notifierGates remembers the gate each test store was built with, so a test +// can mint the token its notification needs without threading it through +// every helper signature. +var notifierGates = map[*identity.Store]gatePool{} + +// tokenFor prepares the notification operation for a held message and runs +// Reserve + ConsumeAttempt, returning the authorization the notifier redeems. +func tokenFor(t *testing.T, store *identity.Store, messageID string) sendingpolicy.ProviderAuthorization { + t.Helper() + gp, ok := notifierGates[store] + if !ok { + t.Fatal("no gate for this store") + } + ctx := context.Background() + tx, err := gp.pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + ref, err := gp.gate.PrepareNotificationTx(ctx, tx, sendingpolicy.NewHITLNotificationRef(messageID)) + if err != nil { + _ = tx.Rollback(ctx) + t.Fatalf("prepare notification: %v", err) + } + if err := tx.Commit(ctx); err != nil { + t.Fatal(err) + } + early, attempt, err := gp.gate.Reserve(ctx, ref) + if err != nil || !early.Allow { + t.Fatalf("reserve: decision=%+v err=%v", early, err) + } + decision, auth, err := gp.gate.ConsumeAttempt(ctx, attempt) + if err != nil || auth == nil { + t.Fatalf("authorize: decision=%+v err=%v", decision, err) + } + return *auth +} + // setupPendingMessage creates a verified HITL-enabled agent with one // pending outbound message. Returns (agent, message). func setupPendingMessage(t *testing.T, store *identity.Store, slug string) (*identity.AgentIdentity, *identity.Message) { @@ -83,7 +129,7 @@ func TestNotifierSendsEmailToOwner(t *testing.T) { n, store, _, smtpDone := newNotifier(t) agent, msg := setupPendingMessage(t, store, "send-email") - if err := n.NotifyPendingApproval(context.Background(), msg, agent); err != nil { + if err := n.NotifyPendingApproval(context.Background(), msg, agent, tokenFor(t, store, msg.ID)); err != nil { t.Fatalf("NotifyPendingApproval: %v", err) } @@ -148,7 +194,7 @@ func TestNotifierMagicLinksAreVerifiable(t *testing.T) { n, store, _, smtpDone := newNotifier(t) agent, msg := setupPendingMessage(t, store, "tok-verify") - if err := n.NotifyPendingApproval(context.Background(), msg, agent); err != nil { + if err := n.NotifyPendingApproval(context.Background(), msg, agent, tokenFor(t, store, msg.ID)); err != nil { t.Fatal(err) } data := smtpDone()[0].Data @@ -191,7 +237,7 @@ func TestNotifierBuildsAbsoluteURLs(t *testing.T) { n, store, _, smtpDone := newNotifier(t) agent, msg := setupPendingMessage(t, store, "abs-url") - if err := n.NotifyPendingApproval(context.Background(), msg, agent); err != nil { + if err := n.NotifyPendingApproval(context.Background(), msg, agent, tokenFor(t, store, msg.ID)); err != nil { t.Fatal(err) } data := smtpDone()[0].Data @@ -213,7 +259,7 @@ func TestNotifierRejectsMessageWithNilApprovalExpiresAt(t *testing.T) { agent, msg := setupPendingMessage(t, store, "nil-exp") msg.ApprovalExpiresAt = nil - err := n.NotifyPendingApproval(context.Background(), msg, agent) + err := n.NotifyPendingApproval(context.Background(), msg, agent, tokenFor(t, store, msg.ID)) if err == nil { t.Fatal("expected error for nil ApprovalExpiresAt") } @@ -231,10 +277,10 @@ func TestNotifierDeterministicMessageID(t *testing.T) { n, store, _, smtpDone := newNotifier(t) agent, msg := setupPendingMessage(t, store, "msgid") - if err := n.NotifyPendingApproval(context.Background(), msg, agent); err != nil { + if err := n.NotifyPendingApproval(context.Background(), msg, agent, tokenFor(t, store, msg.ID)); err != nil { t.Fatal(err) } - if err := n.NotifyPendingApproval(context.Background(), msg, agent); err != nil { + if err := n.NotifyPendingApproval(context.Background(), msg, agent, tokenFor(t, store, msg.ID)); err != nil { t.Fatal(err) } @@ -251,8 +297,11 @@ func TestNotifierDeterministicMessageID(t *testing.T) { if n := strings.Count(m.Data, "Message-ID:"); n != 1 { t.Errorf("message %d has %d Message-ID headers, want exactly 1", i, n) } - if !strings.HasPrefix(m.Data, "Message-ID: delay { + delay = until + } + } + return river.JobSnooze(delay) +} diff --git a/internal/hitlnotify/worker_test.go b/internal/hitlnotify/worker_test.go index 2f75ec8ea..03503d464 100644 --- a/internal/hitlnotify/worker_test.go +++ b/internal/hitlnotify/worker_test.go @@ -2,6 +2,7 @@ package hitlnotify_test import ( "context" + "encoding/json" "errors" "testing" "time" @@ -12,6 +13,7 @@ import ( "github.com/tokencanopy/e2a/internal/hitlnotify" "github.com/tokencanopy/e2a/internal/identity" + "github.com/tokencanopy/e2a/internal/sendingpolicy" ) type fakeStore struct { @@ -36,10 +38,12 @@ func (f *fakeStore) StampNotifyJobIDTx(_ context.Context, _ pgx.Tx, _ string, _ type fakeDeliverer struct { out hitlnotify.DeliverOutcome called int + auths []sendingpolicy.ProviderAuthorization } -func (f *fakeDeliverer) Deliver(_ context.Context, _ *identity.PendingNotify) hitlnotify.DeliverOutcome { +func (f *fakeDeliverer) Deliver(_ context.Context, _ *identity.PendingNotify, auth sendingpolicy.ProviderAuthorization) hitlnotify.DeliverOutcome { f.called++ + f.auths = append(f.auths, auth) return f.out } @@ -212,3 +216,141 @@ func TestNotifyWorker_NextRetryMatchesEnvelope(t *testing.T) { } } } + +// fakeGate is a scriptable sendingpolicy.Gate for the worker-order tests. +type fakeGate struct { + reserve sendingpolicy.Decision + consume sendingpolicy.Decision + reserves int + consumes int + reserveErr error +} + +func allowAll() *fakeGate { + return &fakeGate{reserve: sendingpolicy.Decision{Allow: true}, consume: sendingpolicy.Decision{Allow: true}} +} + +func (g *fakeGate) PrepareExternalTx(context.Context, pgx.Tx, string) (sendingpolicy.AcceptanceDecision, sendingpolicy.OperationRef, error) { + return sendingpolicy.AcceptanceAccept, sendingpolicy.OperationRef{}, nil +} +func (g *fakeGate) PrepareNotificationTx(context.Context, pgx.Tx, sendingpolicy.NotificationRef) (sendingpolicy.OperationRef, error) { + return refFor("op_prepared"), nil +} +func (g *fakeGate) PrepareProtectionNoticeTx(context.Context, pgx.Tx, sendingpolicy.ProtectionNoticeRef) (sendingpolicy.OperationRef, error) { + return sendingpolicy.OperationRef{}, nil +} +func (g *fakeGate) PreparePublicFeedback(context.Context, sendingpolicy.PublicFeedbackRef) (sendingpolicy.OperationRef, error) { + return sendingpolicy.OperationRef{}, nil +} +func (g *fakeGate) Reserve(context.Context, sendingpolicy.OperationRef) (sendingpolicy.Decision, sendingpolicy.AttemptRef, error) { + g.reserves++ + return g.reserve, sendingpolicy.AttemptRef{}, g.reserveErr +} +func (g *fakeGate) ConsumeAttempt(context.Context, sendingpolicy.AttemptRef) (sendingpolicy.Decision, *sendingpolicy.ProviderAuthorization, error) { + g.consumes++ + if !g.consume.Allow { + return g.consume, nil, nil + } + return g.consume, &sendingpolicy.ProviderAuthorization{}, nil +} +func (g *fakeGate) RedeemProviderCall(context.Context, sendingpolicy.ProviderAuthorization) error { + return nil +} +func (g *fakeGate) DeferAttempt(context.Context, sendingpolicy.AttemptRef) error { return nil } +func (g *fakeGate) CancelAttempt(context.Context, sendingpolicy.AttemptRef) error { return nil } +func (g *fakeGate) SettleProvider(context.Context, sendingpolicy.ProviderSettlement) error { + return nil +} +func (g *fakeGate) SettleOperation(context.Context, sendingpolicy.OperationRef, sendingpolicy.SettlementOutcome, string) error { + return nil +} +func (g *fakeGate) LookupOperation(_ context.Context, id string) (sendingpolicy.OperationRef, error) { + return refFor(id), nil +} + +func refFor(id string) sendingpolicy.OperationRef { + var ref sendingpolicy.OperationRef + if err := json.Unmarshal([]byte(`{"v":1,"id":"`+id+`"}`), &ref); err != nil { + panic(err) + } + return ref +} + +func gatedJob(id string, attempt int) *river.Job[hitlnotify.HITLNotifyArgs] { + j := job(id, attempt) + ref := refFor("op_" + id) + j.Args.OperationRef = &ref + return j +} + +func isSnooze(err error) bool { + var snooze *river.JobSnoozeError + return errors.As(err, &snooze) +} + +func isCancel(err error) bool { + var cancel *river.JobCancelError + return errors.As(err, &cancel) +} + +func TestNotifyWorker_GatedPathAuthorizesThenDelivers(t *testing.T) { + st := &fakeStore{pn: pending("msg_gated")} + dl := &fakeDeliverer{} + g := allowAll() + if err := hitlnotify.NewNotifyWorker(st, dl).WithGate(g).Work(context.Background(), gatedJob("msg_gated", 1)); err != nil { + t.Fatalf("Work: %v", err) + } + if g.reserves != 1 || g.consumes != 1 || dl.called != 1 || len(st.notified) != 1 { + t.Fatalf("reserves=%d consumes=%d delivers=%d notified=%d, want 1/1/1/1", g.reserves, g.consumes, dl.called, len(st.notified)) + } +} + +func TestNotifyWorker_GateHoldSnoozesWithoutDelivery(t *testing.T) { + for name, g := range map[string]*fakeGate{ + "early hold": {reserve: sendingpolicy.Decision{Allow: false, Reason: sendingpolicy.ReasonAccountPaused}}, + "late hold": {reserve: sendingpolicy.Decision{Allow: true}, consume: sendingpolicy.Decision{Allow: false, Reason: sendingpolicy.ReasonAccountSharedBudget, RetryAt: time.Now().Add(2 * time.Hour)}}, + "gate error": {reserveErr: errors.New("policy db down")}, + } { + st := &fakeStore{pn: pending("msg_hold")} + dl := &fakeDeliverer{} + err := hitlnotify.NewNotifyWorker(st, dl).WithGate(g).Work(context.Background(), gatedJob("msg_hold", 1)) + if !isSnooze(err) || dl.called != 0 || len(st.notified) != 0 { + t.Fatalf("%s: err=%v delivers=%d notified=%d, want snooze with no I/O", name, err, dl.called, len(st.notified)) + } + } +} + +func TestNotifyWorker_TerminalHoldCancels(t *testing.T) { + st := &fakeStore{pn: pending("msg_terminal")} + dl := &fakeDeliverer{} + g := &fakeGate{reserve: sendingpolicy.Decision{Allow: false, Reason: sendingpolicy.ReasonAccountDeleted, Terminal: true}} + if err := hitlnotify.NewNotifyWorker(st, dl).WithGate(g).Work(context.Background(), gatedJob("msg_terminal", 1)); !isCancel(err) || dl.called != 0 { + t.Fatalf("err=%v delivers=%d, want cancel with no I/O", err, dl.called) + } +} + +func TestNotifyWorker_LegacyJobResolvesAndStampsOnce(t *testing.T) { + st := &fakeStore{pn: pending("msg_legacy")} + dl := &fakeDeliverer{} + resolved, stamped := 0, 0 + w := hitlnotify.NewNotifyWorker(st, dl).WithGate(allowAll()). + WithOperationResolver(func(_ context.Context, id string) (sendingpolicy.OperationRef, error) { + resolved++ + return refFor("op_" + id), nil + }). + WithArgStamper(func(_ context.Context, _ int64, _ sendingpolicy.OperationRef) error { stamped++; return nil }) + if err := w.Work(context.Background(), job("msg_legacy", 1)); err != nil { + t.Fatalf("Work: %v", err) + } + if resolved != 1 || stamped != 1 || dl.called != 1 { + t.Fatalf("resolved=%d stamped=%d delivers=%d, want 1/1/1", resolved, stamped, dl.called) + } + // A legacy job whose source is gone is a no-op, never a retry loop. + w = hitlnotify.NewNotifyWorker(&fakeStore{pn: pending("msg_gone")}, dl).WithGate(allowAll()). + WithOperationResolver(func(context.Context, string) (sendingpolicy.OperationRef, error) { + return sendingpolicy.OperationRef{}, sendingpolicy.ErrSourceUnavailable + }) + if err := w.Work(context.Background(), job("msg_gone", 1)); err != nil || dl.called != 1 { + t.Fatalf("orphan legacy: err=%v delivers=%d, want nil and no new delivery", err, dl.called) + } +} diff --git a/internal/jobs/argstamp.go b/internal/jobs/argstamp.go new file mode 100644 index 000000000..4ea42a39b --- /dev/null +++ b/internal/jobs/argstamp.go @@ -0,0 +1,40 @@ +package jobs + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/jackc/pgx/v5/pgconn" +) + +// Execer is the one method StampJobArg needs; both a pool and a transaction +// satisfy it. +type Execer interface { + Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error) +} + +// StampJobArg adds one key to a River job's args, only when that key is +// absent, leaving every existing field in place. +// +// It exists for the sending-protection compatibility resolvers: a job +// enqueued by a pre-floor slot carries no operation reference, the worker +// derives one through the same Prepare path an enqueue uses, and stamping it +// here makes that derivation happen once per job rather than once per +// execution. Existing fields stay so an older worker can still read the job. +func StampJobArg(ctx context.Context, db Execer, jobID int64, key string, value any) error { + if db == nil { + return fmt.Errorf("stamp job arg: no database") + } + patch, err := json.Marshal(map[string]any{key: value}) + if err != nil { + return fmt.Errorf("stamp job arg: encode %s: %w", key, err) + } + if _, err := db.Exec(ctx, + `UPDATE river_job SET args = args || $2::jsonb WHERE id = $1 AND NOT (args ? $3)`, + jobID, string(patch), key, + ); err != nil { + return fmt.Errorf("stamp job arg %s on job %d: %w", key, jobID, err) + } + return nil +} diff --git a/internal/outbound/provider_authorization_guard_test.go b/internal/outbound/provider_authorization_guard_test.go new file mode 100644 index 000000000..0da9378d2 --- /dev/null +++ b/internal/outbound/provider_authorization_guard_test.go @@ -0,0 +1,140 @@ +package outbound + +import ( + "go/ast" + "go/parser" + "go/token" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// TestEveryProviderCallRequiresAuthorization is the tracked closure guard for +// the provider seam. It parses every tracked production Go file and rejects: +// +// - any import of net/smtp outside the relay itself and the named exceptions; +// - any call to the relay's private socket-opening core outside the one +// authorized adapter; +// - any exported relay method that could open a socket without a token. +// +// Exceptions are exact file paths, never substrings, and each is named here +// with the reason it may exist. Adding a provider-bound caller anywhere else +// fails this test until it goes through ProviderSubmitter.SubmitOnce. +func TestEveryProviderCallRequiresAuthorization(t *testing.T) { + root := moduleRoot(t) + files := trackedGoFiles(t, root) + + // Files that may import net/smtp: the relay (the only SES client) and the + // self-test scenarios, which drive a local SMTP conversation against + // e2a's OWN inbound listener to prove delivery end to end — never the + // provider. + smtpImportAllowed := map[string]string{ + "internal/outbound/smtp_relay.go": "the provider relay itself", + "internal/selftest/scenarios.go": "local inbound self-test client, not provider-bound", + } + // Files that may call the relay's socket-opening core. + socketCallAllowed := map[string]string{ + "internal/outbound/provider_submit.go": "the one authorized adapter", + } + + fset := token.NewFileSet() + for _, rel := range files { + src, err := os.ReadFile(filepath.Join(root, rel)) + if err != nil { + t.Fatalf("read %s: %v", rel, err) + } + f, err := parser.ParseFile(fset, rel, src, parser.ImportsOnly|parser.ParseComments) + if err != nil { + t.Fatalf("parse %s: %v", rel, err) + } + for _, imp := range f.Imports { + if strings.Trim(imp.Path.Value, `"`) == "net/smtp" { + if _, ok := smtpImportAllowed[rel]; !ok { + t.Errorf("%s imports net/smtp: provider I/O must go through outbound.ProviderSubmitter (or be named in the guard's exception list with its reason)", rel) + } + } + } + full, err := parser.ParseFile(fset, rel, src, 0) + if err != nil { + t.Fatalf("parse %s: %v", rel, err) + } + ast.Inspect(full, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + return true + } + if sel.Sel.Name == "sendOnceContext" { + if _, ok := socketCallAllowed[rel]; !ok { + t.Errorf("%s:%s calls the relay's socket-opening core outside the authorized adapter", rel, fset.Position(call.Pos())) + } + } + return true + }) + } + + // The relay's exported surface may not open a socket: Configured is a + // field read, and everything that dials is unexported. A newly exported + // Send* method is exactly the bypass this guard exists to refuse. + relaySrc, err := os.ReadFile(filepath.Join(root, "internal/outbound/smtp_relay.go")) + if err != nil { + t.Fatal(err) + } + relayFile, err := parser.ParseFile(fset, "smtp_relay.go", relaySrc, 0) + if err != nil { + t.Fatal(err) + } + for _, decl := range relayFile.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Recv == nil || len(fn.Recv.List) == 0 { + continue + } + recv := fn.Recv.List[0].Type + if star, ok := recv.(*ast.StarExpr); ok { + recv = star.X + } + if ident, ok := recv.(*ast.Ident); !ok || ident.Name != "SMTPRelay" { + continue + } + if fn.Name.IsExported() && fn.Name.Name != "Configured" { + t.Errorf("SMTPRelay exports %s: the relay must expose no socket-opening method", fn.Name.Name) + } + } +} + +func moduleRoot(t *testing.T) string { + t.Helper() + out, err := exec.Command("git", "rev-parse", "--show-toplevel").Output() + if err != nil { + t.Skipf("not in a git checkout: %v", err) + } + return strings.TrimSpace(string(out)) +} + +// trackedGoFiles lists tracked, non-test Go files under internal/ and cmd/ +// — production code only, by git's own account of what ships. +func trackedGoFiles(t *testing.T, root string) []string { + t.Helper() + cmd := exec.Command("git", "ls-files", "--", "internal/*.go", "internal/**/*.go", "cmd/*.go", "cmd/**/*.go") + cmd.Dir = root + out, err := cmd.Output() + if err != nil { + t.Fatalf("git ls-files: %v", err) + } + var files []string + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + if line == "" || strings.HasSuffix(line, "_test.go") { + continue + } + files = append(files, line) + } + if len(files) < 50 { + t.Fatalf("only %d tracked production files found; the guard is scanning the wrong tree", len(files)) + } + return files +} diff --git a/internal/outbound/provider_submit.go b/internal/outbound/provider_submit.go index fa01de163..982b8f717 100644 --- a/internal/outbound/provider_submit.go +++ b/internal/outbound/provider_submit.go @@ -199,7 +199,7 @@ func (s *ProviderSubmitter) SubmitOnce(ctx context.Context, auth sendingpolicy.P // A failure after the body was fully written (ErrProviderAcceptanceUnknown) // is neither accepted nor rejected here: it is returned unsettled, because // the provider may hold the message and only its feedback can say. - providerID, sendErr := s.relay.SendOnceContext(ctx, env.From, auth.AuthorizedRecipients(), wire) + providerID, sendErr := s.relay.sendOnceContext(ctx, env.From, auth.AuthorizedRecipients(), wire) if sendErr != nil { // IsPermanentSMTPError is the worker's retry classifier: any 5xx, // including one raised before DATA (an AUTH 535, say). Settling such a diff --git a/internal/outbound/sender.go b/internal/outbound/sender.go index 769da13f6..388b21fcd 100644 --- a/internal/outbound/sender.go +++ b/internal/outbound/sender.go @@ -299,54 +299,6 @@ type ComposeResult struct { To, CC, BCC []string } -// Send normalizes recipients, composes, and sends an email via SMTP relay -// (the historical retrying submit). Returns a ValidationError for caller errors -// (bad addresses, no visible recipients) and a plain error for transport failures. -func (s *Sender) Send(agent *identity.AgentIdentity, req SendRequest) (*SendResult, error) { - c, err := s.compose(agent, req) - if err != nil { - return nil, err - } - sesMessageID, err := s.smtpRelay.Send(c.envelopeFrom, c.envelope, c.wire) - if err != nil { - return nil, fmt.Errorf("smtp relay: %w", err) - } - return &SendResult{ - MessageID: sesMessageID, - Method: "smtp", - SentAs: c.sentAs, - To: c.to, - CC: c.cc, - BCC: c.bcc, - Raw: c.sentBody, - }, nil -} - -// SendOnce is Send with a SINGLE SMTP submit and no internal retry loop — the -// entry point for a caller that owns its own retry envelope. Behaviorally -// identical to Send except it calls smtpRelay.SendOnce. (The async pipeline does -// NOT use this — it persists ComposeForAccept's bytes and the River worker -// submits them via SubmitOnce — but it is the direct single-attempt analogue.) -func (s *Sender) SendOnce(agent *identity.AgentIdentity, req SendRequest) (*SendResult, error) { - c, err := s.compose(agent, req) - if err != nil { - return nil, err - } - sesMessageID, err := s.smtpRelay.SendOnce(c.envelopeFrom, c.envelope, c.wire) - if err != nil { - return nil, fmt.Errorf("smtp relay: %w", err) - } - return &SendResult{ - MessageID: sesMessageID, - Method: "smtp", - SentAs: c.sentAs, - To: c.to, - CC: c.cc, - BCC: c.bcc, - Raw: c.sentBody, - }, nil -} - // ComposeForAccept composes an outbound message for the async accept path WITHOUT // submitting it. The accept-tx persists the returned bytes + envelope so the River // worker owns the actual SMTP submit; it reuses Send's exact compose stage (same @@ -368,35 +320,6 @@ func (s *Sender) ComposeForAccept(agent *identity.AgentIdentity, req SendRequest }, nil } -// SubmitOnce submits the persisted Sent-folder bytes in a SINGLE SMTP attempt -// (River owns retries) and returns the provider Message-ID. It attaches two -// wire-time headers post-DKIM (never in the signed header set): -// -// - X-E2A-Message-ID (delivery.MessageIDHeader) — the stable e2a correlation -// marker (async-send-contract §3.1). SES overrides supplied Message-ID/Date -// headers, but echoes original headers back in its notifications -// (mail.headers, when "include original headers" is enabled on the -// configuration set), so this is the value that correlates feedback for -// the SMTP-accept↔mark-sent crash window. Unlike the config-set header SES -// does NOT strip it — recipients see it too; it is deliberately a stable -// public marker. Stamped at submit time (not compose time) so messages -// accepted before this header existed still carry it on re-drive. -// -// - X-SES-CONFIGURATION-SET — re-attached because raw_message is stored -// WITHOUT it (SES strips it before delivery; the recipient/Sent-folder -// copy must not carry it). -// -// Keeping the header logic here (not in the worker) means Send and the async -// path share one source of truth for what SES actually receives. -func (s *Sender) SubmitOnce(messageID, envelopeFrom string, recipients []string, sentBody []byte) (string, error) { - return s.SubmitOnceContext(context.Background(), messageID, envelopeFrom, recipients, sentBody) -} - -// SubmitOnceContext is SubmitOnce with caller cancellation propagated to SMTP. -func (s *Sender) SubmitOnceContext(ctx context.Context, messageID, envelopeFrom string, recipients []string, sentBody []byte) (string, error) { - return s.smtpRelay.SendOnceContext(ctx, envelopeFrom, recipients, s.applySESConfigSet(applyCorrelationHeader(sentBody, messageID))) -} - // applyCorrelationHeader prepends the X-E2A-Message-ID marker. The id is // server-minted, but sanitize anyway — this is a header write. Empty id // (defensive) = no header. diff --git a/internal/outbound/smtp_relay.go b/internal/outbound/smtp_relay.go index b67753423..c6d77184b 100644 --- a/internal/outbound/smtp_relay.go +++ b/internal/outbound/smtp_relay.go @@ -5,7 +5,6 @@ import ( "crypto/tls" "errors" "fmt" - "log" "net" "net/smtp" "net/textproto" @@ -14,11 +13,8 @@ import ( "time" "github.com/tokencanopy/e2a/internal/config" - "github.com/tokencanopy/e2a/internal/logredact" ) -var smtpRetryBackoffs = []time.Duration{1 * time.Second, 5 * time.Second, 15 * time.Second} - // ErrProviderAcceptanceUnknown marks a failure that happened AFTER the whole // message body was handed to the provider: the terminating dot was written and // the 250 never arrived. The provider may have accepted the message. No @@ -40,85 +36,12 @@ func (r *SMTPRelay) Configured() bool { return r.cfg.Host != "" } -// Send sends an email to one or more recipients and returns the Message-ID assigned by the remote server (e.g. SES). -func (r *SMTPRelay) Send(from string, recipients []string, message []byte) (string, error) { - return r.SendWithContext(context.Background(), from, recipients, message) -} - -// SendWithContext sends an email while honoring ctx during SMTP I/O and retry -// backoff. It is intended for request-bound callers that cannot allow the -// relay's normal retry envelope to outlive the request budget. -func (r *SMTPRelay) SendWithContext(ctx context.Context, from string, recipients []string, message []byte) (string, error) { - return r.SendWithEnvelopeContext(ctx, from, recipients, message) -} - -// SendWithEnvelope sends an email using envelopeFrom for SMTP MAIL FROM. -// Issues RCPT TO for each recipient. If any RCPT TO is rejected, the transaction is aborted. -// Returns the Message-ID assigned by the remote SMTP server from the DATA response. -// Retries transient SMTP errors (4xx) up to 3 times with backoff. -func (r *SMTPRelay) SendWithEnvelope(envelopeFrom string, recipients []string, message []byte) (string, error) { - return r.SendWithEnvelopeContext(context.Background(), envelopeFrom, recipients, message) -} - -// SendWithEnvelopeContext is SendWithEnvelope with caller-controlled -// cancellation and deadline propagation. -func (r *SMTPRelay) SendWithEnvelopeContext(ctx context.Context, envelopeFrom string, recipients []string, message []byte) (string, error) { - if !r.Configured() { - return "", fmt.Errorf("outbound SMTP relay not configured") - } - - var lastErr error - for attempt := 0; attempt <= len(smtpRetryBackoffs); attempt++ { - msgID, err := r.sendOnceContext(ctx, envelopeFrom, recipients, message) - if err == nil { - return msgID, nil - } - lastErr = err - if ctx.Err() != nil { - return "", ctx.Err() - } - if !isTransientSMTPError(lastErr) { - return "", lastErr - } - if attempt < len(smtpRetryBackoffs) { - // lastErr is an upstream MTA response and cannot be perfectly - // sanitized: rejections routinely quote the recipient back at us - // ("550 5.1.1 : user unknown"), which would - // otherwise defeat the recipient redaction on this same line. Cap - // it so at most a bounded slice of provider text is retained; the - // full error still reaches the caller and the message row. - log.Printf("[smtp-relay] transient error sending to recipient_count=%d recipient_domains=%v (attempt %d/%d), retrying in %s: %s", - len(recipients), logredact.AddressDomains(recipients), attempt+1, len(smtpRetryBackoffs)+1, smtpRetryBackoffs[attempt], logredact.Truncate(lastErr.Error(), 200)) - select { - case <-time.After(smtpRetryBackoffs[attempt]): - case <-ctx.Done(): - return "", ctx.Err() - } - } - } - return "", lastErr -} - -// SendOnce performs a SINGLE SMTP submit — no internal retry loop — and returns -// the provider Message-ID. This is the entry point for the River outbound worker -// (internal/outboundsend), which owns the retry envelope: River reschedules the -// next attempt per the worker's NextRetry, so the relay must NOT loop (a loop here -// would hide the envelope from river_job and make each Work() run up to ~6.5 min). -// Classify the returned error with IsTransientSMTPError — transient (4xx/throttle) -// → let River retry; permanent (5xx/validation) → fail the message terminally. -func (r *SMTPRelay) SendOnce(envelopeFrom string, recipients []string, message []byte) (string, error) { - return r.SendOnceContext(context.Background(), envelopeFrom, recipients, message) -} - -// SendOnceContext is SendOnce with caller cancellation propagated into the -// SMTP dial/command path. River workers use it so remotely cancelling a running -// job can stop provider I/O promptly. -func (r *SMTPRelay) SendOnceContext(ctx context.Context, envelopeFrom string, recipients []string, message []byte) (string, error) { - if !r.Configured() { - return "", fmt.Errorf("outbound SMTP relay not configured") - } - return r.sendOnceContext(ctx, envelopeFrom, recipients, message) -} +// The relay exposes no socket-opening method. Every provider call is made by +// the ProviderSubmitter in this package through sendOnceContext, after the +// caller's authorization token has been redeemed; there is no in-process +// retry loop either, because a retry is a new charged attempt that only the +// sending-protection gate may allocate. The tracked guard test +// (provider_authorization_guard_test.go) keeps it that way. // IsTransientSMTPError reports whether err is a retryable SMTP failure (4xx / // throttle) vs a permanent one. Exported so the River worker's deliverer can set diff --git a/internal/outbound/smtp_relay_test.go b/internal/outbound/smtp_relay_test.go index ee4cb3f12..e03a2035a 100644 --- a/internal/outbound/smtp_relay_test.go +++ b/internal/outbound/smtp_relay_test.go @@ -12,7 +12,7 @@ import ( "github.com/tokencanopy/e2a/internal/config" ) -func TestSMTPRelaySendWithContextCancelsHangingServer(t *testing.T) { +func TestSMTPRelayCancelsHangingServer(t *testing.T) { listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) @@ -40,24 +40,24 @@ func TestSMTPRelaySendWithContextCancelsHangingServer(t *testing.T) { defer cancel() started := time.Now() - _, err = relay.SendWithContext(ctx, "noreply@example.com", []string{"feedback@example.com"}, []byte("Subject: test\r\n\r\nbody")) + _, err = relay.sendOnceContext(ctx, "noreply@example.com", []string{"feedback@example.com"}, []byte("Subject: test\r\n\r\nbody")) if !errors.Is(err, context.DeadlineExceeded) { - t.Fatalf("SendWithContext error = %v, want context deadline exceeded", err) + t.Fatalf("sendOnceContext error = %v, want context deadline exceeded", err) } if elapsed := time.Since(started); elapsed > time.Second { - t.Fatalf("SendWithContext returned after %s, want cancellation within 1s", elapsed) + t.Fatalf("sendOnceContext returned after %s, want cancellation within 1s", elapsed) } } -func TestSMTPRelaySendOnceContextHonorsCancellation(t *testing.T) { +func TestSMTPRelayHonorsCancellation(t *testing.T) { relay := NewSMTPRelay(&config.OutboundSMTPConfig{Host: "127.0.0.1", Port: 1}) ctx, cancel := context.WithCancel(context.Background()) cancel() - _, err := relay.SendOnceContext(ctx, "noreply@example.com", + _, err := relay.sendOnceContext(ctx, "noreply@example.com", []string{"recipient@example.com"}, []byte("Subject: test\r\n\r\nbody")) if !errors.Is(err, context.Canceled) { - t.Fatalf("SendOnceContext error = %v, want context canceled", err) + t.Fatalf("sendOnceContext error = %v, want context canceled", err) } } diff --git a/internal/testutil/contract_server.go b/internal/testutil/contract_server.go index 63ae44ad5..332603cdd 100644 --- a/internal/testutil/contract_server.go +++ b/internal/testutil/contract_server.go @@ -122,9 +122,10 @@ func StartContractServer(ctx context.Context, dbURL string) (*ContractServer, er // disabled policy (pass-through admission, every attempt still durable) // and the authorized submitter that refuses to dial without its token. sendingGate := sendingpolicy.NewGate(pool, sendingpolicy.Secrets{}, sendingpolicy.PolicySourceConfig, sendingpolicy.DisabledPolicy()) + providerSubmitter := outbound.NewProviderSubmitter(smtpRelay, sendingGate) outboundJobs := outboundsend.NewJobs( outboundSendStore, - agent.NewOutboundDeliverer(outbound.NewProviderSubmitter(smtpRelay, sendingGate)), + agent.NewOutboundDeliverer(providerSubmitter), pool, ).WithGate(sendingGate) jobsClient, err := jobs.New(pool, jobs.Config{OutboundWorkers: 1}, outboundJobs) @@ -137,6 +138,7 @@ func StartContractServer(ctx context.Context, dbURL string) (*ContractServer, er router := mux.NewRouter() api := agent.NewAPI(store, sender, smtpRelay, nil, noopUsage, "e2a.dev", "test.e2a.dev", "agents.e2a.dev", "", false) + api.SetProviderSubmitter(providerSubmitter, sendingGate) api.SetIdempotencyStore(idempotencyStore) api.SetEnforcer(enforcer) api.SetUsageStore(usageStore) diff --git a/internal/testutil/server.go b/internal/testutil/server.go index 22d37e500..489fd7c42 100644 --- a/internal/testutil/server.go +++ b/internal/testutil/server.go @@ -222,9 +222,10 @@ func TestServer(t *testing.T, pool *pgxpool.Pool, opts ...TestServerOption) *E2A // disabled policy (pass-through admission, every attempt still durable) // and the authorized submitter that refuses to dial without its token. sendingGate := sendingpolicy.NewGate(pool, sendingpolicy.Secrets{}, sendingpolicy.PolicySourceConfig, sendingpolicy.DisabledPolicy()) + providerSubmitter := outbound.NewProviderSubmitter(smtpRelay, sendingGate) outboundJobs := outboundsend.NewJobs( outboundSendStore, - agent.NewOutboundDeliverer(outbound.NewProviderSubmitter(smtpRelay, sendingGate)), + agent.NewOutboundDeliverer(providerSubmitter), pool, ).WithGate(sendingGate) jobsClient, err := jobs.New(pool, jobs.Config{OutboundWorkers: 2}, outboundJobs) @@ -247,6 +248,7 @@ func TestServer(t *testing.T, pool *pgxpool.Pool, opts ...TestServerOption) *E2A }, time.Minute) idempotencyStore := idempotency.NewStore(pool) api := agent.NewAPI(store, sender, smtpRelay, nil, noopUsage, "e2a.dev", "test.e2a.dev", "agents.e2a.dev", "", false) + api.SetProviderSubmitter(providerSubmitter, sendingGate) api.SetIdempotencyStore(idempotencyStore) api.SetSubscriberStore(subscriberStore) api.SetOutbox(outbox) diff --git a/internal/webhooknotify/e2e_test.go b/internal/webhooknotify/e2e_test.go index 3f6ed375e..019f7d96f 100644 --- a/internal/webhooknotify/e2e_test.go +++ b/internal/webhooknotify/e2e_test.go @@ -13,6 +13,7 @@ import ( "github.com/tokencanopy/e2a/internal/identity" "github.com/tokencanopy/e2a/internal/jobs" "github.com/tokencanopy/e2a/internal/outbound" + "github.com/tokencanopy/e2a/internal/sendingpolicy" "github.com/tokencanopy/e2a/internal/testutil" "github.com/tokencanopy/e2a/internal/webhooknotify" ) @@ -49,9 +50,10 @@ func newE2EHarness(t *testing.T, replyTo string) *e2eHarness { relay := outbound.NewSMTPRelay(&config.OutboundSMTPConfig{ Host: smtpAddr.Host, Port: smtpAddr.Port, FromDomain: "notify.test", }) - notifier := webhooknotify.New(store, relay, "notify.test", "", replyTo, "https://app.example.test") + gate := sendingpolicy.NewGate(pool, sendingpolicy.Secrets{}, sendingpolicy.PolicySourceConfig, sendingpolicy.DisabledPolicy()) + notifier := webhooknotify.New(store, outbound.NewProviderSubmitter(relay, gate), "notify.test", "", replyTo, "https://app.example.test") - j := webhooknotify.NewJobs(store) + j := webhooknotify.NewJobs(store).WithGate(gate, pool) client, err := jobs.New(pool, jobs.Config{}, j) if err != nil { t.Fatalf("jobs.New: %v", err) diff --git a/internal/webhooknotify/jobs.go b/internal/webhooknotify/jobs.go index 954874d49..0457b1cac 100644 --- a/internal/webhooknotify/jobs.go +++ b/internal/webhooknotify/jobs.go @@ -3,13 +3,16 @@ package webhooknotify import ( "context" "errors" + "fmt" "sync" "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" "github.com/riverqueue/river" "github.com/tokencanopy/e2a/internal/identity" "github.com/tokencanopy/e2a/internal/jobs" + "github.com/tokencanopy/e2a/internal/sendingpolicy" ) // Jobs is the webhook health-notification integration on the shared River @@ -29,6 +32,8 @@ type Jobs struct { store Store enq jobs.Enqueuer metrics Metrics + gate sendingpolicy.Gate + pool *pgxpool.Pool mu sync.RWMutex deliverer Deliverer @@ -38,6 +43,18 @@ type Jobs struct { // deliverer yet). func NewJobs(store Store) *Jobs { return &Jobs{store: store} } +// WithGate injects the sending-protection gate and the pool its legacy +// resolver and arg stamp use. Chainable; nil keeps the gateless default. +func (j *Jobs) WithGate(g sendingpolicy.Gate, pool *pgxpool.Pool) *Jobs { + if g != nil { + j.gate = g + } + if pool != nil { + j.pool = pool + } + return j +} + // SetEnqueuer injects the shared client so the EnqueueTx methods can // insert jobs. func (j *Jobs) SetEnqueuer(e jobs.Enqueuer) { j.enq = e } @@ -55,14 +72,14 @@ func (j *Jobs) SetDeliverer(d Deliverer) { // concrete one set via SetDeliverer. Until that is wired (the brief // startup window before the notifier is built) it returns a retryable // outcome, so a pending job simply retries rather than dropping. -func (j *Jobs) Deliver(ctx context.Context, wh *identity.Webhook, kind string) DeliverOutcome { +func (j *Jobs) Deliver(ctx context.Context, wh *identity.Webhook, kind string, auth sendingpolicy.ProviderAuthorization) DeliverOutcome { j.mu.RLock() d := j.deliverer j.mu.RUnlock() if d == nil { return DeliverOutcome{Err: errors.New("webhook notifier not wired yet — retrying")} } - return d.Deliver(ctx, wh, kind) + return d.Deliver(ctx, wh, kind, auth) } // WithMetrics wires the observability backend the NotifyWorker emits the @@ -76,15 +93,60 @@ func (j *Jobs) WithMetrics(m Metrics) *Jobs { // Deliverer). No periodics — the maintenance sweep is the only producer. // Implements jobs.Registrar. func (j *Jobs) RegisterJobs(w *river.Workers) []*river.PeriodicJob { - river.AddWorker(w, NewNotifyWorker(j.store, j).WithMetrics(j.metrics)) + river.AddWorker(w, j.NotifyWorker()) return nil } +// NotifyWorker builds the fully armed worker RegisterJobs registers. +func (j *Jobs) NotifyWorker() *NotifyWorker { + w := NewNotifyWorker(j.store, j).WithMetrics(j.metrics).WithGate(j.gate).WithOperationResolver(j.ResolveLegacyOperation) + if j.pool != nil { + w = w.WithArgStamper(func(ctx context.Context, jobID int64, ref sendingpolicy.OperationRef) error { + return jobs.StampJobArg(ctx, j.pool, jobID, "operation_ref", ref) + }) + } + return w +} + +// ResolveLegacyOperation prepares the notification operation for a job that +// carries no reference, in its own committed transaction, through the same +// PrepareNotificationTx the sweep's enqueue runs. +func (j *Jobs) ResolveLegacyOperation(ctx context.Context, webhookID string) (sendingpolicy.OperationRef, error) { + if j.gate == nil || j.pool == nil { + return sendingpolicy.OperationRef{}, fmt.Errorf("webhook notify: legacy operation resolver is not wired") + } + tx, err := j.pool.Begin(ctx) + if err != nil { + return sendingpolicy.OperationRef{}, fmt.Errorf("begin legacy resolve: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + ref, err := j.gate.PrepareNotificationTx(ctx, tx, sendingpolicy.NewWebhookHealthNotificationRef(webhookID)) + if err != nil { + return sendingpolicy.OperationRef{}, err + } + if err := tx.Commit(ctx); err != nil { + return sendingpolicy.OperationRef{}, fmt.Errorf("commit legacy resolve: %w", err) + } + return ref, nil +} + // EnqueueWebhookNotifyTx inserts one webhook_notify job in the caller's // transaction — the maintenance sweep's, so the state transition and its // notification job commit atomically (the design's SC2 argument). +// +// With a gate wired the notification's operation is prepared here, against +// the locked webhook row, so the owning account is charged and the worker +// never derives attribution. func (j *Jobs) EnqueueWebhookNotifyTx(ctx context.Context, tx pgx.Tx, webhookID, kind string) (int64, error) { - res, err := j.enq.InsertTx(ctx, tx, WebhookNotifyArgs{WebhookID: webhookID, NotifyKind: kind}, &river.InsertOpts{ + args := WebhookNotifyArgs{WebhookID: webhookID, NotifyKind: kind} + if j.gate != nil { + ref, err := j.gate.PrepareNotificationTx(ctx, tx, sendingpolicy.NewWebhookHealthNotificationRef(webhookID)) + if err != nil { + return 0, fmt.Errorf("prepare notification operation: %w", err) + } + args.OperationRef = &ref + } + res, err := j.enq.InsertTx(ctx, tx, args, &river.InsertOpts{ Queue: jobs.QueueNotify, MaxAttempts: MaxNotifyAttempts, }) diff --git a/internal/webhooknotify/notifier.go b/internal/webhooknotify/notifier.go index 659d24f4b..09e198094 100644 --- a/internal/webhooknotify/notifier.go +++ b/internal/webhooknotify/notifier.go @@ -12,6 +12,7 @@ import ( "github.com/tokencanopy/e2a/internal/identity" "github.com/tokencanopy/e2a/internal/outbound" + "github.com/tokencanopy/e2a/internal/sendingpolicy" ) // notifyLocalPart is the fallback local-part of the sender address, used @@ -51,15 +52,15 @@ type NotifierStore interface { // relay is the narrow send surface (*outbound.SMTPRelay satisfies it). // SendOnce, not Send: this runs inside a River job, so River owns retries. -type relay interface { - SendOnce(envelopeFrom string, recipients []string, message []byte) (string, error) +type submitter interface { + SubmitOnce(ctx context.Context, auth sendingpolicy.ProviderAuthorization, env outbound.Envelope) (outbound.ProviderResult, error) } // Notifier composes and sends the two webhook health emails. Construct // with New; the NotifyWorker drives Deliver. type Notifier struct { - store NotifierStore - relay relay + store NotifierStore + submitter submitter // dkim, when non-nil, signs each email for the From-header domain // before it reaches the relay (see WithDKIM). dkim outbound.DKIMKeyLookup @@ -89,7 +90,7 @@ type Notifier struct { // local part on fromDomain; replyTo is the optional // notifications.reply_to config value, empty = no Reply-To header. // publicURL builds the dashboard link; empty degrades to generic copy. -func New(store NotifierStore, r relay, fromDomain, fromAddress, replyTo, publicURL string) *Notifier { +func New(store NotifierStore, s submitter, fromDomain, fromAddress, replyTo, publicURL string) *Notifier { addr := strings.TrimSpace(fromAddress) if addr == "" { addr = fmt.Sprintf("%s@%s", notifyLocalPart, fromDomain) @@ -100,7 +101,7 @@ func New(store NotifierStore, r relay, fromDomain, fromAddress, replyTo, publicU } return &Notifier{ store: store, - relay: r, + submitter: s, fromAddress: addr, fromDomain: msgIDDomain, replyTo: strings.TrimSpace(replyTo), @@ -131,8 +132,8 @@ func (n *Notifier) WithDKIM(lookup outbound.DKIMKeyLookup) *Notifier { // Deliver composes and sends one health email, classifying the result for // the NotifyWorker. Implements Deliverer. -func (n *Notifier) Deliver(ctx context.Context, wh *identity.Webhook, kind string) DeliverOutcome { - if err := n.send(ctx, wh, kind); err != nil { +func (n *Notifier) Deliver(ctx context.Context, wh *identity.Webhook, kind string, auth sendingpolicy.ProviderAuthorization) DeliverOutcome { + if err := n.send(ctx, wh, kind, auth); err != nil { return DeliverOutcome{ Err: err, Permanent: outbound.IsPermanentSMTPError(err) || errors.Is(err, errNoOwnerEmail), @@ -142,7 +143,7 @@ func (n *Notifier) Deliver(ctx context.Context, wh *identity.Webhook, kind strin return DeliverOutcome{} } -func (n *Notifier) send(ctx context.Context, wh *identity.Webhook, kind string) error { +func (n *Notifier) send(ctx context.Context, wh *identity.Webhook, kind string, auth sendingpolicy.ProviderAuthorization) error { if n == nil { return nil } @@ -235,7 +236,11 @@ func (n *Notifier) send(ctx context.Context, wh *identity.Webhook, kind string) message = signed } - if _, err := n.relay.SendOnce(n.fromAddress, []string{owner.Email}, message); err != nil { + if _, err := n.submitter.SubmitOnce(ctx, auth, outbound.Envelope{ + From: n.fromAddress, + Recipients: []string{owner.Email}, + Message: message, + }); err != nil { return fmt.Errorf("webhook notify: smtp send: %w", err) } diff --git a/internal/webhooknotify/notifier_test.go b/internal/webhooknotify/notifier_test.go index c3913f34c..5a2493be4 100644 --- a/internal/webhooknotify/notifier_test.go +++ b/internal/webhooknotify/notifier_test.go @@ -9,6 +9,8 @@ import ( "github.com/tokencanopy/e2a/internal/dkim" "github.com/tokencanopy/e2a/internal/identity" + "github.com/tokencanopy/e2a/internal/outbound" + "github.com/tokencanopy/e2a/internal/sendingpolicy" ) type stubStore struct { @@ -33,9 +35,14 @@ type captureRelay struct { err error } -func (r *captureRelay) SendOnce(from string, to []string, msg []byte) (string, error) { - r.from, r.to, r.message = from, to, msg - return "queued-id", r.err +// SubmitOnce satisfies the notifier's submitter seam: it captures the envelope +// the notifier hands over and returns the scripted error. +func (r *captureRelay) SubmitOnce(_ context.Context, _ sendingpolicy.ProviderAuthorization, env outbound.Envelope) (outbound.ProviderResult, error) { + r.from, r.to, r.message = env.From, env.Recipients, env.Message + if r.err != nil { + return outbound.ProviderResult{}, r.err + } + return outbound.ProviderResult{ProviderMessageID: "queued-id"}, nil } func testWebhook() *identity.Webhook { @@ -63,7 +70,7 @@ func TestNotifier_DisabledEmailContent(t *testing.T) { relay := &captureRelay{} n := New(okStore(), relay, "send.example.com", "", "", "https://app.example.com") - out := n.Deliver(context.Background(), testWebhook(), KindDisabled) + out := n.Deliver(context.Background(), testWebhook(), KindDisabled, sendingpolicy.ProviderAuthorization{}) if out.Err != nil { t.Fatalf("Deliver: %v", out.Err) } @@ -111,7 +118,7 @@ func TestNotifier_WarningEmailContent(t *testing.T) { wh.Enabled = true wh.AutoDisabledAt = nil wh.AutoDisableReason = "" - out := n.Deliver(context.Background(), wh, KindWarning) + out := n.Deliver(context.Background(), wh, KindWarning, sendingpolicy.ProviderAuthorization{}) if out.Err != nil { t.Fatalf("Deliver: %v", out.Err) } @@ -140,7 +147,7 @@ func TestNotifier_ConfiguredFromAddress(t *testing.T) { if got := n.FromAddress(); got != "support@corp.example" { t.Fatalf("FromAddress = %q", got) } - out := n.Deliver(context.Background(), testWebhook(), KindDisabled) + out := n.Deliver(context.Background(), testWebhook(), KindDisabled, sendingpolicy.ProviderAuthorization{}) if out.Err != nil { t.Fatalf("Deliver: %v", out.Err) } @@ -172,7 +179,7 @@ func TestNotifier_ConfiguredReplyTo(t *testing.T) { relay := &captureRelay{} n := New(okStore(), relay, "send.example.com", "support@send.example.com", "support@agents.example.com", "") - if out := n.Deliver(context.Background(), testWebhook(), KindDisabled); out.Err != nil { + if out := n.Deliver(context.Background(), testWebhook(), KindDisabled, sendingpolicy.ProviderAuthorization{}); out.Err != nil { t.Fatalf("Deliver: %v", out.Err) } msg := string(relay.message) @@ -192,7 +199,7 @@ func TestNotifier_NoOwnerEmailIsPermanent(t *testing.T) { st.owner = &identity.User{ID: "user_1", Email: ""} n := New(st, &captureRelay{}, "send.example.com", "", "", "") - out := n.Deliver(context.Background(), testWebhook(), KindDisabled) + out := n.Deliver(context.Background(), testWebhook(), KindDisabled, sendingpolicy.ProviderAuthorization{}) if out.Err == nil { t.Fatal("expected an error for a missing owner email") } @@ -206,7 +213,7 @@ func TestNotifier_TransientStoreErrorIsRetryable(t *testing.T) { st.statsErr = errors.New("db blip") n := New(st, &captureRelay{}, "send.example.com", "", "", "") - out := n.Deliver(context.Background(), testWebhook(), KindDisabled) + out := n.Deliver(context.Background(), testWebhook(), KindDisabled, sendingpolicy.ProviderAuthorization{}) if out.Err == nil { t.Fatal("expected an error") } @@ -244,7 +251,7 @@ func TestNotifier_SignsWithDKIMWhenKeyExists(t *testing.T) { relay := &captureRelay{} n := New(okStore(), relay, "send.example.com", "support@corp.example", "", "").WithDKIM(lookup) - if out := n.Deliver(context.Background(), testWebhook(), KindDisabled); out.Err != nil { + if out := n.Deliver(context.Background(), testWebhook(), KindDisabled, sendingpolicy.ProviderAuthorization{}); out.Err != nil { t.Fatalf("Deliver: %v", out.Err) } msg := string(relay.message) @@ -264,7 +271,7 @@ func TestNotifier_SendsUnsignedWhenNoDKIMKey(t *testing.T) { relay := &captureRelay{} n := New(okStore(), relay, "send.example.com", "", "", "").WithDKIM(lookup) - if out := n.Deliver(context.Background(), testWebhook(), KindDisabled); out.Err != nil { + if out := n.Deliver(context.Background(), testWebhook(), KindDisabled, sendingpolicy.ProviderAuthorization{}); out.Err != nil { t.Fatalf("Deliver must succeed unsigned: %v", out.Err) } if strings.Contains(string(relay.message), "DKIM-Signature:") { @@ -273,7 +280,7 @@ func TestNotifier_SendsUnsignedWhenNoDKIMKey(t *testing.T) { // And with no lookup wired at all (zero-config self-host). relay2 := &captureRelay{} n2 := New(okStore(), relay2, "send.example.com", "", "", "") - if out := n2.Deliver(context.Background(), testWebhook(), KindDisabled); out.Err != nil { + if out := n2.Deliver(context.Background(), testWebhook(), KindDisabled, sendingpolicy.ProviderAuthorization{}); out.Err != nil { t.Fatalf("Deliver must succeed without a DKIM lookup: %v", out.Err) } } @@ -288,7 +295,7 @@ func TestNotifier_ReasonIsHTMLEscaped(t *testing.T) { wh := testWebhook() wh.AutoDisableReason = "" - if out := n.Deliver(context.Background(), wh, KindDisabled); out.Err != nil { + if out := n.Deliver(context.Background(), wh, KindDisabled, sendingpolicy.ProviderAuthorization{}); out.Err != nil { t.Fatalf("Deliver: %v", out.Err) } // The text/plain part may carry the raw string (harmless in plain diff --git a/internal/webhooknotify/worker.go b/internal/webhooknotify/worker.go index e33e5bf14..3fea56def 100644 --- a/internal/webhooknotify/worker.go +++ b/internal/webhooknotify/worker.go @@ -22,6 +22,7 @@ import ( "github.com/riverqueue/river" "github.com/tokencanopy/e2a/internal/identity" + "github.com/tokencanopy/e2a/internal/sendingpolicy" ) // Notification kinds. One worker, two templates: the guards and the @@ -56,6 +57,10 @@ const notifyOutageSnooze = 5 * time.Minute // of truth) each attempt, so the guards always see current state. type WebhookNotifyArgs struct { WebhookID string `json:"webhook_id"` + // OperationRef is the durable sending operation the sweep's transaction + // prepared; a job from a pre-floor slot carries none and is resolved at + // fire time, then stamped. + OperationRef *sendingpolicy.OperationRef `json:"operation_ref,omitempty"` // NotifyKind ∈ {warning, disabled}. (Named NotifyKind because river's // JobArgs interface reserves the Kind() method name.) NotifyKind string `json:"kind"` @@ -75,9 +80,16 @@ type DeliverOutcome struct { // Deliverer composes and sends one health email. Implemented by *Notifier // (compose + SMTPRelay.SendOnce + classify). type Deliverer interface { - Deliver(ctx context.Context, wh *identity.Webhook, kind string) DeliverOutcome + Deliver(ctx context.Context, wh *identity.Webhook, kind string, auth sendingpolicy.ProviderAuthorization) DeliverOutcome } +// OperationResolver recovers the durable operation for a job that carries no +// reference, through the same Prepare path an enqueue runs. +type OperationResolver func(ctx context.Context, webhookID string) (sendingpolicy.OperationRef, error) + +// ArgStamper persists a resolved reference into the job's args. +type ArgStamper func(ctx context.Context, jobID int64, ref sendingpolicy.OperationRef) error + // Store is the read surface the worker needs. *identity.Store satisfies it. type Store interface { // GetWebhookByIDInternal loads the webhook with no ownership check — @@ -112,6 +124,9 @@ type NotifyWorker struct { river.WorkerDefaults[WebhookNotifyArgs] store Store deliverer Deliverer + gate sendingpolicy.Gate + resolve OperationResolver + stamp ArgStamper metrics Metrics // nil ⇒ no emission (nil-safe via emitNotify) } @@ -121,6 +136,30 @@ func NewNotifyWorker(store Store, deliverer Deliverer) *NotifyWorker { // WithMetrics swaps in a metrics backend. Nil-safe: unset (or nil) means no // emission, so tests and self-host builds don't have to wire anything. +// WithGate injects the sending-protection gate every notification must pass. +func (w *NotifyWorker) WithGate(g sendingpolicy.Gate) *NotifyWorker { + if g != nil { + w.gate = g + } + return w +} + +// WithOperationResolver injects the legacy-argument resolver. +func (w *NotifyWorker) WithOperationResolver(r OperationResolver) *NotifyWorker { + if r != nil { + w.resolve = r + } + return w +} + +// WithArgStamper injects the job-args stamp used after a legacy resolution. +func (w *NotifyWorker) WithArgStamper(s ArgStamper) *NotifyWorker { + if s != nil { + w.stamp = s + } + return w +} + func (w *NotifyWorker) WithMetrics(m Metrics) *NotifyWorker { w.metrics = m return w @@ -185,7 +224,50 @@ func (w *NotifyWorker) Work(ctx context.Context, job *river.Job[WebhookNotifyArg return nil } - out := w.deliverer.Deliver(ctx, wh, kind) + // Every provider call is authorized: Reserve, hold without I/O, then + // ConsumeAttempt as the last decision before the deliverer's submitter + // redeems the token. A health notice has no durable hold class; the + // guards above re-run on every execution and drop a notice that went + // stale while it waited. + auth := sendingpolicy.ProviderAuthorization{} + if w.gate != nil { + ref, err := w.operationFor(ctx, job) + if err != nil { + if errors.Is(err, sendingpolicy.ErrSourceUnavailable) { + w.emitNotify(kind, outcomeSkipped) + return nil + } + w.emitNotify(kind, outcomeRetryable) + return err + } + early, attempt, err := w.gate.Reserve(ctx, ref) + if err != nil { + if errors.Is(err, sendingpolicy.ErrSourceUnavailable) { + w.emitNotify(kind, outcomeSkipped) + return nil + } + w.emitNotify(kind, outcomeOutage) + return river.JobSnooze(notifyOutageSnooze) + } + if !early.Allow { + return w.holdVerdict(kind, early) + } + decision, token, err := w.gate.ConsumeAttempt(ctx, attempt) + if err != nil { + if errors.Is(err, sendingpolicy.ErrSourceUnavailable) { + w.emitNotify(kind, outcomeSkipped) + return nil + } + w.emitNotify(kind, outcomeOutage) + return river.JobSnooze(notifyOutageSnooze) + } + if !decision.Allow || token == nil { + return w.holdVerdict(kind, decision) + } + auth = *token + } + + out := w.deliverer.Deliver(ctx, wh, kind, auth) if out.Err == nil { w.emitNotify(kind, outcomeSent) return nil @@ -208,3 +290,41 @@ func (w *NotifyWorker) Work(ctx context.Context, job *river.Job[WebhookNotifyArg w.emitNotify(kind, outcomeRetryable) return fmt.Errorf("webhook notify attempt %d failed: %w", job.Attempt, out.Err) } + +// operationFor returns the job's durable operation, resolving and stamping a +// legacy job through the sweep's Prepare path. +func (w *NotifyWorker) operationFor(ctx context.Context, job *river.Job[WebhookNotifyArgs]) (sendingpolicy.OperationRef, error) { + if job.Args.OperationRef != nil && !job.Args.OperationRef.IsZero() { + return *job.Args.OperationRef, nil + } + if w.resolve == nil { + return sendingpolicy.OperationRef{}, fmt.Errorf("webhook notify: legacy job %d carries no operation and no resolver is wired", job.ID) + } + ref, err := w.resolve(ctx, job.Args.WebhookID) + if err != nil { + return sendingpolicy.OperationRef{}, err + } + if w.stamp != nil { + if err := w.stamp(ctx, job.ID, ref); err != nil { + log.Printf("[webhook-notify] stamp operation on legacy job %d: %v", job.ID, err) + } + } + return ref, nil +} + +// holdVerdict turns a gate hold into River's answer: a terminal hold cancels +// the job; everything else waits for the gate's retry time or the outage pace. +func (w *NotifyWorker) holdVerdict(kind string, d sendingpolicy.Decision) error { + if d.Terminal { + w.emitNotify(kind, outcomePermanent) + return river.JobCancel(fmt.Errorf("webhook notify: sending policy: %s", d.Reason)) + } + w.emitNotify(kind, outcomeOutage) + delay := notifyOutageSnooze + if !d.RetryAt.IsZero() { + if until := time.Until(d.RetryAt); until > delay { + delay = until + } + } + return river.JobSnooze(delay) +} diff --git a/internal/webhooknotify/worker_test.go b/internal/webhooknotify/worker_test.go index 4990ada0d..4564ca63e 100644 --- a/internal/webhooknotify/worker_test.go +++ b/internal/webhooknotify/worker_test.go @@ -2,15 +2,18 @@ package webhooknotify_test import ( "context" + "encoding/json" "errors" "strings" "testing" "time" + "github.com/jackc/pgx/v5" "github.com/riverqueue/river" "github.com/riverqueue/river/rivertype" "github.com/tokencanopy/e2a/internal/identity" + "github.com/tokencanopy/e2a/internal/sendingpolicy" "github.com/tokencanopy/e2a/internal/webhooknotify" ) @@ -29,7 +32,7 @@ type fakeDeliverer struct { kinds []string } -func (f *fakeDeliverer) Deliver(_ context.Context, _ *identity.Webhook, kind string) webhooknotify.DeliverOutcome { +func (f *fakeDeliverer) Deliver(_ context.Context, _ *identity.Webhook, kind string, _ sendingpolicy.ProviderAuthorization) webhooknotify.DeliverOutcome { f.called++ f.kinds = append(f.kinds, kind) return f.out @@ -232,3 +235,118 @@ func TestNotifyWorker_ErrorTriage(t *testing.T) { fm.only(t, webhooknotify.KindDisabled, "retryable") }) } + +// fakeGate is a scriptable sendingpolicy.Gate for the worker-order tests. +type fakeGate struct { + reserve sendingpolicy.Decision + consume sendingpolicy.Decision + reserveErr error + reserves int + consumes int +} + +func allowAll() *fakeGate { + return &fakeGate{reserve: sendingpolicy.Decision{Allow: true}, consume: sendingpolicy.Decision{Allow: true}} +} + +func (g *fakeGate) PrepareExternalTx(context.Context, pgx.Tx, string) (sendingpolicy.AcceptanceDecision, sendingpolicy.OperationRef, error) { + return sendingpolicy.AcceptanceAccept, sendingpolicy.OperationRef{}, nil +} +func (g *fakeGate) PrepareNotificationTx(context.Context, pgx.Tx, sendingpolicy.NotificationRef) (sendingpolicy.OperationRef, error) { + return refFor("op_prepared"), nil +} +func (g *fakeGate) PrepareProtectionNoticeTx(context.Context, pgx.Tx, sendingpolicy.ProtectionNoticeRef) (sendingpolicy.OperationRef, error) { + return sendingpolicy.OperationRef{}, nil +} +func (g *fakeGate) PreparePublicFeedback(context.Context, sendingpolicy.PublicFeedbackRef) (sendingpolicy.OperationRef, error) { + return sendingpolicy.OperationRef{}, nil +} +func (g *fakeGate) Reserve(context.Context, sendingpolicy.OperationRef) (sendingpolicy.Decision, sendingpolicy.AttemptRef, error) { + g.reserves++ + return g.reserve, sendingpolicy.AttemptRef{}, g.reserveErr +} +func (g *fakeGate) ConsumeAttempt(context.Context, sendingpolicy.AttemptRef) (sendingpolicy.Decision, *sendingpolicy.ProviderAuthorization, error) { + g.consumes++ + if !g.consume.Allow { + return g.consume, nil, nil + } + return g.consume, &sendingpolicy.ProviderAuthorization{}, nil +} +func (g *fakeGate) RedeemProviderCall(context.Context, sendingpolicy.ProviderAuthorization) error { + return nil +} +func (g *fakeGate) DeferAttempt(context.Context, sendingpolicy.AttemptRef) error { return nil } +func (g *fakeGate) CancelAttempt(context.Context, sendingpolicy.AttemptRef) error { return nil } +func (g *fakeGate) SettleProvider(context.Context, sendingpolicy.ProviderSettlement) error { + return nil +} +func (g *fakeGate) SettleOperation(context.Context, sendingpolicy.OperationRef, sendingpolicy.SettlementOutcome, string) error { + return nil +} +func (g *fakeGate) LookupOperation(_ context.Context, id string) (sendingpolicy.OperationRef, error) { + return refFor(id), nil +} + +func refFor(id string) sendingpolicy.OperationRef { + var ref sendingpolicy.OperationRef + if err := json.Unmarshal([]byte(`{"v":1,"id":"`+id+`"}`), &ref); err != nil { + panic(err) + } + return ref +} + +func gatedJob(webhookID, kind string, attempt int) *river.Job[webhooknotify.WebhookNotifyArgs] { + j := job(webhookID, kind, attempt) + ref := refFor("op_" + webhookID) + j.Args.OperationRef = &ref + return j +} + +func isSnooze(err error) bool { + var snooze *river.JobSnoozeError + return errors.As(err, &snooze) +} + +func TestNotifyWorker_GatedPathAuthorizesThenDelivers(t *testing.T) { + fd := &fakeDeliverer{} + fm := &fakeMetrics{} + g := allowAll() + w := webhooknotify.NewNotifyWorker(&fakeStore{wh: hook(false, nil)}, fd).WithMetrics(fm).WithGate(g) + if err := w.Work(context.Background(), gatedJob("wh_1", webhooknotify.KindDisabled, 1)); err != nil { + t.Fatalf("Work: %v", err) + } + if g.reserves != 1 || g.consumes != 1 || fd.called != 1 { + t.Fatalf("reserves=%d consumes=%d delivers=%d, want 1/1/1", g.reserves, g.consumes, fd.called) + } +} + +func TestNotifyWorker_GateHoldSnoozesWithoutDelivery(t *testing.T) { + for name, g := range map[string]*fakeGate{ + "early hold": {reserve: sendingpolicy.Decision{Allow: false, Reason: sendingpolicy.ReasonAccountPaused}}, + "late hold": {reserve: sendingpolicy.Decision{Allow: true}, consume: sendingpolicy.Decision{Allow: false, Reason: sendingpolicy.ReasonGlobalAllBudget, RetryAt: time.Now().Add(time.Hour)}}, + "gate error": {reserveErr: errors.New("policy db down")}, + } { + fd := &fakeDeliverer{} + w := webhooknotify.NewNotifyWorker(&fakeStore{wh: hook(false, nil)}, fd).WithMetrics(&fakeMetrics{}).WithGate(g) + if err := w.Work(context.Background(), gatedJob("wh_1", webhooknotify.KindDisabled, 1)); !isSnooze(err) || fd.called != 0 { + t.Fatalf("%s: err=%v delivers=%d, want snooze with no I/O", name, err, fd.called) + } + } +} + +func TestNotifyWorker_LegacyJobResolvesAndStampsOnce(t *testing.T) { + fd := &fakeDeliverer{} + resolved, stamped := 0, 0 + w := webhooknotify.NewNotifyWorker(&fakeStore{wh: hook(false, nil)}, fd).WithMetrics(&fakeMetrics{}).WithGate(allowAll()). + WithOperationResolver(func(_ context.Context, id string) (sendingpolicy.OperationRef, error) { + resolved++ + return refFor("op_" + id), nil + }). + WithArgStamper(func(context.Context, int64, sendingpolicy.OperationRef) error { stamped++; return nil }) + if err := w.Work(context.Background(), job("wh_legacy", webhooknotify.KindDisabled, 1)); err != nil { + t.Fatalf("Work: %v", err) + } + if resolved != 1 || stamped != 1 || fd.called != 1 { + t.Fatalf("resolved=%d stamped=%d delivers=%d, want 1/1/1", resolved, stamped, fd.called) + } +} From 89a1e84b2851895d62542622e8343e764b2508ce Mon Sep 17 00:00:00 2001 From: jiashuoz <39790535+jiashuoz@users.noreply.github.com> Date: Sat, 5 Sep 2026 00:07:30 -0700 Subject: [PATCH 10/12] fix(outbound): key notification operations by source and charge last Review round 1 of the provider-seam closure (B7). - PrepareNotificationTx derives the operation id from its source (op_hitl_, op_wh___), so a repeat preparation yields one operation and the notify workers cancel a job whose reference names any other operation. - The notify Deliverer is Compose + Submit; workers run compose -> Reserve -> hold -> ConsumeAttempt -> Submit, so a compose failure charges nothing and the token is consumed right before the socket. - Reconcile command re-reads each job under FOR UPDATE and skips one a worker claimed or stamped meanwhile; counts separate paused/skipped. - Feedback loop submits the token's canonical recipients, paces retries to fit the handler budget, keeps the SMTP error on a deadline, releases a reserved attempt when authorize errors, no panic on id mint. - Closure guard never skips, matches method references, exempts the SubmitOnce symbol only, asserts its sentinel, fences the SES v2 import. - Webhook health notices older than seven days are dropped. - Wiring test for the notification bundles and the API seam; StampJobArg tests for the jobs coverage floor. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01AjfGxvXW6fNKWGFHuo68yX --- cmd/e2a/main.go | 20 +- cmd/e2a/outbound_wiring.go | 45 +++++ cmd/e2a/sending_policy_wiring_test.go | 49 +++++ cmd/e2a/sending_reconcile.go | 33 +++- cmd/e2a/sending_reconcile_test.go | 88 ++++++++- docs/design/async-message-pipeline.md | 32 +++- internal/agent/api.go | 38 +++- internal/agent/feedback_seam_test.go | 34 ++++ internal/hitlnotify/jobs.go | 37 +++- internal/hitlnotify/notifier.go | 93 +++++---- internal/hitlnotify/notifier_test.go | 6 +- internal/hitlnotify/worker.go | 78 ++++++-- internal/hitlnotify/worker_test.go | 122 +++++++++++- internal/jobs/argstamp_test.go | 66 +++++++ .../provider_authorization_guard_test.go | 131 +++++++++---- internal/sendingpolicy/operations.go | 31 ++- .../sendingpolicy/store_integration_test.go | 102 +++++++++- internal/sendingpolicy/types.go | 46 ++++- internal/webhooknotify/jobs.go | 43 +++-- internal/webhooknotify/notifier.go | 70 ++++--- internal/webhooknotify/worker.go | 118 ++++++++++-- internal/webhooknotify/worker_test.go | 179 ++++++++++++++++-- 22 files changed, 1246 insertions(+), 215 deletions(-) create mode 100644 internal/jobs/argstamp_test.go diff --git a/cmd/e2a/main.go b/cmd/e2a/main.go index e3d730456..09eefa404 100644 --- a/cmd/e2a/main.go +++ b/cmd/e2a/main.go @@ -393,10 +393,17 @@ func main() { // later via SetDeliverer — mirrors inbound's late-bound Processor. Gated on the // same relay+public-URL config as the notifier itself; when unconfigured, no jobs // register and the hold takes the plain path (no notification). - var notifyJobs *hitlnotify.Jobs notifierEnabled := cfg.OutboundSMTP.FromDomain != "" && cfg.HTTP.PublicURL != "" - if notifierEnabled { - notifyJobs = hitlnotify.NewJobs(store).WithGate(sendingGate, pool) + notification := newNotificationJobs(notificationDeps{ + store: store, + pool: pool, + gate: sendingGate, + metrics: metrics, + hitlEnabled: notifierEnabled, + webhookEnabled: cfg.OutboundSMTP.FromDomain != "", + }) + notifyJobs := notification.hitl + if notifyJobs != nil { registrars = append(registrars, notifyJobs) } @@ -409,9 +416,8 @@ func main() { // (generic dashboard copy instead of a link). When unconfigured, no jobs // register and the sweep transitions state without notifications // (pre-feature behavior). - var webhookNotifyJobs *webhooknotify.Jobs - if cfg.OutboundSMTP.FromDomain != "" { - webhookNotifyJobs = webhooknotify.NewJobs(store).WithMetrics(metrics).WithGate(sendingGate, pool) + webhookNotifyJobs := notification.webhook + if webhookNotifyJobs != nil { registrars = append(registrars, webhookNotifyJobs) } @@ -837,7 +843,7 @@ func main() { // The outbound accept-tx enqueuer is mandatory: DeliverOutbound always // persists+enqueues and returns accepted before provider submission. api.SetOutboundEnqueuer(outboundJobs) - api.SetProviderSubmitter(providerSubmitter, sendingGate) + outboundSending.armAPI(api) // Slices 6 + 7: customer-facing events API needs the raw pool to // query webhook_events and write webhook_subscriber_deliveries on // replay. Kept as a separate setter so a future refactor can route diff --git a/cmd/e2a/outbound_wiring.go b/cmd/e2a/outbound_wiring.go index 462cdea66..7ec9b6b38 100644 --- a/cmd/e2a/outbound_wiring.go +++ b/cmd/e2a/outbound_wiring.go @@ -4,9 +4,12 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "github.com/tokencanopy/e2a/internal/agent" + "github.com/tokencanopy/e2a/internal/hitlnotify" + "github.com/tokencanopy/e2a/internal/identity" "github.com/tokencanopy/e2a/internal/outbound" "github.com/tokencanopy/e2a/internal/outboundsend" "github.com/tokencanopy/e2a/internal/sendingpolicy" + "github.com/tokencanopy/e2a/internal/webhooknotify" ) // outboundSendingDeps is everything the outbound composition root needs. It @@ -49,3 +52,45 @@ func newOutboundSending(d outboundSendingDeps) outboundSending { WithRateGate(d.rate) return outboundSending{gate: gate, submitter: submitter, jobs: jobs} } + +// notificationDeps is what the notification composition needs: the same gate +// and pool the customer path uses, plus the two config gates main applies. +type notificationDeps struct { + store *identity.Store + pool *pgxpool.Pool + gate sendingpolicy.Gate + metrics webhooknotify.Metrics + hitlEnabled bool // outbound_smtp.from_domain and http.public_url set + webhookEnabled bool // outbound_smtp.from_domain set +} + +// notificationJobs are the two notification job bundles, nil when their +// feature is unconfigured (no worker registers, the sweep/hold take the +// plain path). +type notificationJobs struct { + hitl *hitlnotify.Jobs + webhook *webhooknotify.Jobs +} + +// newNotificationJobs composes the notification bundles over the ONE gate. +// Every enqueue prepares a customer_notification operation in the source +// transaction and every worker execution authorizes through the gate; a +// bundle built any other way would fail closed at runtime (empty token) with +// an error that says nothing about wiring, which is why the composition is +// factored here and pinned by the wiring test. +func newNotificationJobs(d notificationDeps) notificationJobs { + var n notificationJobs + if d.hitlEnabled { + n.hitl = hitlnotify.NewJobs(d.store).WithGate(d.gate, d.pool) + } + if d.webhookEnabled { + n.webhook = webhooknotify.NewJobs(d.store).WithMetrics(d.metrics).WithGate(d.gate, d.pool) + } + return n +} + +// armAPI hands the API the authorized seam for the platform mail it sends +// itself (public feedback). +func (s outboundSending) armAPI(api *agent.API) { + api.SetProviderSubmitter(s.submitter, s.gate) +} diff --git a/cmd/e2a/sending_policy_wiring_test.go b/cmd/e2a/sending_policy_wiring_test.go index a7eebc1a4..206113818 100644 --- a/cmd/e2a/sending_policy_wiring_test.go +++ b/cmd/e2a/sending_policy_wiring_test.go @@ -8,10 +8,12 @@ import ( "github.com/riverqueue/river" + "github.com/tokencanopy/e2a/internal/agent" "github.com/tokencanopy/e2a/internal/config" "github.com/tokencanopy/e2a/internal/outbound" "github.com/tokencanopy/e2a/internal/sendingpolicy" "github.com/tokencanopy/e2a/internal/testutil/testdb" + "github.com/tokencanopy/e2a/internal/usage" ) // TestSendingPolicyWiring builds the production outbound composition from @@ -76,3 +78,50 @@ func TestSendingPolicyWiring(t *testing.T) { t.Fatal("a never-prepared operation resolved") } } + +// TestNotificationAndPlatformMailWiring pins the three composition-root +// edges the AST closure guard cannot see: both notification bundles hold the +// gate (so their enqueues prepare operations and their workers authorize), +// and the API holds the submitter + gate for public feedback. Dropping any +// of them fails closed at runtime with an opaque "authorization required" +// error; this is where it fails loudly instead. +func TestNotificationAndPlatformMailWiring(t *testing.T) { + pool := testdb.TestDB(t) + relay := outbound.NewSMTPRelay(&config.OutboundSMTPConfig{Host: "relay.invalid", Port: 587, FromDomain: "test.e2a.dev"}) + composed := newOutboundSending(outboundSendingDeps{ + pool: pool, + relay: relay, + secrets: sendingpolicy.Secrets{}, + source: sendingpolicy.PolicySourceConfig, + policy: sendingpolicy.DisabledPolicy(), + }) + + n := newNotificationJobs(notificationDeps{pool: pool, gate: composed.gate, hitlEnabled: true, webhookEnabled: true}) + if n.hitl == nil || n.hitl.Gate() != composed.gate { + t.Fatal("hitl notification bundle does not hold the composed gate") + } + if n.webhook == nil || n.webhook.Gate() != composed.gate { + t.Fatal("webhook notification bundle does not hold the composed gate") + } + // The registered workers are what run; they must carry the gate too. + if w := n.hitl.NotifyWorker(); w == nil || w.Gate() != composed.gate { + t.Fatal("hitl notify worker registered without the gate") + } + if w := n.webhook.NotifyWorker(); w == nil || w.Gate() != composed.gate { + t.Fatal("webhook notify worker registered without the gate") + } + + off := newNotificationJobs(notificationDeps{pool: pool, gate: composed.gate}) + if off.hitl != nil || off.webhook != nil { + t.Fatal("unconfigured notifications must register nothing") + } + + api := agent.NewAPI(nil, nil, relay, nil, usage.NewNoopUsageTracker(), "e2a.dev", "test.e2a.dev", "agents.e2a.dev", "", false) + if api.ProviderSubmitterWired() { + t.Fatal("a fresh API must not claim a submitter") + } + composed.armAPI(api) + if !api.ProviderSubmitterWired() { + t.Fatal("armAPI did not hand the API the submitter and gate") + } +} diff --git a/cmd/e2a/sending_reconcile.go b/cmd/e2a/sending_reconcile.go index f4c6b8e7c..78476f90e 100644 --- a/cmd/e2a/sending_reconcile.go +++ b/cmd/e2a/sending_reconcile.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "slices" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" @@ -46,6 +47,7 @@ type legacyReconcileCounts struct { Stamped int Cancelled int Paused int + Skipped int // moved on by a worker between the scan and the job's own transaction Failed int } @@ -111,15 +113,18 @@ func runReconcileLegacySendingJobs(ctx context.Context, pool *pgxpool.Pool, gate counts.Cancelled++ case legacyOutcomePaused: counts.Paused++ + case legacyOutcomeSkipped: + counts.Skipped++ } } fmt.Fprintf(stdout, "scanned: %d\n", counts.Scanned) fmt.Fprintf(stdout, "stamped: %d\n", counts.Stamped) fmt.Fprintf(stdout, "cancelled: %d\n", counts.Cancelled) - fmt.Fprintf(stdout, "paused: %d\n", counts.Paused) + fmt.Fprintf(stdout, "paused: %d (left unstamped for the worker's hold path; rerun after the account resumes)\n", counts.Paused) + fmt.Fprintf(stdout, "skipped: %d (picked up by a worker meanwhile; the worker resolves them)\n", counts.Skipped) fmt.Fprintf(stdout, "failed: %d\n", counts.Failed) - fmt.Fprintf(stdout, "remaining: %d\n", counts.remaining()) + fmt.Fprintf(stdout, "remaining: %d (undecided; nonzero exit)\n", counts.remaining()) if counts.remaining() != 0 { return fmt.Errorf("%d legacy sending job(s) could not be reconciled", counts.remaining()) } @@ -132,6 +137,7 @@ const ( legacyOutcomeStamped legacyOutcome = iota + 1 legacyOutcomeCancelled legacyOutcomePaused + legacyOutcomeSkipped ) // reconcileLegacySendingJob decides one job inside one transaction: the @@ -145,6 +151,27 @@ func reconcileLegacySendingJob(ctx context.Context, pool *pgxpool.Pool, client * } defer func() { _ = tx.Rollback(ctx) }() + // Re-read the job under its row lock: the scan ran outside this + // transaction, and a worker may have claimed the job (or resolved and + // stamped it itself) since. Deciding a job a worker now owns would + // prepare beside it and could cancel it mid-flight, so anything that + // left the reconcilable states is skipped and left to that worker. The + // lock also serializes against the worker's own stamp. + var state string + var stamped bool + err = tx.QueryRow(ctx, + `SELECT state, (args ? 'operation_ref') FROM river_job WHERE id = $1 FOR UPDATE`, jobID, + ).Scan(&state, &stamped) + if errors.Is(err, pgx.ErrNoRows) { + return legacyOutcomeSkipped, nil + } + if err != nil { + return 0, fmt.Errorf("lock job: %w", err) + } + if stamped || !slices.Contains(legacyReconcileStates, state) { + return legacyOutcomeSkipped, nil + } + var ref sendingpolicy.OperationRef var cancelReason string switch kind { @@ -186,7 +213,7 @@ func reconcileLegacySendingJob(ctx context.Context, pool *pgxpool.Pool, client * if err := json.Unmarshal(rawArgs, &args); err != nil { return 0, fmt.Errorf("decode args: %w", err) } - ref, cancelReason, err = prepareLegacyNotification(ctx, tx, gate, sendingpolicy.NewWebhookHealthNotificationRef(args.WebhookID)) + ref, cancelReason, err = prepareLegacyNotification(ctx, tx, gate, sendingpolicy.NewWebhookHealthNotificationRef(args.WebhookID, args.NotifyKind)) if err != nil { return 0, err } diff --git a/cmd/e2a/sending_reconcile_test.go b/cmd/e2a/sending_reconcile_test.go index bfe4fc847..ea82fdf71 100644 --- a/cmd/e2a/sending_reconcile_test.go +++ b/cmd/e2a/sending_reconcile_test.go @@ -12,6 +12,7 @@ import ( "github.com/tokencanopy/e2a/internal/jobs" "github.com/tokencanopy/e2a/internal/sendingpolicy" "github.com/tokencanopy/e2a/internal/testutil" + "github.com/tokencanopy/e2a/internal/webhooknotify" ) // insertLegacyJob enqueues a River job the way a pre-floor slot did: the @@ -52,7 +53,7 @@ func legacyJobState(t *testing.T, pool *pgxpool.Pool, id int64) (state, opID str return state, opID } -func seedReconcileSource(t *testing.T, store *identity.Store, slug string) (*identity.Message, *identity.Webhook) { +func seedReconcileSource(t *testing.T, pool *pgxpool.Pool, store *identity.Store, slug string) (*identity.Message, *identity.Webhook) { t.Helper() ctx := context.Background() user, err := store.CreateOrGetUser(ctx, "owner-"+slug+"@reviewer.test", "Owner", "google-reconcile-"+slug) @@ -80,6 +81,15 @@ func seedReconcileSource(t *testing.T, store *identity.Store, slug string) (*ide if err != nil { t.Fatal(err) } + // The sweep stamps the warning episode before it enqueues the notice; + // a legacy warning job's operation is keyed by that stamp. + if _, err := pool.Exec(ctx, `UPDATE webhooks SET warn_notified_at = now() WHERE id = $1`, wh.ID); err != nil { + t.Fatal(err) + } + wh, err = store.GetWebhookByIDInternal(ctx, wh.ID) + if err != nil { + t.Fatal(err) + } return msg, wh } @@ -93,7 +103,7 @@ func TestReconcileLegacySendingJobs(t *testing.T) { resetRiverJobs(t, pool) store := identity.NewStore(pool) gate := sendingpolicy.NewGate(pool, sendingpolicy.Secrets{}, sendingpolicy.PolicySourceConfig, sendingpolicy.DisabledPolicy()) - msg, wh := seedReconcileSource(t, store, "reconcile") + msg, wh := seedReconcileSource(t, pool, store, "reconcile") sendLive := insertLegacyJob(t, pool, "outbound_send", `{"message_id":"`+msg.ID+`"}`) sendGone := insertLegacyJob(t, pool, "outbound_send", `{"message_id":"msg_does_not_exist"}`) @@ -119,11 +129,11 @@ func TestReconcileLegacySendingJobs(t *testing.T) { if state, op := legacyJobState(t, pool, sendLive); state != "available" || op != msg.ID { t.Errorf("live send job: state=%s op=%q, want available with the message id", state, op) } - if state, op := legacyJobState(t, pool, hitlLive); state != "available" || !strings.HasPrefix(op, "op_") { - t.Errorf("live hitl job: state=%s op=%q, want available with a notification operation", state, op) + if state, op := legacyJobState(t, pool, hitlLive); state != "available" || op != sendingpolicy.HITLNotificationOperationID(msg.ID) { + t.Errorf("live hitl job: state=%s op=%q, want available with the message's notification operation", state, op) } - if state, op := legacyJobState(t, pool, whLive); state != "available" || !strings.HasPrefix(op, "op_") { - t.Errorf("live webhook job: state=%s op=%q, want available with a notification operation", state, op) + if state, op := legacyJobState(t, pool, whLive); state != "available" || op != webhooknotify.ExpectedOperationID(wh, webhooknotify.KindWarning) { + t.Errorf("live webhook job: state=%s op=%q, want available with the warning episode's operation", state, op) } for name, id := range map[string]int64{"send": sendGone, "webhook": whGone} { if state, op := legacyJobState(t, pool, id); state != "cancelled" || op != "" { @@ -181,3 +191,69 @@ func TestReconcileLegacySendingJobsReportsUndecided(t *testing.T) { t.Errorf("undecided job touched: state=%s op=%q", state, op) } } + +// TestReconcileLegacySendingJobsLeavesClaimedJobsToTheirWorker: a job that +// left the reconcilable states (a worker claimed it) or was stamped by its +// worker between the scan and its own transaction is skipped untouched — no +// second operation, no cancel under a running worker. +func TestReconcileLegacySendingJobsLeavesClaimedJobsToTheirWorker(t *testing.T) { + ctx := context.Background() + pool := testutil.TestDB(t) + resetRiverJobs(t, pool) + store := identity.NewStore(pool) + gate := sendingpolicy.NewGate(pool, sendingpolicy.Secrets{}, sendingpolicy.PolicySourceConfig, sendingpolicy.DisabledPolicy()) + msg, _ := seedReconcileSource(t, pool, store, "claimed") + + running := insertLegacyJob(t, pool, "hitl_notify", `{"message_id":"`+msg.ID+`"}`) + if _, err := pool.Exec(ctx, `UPDATE river_job SET state = 'running', attempted_at = now() WHERE id = $1`, running); err != nil { + t.Fatal(err) + } + orphanRunning := insertLegacyJob(t, pool, "outbound_send", `{"message_id":"msg_gone"}`) + if _, err := pool.Exec(ctx, `UPDATE river_job SET state = 'running', attempted_at = now() WHERE id = $1`, orphanRunning); err != nil { + t.Fatal(err) + } + + var ops int + if err := pool.QueryRow(ctx, `SELECT count(*) FROM sending_provider_operations`).Scan(&ops); err != nil { + t.Fatal(err) + } + var out bytes.Buffer + if err := runReconcileLegacySendingJobs(ctx, pool, gate, &out); err != nil { + t.Fatalf("reconcile: %v\n%s", err, out.String()) + } + if !strings.Contains(out.String(), "scanned: 0") { + t.Fatalf("running jobs must not be scanned:\n%s", out.String()) + } + for name, id := range map[string]int64{"running": running, "orphan running": orphanRunning} { + if state, op := legacyJobState(t, pool, id); state != "running" || op != "" { + t.Errorf("%s job touched: state=%s op=%q", name, state, op) + } + } + var after int + if err := pool.QueryRow(ctx, `SELECT count(*) FROM sending_provider_operations`).Scan(&after); err != nil { + t.Fatal(err) + } + if after != ops { + t.Errorf("operations minted for jobs the command did not own: %d → %d", ops, after) + } + + // The per-job transaction re-checks under lock: simulate a worker that + // claimed the job after the scan by driving the per-job step directly. + claimed := insertLegacyJob(t, pool, "hitl_notify", `{"message_id":"`+msg.ID+`"}`) + if _, err := pool.Exec(ctx, `UPDATE river_job SET state = 'running', attempted_at = now() WHERE id = $1`, claimed); err != nil { + t.Fatal(err) + } + client, err := jobs.New(pool, jobs.Config{}) + if err != nil { + t.Fatal(err) + } + outcome, err := reconcileLegacySendingJob(ctx, pool, client, gate, claimed, "hitl_notify", []byte(`{"message_id":"`+msg.ID+`"}`)) + if err != nil || outcome != legacyOutcomeSkipped { + t.Fatalf("claimed job: outcome=%v err=%v, want skipped", outcome, err) + } + stampedByWorker := insertLegacyJob(t, pool, "hitl_notify", `{"message_id":"`+msg.ID+`","operation_ref":{"v":1,"id":"op_hitl_`+msg.ID+`"}}`) + outcome, err = reconcileLegacySendingJob(ctx, pool, client, gate, stampedByWorker, "hitl_notify", []byte(`{"message_id":"`+msg.ID+`"}`)) + if err != nil || outcome != legacyOutcomeSkipped { + t.Fatalf("already stamped job: outcome=%v err=%v, want skipped", outcome, err) + } +} diff --git a/docs/design/async-message-pipeline.md b/docs/design/async-message-pipeline.md index e6ba9649b..b6cee6c6b 100644 --- a/docs/design/async-message-pipeline.md +++ b/docs/design/async-message-pipeline.md @@ -307,11 +307,20 @@ cross it: notices** (`internal/webhooknotify`): the enqueue prepares a `customer_notification` operation in the same transaction as the source row (`PrepareNotificationTx`, charged to the triggering account, shared - reputation class) and stamps it on the job; the worker runs the same - Reserve → early hold → ConsumeAttempt → authorized submit order as the - message worker, snoozing on a hold without provider I/O. A job from a - pre-floor slot resolves its operation at fire time and stamps it once - (`jobs.StampJobArg`), so the derivation never repeats. + reputation class) and stamps it on the job. The operation id is derived + from the source — `op_hitl_` for an approval request, + `op_wh___` for a health notice, where the + episode is the `warn_notified_at` / `auto_disabled_at` stamp the sweep + wrote in the same transaction — so preparing the same source twice yields + one operation, and the worker cancels a job whose reference names any + other operation (the binding the message worker enforces). The worker + order is compose → Reserve → early hold → ConsumeAttempt → authorized + submit: every fallible, provider-free step (owner lookup, token signing, + MIME, DKIM) runs before an ordinal is charged, and the token is consumed + immediately before the socket opens. A job from a pre-floor slot resolves + its operation at fire time and stamps it once (`jobs.StampJobArg`); with a + source-derived id a repeat resolve is harmless. A health notice older than + seven days is dropped rather than left snoozing behind a pause. - **Public feedback mail** (`POST /api/feedback`): the operation is keyed by a server-minted submission id and its envelope is the configured notify set, never the request, so the form cannot become a relay. No queue owns @@ -324,5 +333,16 @@ Operators cutting over a slot with a queued backlog run pending `outbound_send` / `hitl_notify` / `webhook_notify` job that has none, through exactly the Prepare path its enqueue would have used, cancels the ones whose source row is gone, and exits nonzero unless every scanned job was -decided. The workers resolve legacy jobs themselves, so the command is a +decided. Each job is re-read under its row lock inside its own transaction, +so one a worker claimed after the scan is skipped and left to that worker; +a paused account's message job is also left unstamped for the worker's hold +path. The workers resolve legacy jobs themselves, so the command is a convenience for a clean cutover, not a prerequisite. + +Two consequences worth knowing. Notification and feedback mail now cross the +same submitter as customer mail, so it carries `X-SES-CONFIGURATION-SET` +and SES publishes delivery feedback for it; none of it correlates to a +message row, and the SNS consumer acks it as unknown (a log line, no +suppression). And the closure guard fences `net/smtp` and the SES v2 SDK +import; a send through some other HTTP provider API would be a new +dependency, which is where review catches it. diff --git a/internal/agent/api.go b/internal/agent/api.go index 570b4dcdd..8a450f467 100644 --- a/internal/agent/api.go +++ b/internal/agent/api.go @@ -1644,6 +1644,10 @@ func (a *API) SetProviderSubmitter(submitter *outbound.ProviderSubmitter, gate s a.gate = gate } +// ProviderSubmitterWired reports whether the platform-mail seam is armed, for +// the composition root's wiring test. +func (a *API) ProviderSubmitterWired() bool { return a.submitter != nil && a.gate != nil } + // SendTestCore accepts (or HITL-holds) a platform test email to the agent's // own address. HTTP-free; shared by the legacy handler and the v1 layer. The // caller has already authed, resolved + owned the agent, domain-verified, @@ -1971,8 +1975,14 @@ func (a *API) sendFeedbackEmail(ctx context.Context, title, category, message, s // own charged ordinal: Reserve, ConsumeAttempt, one authorized submit. // The operation is keyed by a server-minted submission id and its // envelope is configuration, never the request, so the form cannot - // become an open relay however it is retried. - submissionID := feedbackSubmissionID() + // become an open relay however it is retried. What goes on the wire is + // the token's canonical recipient set: the configured TO/CC lists may + // overlap or differ in case, and the seam refuses an envelope whose raw + // count disagrees with its normalized one. + submissionID, err := feedbackSubmissionID() + if err != nil { + return err + } ref, err := a.gate.PreparePublicFeedback(ctx, sendingpolicy.NewPublicFeedbackRef(submissionID, rcpts)) if err != nil { return fmt.Errorf("prepare feedback operation: %w", err) @@ -1982,7 +1992,7 @@ func (a *API) sendFeedbackEmail(ctx context.Context, title, category, message, s if attempt > 0 { select { case <-ctx.Done(): - return ctx.Err() + return errors.Join(ctx.Err(), last) case <-time.After(feedbackRetryBackoff[attempt-1]): } } @@ -1995,12 +2005,19 @@ func (a *API) sendFeedbackEmail(ctx context.Context, title, category, message, s } decision, auth, err := a.gate.ConsumeAttempt(ctx, attemptRef) if err != nil { + // The ordinal is reserved and nothing will Reserve it again on + // this path (the request ends here), so give its units back + // rather than leave them charged until midnight. Best effort: + // the gate's day-scoped expiry is the backstop. + if cerr := a.gate.CancelAttempt(context.WithoutCancel(ctx), attemptRef); cerr != nil { + log.Printf("[feedback] release reserved attempt after authorize error: %v", cerr) + } return fmt.Errorf("authorize feedback attempt: %w", err) } if !decision.Allow || auth == nil { return fmt.Errorf("feedback send held by sending policy: %s", decision.Reason) } - _, err = a.submitter.SubmitOnce(ctx, *auth, outbound.Envelope{From: from, Recipients: rcpts, Message: raw}) + _, err = a.submitter.SubmitOnce(ctx, *auth, outbound.Envelope{From: from, Recipients: auth.AuthorizedRecipients(), Message: raw}) if err == nil { return nil } @@ -2016,20 +2033,21 @@ func (a *API) sendFeedbackEmail(ctx context.Context, title, category, message, s } // feedbackSendAttempts bounds the physical submissions one feedback request -// may make; feedbackRetryBackoff paces them. Each is a distinct charged -// attempt on the feedback operation. +// may make; feedbackRetryBackoff paces them so every attempt fits inside +// feedbackEmailTimeout. Each is a distinct charged attempt on the feedback +// operation. const feedbackSendAttempts = 4 -var feedbackRetryBackoff = []time.Duration{time.Second, 5 * time.Second, 15 * time.Second} +var feedbackRetryBackoff = []time.Duration{time.Second, 2 * time.Second, 3 * time.Second} // feedbackSubmissionID mints the server-side identity one feedback request's // operation is keyed by. -func feedbackSubmissionID() string { +func feedbackSubmissionID() (string, error) { var b [12]byte if _, err := rand.Read(b[:]); err != nil { - panic(fmt.Sprintf("feedback submission id: %v", err)) + return "", fmt.Errorf("feedback submission id: %w", err) } - return hex.EncodeToString(b[:]) + return hex.EncodeToString(b[:]), nil } // splitFeedbackAddrs parses a comma-separated address list from env config, diff --git a/internal/agent/feedback_seam_test.go b/internal/agent/feedback_seam_test.go index f2c4ac340..3318963d9 100644 --- a/internal/agent/feedback_seam_test.go +++ b/internal/agent/feedback_seam_test.go @@ -30,6 +30,7 @@ type scriptedSMTP struct { mu sync.Mutex messages []string + rcpts [][]string // RCPT TO per connection, in wire order conns int } @@ -63,6 +64,7 @@ func (s *scriptedSMTP) serve(conn net.Conn, reply string) { reader := bufio.NewReader(conn) fmt.Fprint(conn, "220 scripted ready\r\n") var data []string + var rcpts []string inData := false for { line, err := reader.ReadString('\n') @@ -77,6 +79,7 @@ func (s *scriptedSMTP) serve(conn net.Conn, reply string) { } s.mu.Lock() s.messages = append(s.messages, strings.Join(data, "\n")) + s.rcpts = append(s.rcpts, rcpts) s.mu.Unlock() if reply == "drop" { return @@ -86,6 +89,9 @@ func (s *scriptedSMTP) serve(conn net.Conn, reply string) { continue } switch { + case len(line) > 8 && strings.EqualFold(line[:8], "RCPT TO:"): + rcpts = append(rcpts, strings.Trim(strings.TrimSpace(line[8:]), "<>")) + fmt.Fprint(conn, "250 OK\r\n") case strings.EqualFold(line, "DATA"): inData = true fmt.Fprint(conn, "354 Go ahead\r\n") @@ -104,6 +110,12 @@ func (s *scriptedSMTP) received() ([]string, int) { return append([]string(nil), s.messages...), s.conns } +func (s *scriptedSMTP) recipients() [][]string { + s.mu.Lock() + defer s.mu.Unlock() + return append([][]string(nil), s.rcpts...) +} + func attemptHeader(wire string) string { for _, line := range strings.Split(wire, "\n") { if strings.HasPrefix(line, outbound.ProviderAttemptHeader+": ") { @@ -245,4 +257,26 @@ func TestFeedbackSeam_EnvelopeIsConfigurationNotRequest(t *testing.T) { if attemptHeader(msgs[0]) == "" { t.Errorf("feedback mail left without the provider attempt header: it did not cross the authorized seam") } + got := s.recipients() + if len(got) != 1 || strings.Join(got[0], ",") != "feedback@example.test,ops@example.test" { + t.Errorf("RCPT TO = %v, want exactly the configured notify set", got) + } +} + +// TestFeedbackSeam_OverlappingNotifyConfigStillSends: TO and CC naming the +// same mailbox (in any case) is a legal configuration that used to send one +// copy; the seam's canonical recipient set keeps it that way instead of +// refusing every attempt. +func TestFeedbackSeam_OverlappingNotifyConfigStillSends(t *testing.T) { + s := startScriptedSMTP(t, "250") + api, _ := newFeedbackSeamAPI(t, s) + + err := api.sendFeedbackEmail(context.Background(), "t", "bug", "m", "", "", []string{"ops@example.test"}, []string{"Ops@example.test"}) + if err != nil { + t.Fatalf("sendFeedbackEmail: %v", err) + } + got := s.recipients() + if len(got) != 1 || len(got[0]) != 1 || !strings.EqualFold(got[0][0], "ops@example.test") { + t.Fatalf("RCPT TO = %v, want the one mailbox once", got) + } } diff --git a/internal/hitlnotify/jobs.go b/internal/hitlnotify/jobs.go index 7702d14d6..177c1dec1 100644 --- a/internal/hitlnotify/jobs.go +++ b/internal/hitlnotify/jobs.go @@ -12,6 +12,7 @@ import ( "github.com/tokencanopy/e2a/internal/identity" "github.com/tokencanopy/e2a/internal/jobs" + "github.com/tokencanopy/e2a/internal/outbound" "github.com/tokencanopy/e2a/internal/sendingpolicy" ) @@ -61,20 +62,38 @@ func (j *Jobs) SetDeliverer(d Deliverer) { j.mu.Unlock() } -// Deliver makes Jobs itself the worker's Deliverer, delegating to the concrete one -// set via SetDeliverer. Until that is wired (the brief startup window before the -// notifier is built) it returns a retryable outcome, so a pending job simply -// retries rather than dropping on a nil deliverer. -func (j *Jobs) Deliver(ctx context.Context, pn *identity.PendingNotify, auth sendingpolicy.ProviderAuthorization) DeliverOutcome { - j.mu.RLock() - d := j.deliverer - j.mu.RUnlock() +// Compose makes Jobs itself the worker's Deliverer, delegating to the +// concrete one set via SetDeliverer. Until that is wired (the brief startup +// window before the notifier is built) it returns a retryable outcome — and +// because Compose runs before any attempt is charged, that window costs +// nothing. +func (j *Jobs) Compose(ctx context.Context, pn *identity.PendingNotify) (outbound.Envelope, DeliverOutcome) { + d := j.currentDeliverer() + if d == nil { + return outbound.Envelope{}, DeliverOutcome{Err: errors.New("hitl notifier not wired yet — retrying")} + } + return d.Compose(ctx, pn) +} + +// Submit delegates the authorized submission to the concrete Deliverer. +func (j *Jobs) Submit(ctx context.Context, env outbound.Envelope, auth sendingpolicy.ProviderAuthorization) DeliverOutcome { + d := j.currentDeliverer() if d == nil { return DeliverOutcome{Err: errors.New("hitl notifier not wired yet — retrying")} } - return d.Deliver(ctx, pn, auth) + return d.Submit(ctx, env, auth) } +func (j *Jobs) currentDeliverer() Deliverer { + j.mu.RLock() + defer j.mu.RUnlock() + return j.deliverer +} + +// Gate exposes the wired sending-protection gate (nil when gateless), so the +// composition root's wiring test can prove the production bundle is armed. +func (j *Jobs) Gate() sendingpolicy.Gate { return j.gate } + // RegisterJobs adds the NotifyWorker (with Jobs as the late-binding Deliverer). // No periodics — the reconciler is a one-shot startup cutover. Implements // jobs.Registrar. diff --git a/internal/hitlnotify/notifier.go b/internal/hitlnotify/notifier.go index ac89c19bd..2eb409968 100644 --- a/internal/hitlnotify/notifier.go +++ b/internal/hitlnotify/notifier.go @@ -18,7 +18,6 @@ import ( "context" "fmt" "html" - "log" "net/url" "strings" "time" @@ -111,26 +110,36 @@ func (n *Notifier) WithDKIM(lookup outbound.DKIMKeyLookup) *Notifier { } // NotifyPendingApproval composes and sends the notification email for a held -// message, submitting once (SendOnce). It is the compose+send core the River -// NotifyWorker drives via Deliver; the returned error is classified there into -// retry/permanent/outage. +// message with an already-authorized attempt: Compose then Submit in one call, +// for callers that hold the token up front (tests, the reconciler drill). The +// worker calls the two phases itself so the token is consumed last. func (n *Notifier) NotifyPendingApproval(ctx context.Context, msg *identity.Message, agent *identity.AgentIdentity, auth sendingpolicy.ProviderAuthorization) error { if n == nil { return nil } + env, err := n.compose(ctx, msg, agent) + if err != nil { + return err + } + return n.submit(ctx, env, auth) +} + +// compose builds the approval email: owner lookup, magic-link tokens, MIME, +// deterministic Message-ID and DKIM. It touches no provider. +func (n *Notifier) compose(ctx context.Context, msg *identity.Message, agent *identity.AgentIdentity) (outbound.Envelope, error) { if msg == nil || agent == nil { - return fmt.Errorf("notify: msg or agent is nil") + return outbound.Envelope{}, fmt.Errorf("notify: msg or agent is nil") } if msg.ApprovalExpiresAt == nil { - return fmt.Errorf("notify: approval_expires_at is nil on msg %s", msg.ID) + return outbound.Envelope{}, fmt.Errorf("notify: approval_expires_at is nil on msg %s", msg.ID) } owner, err := n.store.GetUserByID(ctx, agent.UserID) if err != nil { - return fmt.Errorf("notify: lookup owner: %w", err) + return outbound.Envelope{}, fmt.Errorf("notify: lookup owner: %w", err) } if owner.Email == "" { - return fmt.Errorf("notify: owner %s has no email on record", owner.ID) + return outbound.Envelope{}, fmt.Errorf("notify: owner %s has no email on record", owner.ID) } tokenExp := msg.ApprovalExpiresAt.Add(tokenGraceAfterTTL) @@ -143,11 +152,11 @@ func (n *Notifier) NotifyPendingApproval(ctx context.Context, msg *identity.Mess approveTok, err := signFn(approvaltoken.ActionApprove) if err != nil { - return fmt.Errorf("notify: sign approve token: %w", err) + return outbound.Envelope{}, fmt.Errorf("notify: sign approve token: %w", err) } rejectTok, err := signFn(approvaltoken.ActionReject) if err != nil { - return fmt.Errorf("notify: sign reject token: %w", err) + return outbound.Envelope{}, fmt.Errorf("notify: sign reject token: %w", err) } subject := fmt.Sprintf("[e2a] approve outbound from %s: %s", @@ -184,7 +193,7 @@ func (n *Notifier) NotifyPendingApproval(ctx context.Context, msg *identity.Mess "", // no conversation_id ) if err != nil { - return fmt.Errorf("notify: compose: %w", err) + return outbound.Envelope{}, fmt.Errorf("notify: compose: %w", err) } // Prepend a DETERMINISTIC Message-ID so a re-sent notification collapses at @@ -226,40 +235,52 @@ func (n *Notifier) NotifyPendingApproval(ctx context.Context, msg *identity.Mess message = signed } - // One authorized submission: the submitter redeems the token immediately - // before the socket opens and settles the provider's answer; River (not - // the relay's in-process loop) owns retries, each as a fresh attempt. The - // %w keeps the SMTP error classifiable by Deliver via internal/outbound's - // IsPermanentSMTPError / IsConnectionError. - if _, err := n.submitter.SubmitOnce(ctx, auth, outbound.Envelope{ - From: fromAddr, - Recipients: []string{owner.Email}, - Message: message, - }); err != nil { + return outbound.Envelope{From: fromAddr, Recipients: []string{owner.Email}, Message: message}, nil +} + +// submit is the one authorized submission: the submitter redeems the token +// immediately before the socket opens and settles the provider's answer; +// River (not the relay's in-process loop) owns retries, each as a fresh +// attempt. The %w keeps the SMTP error classifiable via internal/outbound's +// IsPermanentSMTPError / IsConnectionError. +func (n *Notifier) submit(ctx context.Context, env outbound.Envelope, auth sendingpolicy.ProviderAuthorization) error { + if _, err := n.submitter.SubmitOnce(ctx, auth, env); err != nil { return fmt.Errorf("notify: smtp send: %w", err) } - - log.Printf("[hitl-notify] sent approval email: msg=%s owner=%s agent=%s", - msg.ID, owner.ID, agent.ID) return nil } -// Deliver composes and sends the approval email for one held message, classifying -// the result for the River NotifyWorker: a 5xx / validation reject is Permanent -// (no retry), an unreachable relay is an Outage (snooze), everything else retries. -// Implements hitlnotify.Deliverer. The classifiers key on the SMTP code / net -// error preserved through NotifyPendingApproval's %w wrapping. -func (n *Notifier) Deliver(ctx context.Context, pn *identity.PendingNotify, auth sendingpolicy.ProviderAuthorization) DeliverOutcome { - if err := n.NotifyPendingApproval(ctx, pn.Message, pn.Agent, auth); err != nil { - return DeliverOutcome{ - Err: err, - Permanent: outbound.IsPermanentSMTPError(err), - Outage: outbound.IsConnectionError(err), - } +// Compose implements Deliverer: the provider-free half, classified like a +// send so the worker treats a permanent compose failure the same way. +func (n *Notifier) Compose(ctx context.Context, pn *identity.PendingNotify) (outbound.Envelope, DeliverOutcome) { + if pn == nil { + return outbound.Envelope{}, DeliverOutcome{Err: fmt.Errorf("notify: nothing to compose"), Permanent: true} + } + env, err := n.compose(ctx, pn.Message, pn.Agent) + if err != nil { + return outbound.Envelope{}, classify(err) + } + return env, DeliverOutcome{} +} + +// Submit implements Deliverer: one authorized submission, classified for the +// River NotifyWorker — a 5xx / validation reject is Permanent (no retry), an +// unreachable relay is an Outage (snooze), everything else retries. +func (n *Notifier) Submit(ctx context.Context, env outbound.Envelope, auth sendingpolicy.ProviderAuthorization) DeliverOutcome { + if err := n.submit(ctx, env, auth); err != nil { + return classify(err) } return DeliverOutcome{} } +func classify(err error) DeliverOutcome { + return DeliverOutcome{ + Err: err, + Permanent: outbound.IsPermanentSMTPError(err), + Outage: outbound.IsConnectionError(err), + } +} + func (n *Notifier) magicURL(path, token string) string { if n.publicURL == "" { return path + "?t=" + url.QueryEscape(token) diff --git a/internal/hitlnotify/notifier_test.go b/internal/hitlnotify/notifier_test.go index 906fba2a1..59edeed62 100644 --- a/internal/hitlnotify/notifier_test.go +++ b/internal/hitlnotify/notifier_test.go @@ -312,7 +312,11 @@ func TestNotifierDeliver(t *testing.T) { // Deliver is what the River NotifyWorker calls: it composes + sends once and // classifies the result. A healthy send returns a zero-value outcome. - out := n.Deliver(context.Background(), &identity.PendingNotify{Message: msg, Agent: agent}, tokenFor(t, store, msg.ID)) + env, out := n.Compose(context.Background(), &identity.PendingNotify{Message: msg, Agent: agent}) + if out.Err != nil { + t.Fatalf("Compose: unexpected err = %v", out.Err) + } + out = n.Submit(context.Background(), env, tokenFor(t, store, msg.ID)) if out.Err != nil { t.Fatalf("Deliver: unexpected err = %v", out.Err) } diff --git a/internal/hitlnotify/worker.go b/internal/hitlnotify/worker.go index 060005b7f..fc7656b5f 100644 --- a/internal/hitlnotify/worker.go +++ b/internal/hitlnotify/worker.go @@ -22,6 +22,7 @@ import ( "github.com/riverqueue/river" "github.com/tokencanopy/e2a/internal/identity" + "github.com/tokencanopy/e2a/internal/outbound" "github.com/tokencanopy/e2a/internal/sendingpolicy" ) @@ -67,12 +68,23 @@ type DeliverOutcome struct { Outage bool // relay unreachable — snooze without spending an attempt } -// Deliverer composes and sends the approval email for one held message. Implemented -// by *Notifier (compose + SMTPRelay.SendOnce + classify). +// Deliverer is the two-phase send of one approval email. Compose does every +// fallible, provider-free step (owner lookup, token signing, MIME, DKIM) and +// returns the envelope; Submit hands that envelope and a freshly consumed +// authorization to the provider seam. The split exists so the worker can +// ConsumeAttempt immediately before the socket opens: a compose failure then +// costs nothing, instead of a charged ordinal that never reached the +// provider. Implemented by *Notifier. type Deliverer interface { - Deliver(ctx context.Context, pn *identity.PendingNotify, auth sendingpolicy.ProviderAuthorization) DeliverOutcome + Compose(ctx context.Context, pn *identity.PendingNotify) (outbound.Envelope, DeliverOutcome) + Submit(ctx context.Context, env outbound.Envelope, auth sendingpolicy.ProviderAuthorization) DeliverOutcome } +// errOperationMismatch marks a job whose operation reference names another +// held message's operation. Authorizing it would charge that operation's +// account, so the job is cancelled, never retried. +var errOperationMismatch = errors.New("hitl notify: job operation reference does not name this message") + // OperationResolver recovers the durable operation for a job that carries no // reference, through the same Prepare path an enqueue runs. type OperationResolver func(ctx context.Context, messageID string) (sendingpolicy.OperationRef, error) @@ -171,12 +183,21 @@ func (w *NotifyWorker) Work(ctx context.Context, job *river.Job[HITLNotifyArgs]) return nil // agent opted out of approval notifications } + // Compose first: owner lookup, magic-link signing, MIME and DKIM are all + // fallible and none of them touches the provider, so they run before any + // attempt is charged. A failure here is classified exactly like a send + // failure but costs no ordinal. + env, out := w.deliverer.Compose(ctx, pn) + if out.Err != nil { + return w.verdict(job, msg.ID, "compose", out) + } + // Every provider call is authorized: Reserve the durable attempt, hold - // without I/O when the gate says so, ConsumeAttempt as the last decision, - // then hand the token to the deliverer, whose submitter redeems it before - // the socket opens. Notifications carry no durable hold class of their - // own — the approval TTL guard above already bounds how long one can - // wait, and a hold past it becomes the no-op the guard returns. + // without I/O when the gate says so, ConsumeAttempt as the LAST decision + // before Submit, whose submitter redeems the token immediately before the + // socket opens. Notifications carry no durable hold class of their own — + // the approval TTL guard above already bounds how long one can wait, and + // a hold past it becomes the no-op the guard returns. auth := sendingpolicy.ProviderAuthorization{} if w.gate != nil { ref, err := w.operationFor(ctx, job) @@ -184,6 +205,9 @@ func (w *NotifyWorker) Work(ctx context.Context, job *river.Job[HITLNotifyArgs]) if errors.Is(err, sendingpolicy.ErrSourceUnavailable) { return nil // the hold is gone — nothing to notify } + if errors.Is(err, errOperationMismatch) { + return river.JobCancel(err) + } return err } early, attempt, err := w.gate.Reserve(ctx, ref) @@ -209,8 +233,9 @@ func (w *NotifyWorker) Work(ctx context.Context, job *river.Job[HITLNotifyArgs]) auth = *token } - out := w.deliverer.Deliver(ctx, pn, auth) + out = w.deliverer.Submit(ctx, env, auth) if out.Err == nil { + log.Printf("[hitl-notify] sent approval email: msg=%s", msg.ID) if merr := w.store.MarkMessageNotified(ctx, msg.ID); merr != nil { // The email is already out; only the dedup marker failed to persist. Do // NOT return an error — a retry would re-send. Completing the job leaves @@ -219,27 +244,37 @@ func (w *NotifyWorker) Work(ctx context.Context, job *river.Job[HITLNotifyArgs]) } return nil } + return w.verdict(job, msg.ID, "send", out) +} + +// verdict turns a classified failure into River's answer: a permanent one +// cancels (the hold still finalizes on its TTL), an outage snoozes without +// spending a River attempt, everything else retries per NextRetry until +// MaxNotifyAttempts. +func (w *NotifyWorker) verdict(job *river.Job[HITLNotifyArgs], messageID, phase string, out DeliverOutcome) error { if out.Permanent { - // e.g. the owner address is rejected 5xx. Unavoidable — the hold still - // finalizes on its TTL. Cancel (no retry) rather than churn the tail. - log.Printf("[hitl-notify] permanent send failure for %s (no retry): %v", msg.ID, out.Err) + log.Printf("[hitl-notify] permanent %s failure for %s (no retry): %v", phase, messageID, out.Err) return river.JobCancel(out.Err) } if out.Outage { - // Relay unreachable. Snooze without burning an attempt. If the hold has - // since passed its TTL, the next attempt's expiry guard above short-circuits - // to a no-op — no need to special-case it here. + // Relay unreachable. If the hold has since passed its TTL, the next + // attempt's expiry guard short-circuits to a no-op. return river.JobSnooze(notifyOutageSnooze) } - // Transient (relay throttle, owner lookup blip, compose error): let River - // reschedule per NextRetry until MaxNotifyAttempts, then discard. - return fmt.Errorf("hitl notify attempt %d failed: %w", job.Attempt, out.Err) + return fmt.Errorf("hitl notify attempt %d %s failed: %w", job.Attempt, phase, out.Err) } // operationFor returns the job's durable operation, resolving and stamping a // legacy job through the accept path. func (w *NotifyWorker) operationFor(ctx context.Context, job *river.Job[HITLNotifyArgs]) (sendingpolicy.OperationRef, error) { + // The approval request's operation IS derived from the message id, so a + // reference naming any other operation would charge another account: the + // same binding the message worker enforces, checked before Reserve. + want := sendingpolicy.HITLNotificationOperationID(job.Args.MessageID) if job.Args.OperationRef != nil && !job.Args.OperationRef.IsZero() { + if job.Args.OperationRef.ID() != want { + return sendingpolicy.OperationRef{}, errOperationMismatch + } return *job.Args.OperationRef, nil } if w.resolve == nil { @@ -249,6 +284,9 @@ func (w *NotifyWorker) operationFor(ctx context.Context, job *river.Job[HITLNoti if err != nil { return sendingpolicy.OperationRef{}, err } + if ref.ID() != want { + return sendingpolicy.OperationRef{}, errOperationMismatch + } if w.stamp != nil { if err := w.stamp(ctx, job.ID, ref); err != nil { // Not fatal: the reference is valid for this execution; a retry @@ -273,3 +311,7 @@ func holdVerdict(d sendingpolicy.Decision) error { } return river.JobSnooze(delay) } + +// Gate exposes the wired gate (nil when gateless), for the composition +// root's wiring test. +func (w *NotifyWorker) Gate() sendingpolicy.Gate { return w.gate } diff --git a/internal/hitlnotify/worker_test.go b/internal/hitlnotify/worker_test.go index 03503d464..638dc64e5 100644 --- a/internal/hitlnotify/worker_test.go +++ b/internal/hitlnotify/worker_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "strings" "testing" "time" @@ -13,6 +14,7 @@ import ( "github.com/tokencanopy/e2a/internal/hitlnotify" "github.com/tokencanopy/e2a/internal/identity" + "github.com/tokencanopy/e2a/internal/outbound" "github.com/tokencanopy/e2a/internal/sendingpolicy" ) @@ -36,17 +38,36 @@ func (f *fakeStore) StampNotifyJobIDTx(_ context.Context, _ pgx.Tx, _ string, _ } type fakeDeliverer struct { - out hitlnotify.DeliverOutcome - called int - auths []sendingpolicy.ProviderAuthorization + out hitlnotify.DeliverOutcome // Submit's outcome + composeOut hitlnotify.DeliverOutcome // Compose's outcome + called int // Submit calls + composed int + auths []sendingpolicy.ProviderAuthorization + trace *[]string // shared with fakeGate to pin ordering } -func (f *fakeDeliverer) Deliver(_ context.Context, _ *identity.PendingNotify, auth sendingpolicy.ProviderAuthorization) hitlnotify.DeliverOutcome { +func (f *fakeDeliverer) Compose(_ context.Context, _ *identity.PendingNotify) (outbound.Envelope, hitlnotify.DeliverOutcome) { + f.composed++ + f.record("compose") + if f.composeOut.Err != nil { + return outbound.Envelope{}, f.composeOut + } + return outbound.Envelope{From: "e2a@notify.test", Recipients: []string{"owner@reviewer.test"}, Message: []byte("Subject: x\r\n\r\nbody")}, hitlnotify.DeliverOutcome{} +} + +func (f *fakeDeliverer) Submit(_ context.Context, _ outbound.Envelope, auth sendingpolicy.ProviderAuthorization) hitlnotify.DeliverOutcome { f.called++ + f.record("submit") f.auths = append(f.auths, auth) return f.out } +func (f *fakeDeliverer) record(step string) { + if f.trace != nil { + *f.trace = append(*f.trace, step) + } +} + func job(id string, attempt int) *river.Job[hitlnotify.HITLNotifyArgs] { return &river.Job[hitlnotify.HITLNotifyArgs]{ JobRow: &rivertype.JobRow{Attempt: attempt, MaxAttempts: hitlnotify.MaxNotifyAttempts, Kind: hitlnotify.HITLNotifyArgs{}.Kind()}, @@ -219,6 +240,7 @@ func TestNotifyWorker_NextRetryMatchesEnvelope(t *testing.T) { // fakeGate is a scriptable sendingpolicy.Gate for the worker-order tests. type fakeGate struct { + trace *[]string reserve sendingpolicy.Decision consume sendingpolicy.Decision reserves int @@ -244,10 +266,12 @@ func (g *fakeGate) PreparePublicFeedback(context.Context, sendingpolicy.PublicFe } func (g *fakeGate) Reserve(context.Context, sendingpolicy.OperationRef) (sendingpolicy.Decision, sendingpolicy.AttemptRef, error) { g.reserves++ + g.record("reserve") return g.reserve, sendingpolicy.AttemptRef{}, g.reserveErr } func (g *fakeGate) ConsumeAttempt(context.Context, sendingpolicy.AttemptRef) (sendingpolicy.Decision, *sendingpolicy.ProviderAuthorization, error) { g.consumes++ + g.record("consume") if !g.consume.Allow { return g.consume, nil, nil } @@ -278,7 +302,7 @@ func refFor(id string) sendingpolicy.OperationRef { func gatedJob(id string, attempt int) *river.Job[hitlnotify.HITLNotifyArgs] { j := job(id, attempt) - ref := refFor("op_" + id) + ref := refFor(sendingpolicy.HITLNotificationOperationID(id)) j.Args.OperationRef = &ref return j } @@ -336,7 +360,7 @@ func TestNotifyWorker_LegacyJobResolvesAndStampsOnce(t *testing.T) { w := hitlnotify.NewNotifyWorker(st, dl).WithGate(allowAll()). WithOperationResolver(func(_ context.Context, id string) (sendingpolicy.OperationRef, error) { resolved++ - return refFor("op_" + id), nil + return refFor(sendingpolicy.HITLNotificationOperationID(id)), nil }). WithArgStamper(func(_ context.Context, _ int64, _ sendingpolicy.OperationRef) error { stamped++; return nil }) if err := w.Work(context.Background(), job("msg_legacy", 1)); err != nil { @@ -354,3 +378,89 @@ func TestNotifyWorker_LegacyJobResolvesAndStampsOnce(t *testing.T) { t.Fatalf("orphan legacy: err=%v delivers=%d, want nil and no new delivery", err, dl.called) } } + +func (g *fakeGate) record(step string) { + if g.trace != nil { + *g.trace = append(*g.trace, step) + } +} + +// TestNotifyWorker_ComposeRunsBeforeAnyChargeAndConsumeIsLast pins the order +// the seam depends on: compose (every fallible, provider-free step) precedes +// Reserve, and ConsumeAttempt is the last call before Submit. +func TestNotifyWorker_ComposeRunsBeforeAnyChargeAndConsumeIsLast(t *testing.T) { + var trace []string + fd := &fakeDeliverer{trace: &trace} + g := allowAll() + g.trace = &trace + st := &fakeStore{pn: pending("msg_1")} + w := hitlnotify.NewNotifyWorker(st, fd).WithGate(g) + if err := w.Work(context.Background(), gatedJob("msg_1", 1)); err != nil { + t.Fatalf("Work: %v", err) + } + if got := strings.Join(trace, ","); got != "compose,reserve,consume,submit" { + t.Fatalf("order = %s, want compose,reserve,consume,submit", got) + } +} + +// TestNotifyWorker_ComposeFailureChargesNothing: a compose failure (owner +// lookup, signing, MIME) happens before Reserve, so it burns no ordinal; it +// is classified exactly like a send failure. +func TestNotifyWorker_ComposeFailureChargesNothing(t *testing.T) { + for name, tc := range map[string]struct { + out hitlnotify.DeliverOutcome + wantErr func(error) bool + wantMsgID bool + }{ + "transient": {out: hitlnotify.DeliverOutcome{Err: errors.New("owner lookup blip")}, wantErr: func(err error) bool { return err != nil && !isCancel(err) && !isSnooze(err) }}, + "permanent": {out: hitlnotify.DeliverOutcome{Err: errors.New("no owner email"), Permanent: true}, wantErr: isCancel}, + "outage": {out: hitlnotify.DeliverOutcome{Err: errors.New("dkim store down"), Outage: true}, wantErr: isSnooze}, + } { + fd := &fakeDeliverer{composeOut: tc.out} + g := allowAll() + st := &fakeStore{pn: pending("msg_1")} + w := hitlnotify.NewNotifyWorker(st, fd).WithGate(g) + err := w.Work(context.Background(), gatedJob("msg_1", 1)) + if !tc.wantErr(err) { + t.Fatalf("%s: err = %v", name, err) + } + if g.reserves != 0 || g.consumes != 0 || fd.called != 0 { + t.Fatalf("%s: reserves=%d consumes=%d submits=%d, want 0/0/0", name, g.reserves, g.consumes, fd.called) + } + if len(st.notified) != 0 { + t.Fatalf("%s: marked notified without a send", name) + } + } +} + +// TestNotifyWorker_ForeignOperationReferenceIsCancelled: a job whose +// reference names another message's operation would charge that operation's +// account; it is cancelled before Reserve, never retried. +func TestNotifyWorker_ForeignOperationReferenceIsCancelled(t *testing.T) { + fd := &fakeDeliverer{} + g := allowAll() + st := &fakeStore{pn: pending("msg_1")} + w := hitlnotify.NewNotifyWorker(st, fd).WithGate(g) + j := job("msg_1", 1) + ref := refFor(sendingpolicy.HITLNotificationOperationID("msg_other")) + j.Args.OperationRef = &ref + if err := w.Work(context.Background(), j); !isCancel(err) { + t.Fatalf("err = %v, want cancel", err) + } + if g.reserves != 0 || fd.called != 0 { + t.Fatalf("reserves=%d submits=%d, want 0/0", g.reserves, fd.called) + } + + // The same binding applies to a legacy resolve that returns a foreign id. + fd, g = &fakeDeliverer{}, allowAll() + w = hitlnotify.NewNotifyWorker(&fakeStore{pn: pending("msg_1")}, fd).WithGate(g). + WithOperationResolver(func(context.Context, string) (sendingpolicy.OperationRef, error) { + return refFor(sendingpolicy.HITLNotificationOperationID("msg_other")), nil + }) + if err := w.Work(context.Background(), job("msg_1", 1)); !isCancel(err) { + t.Fatalf("legacy: err = %v, want cancel", err) + } + if g.reserves != 0 || fd.called != 0 { + t.Fatalf("legacy: reserves=%d submits=%d, want 0/0", g.reserves, fd.called) + } +} diff --git a/internal/jobs/argstamp_test.go b/internal/jobs/argstamp_test.go new file mode 100644 index 000000000..8a56afe5d --- /dev/null +++ b/internal/jobs/argstamp_test.go @@ -0,0 +1,66 @@ +package jobs_test + +import ( + "context" + "testing" + + "github.com/tokencanopy/e2a/internal/jobs" + "github.com/tokencanopy/e2a/internal/testutil" +) + +// TestStampJobArg: the key is added once, existing fields survive, a present +// key is never overwritten, and a missing job is a no-op rather than an +// error (River may have pruned it). +func TestStampJobArg(t *testing.T) { + ctx := context.Background() + pool := testutil.TestDB(t) + if err := jobs.Migrate(ctx, pool); err != nil { + t.Fatalf("Migrate: %v", err) + } + var id int64 + if err := pool.QueryRow(ctx, + `INSERT INTO river_job (args, kind, max_attempts) VALUES ('{"message_id":"msg_1"}'::jsonb, 'argstamp_test', 3) RETURNING id`, + ).Scan(&id); err != nil { + t.Fatal(err) + } + + if err := jobs.StampJobArg(ctx, pool, id, "operation_ref", map[string]any{"v": 1, "id": "op_1"}); err != nil { + t.Fatalf("stamp: %v", err) + } + if err := jobs.StampJobArg(ctx, pool, id, "operation_ref", map[string]any{"v": 1, "id": "op_2"}); err != nil { + t.Fatalf("second stamp: %v", err) + } + var messageID, opID string + if err := pool.QueryRow(ctx, + `SELECT args->>'message_id', args->'operation_ref'->>'id' FROM river_job WHERE id = $1`, id, + ).Scan(&messageID, &opID); err != nil { + t.Fatal(err) + } + if messageID != "msg_1" || opID != "op_1" { + t.Fatalf("args = message_id=%q operation_ref.id=%q, want msg_1 / op_1 (first stamp wins, existing field kept)", messageID, opID) + } + + if err := jobs.StampJobArg(ctx, pool, id+1000, "operation_ref", "x"); err != nil { + t.Fatalf("missing job must be a no-op, got %v", err) + } +} + +// TestStampJobArgRefusesBadInputs: no database and an unencodable value are +// errors before any SQL runs; a failed statement is reported, not swallowed. +func TestStampJobArgRefusesBadInputs(t *testing.T) { + ctx := context.Background() + if err := jobs.StampJobArg(ctx, nil, 1, "k", "v"); err == nil { + t.Fatal("nil database must be refused") + } + pool := testutil.TestDB(t) + if err := jobs.StampJobArg(ctx, pool, 1, "k", make(chan int)); err == nil { + t.Fatal("unencodable value must be refused") + } + if err := jobs.StampJobArg(ctx, pool, 1, "k", "v"); err == nil { + // river_job may not exist on this fresh pool (no Migrate): the + // statement fails and the error must surface. + if _, qerr := pool.Exec(ctx, `SELECT 1 FROM river_job LIMIT 1`); qerr != nil { + t.Fatal("statement failure must be reported") + } + } +} diff --git a/internal/outbound/provider_authorization_guard_test.go b/internal/outbound/provider_authorization_guard_test.go index 0da9378d2..86af2f0ba 100644 --- a/internal/outbound/provider_authorization_guard_test.go +++ b/internal/outbound/provider_authorization_guard_test.go @@ -34,10 +34,23 @@ func TestEveryProviderCallRequiresAuthorization(t *testing.T) { "internal/outbound/smtp_relay.go": "the provider relay itself", "internal/selftest/scenarios.go": "local inbound self-test client, not provider-bound", } - // Files that may call the relay's socket-opening core. + // The ONE function that may reference the relay's socket-opening core: + // the authorized adapter's SubmitOnce. The exception is a symbol, not a + // file, so a second function added beside it is not exempt. socketCallAllowed := map[string]string{ - "internal/outbound/provider_submit.go": "the one authorized adapter", + "internal/outbound/provider_submit.go:SubmitOnce": "the one authorized adapter method", } + // Provider SDKs that can send mail without SMTP, and the one package + // that may import each: sender-identity provisioning uses SES v2 for + // identities and tags, never SendEmail. A send through an HTTP provider + // API is invisible to the socket check, so the import is fenced instead. + providerSDKAllowed := map[string]map[string]string{ + "github.com/aws/aws-sdk-go-v2/service/sesv2": { + "internal/senderidentity/ses.go": "SES identity provisioning", + "internal/senderidentity/tags.go": "SES identity tagging", + }, + } + allowedSocketCalls := 0 fset := token.NewFileSet() for _, rel := range files { @@ -50,32 +63,54 @@ func TestEveryProviderCallRequiresAuthorization(t *testing.T) { t.Fatalf("parse %s: %v", rel, err) } for _, imp := range f.Imports { - if strings.Trim(imp.Path.Value, `"`) == "net/smtp" { + path := strings.Trim(imp.Path.Value, `"`) + if path == "net/smtp" { if _, ok := smtpImportAllowed[rel]; !ok { t.Errorf("%s imports net/smtp: provider I/O must go through outbound.ProviderSubmitter (or be named in the guard's exception list with its reason)", rel) } } + if files, fenced := providerSDKAllowed[path]; fenced { + if _, ok := files[rel]; !ok { + t.Errorf("%s imports %s: a provider SDK may only be used where the guard names it, and never to send", rel, path) + } + } } full, err := parser.ParseFile(fset, rel, src, 0) if err != nil { t.Fatalf("parse %s: %v", rel, err) } - ast.Inspect(full, func(n ast.Node) bool { - call, ok := n.(*ast.CallExpr) - if !ok { - return true + // Any reference to the socket core counts, not only a direct call: + // a method value (`f := r.sendOnceContext`) or a method expression + // (`(*SMTPRelay).sendOnceContext`) is a SelectorExpr too, and either + // would otherwise let a caller open the socket one hop away from the + // name this guard looks for. + for _, decl := range full.Decls { + fn, isFunc := decl.(*ast.FuncDecl) + var enclosing string + if isFunc { + enclosing = rel + ":" + fn.Name.Name } - sel, ok := call.Fun.(*ast.SelectorExpr) - if !ok { - return true - } - if sel.Sel.Name == "sendOnceContext" { - if _, ok := socketCallAllowed[rel]; !ok { - t.Errorf("%s:%s calls the relay's socket-opening core outside the authorized adapter", rel, fset.Position(call.Pos())) + ast.Inspect(decl, func(n ast.Node) bool { + sel, ok := n.(*ast.SelectorExpr) + if !ok || sel.Sel.Name != "sendOnceContext" { + return true } - } - return true - }) + if rel == "internal/outbound/smtp_relay.go" && isFunc && fn.Name.Name == "sendOnceContext" { + return true // the definition's own receiver method is not a reference + } + if _, ok := socketCallAllowed[enclosing]; ok { + allowedSocketCalls++ + return true + } + t.Errorf("%s:%s references the relay's socket-opening core outside ProviderSubmitter.SubmitOnce", rel, fset.Position(sel.Pos())) + return true + }) + } + } + // The sentinel must be real: renaming the socket core would otherwise + // turn the whole reference check into a no-op that still passes. + if allowedSocketCalls == 0 { + t.Fatal("ProviderSubmitter.SubmitOnce no longer references sendOnceContext: the guard's sentinel is stale, update both together") } // The relay's exported surface may not open a socket: Configured is a @@ -107,34 +142,66 @@ func TestEveryProviderCallRequiresAuthorization(t *testing.T) { } } +// moduleRoot walks up from the package directory to the module's go.mod. +// It needs no git: a guard that skipped itself wherever git was absent (a +// source tarball, a container without the binary, a prebuilt test binary) +// would report green exactly where nobody was looking. func moduleRoot(t *testing.T) string { t.Helper() - out, err := exec.Command("git", "rev-parse", "--show-toplevel").Output() + dir, err := os.Getwd() if err != nil { - t.Skipf("not in a git checkout: %v", err) + t.Fatal(err) + } + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir + } + parent := filepath.Dir(dir) + if parent == dir { + t.Fatal("go.mod not found above the package directory") + } + dir = parent } - return strings.TrimSpace(string(out)) } -// trackedGoFiles lists tracked, non-test Go files under internal/ and cmd/ -// — production code only, by git's own account of what ships. +// trackedGoFiles lists the production (non-test) Go files under internal/ +// and cmd/. Git's index is the authority when available — it is what ships — +// and a filesystem walk is the fallback so the guard never skips. func trackedGoFiles(t *testing.T, root string) []string { t.Helper() + var files []string cmd := exec.Command("git", "ls-files", "--", "internal/*.go", "internal/**/*.go", "cmd/*.go", "cmd/**/*.go") cmd.Dir = root - out, err := cmd.Output() - if err != nil { - t.Fatalf("git ls-files: %v", err) - } - var files []string - for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { - if line == "" || strings.HasSuffix(line, "_test.go") { - continue + if out, err := cmd.Output(); err == nil { + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + if line == "" || strings.HasSuffix(line, "_test.go") { + continue + } + files = append(files, line) + } + } else { + for _, top := range []string{"internal", "cmd"} { + err := filepath.WalkDir(filepath.Join(root, top), func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + files = append(files, filepath.ToSlash(rel)) + return nil + }) + if err != nil { + t.Fatalf("walk %s: %v", top, err) + } } - files = append(files, line) } if len(files) < 50 { - t.Fatalf("only %d tracked production files found; the guard is scanning the wrong tree", len(files)) + t.Fatalf("only %d production files found; the guard is scanning the wrong tree", len(files)) } return files } diff --git a/internal/sendingpolicy/operations.go b/internal/sendingpolicy/operations.go index 237ad600b..e0781a498 100644 --- a/internal/sendingpolicy/operations.go +++ b/internal/sendingpolicy/operations.go @@ -281,18 +281,41 @@ func (m *Module) PrepareNotificationTx(ctx context.Context, tx pgx.Tx, ref Notif return OperationRef{}, ErrSourceUnavailable } - var userID string + var userID, operationID string var err error switch ref.source { case NotificationHITLMessage: userID, err = lockHITLSourceOwner(ctx, tx, ref.id) + operationID = HITLNotificationOperationID(ref.id) case NotificationWebhookHealth: + // The operation is keyed by the episode the sweep stamped in the + // same transaction that enqueues the notice, so preparing the same + // episode twice (an enqueue and a later legacy resolve, or two + // resolvers racing) yields one operation, and a job whose reference + // names another episode is detectably stale. + var warnedAt, disabledAt *time.Time err = tx.QueryRow(ctx, - `SELECT user_id FROM webhooks WHERE id = $1 FOR UPDATE`, ref.id, - ).Scan(&userID) + `SELECT user_id, warn_notified_at, auto_disabled_at FROM webhooks WHERE id = $1 FOR UPDATE`, ref.id, + ).Scan(&userID, &warnedAt, &disabledAt) if errors.Is(err, pgx.ErrNoRows) { err = ErrSourceUnavailable } + if err == nil { + var episode *time.Time + switch ref.kind { + case WebhookHealthKindWarning: + episode = warnedAt + case WebhookHealthKindDisabled: + episode = disabledAt + } + if episode == nil { + // Unknown kind, or an episode the sweep never stamped: + // there is no notice to send, so there is nothing to + // authorize. + return OperationRef{}, ErrSourceUnavailable + } + operationID = WebhookHealthOperationID(ref.id, ref.kind, *episode) + } default: return OperationRef{}, ErrSourceUnavailable } @@ -308,7 +331,7 @@ func (m *Module) PrepareNotificationTx(ctx context.Context, tx pgx.Tx, ref Notif } row, err := insertOperation(ctx, tx, operationRow{ - OperationID: randomID("op_"), + OperationID: operationID, SourceAccountRef: &userID, PolicySubjectRef: userID, Purpose: PurposeCustomerNotification, diff --git a/internal/sendingpolicy/store_integration_test.go b/internal/sendingpolicy/store_integration_test.go index 2d160a733..d2a53f432 100644 --- a/internal/sendingpolicy/store_integration_test.go +++ b/internal/sendingpolicy/store_integration_test.go @@ -172,8 +172,8 @@ func (f *fixture) webhook(userID string) string { id := fmt.Sprintf("wh_gate_%d", messageSeq+1000) messageSeq++ if _, err := f.pool.Exec(f.ctx, - `INSERT INTO webhooks (id, user_id, url, signing_secret, events) - VALUES ($1, $2, $3, $4, ARRAY['message.received'])`, + `INSERT INTO webhooks (id, user_id, url, signing_secret, events, enabled, auto_disabled_at) + VALUES ($1, $2, $3, $4, ARRAY['message.received'], false, now())`, id, userID, "https://hook.example.test/"+id, "secret", ); err != nil { f.t.Fatalf("insert webhook: %v", err) @@ -421,7 +421,7 @@ func TestSharedMailboxAndNotificationsShareOneAccountCounter(t *testing.T) { var hookRef sendingpolicy.OperationRef f.inTx(func(tx pgx.Tx) error { var err error - hookRef, err = g.PrepareNotificationTx(f.ctx, tx, sendingpolicy.NewWebhookHealthNotificationRef(hook)) + hookRef, err = g.PrepareNotificationTx(f.ctx, tx, sendingpolicy.NewWebhookHealthNotificationRef(hook, sendingpolicy.WebhookHealthKindDisabled)) return err }) d := f.authorize(g, hookRef) @@ -1126,3 +1126,99 @@ func TestReputationClassCannotBecomeCheaperAfterPreparation(t *testing.T) { t.Fatalf("tightening in the safe direction must still send: %q", d.Reason) } } + +// TestNotificationOperationsAreKeyedBySource: preparing the same held +// message or the same webhook health episode twice yields ONE operation, so +// an enqueue and a later legacy resolve (or two resolvers racing) cannot mint +// a second operation that nothing settles; a different episode is a +// different operation; an episode the sweep never stamped has nothing to +// authorize. +func TestNotificationOperationsAreKeyedBySource(t *testing.T) { + f := newFixture(t) + g := f.gate(enforcingPolicy(nil)) + user := f.user("standard") + agent := f.agent(user) + held := f.pendingMessage(agent, "relay") + + var first, second sendingpolicy.OperationRef + f.inTx(func(tx pgx.Tx) error { + var err error + first, err = g.PrepareNotificationTx(f.ctx, tx, sendingpolicy.NewHITLNotificationRef(held)) + return err + }) + f.inTx(func(tx pgx.Tx) error { + var err error + second, err = g.PrepareNotificationTx(f.ctx, tx, sendingpolicy.NewHITLNotificationRef(held)) + return err + }) + if first.ID() != sendingpolicy.HITLNotificationOperationID(held) || first.ID() != second.ID() { + t.Fatalf("hitl operation ids = %q / %q, want both %q", first.ID(), second.ID(), sendingpolicy.HITLNotificationOperationID(held)) + } + + hook := f.webhook(user) + var episode time.Time + if err := f.pool.QueryRow(f.ctx, `SELECT auto_disabled_at FROM webhooks WHERE id = $1`, hook).Scan(&episode); err != nil { + t.Fatal(err) + } + var disabled1, disabled2 sendingpolicy.OperationRef + f.inTx(func(tx pgx.Tx) error { + var err error + disabled1, err = g.PrepareNotificationTx(f.ctx, tx, sendingpolicy.NewWebhookHealthNotificationRef(hook, sendingpolicy.WebhookHealthKindDisabled)) + return err + }) + f.inTx(func(tx pgx.Tx) error { + var err error + disabled2, err = g.PrepareNotificationTx(f.ctx, tx, sendingpolicy.NewWebhookHealthNotificationRef(hook, sendingpolicy.WebhookHealthKindDisabled)) + return err + }) + want := sendingpolicy.WebhookHealthOperationID(hook, sendingpolicy.WebhookHealthKindDisabled, episode) + if disabled1.ID() != want || disabled2.ID() != want { + t.Fatalf("webhook operation ids = %q / %q, want both %q", disabled1.ID(), disabled2.ID(), want) + } + + // No warning episode was ever stamped: nothing to authorize. + err := f.tryTx(func(tx pgx.Tx) error { + _, err := g.PrepareNotificationTx(f.ctx, tx, sendingpolicy.NewWebhookHealthNotificationRef(hook, sendingpolicy.WebhookHealthKindWarning)) + return err + }) + if !errors.Is(err, sendingpolicy.ErrSourceUnavailable) { + t.Fatalf("unstamped warning episode: err = %v, want ErrSourceUnavailable", err) + } + err = f.tryTx(func(tx pgx.Tx) error { + _, err := g.PrepareNotificationTx(f.ctx, tx, sendingpolicy.NewWebhookHealthNotificationRef(hook, "bogus")) + return err + }) + if !errors.Is(err, sendingpolicy.ErrSourceUnavailable) { + t.Fatalf("unknown kind: err = %v, want ErrSourceUnavailable", err) + } + + // A later episode (the webhook recovered and was disabled again) is a + // new operation. + if _, err := f.pool.Exec(f.ctx, `UPDATE webhooks SET auto_disabled_at = auto_disabled_at + interval '1 hour' WHERE id = $1`, hook); err != nil { + t.Fatal(err) + } + var disabled3 sendingpolicy.OperationRef + f.inTx(func(tx pgx.Tx) error { + var err error + disabled3, err = g.PrepareNotificationTx(f.ctx, tx, sendingpolicy.NewWebhookHealthNotificationRef(hook, sendingpolicy.WebhookHealthKindDisabled)) + return err + }) + if disabled3.ID() == disabled1.ID() { + t.Fatalf("a new episode must be a new operation, got %q twice", disabled3.ID()) + } +} + +// tryTx runs fn in a transaction that is rolled back on error and returns +// fn's error, for the paths a fixture expects to be refused. +func (f *fixture) tryTx(fn func(tx pgx.Tx) error) error { + f.t.Helper() + tx, err := f.pool.Begin(f.ctx) + if err != nil { + f.t.Fatalf("begin: %v", err) + } + defer func() { _ = tx.Rollback(f.ctx) }() + if err := fn(tx); err != nil { + return err + } + return tx.Commit(f.ctx) +} diff --git a/internal/sendingpolicy/types.go b/internal/sendingpolicy/types.go index dad75ab5e..6886571bb 100644 --- a/internal/sendingpolicy/types.go +++ b/internal/sendingpolicy/types.go @@ -348,6 +348,41 @@ const ( type NotificationRef struct { source NotificationSource id string + // kind is the webhook health episode kind (WebhookHealthKindWarning or + // WebhookHealthKindDisabled); empty for every other source. + kind string +} + +// Source exposes the notification source, for tests and logging. +func (r NotificationRef) Source() NotificationSource { return r.source } + +// SourceID exposes the source row id. +func (r NotificationRef) SourceID() string { return r.id } + +// Kind exposes the webhook health episode kind; empty for other sources. +func (r NotificationRef) Kind() string { return r.kind } + +// Webhook health episode kinds. They mirror the notification job's own +// vocabulary; the notify package asserts the two agree. +const ( + WebhookHealthKindWarning = "warning" + WebhookHealthKindDisabled = "disabled" +) + +// HITLNotificationOperationID is the operation id of the approval request +// for one held message. Deriving it from the message makes +// PrepareNotificationTx idempotent per hold and lets the worker bind a +// job's reference to its source the way the message worker does. +func HITLNotificationOperationID(messageID string) string { + return "op_hitl_" + messageID +} + +// WebhookHealthOperationID is the operation id of one webhook health +// episode: the kind plus the timestamp the sweep stamped when it flipped +// the state (warn_notified_at or auto_disabled_at). A webhook that recovers +// and fails again is a new episode with a new operation. +func WebhookHealthOperationID(webhookID, kind string, episode time.Time) string { + return fmt.Sprintf("op_wh_%s_%s_%d", kind, webhookID, episode.UTC().Unix()) } // NewHITLNotificationRef references a pending outbound message whose approval @@ -358,10 +393,13 @@ func NewHITLNotificationRef(messageID string) NotificationRef { return NotificationRef{source: NotificationHITLMessage, id: messageID} } -// NewWebhookHealthNotificationRef references a webhook whose health episode is -// being reported to its owner. -func NewWebhookHealthNotificationRef(webhookID string) NotificationRef { - return NotificationRef{source: NotificationWebhookHealth, id: webhookID} +// NewWebhookHealthNotificationRef references a webhook whose health episode +// of the given kind (WebhookHealthKindWarning / WebhookHealthKindDisabled) is +// being reported to its owner. PrepareNotificationTx reads the episode's +// timestamp from the locked webhook row; an unknown kind or an episode the +// sweep never stamped is ErrSourceUnavailable. +func NewWebhookHealthNotificationRef(webhookID, kind string) NotificationRef { + return NotificationRef{source: NotificationWebhookHealth, id: webhookID, kind: kind} } // ProtectionNoticeRef names one already-committed notice event and audience. diff --git a/internal/webhooknotify/jobs.go b/internal/webhooknotify/jobs.go index 0457b1cac..ef17d8bee 100644 --- a/internal/webhooknotify/jobs.go +++ b/internal/webhooknotify/jobs.go @@ -12,6 +12,7 @@ import ( "github.com/tokencanopy/e2a/internal/identity" "github.com/tokencanopy/e2a/internal/jobs" + "github.com/tokencanopy/e2a/internal/outbound" "github.com/tokencanopy/e2a/internal/sendingpolicy" ) @@ -68,20 +69,38 @@ func (j *Jobs) SetDeliverer(d Deliverer) { j.mu.Unlock() } -// Deliver makes Jobs itself the worker's Deliverer, delegating to the -// concrete one set via SetDeliverer. Until that is wired (the brief -// startup window before the notifier is built) it returns a retryable -// outcome, so a pending job simply retries rather than dropping. -func (j *Jobs) Deliver(ctx context.Context, wh *identity.Webhook, kind string, auth sendingpolicy.ProviderAuthorization) DeliverOutcome { - j.mu.RLock() - d := j.deliverer - j.mu.RUnlock() +// Compose makes Jobs itself the worker's Deliverer, delegating to the +// concrete one set via SetDeliverer. Until that is wired (the brief startup +// window before the notifier is built) it returns a retryable outcome — and +// because Compose runs before any attempt is charged, that window costs +// nothing. +func (j *Jobs) Compose(ctx context.Context, wh *identity.Webhook, kind string) (outbound.Envelope, DeliverOutcome) { + d := j.currentDeliverer() + if d == nil { + return outbound.Envelope{}, DeliverOutcome{Err: errors.New("webhook notifier not wired yet — retrying")} + } + return d.Compose(ctx, wh, kind) +} + +// Submit delegates the authorized submission to the concrete Deliverer. +func (j *Jobs) Submit(ctx context.Context, env outbound.Envelope, auth sendingpolicy.ProviderAuthorization) DeliverOutcome { + d := j.currentDeliverer() if d == nil { return DeliverOutcome{Err: errors.New("webhook notifier not wired yet — retrying")} } - return d.Deliver(ctx, wh, kind, auth) + return d.Submit(ctx, env, auth) } +func (j *Jobs) currentDeliverer() Deliverer { + j.mu.RLock() + defer j.mu.RUnlock() + return j.deliverer +} + +// Gate exposes the wired sending-protection gate (nil when gateless), so the +// composition root's wiring test can prove the production bundle is armed. +func (j *Jobs) Gate() sendingpolicy.Gate { return j.gate } + // WithMetrics wires the observability backend the NotifyWorker emits the // notification-outcome counter on. Nil-safe; call before RegisterJobs. func (j *Jobs) WithMetrics(m Metrics) *Jobs { @@ -111,7 +130,7 @@ func (j *Jobs) NotifyWorker() *NotifyWorker { // ResolveLegacyOperation prepares the notification operation for a job that // carries no reference, in its own committed transaction, through the same // PrepareNotificationTx the sweep's enqueue runs. -func (j *Jobs) ResolveLegacyOperation(ctx context.Context, webhookID string) (sendingpolicy.OperationRef, error) { +func (j *Jobs) ResolveLegacyOperation(ctx context.Context, webhookID, kind string) (sendingpolicy.OperationRef, error) { if j.gate == nil || j.pool == nil { return sendingpolicy.OperationRef{}, fmt.Errorf("webhook notify: legacy operation resolver is not wired") } @@ -120,7 +139,7 @@ func (j *Jobs) ResolveLegacyOperation(ctx context.Context, webhookID string) (se return sendingpolicy.OperationRef{}, fmt.Errorf("begin legacy resolve: %w", err) } defer func() { _ = tx.Rollback(ctx) }() - ref, err := j.gate.PrepareNotificationTx(ctx, tx, sendingpolicy.NewWebhookHealthNotificationRef(webhookID)) + ref, err := j.gate.PrepareNotificationTx(ctx, tx, sendingpolicy.NewWebhookHealthNotificationRef(webhookID, kind)) if err != nil { return sendingpolicy.OperationRef{}, err } @@ -140,7 +159,7 @@ func (j *Jobs) ResolveLegacyOperation(ctx context.Context, webhookID string) (se func (j *Jobs) EnqueueWebhookNotifyTx(ctx context.Context, tx pgx.Tx, webhookID, kind string) (int64, error) { args := WebhookNotifyArgs{WebhookID: webhookID, NotifyKind: kind} if j.gate != nil { - ref, err := j.gate.PrepareNotificationTx(ctx, tx, sendingpolicy.NewWebhookHealthNotificationRef(webhookID)) + ref, err := j.gate.PrepareNotificationTx(ctx, tx, sendingpolicy.NewWebhookHealthNotificationRef(webhookID, kind)) if err != nil { return 0, fmt.Errorf("prepare notification operation: %w", err) } diff --git a/internal/webhooknotify/notifier.go b/internal/webhooknotify/notifier.go index 09e198094..7771f1cc9 100644 --- a/internal/webhooknotify/notifier.go +++ b/internal/webhooknotify/notifier.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "html" - "log" "net/url" "strings" "time" @@ -130,33 +129,57 @@ func (n *Notifier) WithDKIM(lookup outbound.DKIMKeyLookup) *Notifier { return n } -// Deliver composes and sends one health email, classifying the result for -// the NotifyWorker. Implements Deliverer. -func (n *Notifier) Deliver(ctx context.Context, wh *identity.Webhook, kind string, auth sendingpolicy.ProviderAuthorization) DeliverOutcome { - if err := n.send(ctx, wh, kind, auth); err != nil { - return DeliverOutcome{ - Err: err, - Permanent: outbound.IsPermanentSMTPError(err) || errors.Is(err, errNoOwnerEmail), - Outage: outbound.IsConnectionError(err), - } +// Compose implements Deliverer: the provider-free half (owner lookup, failure +// stats, MIME, Message-ID, DKIM), classified like a send so the worker +// treats a permanent compose failure the same way. +func (n *Notifier) Compose(ctx context.Context, wh *identity.Webhook, kind string) (outbound.Envelope, DeliverOutcome) { + env, err := n.compose(ctx, wh, kind) + if err != nil { + return outbound.Envelope{}, classify(err) + } + return env, DeliverOutcome{} +} + +// Submit implements Deliverer: one authorized submission, classified for the +// NotifyWorker. +func (n *Notifier) Submit(ctx context.Context, env outbound.Envelope, auth sendingpolicy.ProviderAuthorization) DeliverOutcome { + if _, err := n.submitter.SubmitOnce(ctx, auth, env); err != nil { + return classify(fmt.Errorf("webhook notify: smtp send: %w", err)) } return DeliverOutcome{} } -func (n *Notifier) send(ctx context.Context, wh *identity.Webhook, kind string, auth sendingpolicy.ProviderAuthorization) error { - if n == nil { - return nil +// Deliver composes and sends one health email with an already-authorized +// attempt: Compose then Submit in one call, for callers that hold the token +// up front (tests). The worker runs the two phases itself so the token is +// consumed last. +func (n *Notifier) Deliver(ctx context.Context, wh *identity.Webhook, kind string, auth sendingpolicy.ProviderAuthorization) DeliverOutcome { + env, out := n.Compose(ctx, wh, kind) + if out.Err != nil { + return out + } + return n.Submit(ctx, env, auth) +} + +func classify(err error) DeliverOutcome { + return DeliverOutcome{ + Err: err, + Permanent: outbound.IsPermanentSMTPError(err) || errors.Is(err, errNoOwnerEmail), + Outage: outbound.IsConnectionError(err), } +} + +func (n *Notifier) compose(ctx context.Context, wh *identity.Webhook, kind string) (outbound.Envelope, error) { if wh == nil { - return fmt.Errorf("webhook notify: webhook is nil") + return outbound.Envelope{}, fmt.Errorf("webhook notify: webhook is nil") } owner, err := n.store.GetUserByID(ctx, wh.UserID) if err != nil { - return fmt.Errorf("webhook notify: lookup owner: %w", err) + return outbound.Envelope{}, fmt.Errorf("webhook notify: lookup owner: %w", err) } if owner.Email == "" { - return fmt.Errorf("webhook notify: owner %s: %w", owner.ID, errNoOwnerEmail) + return outbound.Envelope{}, fmt.Errorf("webhook notify: owner %s: %w", owner.ID, errNoOwnerEmail) } window := identity.WarnWindow @@ -165,7 +188,7 @@ func (n *Notifier) send(ctx context.Context, wh *identity.Webhook, kind string, } stats, err := n.store.RecentWebhookFailureStats(ctx, wh.ID, window) if err != nil { - return fmt.Errorf("webhook notify: failure stats: %w", err) + return outbound.Envelope{}, fmt.Errorf("webhook notify: failure stats: %w", err) } reason := stats.LastError @@ -208,7 +231,7 @@ func (n *Notifier) send(ctx context.Context, wh *identity.Webhook, kind string, "", // no conversation_id ) if err != nil { - return fmt.Errorf("webhook notify: compose: %w", err) + return outbound.Envelope{}, fmt.Errorf("webhook notify: compose: %w", err) } // Deterministic Message-ID so a crash-after-send re-drive collapses at @@ -236,16 +259,7 @@ func (n *Notifier) send(ctx context.Context, wh *identity.Webhook, kind string, message = signed } - if _, err := n.submitter.SubmitOnce(ctx, auth, outbound.Envelope{ - From: n.fromAddress, - Recipients: []string{owner.Email}, - Message: message, - }); err != nil { - return fmt.Errorf("webhook notify: smtp send: %w", err) - } - - log.Printf("[webhook-notify] sent %s email: webhook=%s owner=%s", kind, wh.ID, owner.ID) - return nil + return outbound.Envelope{From: n.fromAddress, Recipients: []string{owner.Email}, Message: message}, nil } // endpointLabel condenses the webhook URL for the subject line: host when diff --git a/internal/webhooknotify/worker.go b/internal/webhooknotify/worker.go index 3fea56def..c7bd56897 100644 --- a/internal/webhooknotify/worker.go +++ b/internal/webhooknotify/worker.go @@ -22,6 +22,7 @@ import ( "github.com/riverqueue/river" "github.com/tokencanopy/e2a/internal/identity" + "github.com/tokencanopy/e2a/internal/outbound" "github.com/tokencanopy/e2a/internal/sendingpolicy" ) @@ -77,15 +78,32 @@ type DeliverOutcome struct { Outage bool // relay unreachable — snooze without spending an attempt } -// Deliverer composes and sends one health email. Implemented by *Notifier -// (compose + SMTPRelay.SendOnce + classify). +// Deliverer is the two-phase send of one health email. Compose does every +// fallible, provider-free step (owner lookup, failure stats, MIME, DKIM) and +// returns the envelope; Submit hands that envelope and a freshly consumed +// authorization to the provider seam. The split lets the worker +// ConsumeAttempt immediately before the socket opens, so a compose failure +// costs no charged ordinal. Implemented by *Notifier. type Deliverer interface { - Deliver(ctx context.Context, wh *identity.Webhook, kind string, auth sendingpolicy.ProviderAuthorization) DeliverOutcome + Compose(ctx context.Context, wh *identity.Webhook, kind string) (outbound.Envelope, DeliverOutcome) + Submit(ctx context.Context, env outbound.Envelope, auth sendingpolicy.ProviderAuthorization) DeliverOutcome } // OperationResolver recovers the durable operation for a job that carries no -// reference, through the same Prepare path an enqueue runs. -type OperationResolver func(ctx context.Context, webhookID string) (sendingpolicy.OperationRef, error) +// reference, through the same Prepare path the sweep's enqueue runs. The kind +// selects the episode (warning or disable) the operation is keyed by. +type OperationResolver func(ctx context.Context, webhookID, kind string) (sendingpolicy.OperationRef, error) + +// errOperationMismatch marks a job whose operation reference names another +// episode's (or another webhook's) operation: authorizing it would charge the +// wrong operation, and a reference for a superseded episode is stale anyway. +var errOperationMismatch = errors.New("webhook notify: job operation reference does not name this episode") + +// maxNotifyAge bounds how long a health notice may wait behind a gate hold. +// A pause has no clock of its own, and a disabled webhook never self-clears, +// so without this a held notice would snooze forever; a week-old health +// notice is stale by any reading. +const maxNotifyAge = 7 * 24 * time.Hour // ArgStamper persists a resolved reference into the job's args. type ArgStamper func(ctx context.Context, jobID int64, ref sendingpolicy.OperationRef) error @@ -223,20 +241,44 @@ func (w *NotifyWorker) Work(ctx context.Context, job *river.Job[WebhookNotifyArg w.emitNotify(kind, outcomeSkipped) return nil } + if kind == KindDisabled && wh.AutoDisabledAt == nil { + // Guard 5: disabled by hand, not by the breaker — there is no + // auto-disable episode to report. + w.emitNotify(kind, outcomeSkipped) + return nil + } + if !job.CreatedAt.IsZero() && time.Since(job.CreatedAt) > maxNotifyAge { + // Guard 6: a notice that waited a week behind a hold is stale; drop + // it rather than snooze forever behind a paused account. + log.Printf("[webhook-notify] dropping %s notice for %s: older than %s", kind, wh.ID, maxNotifyAge) + w.emitNotify(kind, outcomeSkipped) + return nil + } + + // Compose first: the owner lookup, failure stats, MIME and DKIM are + // fallible and provider-free, so they run before any attempt is charged. + env, out := w.deliverer.Compose(ctx, wh, kind) + if out.Err != nil { + return w.verdict(job, wh.ID, kind, "compose", out) + } // Every provider call is authorized: Reserve, hold without I/O, then - // ConsumeAttempt as the last decision before the deliverer's submitter - // redeems the token. A health notice has no durable hold class; the - // guards above re-run on every execution and drop a notice that went - // stale while it waited. + // ConsumeAttempt as the LAST decision before Submit, whose submitter + // redeems the token immediately before the socket opens. A health notice + // has no durable hold class; the guards above re-run on every execution + // and drop a notice that went stale while it waited. auth := sendingpolicy.ProviderAuthorization{} if w.gate != nil { - ref, err := w.operationFor(ctx, job) + ref, err := w.operationFor(ctx, job, wh) if err != nil { if errors.Is(err, sendingpolicy.ErrSourceUnavailable) { w.emitNotify(kind, outcomeSkipped) return nil } + if errors.Is(err, errOperationMismatch) { + w.emitNotify(kind, outcomeSkipped) + return river.JobCancel(err) + } w.emitNotify(kind, outcomeRetryable) return err } @@ -267,43 +309,61 @@ func (w *NotifyWorker) Work(ctx context.Context, job *river.Job[WebhookNotifyArg auth = *token } - out := w.deliverer.Deliver(ctx, wh, kind, auth) + out = w.deliverer.Submit(ctx, env, auth) if out.Err == nil { + log.Printf("[webhook-notify] sent %s email: webhook=%s", kind, wh.ID) w.emitNotify(kind, outcomeSent) return nil } + return w.verdict(job, wh.ID, kind, "send", out) +} + +// verdict turns a classified failure into River's answer. +func (w *NotifyWorker) verdict(job *river.Job[WebhookNotifyArgs], webhookID, kind, phase string, out DeliverOutcome) error { if out.Permanent { // e.g. the owner address is rejected 5xx, or there is no owner email // on record. Cancel (no retry) rather than churn the tail. - log.Printf("[webhook-notify] permanent send failure for %s (%s, no retry): %v", wh.ID, kind, out.Err) + log.Printf("[webhook-notify] permanent %s failure for %s (%s, no retry): %v", phase, webhookID, kind, out.Err) w.emitNotify(kind, outcomePermanent) return river.JobCancel(out.Err) } if out.Outage { // Relay unreachable — snooze without burning an attempt. The guards - // above re-run on the next attempt, so a notification that goes - // stale during the outage still drops correctly. + // re-run on the next attempt, so a notification that goes stale + // during the outage still drops correctly. w.emitNotify(kind, outcomeOutage) return river.JobSnooze(notifyOutageSnooze) } // Transient: let River reschedule per NextRetry until MaxNotifyAttempts. w.emitNotify(kind, outcomeRetryable) - return fmt.Errorf("webhook notify attempt %d failed: %w", job.Attempt, out.Err) + return fmt.Errorf("webhook notify attempt %d %s failed: %w", job.Attempt, phase, out.Err) } // operationFor returns the job's durable operation, resolving and stamping a // legacy job through the sweep's Prepare path. -func (w *NotifyWorker) operationFor(ctx context.Context, job *river.Job[WebhookNotifyArgs]) (sendingpolicy.OperationRef, error) { +func (w *NotifyWorker) operationFor(ctx context.Context, job *river.Job[WebhookNotifyArgs], wh *identity.Webhook) (sendingpolicy.OperationRef, error) { + // The episode's operation is derived from the webhook, the kind and the + // timestamp the sweep stamped, so a reference naming any other operation + // is either another account's (never authorize it) or a superseded + // episode's (nothing left to say): the binding the message worker + // enforces, checked before Reserve. + want := ExpectedOperationID(wh, job.Args.NotifyKind) if job.Args.OperationRef != nil && !job.Args.OperationRef.IsZero() { + if job.Args.OperationRef.ID() != want { + return sendingpolicy.OperationRef{}, errOperationMismatch + } return *job.Args.OperationRef, nil } if w.resolve == nil { return sendingpolicy.OperationRef{}, fmt.Errorf("webhook notify: legacy job %d carries no operation and no resolver is wired", job.ID) } - ref, err := w.resolve(ctx, job.Args.WebhookID) + ref, err := w.resolve(ctx, job.Args.WebhookID, job.Args.NotifyKind) if err != nil { return sendingpolicy.OperationRef{}, err } + if ref.ID() != want { + return sendingpolicy.OperationRef{}, errOperationMismatch + } if w.stamp != nil { if err := w.stamp(ctx, job.ID, ref); err != nil { log.Printf("[webhook-notify] stamp operation on legacy job %d: %v", job.ID, err) @@ -328,3 +388,27 @@ func (w *NotifyWorker) holdVerdict(kind string, d sendingpolicy.Decision) error } return river.JobSnooze(delay) } + +// ExpectedOperationID is the operation a notice of the given kind for this +// webhook's current episode must carry: the same derivation the gate's +// PrepareNotificationTx uses. Empty when the episode was never stamped. +func ExpectedOperationID(wh *identity.Webhook, kind string) string { + if wh == nil { + return "" + } + var episode *time.Time + switch kind { + case KindWarning: + episode = wh.WarnNotifiedAt + case KindDisabled: + episode = wh.AutoDisabledAt + } + if episode == nil { + return "" + } + return sendingpolicy.WebhookHealthOperationID(wh.ID, kind, *episode) +} + +// Gate exposes the wired gate (nil when gateless), for the composition +// root's wiring test. +func (w *NotifyWorker) Gate() sendingpolicy.Gate { return w.gate } diff --git a/internal/webhooknotify/worker_test.go b/internal/webhooknotify/worker_test.go index 4564ca63e..050242e33 100644 --- a/internal/webhooknotify/worker_test.go +++ b/internal/webhooknotify/worker_test.go @@ -13,6 +13,7 @@ import ( "github.com/riverqueue/river/rivertype" "github.com/tokencanopy/e2a/internal/identity" + "github.com/tokencanopy/e2a/internal/outbound" "github.com/tokencanopy/e2a/internal/sendingpolicy" "github.com/tokencanopy/e2a/internal/webhooknotify" ) @@ -27,17 +28,38 @@ func (f *fakeStore) GetWebhookByIDInternal(_ context.Context, _ string) (*identi } type fakeDeliverer struct { - out webhooknotify.DeliverOutcome - called int - kinds []string + out webhooknotify.DeliverOutcome // Submit's outcome + composeOut webhooknotify.DeliverOutcome // Compose's outcome + called int // Submit calls + composed int + kinds []string + auths []sendingpolicy.ProviderAuthorization + trace *[]string } -func (f *fakeDeliverer) Deliver(_ context.Context, _ *identity.Webhook, kind string, _ sendingpolicy.ProviderAuthorization) webhooknotify.DeliverOutcome { - f.called++ +func (f *fakeDeliverer) Compose(_ context.Context, _ *identity.Webhook, kind string) (outbound.Envelope, webhooknotify.DeliverOutcome) { + f.composed++ f.kinds = append(f.kinds, kind) + f.record("compose") + if f.composeOut.Err != nil { + return outbound.Envelope{}, f.composeOut + } + return outbound.Envelope{From: "e2a@notify.test", Recipients: []string{"owner@reviewer.test"}, Message: []byte("Subject: x\r\n\r\nbody")}, webhooknotify.DeliverOutcome{} +} + +func (f *fakeDeliverer) Submit(_ context.Context, _ outbound.Envelope, auth sendingpolicy.ProviderAuthorization) webhooknotify.DeliverOutcome { + f.called++ + f.record("submit") + f.auths = append(f.auths, auth) return f.out } +func (f *fakeDeliverer) record(step string) { + if f.trace != nil { + *f.trace = append(*f.trace, step) + } +} + func job(webhookID, kind string, attempt int) *river.Job[webhooknotify.WebhookNotifyArgs] { return &river.Job[webhooknotify.WebhookNotifyArgs]{ JobRow: &rivertype.JobRow{Attempt: attempt, MaxAttempts: webhooknotify.MaxNotifyAttempts, Kind: webhooknotify.WebhookNotifyArgs{}.Kind()}, @@ -45,14 +67,24 @@ func job(webhookID, kind string, attempt int) *river.Job[webhooknotify.WebhookNo } } +// episodeAt is the fixed auto-disable timestamp every disabled fixture +// carries: the breaker stamps it when it flips a webhook, and the operation +// a disable notice authorizes under is keyed by it. +var episodeAt = time.Date(2026, 9, 1, 12, 0, 0, 0, time.UTC) + func hook(enabled bool, warnedAt *time.Time) *identity.Webhook { - return &identity.Webhook{ + wh := &identity.Webhook{ ID: "wh_test", UserID: "user_test", URL: "https://hooks.example.com/inbox", Enabled: enabled, WarnNotifiedAt: warnedAt, } + if !enabled { + at := episodeAt + wh.AutoDisabledAt = &at + } + return wh } func now() *time.Time { t := time.Now(); return &t } @@ -238,6 +270,7 @@ func TestNotifyWorker_ErrorTriage(t *testing.T) { // fakeGate is a scriptable sendingpolicy.Gate for the worker-order tests. type fakeGate struct { + trace *[]string reserve sendingpolicy.Decision consume sendingpolicy.Decision reserveErr error @@ -263,10 +296,12 @@ func (g *fakeGate) PreparePublicFeedback(context.Context, sendingpolicy.PublicFe } func (g *fakeGate) Reserve(context.Context, sendingpolicy.OperationRef) (sendingpolicy.Decision, sendingpolicy.AttemptRef, error) { g.reserves++ + g.record("reserve") return g.reserve, sendingpolicy.AttemptRef{}, g.reserveErr } func (g *fakeGate) ConsumeAttempt(context.Context, sendingpolicy.AttemptRef) (sendingpolicy.Decision, *sendingpolicy.ProviderAuthorization, error) { g.consumes++ + g.record("consume") if !g.consume.Allow { return g.consume, nil, nil } @@ -295,9 +330,22 @@ func refFor(id string) sendingpolicy.OperationRef { return ref } +// gatedJob carries the operation a notice of this kind for the disabled +// fixture (hook(false, …)) is keyed by; a warning fixture passes its own +// webhook through gatedJobFor. func gatedJob(webhookID, kind string, attempt int) *river.Job[webhooknotify.WebhookNotifyArgs] { - j := job(webhookID, kind, attempt) - ref := refFor("op_" + webhookID) + wh := hook(false, nil) + wh.ID = webhookID + if kind == webhooknotify.KindWarning { + wh.Enabled = true + wh.WarnNotifiedAt = now() + } + return gatedJobFor(wh, kind, attempt) +} + +func gatedJobFor(wh *identity.Webhook, kind string, attempt int) *river.Job[webhooknotify.WebhookNotifyArgs] { + j := job(wh.ID, kind, attempt) + ref := refFor(webhooknotify.ExpectedOperationID(wh, kind)) j.Args.OperationRef = &ref return j } @@ -312,7 +360,7 @@ func TestNotifyWorker_GatedPathAuthorizesThenDelivers(t *testing.T) { fm := &fakeMetrics{} g := allowAll() w := webhooknotify.NewNotifyWorker(&fakeStore{wh: hook(false, nil)}, fd).WithMetrics(fm).WithGate(g) - if err := w.Work(context.Background(), gatedJob("wh_1", webhooknotify.KindDisabled, 1)); err != nil { + if err := w.Work(context.Background(), gatedJob("wh_test", webhooknotify.KindDisabled, 1)); err != nil { t.Fatalf("Work: %v", err) } if g.reserves != 1 || g.consumes != 1 || fd.called != 1 { @@ -328,7 +376,7 @@ func TestNotifyWorker_GateHoldSnoozesWithoutDelivery(t *testing.T) { } { fd := &fakeDeliverer{} w := webhooknotify.NewNotifyWorker(&fakeStore{wh: hook(false, nil)}, fd).WithMetrics(&fakeMetrics{}).WithGate(g) - if err := w.Work(context.Background(), gatedJob("wh_1", webhooknotify.KindDisabled, 1)); !isSnooze(err) || fd.called != 0 { + if err := w.Work(context.Background(), gatedJob("wh_test", webhooknotify.KindDisabled, 1)); !isSnooze(err) || fd.called != 0 { t.Fatalf("%s: err=%v delivers=%d, want snooze with no I/O", name, err, fd.called) } } @@ -338,15 +386,120 @@ func TestNotifyWorker_LegacyJobResolvesAndStampsOnce(t *testing.T) { fd := &fakeDeliverer{} resolved, stamped := 0, 0 w := webhooknotify.NewNotifyWorker(&fakeStore{wh: hook(false, nil)}, fd).WithMetrics(&fakeMetrics{}).WithGate(allowAll()). - WithOperationResolver(func(_ context.Context, id string) (sendingpolicy.OperationRef, error) { + WithOperationResolver(func(_ context.Context, id, kind string) (sendingpolicy.OperationRef, error) { resolved++ - return refFor("op_" + id), nil + wh := hook(false, nil) + wh.ID = id + return refFor(webhooknotify.ExpectedOperationID(wh, kind)), nil }). WithArgStamper(func(context.Context, int64, sendingpolicy.OperationRef) error { stamped++; return nil }) - if err := w.Work(context.Background(), job("wh_legacy", webhooknotify.KindDisabled, 1)); err != nil { + if err := w.Work(context.Background(), job("wh_test", webhooknotify.KindDisabled, 1)); err != nil { t.Fatalf("Work: %v", err) } if resolved != 1 || stamped != 1 || fd.called != 1 { t.Fatalf("resolved=%d stamped=%d delivers=%d, want 1/1/1", resolved, stamped, fd.called) } } + +func (g *fakeGate) record(step string) { + if g.trace != nil { + *g.trace = append(*g.trace, step) + } +} + +// TestNotifyWorker_ComposeRunsBeforeAnyChargeAndConsumeIsLast pins the order +// the seam depends on: compose precedes Reserve, ConsumeAttempt is the last +// call before Submit. +func TestNotifyWorker_ComposeRunsBeforeAnyChargeAndConsumeIsLast(t *testing.T) { + var trace []string + fd := &fakeDeliverer{trace: &trace} + g := allowAll() + g.trace = &trace + w := webhooknotify.NewNotifyWorker(&fakeStore{wh: hook(false, nil)}, fd).WithMetrics(&fakeMetrics{}).WithGate(g) + if err := w.Work(context.Background(), gatedJob("wh_test", webhooknotify.KindDisabled, 1)); err != nil { + t.Fatalf("Work: %v", err) + } + if got := strings.Join(trace, ","); got != "compose,reserve,consume,submit" { + t.Fatalf("order = %s, want compose,reserve,consume,submit", got) + } +} + +// TestNotifyWorker_ComposeFailureChargesNothing: a compose failure precedes +// Reserve, so it burns no ordinal. +func TestNotifyWorker_ComposeFailureChargesNothing(t *testing.T) { + for name, tc := range map[string]struct { + out webhooknotify.DeliverOutcome + wantErr func(error) bool + }{ + "transient": {out: webhooknotify.DeliverOutcome{Err: errors.New("stats blip")}, wantErr: func(err error) bool { return err != nil && !isSnooze(err) && !isCancel(err) }}, + "permanent": {out: webhooknotify.DeliverOutcome{Err: errors.New("no owner email"), Permanent: true}, wantErr: isCancel}, + "outage": {out: webhooknotify.DeliverOutcome{Err: errors.New("dkim store down"), Outage: true}, wantErr: isSnooze}, + } { + fd := &fakeDeliverer{composeOut: tc.out} + g := allowAll() + w := webhooknotify.NewNotifyWorker(&fakeStore{wh: hook(false, nil)}, fd).WithMetrics(&fakeMetrics{}).WithGate(g) + err := w.Work(context.Background(), gatedJob("wh_test", webhooknotify.KindDisabled, 1)) + if !tc.wantErr(err) { + t.Fatalf("%s: err = %v", name, err) + } + if g.reserves != 0 || g.consumes != 0 || fd.called != 0 { + t.Fatalf("%s: reserves=%d consumes=%d submits=%d, want 0/0/0", name, g.reserves, g.consumes, fd.called) + } + } +} + +// TestNotifyWorker_ForeignOrStaleOperationReferenceIsCancelled: a reference +// naming another webhook's operation, or a superseded episode of this one, +// is cancelled before Reserve. +func TestNotifyWorker_ForeignOrStaleOperationReferenceIsCancelled(t *testing.T) { + other := hook(false, nil) + other.ID = "wh_other" + stale := hook(false, nil) + at := episodeAt.Add(-time.Hour) + stale.AutoDisabledAt = &at + for name, ref := range map[string]sendingpolicy.OperationRef{ + "foreign webhook": refFor(webhooknotify.ExpectedOperationID(other, webhooknotify.KindDisabled)), + "stale episode": refFor(webhooknotify.ExpectedOperationID(stale, webhooknotify.KindDisabled)), + } { + fd := &fakeDeliverer{} + g := allowAll() + w := webhooknotify.NewNotifyWorker(&fakeStore{wh: hook(false, nil)}, fd).WithMetrics(&fakeMetrics{}).WithGate(g) + j := job("wh_test", webhooknotify.KindDisabled, 1) + r := ref + j.Args.OperationRef = &r + if err := w.Work(context.Background(), j); !isCancel(err) { + t.Fatalf("%s: err = %v, want cancel", name, err) + } + if g.reserves != 0 || fd.called != 0 { + t.Fatalf("%s: reserves=%d submits=%d, want 0/0", name, g.reserves, fd.called) + } + } +} + +// TestNotifyWorker_StaleNoticeIsDropped: a notice older than the age bound +// is dropped instead of snoozing forever behind a hold. +func TestNotifyWorker_StaleNoticeIsDropped(t *testing.T) { + fd := &fakeDeliverer{} + g := &fakeGate{reserve: sendingpolicy.Decision{Allow: false, Reason: sendingpolicy.ReasonAccountPaused}} + w := webhooknotify.NewNotifyWorker(&fakeStore{wh: hook(false, nil)}, fd).WithMetrics(&fakeMetrics{}).WithGate(g) + j := gatedJob("wh_test", webhooknotify.KindDisabled, 1) + j.CreatedAt = time.Now().Add(-8 * 24 * time.Hour) + if err := w.Work(context.Background(), j); err != nil { + t.Fatalf("err = %v, want a silent drop", err) + } + if g.reserves != 0 || fd.composed != 0 || fd.called != 0 { + t.Fatalf("reserves=%d composes=%d submits=%d, want 0/0/0", g.reserves, fd.composed, fd.called) + } +} + +// TestKindVocabularyMatchesGate: the job's kinds are the gate's episode kinds. +func TestKindVocabularyMatchesGate(t *testing.T) { + if webhooknotify.KindWarning != sendingpolicy.WebhookHealthKindWarning || webhooknotify.KindDisabled != sendingpolicy.WebhookHealthKindDisabled { + t.Fatal("webhooknotify kinds and sendingpolicy webhook health kinds disagree") + } +} + +func isCancel(err error) bool { + var cancel *river.JobCancelError + return errors.As(err, &cancel) +} From bec4437435f1265e98cb9ee690604b8a1f00a83e Mon Sep 17 00:00:00 2001 From: jiashuoz <39790535+jiashuoz@users.noreply.github.com> Date: Sat, 5 Sep 2026 00:31:01 -0700 Subject: [PATCH 11/12] fix(outbound): re-key pre-derivation notification references Re-review of the provider-seam closure (B7) found that migration 113 stamped adopted notify jobs with op_ references, which the new source binding would have cancelled on any upgrade crossing v1.8.7. - A reference that is not a derived id is treated as pre-derivation: the notify workers re-resolve it through the Prepare path and replace it once (jobs.SetJobArg); a derived id for another source still cancels. - The reconcile command scans and re-keys those references too. - Bounded the feedback attempt release (2s); symmetric 7-day age guard on HITL notices; episode key in microseconds; nil-receiver guards on the compose path; doc and comment corrections; the closure guard states its residual scope. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01AjfGxvXW6fNKWGFHuo68yX --- cmd/e2a/sending_reconcile.go | 28 +++++++--- cmd/e2a/sending_reconcile_test.go | 37 +++++++++++++ docs/design/async-message-pipeline.md | 14 +++-- internal/agent/api.go | 19 +++++-- internal/hitlnotify/jobs.go | 2 + internal/hitlnotify/notifier.go | 6 +++ internal/hitlnotify/worker.go | 45 +++++++++++++--- internal/hitlnotify/worker_test.go | 52 +++++++++++++++++++ internal/jobs/argstamp.go | 22 ++++++++ internal/jobs/argstamp_test.go | 19 +++++++ .../provider_authorization_guard_test.go | 16 ++++-- internal/sendingpolicy/types.go | 36 ++++++++++--- internal/webhooknotify/jobs.go | 2 + internal/webhooknotify/notifier.go | 6 +++ internal/webhooknotify/worker.go | 31 +++++++++-- internal/webhooknotify/worker_test.go | 36 +++++++++++++ 16 files changed, 334 insertions(+), 37 deletions(-) diff --git a/cmd/e2a/sending_reconcile.go b/cmd/e2a/sending_reconcile.go index 78476f90e..8ab57474a 100644 --- a/cmd/e2a/sending_reconcile.go +++ b/cmd/e2a/sending_reconcile.go @@ -41,6 +41,17 @@ var legacyReconcileStates = []string{ string(rivertype.JobStateScheduled), } +// conformingReferenceSQL is true for a river_job row whose operation_ref +// already has the shape its worker derives: the message id for a send, the +// op_hitl_ / op_wh_ derivations for the two notice kinds. +const conformingReferenceSQL = `( + (args ? 'operation_ref') AND ( + (kind = 'outbound_send') + OR (kind = 'hitl_notify' AND args->'operation_ref'->>'id' LIKE 'op\_hitl\_%') + OR (kind = 'webhook_notify' AND args->'operation_ref'->>'id' LIKE 'op\_wh\_%') + ) +)` + // legacyReconcileCounts is the operator-facing summary of one reconcile pass. type legacyReconcileCounts struct { Scanned int @@ -68,12 +79,15 @@ func runReconcileLegacySendingJobs(ctx context.Context, pool *pgxpool.Pool, gate if err != nil { return fmt.Errorf("river client: %w", err) } + // A job is legacy when it carries no reference, or a pre-derivation one: + // migration 113 stamped adopted notify jobs with op_, which the + // workers now re-key at fire time; this command does the same up front. rows, err := pool.Query(ctx, ` SELECT id, kind, args FROM river_job WHERE kind = ANY($1) AND state = ANY($2) - AND NOT (args ? 'operation_ref') + AND NOT `+conformingReferenceSQL+` ORDER BY id`, legacySendingJobKinds, legacyReconcileStates) if err != nil { return fmt.Errorf("scan legacy sending jobs: %w", err) @@ -158,17 +172,17 @@ func reconcileLegacySendingJob(ctx context.Context, pool *pgxpool.Pool, client * // left the reconcilable states is skipped and left to that worker. The // lock also serializes against the worker's own stamp. var state string - var stamped bool + var conforming bool err = tx.QueryRow(ctx, - `SELECT state, (args ? 'operation_ref') FROM river_job WHERE id = $1 FOR UPDATE`, jobID, - ).Scan(&state, &stamped) + `SELECT state, `+conformingReferenceSQL+` FROM river_job WHERE id = $1 FOR UPDATE`, jobID, + ).Scan(&state, &conforming) if errors.Is(err, pgx.ErrNoRows) { return legacyOutcomeSkipped, nil } if err != nil { return 0, fmt.Errorf("lock job: %w", err) } - if stamped || !slices.Contains(legacyReconcileStates, state) { + if conforming || !slices.Contains(legacyReconcileStates, state) { return legacyOutcomeSkipped, nil } @@ -227,7 +241,9 @@ func reconcileLegacySendingJob(ctx context.Context, pool *pgxpool.Pool, client * return 0, fmt.Errorf("cancel (%s): %w", cancelReason, err) } outcome = legacyOutcomeCancelled - } else if err := jobs.StampJobArg(ctx, tx, jobID, "operation_ref", ref); err != nil { + } else if err := jobs.SetJobArg(ctx, tx, jobID, "operation_ref", ref); err != nil { + // Unconditional: the row is locked and known non-conforming, and a + // pre-derivation reference must be replaced, not kept. return 0, err } if err := tx.Commit(ctx); err != nil { diff --git a/cmd/e2a/sending_reconcile_test.go b/cmd/e2a/sending_reconcile_test.go index ea82fdf71..677512792 100644 --- a/cmd/e2a/sending_reconcile_test.go +++ b/cmd/e2a/sending_reconcile_test.go @@ -257,3 +257,40 @@ func TestReconcileLegacySendingJobsLeavesClaimedJobsToTheirWorker(t *testing.T) t.Fatalf("already stamped job: outcome=%v err=%v, want skipped", outcome, err) } } + +// TestReconcileLegacySendingJobsReKeysPreDerivationReferences: a notify job +// migration 113 stamped with op_ is scanned, re-resolved through the +// Prepare path and re-keyed to the derived id; a conforming one is left alone. +func TestReconcileLegacySendingJobsReKeysPreDerivationReferences(t *testing.T) { + ctx := context.Background() + pool := testutil.TestDB(t) + resetRiverJobs(t, pool) + store := identity.NewStore(pool) + gate := sendingpolicy.NewGate(pool, sendingpolicy.Secrets{}, sendingpolicy.PolicySourceConfig, sendingpolicy.DisabledPolicy()) + msg, wh := seedReconcileSource(t, pool, store, "rekey") + + md5Hitl := insertLegacyJob(t, pool, "hitl_notify", `{"message_id":"`+msg.ID+`","operation_ref":{"v":1,"id":"op_0123456789abcdef0123456789abcdef"}}`) + md5Wh := insertLegacyJob(t, pool, "webhook_notify", `{"webhook_id":"`+wh.ID+`","kind":"warning","operation_ref":{"v":1,"id":"op_fedcba9876543210fedcba9876543210"}}`) + conforming := insertLegacyJob(t, pool, "hitl_notify", `{"message_id":"`+msg.ID+`","operation_ref":{"v":1,"id":"`+sendingpolicy.HITLNotificationOperationID(msg.ID)+`"}}`) + send := insertLegacyJob(t, pool, "outbound_send", `{"message_id":"`+msg.ID+`","operation_ref":{"v":1,"id":"`+msg.ID+`"}}`) + + var out bytes.Buffer + if err := runReconcileLegacySendingJobs(ctx, pool, gate, &out); err != nil { + t.Fatalf("reconcile: %v\n%s", err, out.String()) + } + if !strings.Contains(out.String(), "scanned: 2") || !strings.Contains(out.String(), "stamped: 2") { + t.Fatalf("want the two md5-keyed jobs scanned and re-keyed:\n%s", out.String()) + } + if _, op := legacyJobState(t, pool, md5Hitl); op != sendingpolicy.HITLNotificationOperationID(msg.ID) { + t.Errorf("hitl job op = %q, want the derived id", op) + } + if _, op := legacyJobState(t, pool, md5Wh); op != webhooknotify.ExpectedOperationID(wh, webhooknotify.KindWarning) { + t.Errorf("webhook job op = %q, want the warning episode's derived id", op) + } + if _, op := legacyJobState(t, pool, conforming); op != sendingpolicy.HITLNotificationOperationID(msg.ID) { + t.Errorf("conforming hitl job touched: %q", op) + } + if _, op := legacyJobState(t, pool, send); op != msg.ID { + t.Errorf("conforming send job touched: %q", op) + } +} diff --git a/docs/design/async-message-pipeline.md b/docs/design/async-message-pipeline.md index b6cee6c6b..89c4f08f0 100644 --- a/docs/design/async-message-pipeline.md +++ b/docs/design/async-message-pipeline.md @@ -312,8 +312,13 @@ cross it: `op_wh___` for a health notice, where the episode is the `warn_notified_at` / `auto_disabled_at` stamp the sweep wrote in the same transaction — so preparing the same source twice yields - one operation, and the worker cancels a job whose reference names any - other operation (the binding the message worker enforces). The worker + one operation, and the worker cancels a job whose reference is a derived + id for any other source (the binding the message worker enforces). A + reference of any other shape — migration 113 stamped adopted notify jobs + with `op_` — is a pre-derivation reference for the job's own source: + the worker re-resolves it through the same Prepare path and replaces it + once (`jobs.SetJobArg`), so an upgrade that crosses v1.8.7 drains its + backlog instead of cancelling it. The worker order is compose → Reserve → early hold → ConsumeAttempt → authorized submit: every fallible, provider-free step (owner lookup, token signing, MIME, DKIM) runs before an ordinal is charged, and the token is consumed @@ -330,8 +335,9 @@ cross it: Operators cutting over a slot with a queued backlog run `e2a -reconcile-legacy-sending-jobs`: it stamps an operation onto every -pending `outbound_send` / `hitl_notify` / `webhook_notify` job that has none, -through exactly the Prepare path its enqueue would have used, cancels the +pending `outbound_send` / `hitl_notify` / `webhook_notify` job that has none +or a pre-derivation one, through exactly the Prepare path its enqueue would +have used, cancels the ones whose source row is gone, and exits nonzero unless every scanned job was decided. Each job is re-read under its row lock inside its own transaction, so one a worker claimed after the scan is skipped and left to that worker; diff --git a/internal/agent/api.go b/internal/agent/api.go index 8a450f467..c94be1ed7 100644 --- a/internal/agent/api.go +++ b/internal/agent/api.go @@ -2009,7 +2009,10 @@ func (a *API) sendFeedbackEmail(ctx context.Context, title, category, message, s // this path (the request ends here), so give its units back // rather than leave them charged until midnight. Best effort: // the gate's day-scoped expiry is the backstop. - if cerr := a.gate.CancelAttempt(context.WithoutCancel(ctx), attemptRef); cerr != nil { + releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), feedbackReleaseTimeout) + cerr := a.gate.CancelAttempt(releaseCtx, attemptRef) + cancel() + if cerr != nil { log.Printf("[feedback] release reserved attempt after authorize error: %v", cerr) } return fmt.Errorf("authorize feedback attempt: %w", err) @@ -2033,13 +2036,21 @@ func (a *API) sendFeedbackEmail(ctx context.Context, title, category, message, s } // feedbackSendAttempts bounds the physical submissions one feedback request -// may make; feedbackRetryBackoff paces them so every attempt fits inside -// feedbackEmailTimeout. Each is a distinct charged attempt on the feedback -// operation. +// may make; feedbackRetryBackoff paces them. The sleeps total six of the ten +// seconds feedbackEmailTimeout allows, so all four attempts fit only when +// the relay answers quickly (a refused connection, a fast 4xx); a relay that +// hangs consumes the budget on its first attempt and the deadline exit +// reports that attempt's error. Each attempt is a distinct charged ordinal +// on the feedback operation. const feedbackSendAttempts = 4 var feedbackRetryBackoff = []time.Duration{time.Second, 2 * time.Second, 3 * time.Second} +// feedbackReleaseTimeout bounds the best-effort release of a reserved +// attempt after an authorize error, so a database that is already failing +// cannot park the handler goroutine. +const feedbackReleaseTimeout = 2 * time.Second + // feedbackSubmissionID mints the server-side identity one feedback request's // operation is keyed by. func feedbackSubmissionID() (string, error) { diff --git a/internal/hitlnotify/jobs.go b/internal/hitlnotify/jobs.go index 177c1dec1..554f28820 100644 --- a/internal/hitlnotify/jobs.go +++ b/internal/hitlnotify/jobs.go @@ -108,6 +108,8 @@ func (j *Jobs) NotifyWorker() *NotifyWorker { if j.pool != nil { w = w.WithArgStamper(func(ctx context.Context, jobID int64, ref sendingpolicy.OperationRef) error { return jobs.StampJobArg(ctx, j.pool, jobID, "operation_ref", ref) + }).WithArgRestamper(func(ctx context.Context, jobID int64, ref sendingpolicy.OperationRef) error { + return jobs.SetJobArg(ctx, j.pool, jobID, "operation_ref", ref) }) } return w diff --git a/internal/hitlnotify/notifier.go b/internal/hitlnotify/notifier.go index 2eb409968..c8503c1ae 100644 --- a/internal/hitlnotify/notifier.go +++ b/internal/hitlnotify/notifier.go @@ -253,6 +253,9 @@ func (n *Notifier) submit(ctx context.Context, env outbound.Envelope, auth sendi // Compose implements Deliverer: the provider-free half, classified like a // send so the worker treats a permanent compose failure the same way. func (n *Notifier) Compose(ctx context.Context, pn *identity.PendingNotify) (outbound.Envelope, DeliverOutcome) { + if n == nil { + return outbound.Envelope{}, DeliverOutcome{Err: fmt.Errorf("notify: notifier is nil")} + } if pn == nil { return outbound.Envelope{}, DeliverOutcome{Err: fmt.Errorf("notify: nothing to compose"), Permanent: true} } @@ -267,6 +270,9 @@ func (n *Notifier) Compose(ctx context.Context, pn *identity.PendingNotify) (out // River NotifyWorker — a 5xx / validation reject is Permanent (no retry), an // unreachable relay is an Outage (snooze), everything else retries. func (n *Notifier) Submit(ctx context.Context, env outbound.Envelope, auth sendingpolicy.ProviderAuthorization) DeliverOutcome { + if n == nil { + return DeliverOutcome{Err: fmt.Errorf("notify: notifier is nil")} + } if err := n.submit(ctx, env, auth); err != nil { return classify(err) } diff --git a/internal/hitlnotify/worker.go b/internal/hitlnotify/worker.go index fc7656b5f..7245737bb 100644 --- a/internal/hitlnotify/worker.go +++ b/internal/hitlnotify/worker.go @@ -85,6 +85,10 @@ type Deliverer interface { // account, so the job is cancelled, never retried. var errOperationMismatch = errors.New("hitl notify: job operation reference does not name this message") +// maxNotifyAge bounds how long an approval request may wait behind a gate +// hold, independent of the hold's own TTL. +const maxNotifyAge = 7 * 24 * time.Hour + // OperationResolver recovers the durable operation for a job that carries no // reference, through the same Prepare path an enqueue runs. type OperationResolver func(ctx context.Context, messageID string) (sendingpolicy.OperationRef, error) @@ -114,6 +118,7 @@ type NotifyWorker struct { gate sendingpolicy.Gate resolve OperationResolver stamp ArgStamper + restamp ArgStamper } // NewNotifyWorker builds a worker with no sending-protection gate. Without one @@ -140,7 +145,8 @@ func (w *NotifyWorker) WithOperationResolver(r OperationResolver) *NotifyWorker return w } -// WithArgStamper injects the job-args stamp used after a legacy resolution. +// WithArgStamper injects the job-args stamp used after a legacy resolution +// (adds the reference only when absent). func (w *NotifyWorker) WithArgStamper(s ArgStamper) *NotifyWorker { if s != nil { w.stamp = s @@ -148,6 +154,15 @@ func (w *NotifyWorker) WithArgStamper(s ArgStamper) *NotifyWorker { return w } +// WithArgRestamper injects the unconditional re-key used when a job carries +// a pre-derivation reference. +func (w *NotifyWorker) WithArgRestamper(s ArgStamper) *NotifyWorker { + if s != nil { + w.restamp = s + } + return w +} + // NextRetry overrides River's default backoff with the decided notify envelope. func (w *NotifyWorker) NextRetry(job *river.Job[HITLNotifyArgs]) time.Time { i := job.Attempt @@ -176,6 +191,13 @@ func (w *NotifyWorker) Work(ctx context.Context, job *river.Job[HITLNotifyArgs]) if msg.ApprovalExpiresAt != nil && msg.ApprovalExpiresAt.Before(time.Now()) { return nil // hold already past TTL — a review email is now useless } + if !job.CreatedAt.IsZero() && time.Since(job.CreatedAt) > maxNotifyAge { + // A hold with no TTL on record behind a paused account would otherwise + // snooze forever (River's snooze spends no attempt); a week-old + // approval request is stale by any reading. + log.Printf("[hitl-notify] dropping notice for %s: older than %s", msg.ID, maxNotifyAge) + return nil + } if pn.Notified { return nil // a prior attempt already sent it (crash-after-send re-drive) } @@ -271,11 +293,22 @@ func (w *NotifyWorker) operationFor(ctx context.Context, job *river.Job[HITLNoti // reference naming any other operation would charge another account: the // same binding the message worker enforces, checked before Reserve. want := sendingpolicy.HITLNotificationOperationID(job.Args.MessageID) + stamp := w.stamp if job.Args.OperationRef != nil && !job.Args.OperationRef.IsZero() { - if job.Args.OperationRef.ID() != want { + stored := job.Args.OperationRef.ID() + if stored == want { + return *job.Args.OperationRef, nil + } + if sendingpolicy.IsHITLNotificationOperationID(stored) { + // A derived id for a different message: foreign, never authorize. return sendingpolicy.OperationRef{}, errOperationMismatch } - return *job.Args.OperationRef, nil + // A pre-derivation reference — migration 113 stamped adopted jobs + // with op_, and the first build of this seam minted op_. + // Its source is still this job's own message, so re-derive through + // the same Prepare path and replace the reference, once. + log.Printf("[hitl-notify] job %d carries a pre-derivation operation reference %s; re-keying", job.ID, stored) + stamp = w.restamp } if w.resolve == nil { return sendingpolicy.OperationRef{}, fmt.Errorf("hitl notify: legacy job %d carries no operation and no resolver is wired", job.ID) @@ -287,10 +320,10 @@ func (w *NotifyWorker) operationFor(ctx context.Context, job *river.Job[HITLNoti if ref.ID() != want { return sendingpolicy.OperationRef{}, errOperationMismatch } - if w.stamp != nil { - if err := w.stamp(ctx, job.ID, ref); err != nil { + if stamp != nil { + if err := stamp(ctx, job.ID, ref); err != nil { // Not fatal: the reference is valid for this execution; a retry - // resolves again and stamps then. + // resolves again (idempotently) and stamps then. log.Printf("[hitl-notify] stamp operation on legacy job %d: %v", job.ID, err) } } diff --git a/internal/hitlnotify/worker_test.go b/internal/hitlnotify/worker_test.go index 638dc64e5..e8c71a0cc 100644 --- a/internal/hitlnotify/worker_test.go +++ b/internal/hitlnotify/worker_test.go @@ -464,3 +464,55 @@ func TestNotifyWorker_ForeignOperationReferenceIsCancelled(t *testing.T) { t.Fatalf("legacy: reserves=%d submits=%d, want 0/0", g.reserves, fd.called) } } + +// TestNotifyWorker_PreDerivationReferenceIsReKeyed: a job stamped before the +// source-derived ids existed (migration 113's op_, or the first build +// of this seam) is re-resolved through the Prepare path and its reference +// replaced, not cancelled — its source is still this job's own message. +func TestNotifyWorker_PreDerivationReferenceIsReKeyed(t *testing.T) { + fd := &fakeDeliverer{} + g := allowAll() + resolved, stamped, restamped := 0, 0, 0 + var restampedWith string + w := hitlnotify.NewNotifyWorker(&fakeStore{pn: pending("msg_1")}, fd).WithGate(g). + WithOperationResolver(func(_ context.Context, id string) (sendingpolicy.OperationRef, error) { + resolved++ + return refFor(sendingpolicy.HITLNotificationOperationID(id)), nil + }). + WithArgStamper(func(context.Context, int64, sendingpolicy.OperationRef) error { stamped++; return nil }). + WithArgRestamper(func(_ context.Context, _ int64, ref sendingpolicy.OperationRef) error { + restamped++ + restampedWith = ref.ID() + return nil + }) + j := job("msg_1", 1) + legacy := refFor("op_0123456789abcdef0123456789abcdef") + j.Args.OperationRef = &legacy + if err := w.Work(context.Background(), j); err != nil { + t.Fatalf("Work: %v", err) + } + if resolved != 1 || restamped != 1 || stamped != 0 || restampedWith != sendingpolicy.HITLNotificationOperationID("msg_1") { + t.Fatalf("resolved=%d restamped=%d stamped=%d with=%q, want 1/1/0 with the derived id", resolved, restamped, stamped, restampedWith) + } + if g.reserves != 1 || fd.called != 1 { + t.Fatalf("reserves=%d submits=%d, want 1/1", g.reserves, fd.called) + } +} + +// TestNotifyWorker_StaleNoticeIsDropped: a request older than the age bound +// is dropped instead of snoozing forever behind a hold. +func TestNotifyWorker_StaleNoticeIsDropped(t *testing.T) { + fd := &fakeDeliverer{} + g := &fakeGate{reserve: sendingpolicy.Decision{Allow: false, Reason: sendingpolicy.ReasonAccountPaused}} + pn := pending("msg_1") + pn.Message.ApprovalExpiresAt = nil + w := hitlnotify.NewNotifyWorker(&fakeStore{pn: pn}, fd).WithGate(g) + j := gatedJob("msg_1", 1) + j.CreatedAt = time.Now().Add(-8 * 24 * time.Hour) + if err := w.Work(context.Background(), j); err != nil { + t.Fatalf("err = %v, want a silent drop", err) + } + if g.reserves != 0 || fd.composed != 0 || fd.called != 0 { + t.Fatalf("reserves=%d composes=%d submits=%d, want 0/0/0", g.reserves, fd.composed, fd.called) + } +} diff --git a/internal/jobs/argstamp.go b/internal/jobs/argstamp.go index 4ea42a39b..569b0018c 100644 --- a/internal/jobs/argstamp.go +++ b/internal/jobs/argstamp.go @@ -38,3 +38,25 @@ func StampJobArg(ctx context.Context, db Execer, jobID int64, key string, value } return nil } + +// SetJobArg writes one key into a River job's args unconditionally, leaving +// every other field in place. It is the re-key half of the compatibility +// story: a job whose reference predates the source-derived ids (migration +// 113 stamped `op_`) is re-resolved through the same Prepare path and +// its reference replaced, once. +func SetJobArg(ctx context.Context, db Execer, jobID int64, key string, value any) error { + if db == nil { + return fmt.Errorf("set job arg: no database") + } + patch, err := json.Marshal(map[string]any{key: value}) + if err != nil { + return fmt.Errorf("set job arg: encode %s: %w", key, err) + } + if _, err := db.Exec(ctx, + `UPDATE river_job SET args = args || $2::jsonb WHERE id = $1`, + jobID, string(patch), + ); err != nil { + return fmt.Errorf("set job arg %s on job %d: %w", key, jobID, err) + } + return nil +} diff --git a/internal/jobs/argstamp_test.go b/internal/jobs/argstamp_test.go index 8a56afe5d..45dc45c42 100644 --- a/internal/jobs/argstamp_test.go +++ b/internal/jobs/argstamp_test.go @@ -43,6 +43,25 @@ func TestStampJobArg(t *testing.T) { if err := jobs.StampJobArg(ctx, pool, id+1000, "operation_ref", "x"); err != nil { t.Fatalf("missing job must be a no-op, got %v", err) } + + // SetJobArg replaces the key and keeps the rest. + if err := jobs.SetJobArg(ctx, pool, id, "operation_ref", map[string]any{"v": 1, "id": "op_3"}); err != nil { + t.Fatalf("set: %v", err) + } + if err := pool.QueryRow(ctx, + `SELECT args->>'message_id', args->'operation_ref'->>'id' FROM river_job WHERE id = $1`, id, + ).Scan(&messageID, &opID); err != nil { + t.Fatal(err) + } + if messageID != "msg_1" || opID != "op_3" { + t.Fatalf("after set: message_id=%q operation_ref.id=%q, want msg_1 / op_3", messageID, opID) + } + if err := jobs.SetJobArg(ctx, nil, id, "k", "v"); err == nil { + t.Fatal("nil database must be refused") + } + if err := jobs.SetJobArg(ctx, pool, id, "k", make(chan int)); err == nil { + t.Fatal("unencodable value must be refused") + } } // TestStampJobArgRefusesBadInputs: no database and an unencodable value are diff --git a/internal/outbound/provider_authorization_guard_test.go b/internal/outbound/provider_authorization_guard_test.go index 86af2f0ba..b8f50bbad 100644 --- a/internal/outbound/provider_authorization_guard_test.go +++ b/internal/outbound/provider_authorization_guard_test.go @@ -19,9 +19,17 @@ import ( // authorized adapter; // - any exported relay method that could open a socket without a token. // -// Exceptions are exact file paths, never substrings, and each is named here -// with the reason it may exist. Adding a provider-bound caller anywhere else -// fails this test until it goes through ProviderSubmitter.SubmitOnce. +// Exceptions are exact file paths (or one exact symbol), never substrings, +// and each is named here with the reason it may exist. Adding a +// provider-bound caller anywhere else fails this test until it goes through +// ProviderSubmitter.SubmitOnce. +// +// What it does not see, stated so nobody over-reads it: a second unexported +// dialer added inside smtp_relay.go under another name (that file may import +// net/smtp; the sentinel check catches a rename of the core, not an addition +// beside it), a mail-capable SDK other than the ones fenced below, and any +// provider reached over plain net/http. Those arrive as a new import or a new +// dependency, which is where review catches them. func TestEveryProviderCallRequiresAuthorization(t *testing.T) { root := moduleRoot(t) files := trackedGoFiles(t, root) @@ -102,7 +110,7 @@ func TestEveryProviderCallRequiresAuthorization(t *testing.T) { allowedSocketCalls++ return true } - t.Errorf("%s:%s references the relay's socket-opening core outside ProviderSubmitter.SubmitOnce", rel, fset.Position(sel.Pos())) + t.Errorf("%s references the relay's socket-opening core outside ProviderSubmitter.SubmitOnce", fset.Position(sel.Pos())) return true }) } diff --git a/internal/sendingpolicy/types.go b/internal/sendingpolicy/types.go index 6886571bb..013c2bc9f 100644 --- a/internal/sendingpolicy/types.go +++ b/internal/sendingpolicy/types.go @@ -374,7 +374,26 @@ const ( // PrepareNotificationTx idempotent per hold and lets the worker bind a // job's reference to its source the way the message worker does. func HITLNotificationOperationID(messageID string) string { - return "op_hitl_" + messageID + return hitlOperationPrefix + messageID +} + +const ( + hitlOperationPrefix = "op_hitl_" + webhookHealthOperationPrefix = "op_wh_" +) + +// IsHITLNotificationOperationID reports whether an id has the source-derived +// shape above. An id of any other shape — migration 113 stamped adopted +// notify jobs with `op_` — is a pre-derivation reference: its source is +// still the job's own, so a worker re-derives rather than refuses it. +func IsHITLNotificationOperationID(id string) bool { + return strings.HasPrefix(id, hitlOperationPrefix) +} + +// IsWebhookHealthOperationID reports whether an id has the episode-derived +// shape; see IsHITLNotificationOperationID for what any other shape means. +func IsWebhookHealthOperationID(id string) bool { + return strings.HasPrefix(id, webhookHealthOperationPrefix) } // WebhookHealthOperationID is the operation id of one webhook health @@ -382,7 +401,7 @@ func HITLNotificationOperationID(messageID string) string { // the state (warn_notified_at or auto_disabled_at). A webhook that recovers // and fails again is a new episode with a new operation. func WebhookHealthOperationID(webhookID, kind string, episode time.Time) string { - return fmt.Sprintf("op_wh_%s_%s_%d", kind, webhookID, episode.UTC().Unix()) + return fmt.Sprintf("%s%s_%s_%d", webhookHealthOperationPrefix, kind, webhookID, episode.UTC().UnixMicro()) } // NewHITLNotificationRef references a pending outbound message whose approval @@ -562,12 +581,13 @@ func (a ProviderAuthorization) Attempt() AttemptRef { return a.attempt } // Purpose exposes the derived purpose, for metrics. func (a ProviderAuthorization) Purpose() Purpose { return a.purpose } -// AuthorizedRecipients returns a defensive copy of the exact final envelope. -// -// Only the protection notifier uses it: that path is the one caller that does -// not already know its recipient, because the address is resolved under lock at -// final authorization and deliberately never persisted in plaintext. Every -// other caller composed its own envelope and must not re-derive one here. +// AuthorizedRecipients is the normalized recipient set this token permits, +// in canonical order. The protection notifier and public feedback compose +// their envelope from it — their recipients are configuration the gate +// already resolved, never a customer-controlled list. Every other caller +// composed its own envelope from the source row and hands that to the seam, +// which proves it names exactly these mailboxes (ValidateEnvelope) before it +// dials; a mismatch there fails closed rather than being re-derived here. func (a ProviderAuthorization) AuthorizedRecipients() []string { out := make([]string, len(a.recipients)) copy(out, a.recipients) diff --git a/internal/webhooknotify/jobs.go b/internal/webhooknotify/jobs.go index ef17d8bee..ce8f9cb7f 100644 --- a/internal/webhooknotify/jobs.go +++ b/internal/webhooknotify/jobs.go @@ -122,6 +122,8 @@ func (j *Jobs) NotifyWorker() *NotifyWorker { if j.pool != nil { w = w.WithArgStamper(func(ctx context.Context, jobID int64, ref sendingpolicy.OperationRef) error { return jobs.StampJobArg(ctx, j.pool, jobID, "operation_ref", ref) + }).WithArgRestamper(func(ctx context.Context, jobID int64, ref sendingpolicy.OperationRef) error { + return jobs.SetJobArg(ctx, j.pool, jobID, "operation_ref", ref) }) } return w diff --git a/internal/webhooknotify/notifier.go b/internal/webhooknotify/notifier.go index 7771f1cc9..dc095e8d3 100644 --- a/internal/webhooknotify/notifier.go +++ b/internal/webhooknotify/notifier.go @@ -133,6 +133,9 @@ func (n *Notifier) WithDKIM(lookup outbound.DKIMKeyLookup) *Notifier { // stats, MIME, Message-ID, DKIM), classified like a send so the worker // treats a permanent compose failure the same way. func (n *Notifier) Compose(ctx context.Context, wh *identity.Webhook, kind string) (outbound.Envelope, DeliverOutcome) { + if n == nil { + return outbound.Envelope{}, DeliverOutcome{Err: fmt.Errorf("webhook notify: notifier is nil")} + } env, err := n.compose(ctx, wh, kind) if err != nil { return outbound.Envelope{}, classify(err) @@ -143,6 +146,9 @@ func (n *Notifier) Compose(ctx context.Context, wh *identity.Webhook, kind strin // Submit implements Deliverer: one authorized submission, classified for the // NotifyWorker. func (n *Notifier) Submit(ctx context.Context, env outbound.Envelope, auth sendingpolicy.ProviderAuthorization) DeliverOutcome { + if n == nil { + return DeliverOutcome{Err: fmt.Errorf("webhook notify: notifier is nil")} + } if _, err := n.submitter.SubmitOnce(ctx, auth, env); err != nil { return classify(fmt.Errorf("webhook notify: smtp send: %w", err)) } diff --git a/internal/webhooknotify/worker.go b/internal/webhooknotify/worker.go index c7bd56897..50de3d0e3 100644 --- a/internal/webhooknotify/worker.go +++ b/internal/webhooknotify/worker.go @@ -145,6 +145,7 @@ type NotifyWorker struct { gate sendingpolicy.Gate resolve OperationResolver stamp ArgStamper + restamp ArgStamper metrics Metrics // nil ⇒ no emission (nil-safe via emitNotify) } @@ -170,7 +171,8 @@ func (w *NotifyWorker) WithOperationResolver(r OperationResolver) *NotifyWorker return w } -// WithArgStamper injects the job-args stamp used after a legacy resolution. +// WithArgStamper injects the job-args stamp used after a legacy resolution +// (adds the reference only when absent). func (w *NotifyWorker) WithArgStamper(s ArgStamper) *NotifyWorker { if s != nil { w.stamp = s @@ -178,6 +180,15 @@ func (w *NotifyWorker) WithArgStamper(s ArgStamper) *NotifyWorker { return w } +// WithArgRestamper injects the unconditional re-key used when a job carries +// a pre-derivation reference. +func (w *NotifyWorker) WithArgRestamper(s ArgStamper) *NotifyWorker { + if s != nil { + w.restamp = s + } + return w +} + func (w *NotifyWorker) WithMetrics(m Metrics) *NotifyWorker { w.metrics = m return w @@ -348,11 +359,21 @@ func (w *NotifyWorker) operationFor(ctx context.Context, job *river.Job[WebhookN // episode's (nothing left to say): the binding the message worker // enforces, checked before Reserve. want := ExpectedOperationID(wh, job.Args.NotifyKind) + stamp := w.stamp if job.Args.OperationRef != nil && !job.Args.OperationRef.IsZero() { - if job.Args.OperationRef.ID() != want { + stored := job.Args.OperationRef.ID() + if stored == want { + return *job.Args.OperationRef, nil + } + if sendingpolicy.IsWebhookHealthOperationID(stored) { + // A derived id for another webhook or a superseded episode. return sendingpolicy.OperationRef{}, errOperationMismatch } - return *job.Args.OperationRef, nil + // A pre-derivation reference (migration 113's op_, or the first + // build of this seam): its source is still this job's own webhook, + // so re-derive through the same Prepare path and replace it, once. + log.Printf("[webhook-notify] job %d carries a pre-derivation operation reference %s; re-keying", job.ID, stored) + stamp = w.restamp } if w.resolve == nil { return sendingpolicy.OperationRef{}, fmt.Errorf("webhook notify: legacy job %d carries no operation and no resolver is wired", job.ID) @@ -364,8 +385,8 @@ func (w *NotifyWorker) operationFor(ctx context.Context, job *river.Job[WebhookN if ref.ID() != want { return sendingpolicy.OperationRef{}, errOperationMismatch } - if w.stamp != nil { - if err := w.stamp(ctx, job.ID, ref); err != nil { + if stamp != nil { + if err := stamp(ctx, job.ID, ref); err != nil { log.Printf("[webhook-notify] stamp operation on legacy job %d: %v", job.ID, err) } } diff --git a/internal/webhooknotify/worker_test.go b/internal/webhooknotify/worker_test.go index 050242e33..d40909125 100644 --- a/internal/webhooknotify/worker_test.go +++ b/internal/webhooknotify/worker_test.go @@ -503,3 +503,39 @@ func isCancel(err error) bool { var cancel *river.JobCancelError return errors.As(err, &cancel) } + +// TestNotifyWorker_PreDerivationReferenceIsReKeyed: a job stamped before the +// episode-derived ids existed (migration 113's op_) is re-resolved and +// its reference replaced, not cancelled. +func TestNotifyWorker_PreDerivationReferenceIsReKeyed(t *testing.T) { + fd := &fakeDeliverer{} + g := allowAll() + resolved, stamped, restamped := 0, 0, 0 + var restampedWith string + w := webhooknotify.NewNotifyWorker(&fakeStore{wh: hook(false, nil)}, fd).WithMetrics(&fakeMetrics{}).WithGate(g). + WithOperationResolver(func(_ context.Context, id, kind string) (sendingpolicy.OperationRef, error) { + resolved++ + wh := hook(false, nil) + wh.ID = id + return refFor(webhooknotify.ExpectedOperationID(wh, kind)), nil + }). + WithArgStamper(func(context.Context, int64, sendingpolicy.OperationRef) error { stamped++; return nil }). + WithArgRestamper(func(_ context.Context, _ int64, ref sendingpolicy.OperationRef) error { + restamped++ + restampedWith = ref.ID() + return nil + }) + j := job("wh_test", webhooknotify.KindDisabled, 1) + legacy := refFor("op_0123456789abcdef0123456789abcdef") + j.Args.OperationRef = &legacy + if err := w.Work(context.Background(), j); err != nil { + t.Fatalf("Work: %v", err) + } + want := webhooknotify.ExpectedOperationID(hook(false, nil), webhooknotify.KindDisabled) + if resolved != 1 || restamped != 1 || stamped != 0 || restampedWith != want { + t.Fatalf("resolved=%d restamped=%d stamped=%d with=%q, want 1/1/0 with %q", resolved, restamped, stamped, restampedWith, want) + } + if g.reserves != 1 || fd.called != 1 { + t.Fatalf("reserves=%d submits=%d, want 1/1", g.reserves, fd.called) + } +} From 15425d4d826503c691d2452dd1b5428f2f70f10f Mon Sep 17 00:00:00 2001 From: jiashuoz <39790535+jiashuoz@users.noreply.github.com> Date: Sat, 5 Sep 2026 00:44:29 -0700 Subject: [PATCH 12/12] fix(cmd): keep the reconcile scan two-valued and replaceable Round-3 re-review nits: COALESCE the conforming-reference predicate so a reference with no id cannot fall out of a NOT scan, decode only the source fields so such a reference is replaced rather than failing to decode, and state in the workers that any non-derived shape re-derives from the job's own source. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01AjfGxvXW6fNKWGFHuo68yX --- cmd/e2a/sending_reconcile.go | 23 +++++++++++++++++------ cmd/e2a/sending_reconcile_test.go | 10 ++++++++-- internal/hitlnotify/worker.go | 3 +++ internal/webhooknotify/worker.go | 4 +++- 4 files changed, 31 insertions(+), 9 deletions(-) diff --git a/cmd/e2a/sending_reconcile.go b/cmd/e2a/sending_reconcile.go index 8ab57474a..b56a888b4 100644 --- a/cmd/e2a/sending_reconcile.go +++ b/cmd/e2a/sending_reconcile.go @@ -44,13 +44,15 @@ var legacyReconcileStates = []string{ // conformingReferenceSQL is true for a river_job row whose operation_ref // already has the shape its worker derives: the message id for a send, the // op_hitl_ / op_wh_ derivations for the two notice kinds. -const conformingReferenceSQL = `( +// +// COALESCE keeps the predicate two-valued: a reference with no id (or a JSON +// null) would otherwise make the LIKE NULL and drop the row from a NOT scan. +const conformingReferenceSQL = `COALESCE( (args ? 'operation_ref') AND ( (kind = 'outbound_send') OR (kind = 'hitl_notify' AND args->'operation_ref'->>'id' LIKE 'op\_hitl\_%') OR (kind = 'webhook_notify' AND args->'operation_ref'->>'id' LIKE 'op\_wh\_%') - ) -)` + ), false)` // legacyReconcileCounts is the operator-facing summary of one reconcile pass. type legacyReconcileCounts struct { @@ -190,7 +192,11 @@ func reconcileLegacySendingJob(ctx context.Context, pool *pgxpool.Pool, client * var cancelReason string switch kind { case outboundsend.OutboundSendArgs{}.Kind(): - var args outboundsend.OutboundSendArgs + // Decode only the source fields: a malformed stored reference is + // exactly what this command replaces, so it must not fail decoding. + var args struct { + MessageID string `json:"message_id"` + } if err := json.Unmarshal(rawArgs, &args); err != nil { return 0, fmt.Errorf("decode args: %w", err) } @@ -214,7 +220,9 @@ func reconcileLegacySendingJob(ctx context.Context, pool *pgxpool.Pool, client * ref = prepared } case hitlnotify.HITLNotifyArgs{}.Kind(): - var args hitlnotify.HITLNotifyArgs + var args struct { + MessageID string `json:"message_id"` + } if err := json.Unmarshal(rawArgs, &args); err != nil { return 0, fmt.Errorf("decode args: %w", err) } @@ -223,7 +231,10 @@ func reconcileLegacySendingJob(ctx context.Context, pool *pgxpool.Pool, client * return 0, err } case webhooknotify.WebhookNotifyArgs{}.Kind(): - var args webhooknotify.WebhookNotifyArgs + var args struct { + WebhookID string `json:"webhook_id"` + NotifyKind string `json:"kind"` + } if err := json.Unmarshal(rawArgs, &args); err != nil { return 0, fmt.Errorf("decode args: %w", err) } diff --git a/cmd/e2a/sending_reconcile_test.go b/cmd/e2a/sending_reconcile_test.go index 677512792..213c3d1a0 100644 --- a/cmd/e2a/sending_reconcile_test.go +++ b/cmd/e2a/sending_reconcile_test.go @@ -272,14 +272,20 @@ func TestReconcileLegacySendingJobsReKeysPreDerivationReferences(t *testing.T) { md5Hitl := insertLegacyJob(t, pool, "hitl_notify", `{"message_id":"`+msg.ID+`","operation_ref":{"v":1,"id":"op_0123456789abcdef0123456789abcdef"}}`) md5Wh := insertLegacyJob(t, pool, "webhook_notify", `{"webhook_id":"`+wh.ID+`","kind":"warning","operation_ref":{"v":1,"id":"op_fedcba9876543210fedcba9876543210"}}`) conforming := insertLegacyJob(t, pool, "hitl_notify", `{"message_id":"`+msg.ID+`","operation_ref":{"v":1,"id":"`+sendingpolicy.HITLNotificationOperationID(msg.ID)+`"}}`) + // A malformed reference (no id) must be scanned and re-keyed, not hidden + // by three-valued logic in the scan predicate. + noID := insertLegacyJob(t, pool, "hitl_notify", `{"message_id":"`+msg.ID+`","operation_ref":{"v":1}}`) send := insertLegacyJob(t, pool, "outbound_send", `{"message_id":"`+msg.ID+`","operation_ref":{"v":1,"id":"`+msg.ID+`"}}`) var out bytes.Buffer if err := runReconcileLegacySendingJobs(ctx, pool, gate, &out); err != nil { t.Fatalf("reconcile: %v\n%s", err, out.String()) } - if !strings.Contains(out.String(), "scanned: 2") || !strings.Contains(out.String(), "stamped: 2") { - t.Fatalf("want the two md5-keyed jobs scanned and re-keyed:\n%s", out.String()) + if !strings.Contains(out.String(), "scanned: 3") || !strings.Contains(out.String(), "stamped: 3") { + t.Fatalf("want the two md5-keyed jobs and the id-less one scanned and re-keyed:\n%s", out.String()) + } + if _, op := legacyJobState(t, pool, noID); op != sendingpolicy.HITLNotificationOperationID(msg.ID) { + t.Errorf("id-less hitl job op = %q, want the derived id", op) } if _, op := legacyJobState(t, pool, md5Hitl); op != sendingpolicy.HITLNotificationOperationID(msg.ID) { t.Errorf("hitl job op = %q, want the derived id", op) diff --git a/internal/hitlnotify/worker.go b/internal/hitlnotify/worker.go index 7245737bb..db1492662 100644 --- a/internal/hitlnotify/worker.go +++ b/internal/hitlnotify/worker.go @@ -301,6 +301,9 @@ func (w *NotifyWorker) operationFor(ctx context.Context, job *river.Job[HITLNoti } if sendingpolicy.IsHITLNotificationOperationID(stored) { // A derived id for a different message: foreign, never authorize. + // (Any other shape, a wrong-kind derivation included, is re-derived + // from this job's own source below, so no stored id can redirect + // attribution.) return sendingpolicy.OperationRef{}, errOperationMismatch } // A pre-derivation reference — migration 113 stamped adopted jobs diff --git a/internal/webhooknotify/worker.go b/internal/webhooknotify/worker.go index 50de3d0e3..4d5ad400f 100644 --- a/internal/webhooknotify/worker.go +++ b/internal/webhooknotify/worker.go @@ -366,7 +366,9 @@ func (w *NotifyWorker) operationFor(ctx context.Context, job *river.Job[WebhookN return *job.Args.OperationRef, nil } if sendingpolicy.IsWebhookHealthOperationID(stored) { - // A derived id for another webhook or a superseded episode. + // A derived id for another webhook or a superseded episode. (Any + // other shape is re-derived from this job's own source below, so no + // stored id can redirect attribution.) return sendingpolicy.OperationRef{}, errOperationMismatch } // A pre-derivation reference (migration 113's op_, or the first