From dc9acf55cee7c7c9c06002ee6b095939f7eaa173 Mon Sep 17 00:00:00 2001 From: Umang Date: Sun, 2 Aug 2026 11:04:01 -0500 Subject: [PATCH 01/12] Make enrichment backlog durable --- .env.example | 35 ++- CLAUDE.md | 19 +- IMPLEMENTATION_PLAN.md | 182 +++++++++++---- README.md | 213 ++++++++++-------- app/jobs/enrich_actor_job.rb | 15 +- app/jobs/enrich_repository_job.rb | 4 + app/jobs/poll_event_source_job.rb | 5 + app/jobs/reconcile_pending_enrichments_job.rb | 6 + app/models/concerns/enrichable.rb | 51 +---- app/models/push_event.rb | 3 +- app/services/github/allowances.rb | 6 +- app/services/github/configuration.rb | 15 +- app/services/github/enrichment/age_out.rb | 93 -------- .../github/enrichment/backlog_metrics.rb | 77 +++++++ app/services/github/enrichment/backoff.rb | 10 +- .../github/enrichment/candidate_selector.rb | 115 +++------- app/services/github/enrichment/claim.rb | 10 +- app/services/github/enrichment/coverage.rb | 20 +- app/services/github/enrichment/dispatch.rb | 34 ++- .../github/enrichment/entity_state.rb | 5 +- app/services/github/enrichment/fairness.rb | 35 ++- app/services/github/enrichment/summary.rb | 143 ++++++++---- app/services/github/enrichment/tally.rb | 11 +- app/services/github/enrichment_runner.rb | 53 ++--- .../github/events/quarantine_reasons.rb | 2 +- app/services/github/ingestion/page_writer.rb | 27 +-- .../github/ingestion/state_summary.rb | 10 +- app/services/github/status/snapshot.rb | 49 ++-- config/queue.yml | 25 +- config/recurring.yml | 3 + ...0_remove_skipped_budget_from_enrichment.rb | 60 +++++ db/schema.rb | 12 +- docker-compose.yml | 4 +- docs/DESIGN_BRIEF.md | 42 ++-- docs/SUBMISSION_CHECKLIST.md | 42 ++-- ...05-at-least-once-with-idempotent-writes.md | 11 +- ...nrichment-fairness-shares-and-borrowing.md | 46 +++- ...nqueue-and-entity-scoped-reconciliation.md | 12 +- ...it-escalation-and-refresh-pool-fairness.md | 35 +-- docs/adr/0012-solid-queue-over-kafka.md | 4 +- .../2026-07-31-clean-checkout-verification.md | 10 +- .../2026-07-31-container-kill-recovery.md | 6 + .../2026-08-01-post-merge-verification.md | 6 + ...ove_skipped_budget_from_enrichment_spec.rb | 102 +++++++++ spec/db/schema_spec.rb | 7 +- spec/docker_compose_spec.rb | 9 +- spec/jobs/poll_event_source_job_spec.rb | 4 + .../reconcile_pending_enrichments_job_spec.rb | 6 +- spec/models/github_actor_spec.rb | 15 +- spec/models/github_repository_spec.rb | 13 +- spec/queue/configuration_spec.rb | 27 ++- spec/queue/solid_queue_integration_spec.rb | 13 +- spec/recovery/concurrent_write_spec.rb | 21 +- spec/recovery/crash_window_spec.rb | 36 +-- spec/recovery/duplicate_job_execution_spec.rb | 2 +- .../pending_enrichment_recovery_spec.rb | 18 +- spec/services/github/configuration_spec.rb | 7 - .../github/enrichment/age_out_spec.rb | 149 ------------ .../github/enrichment/backlog_metrics_spec.rb | 80 +++++++ .../github/enrichment/backoff_spec.rb | 8 +- .../enrichment/candidate_selector_spec.rb | 130 +++++------ .../github/enrichment/coverage_spec.rb | 6 +- .../github/enrichment/dispatch_spec.rb | 32 ++- .../github/enrichment/end_to_end_spec.rb | 40 +--- .../github/enrichment/entity_state_spec.rb | 12 +- .../github/enrichment/fairness_spec.rb | 16 +- .../github/enrichment/fairness_stress_spec.rb | 42 ++-- .../github/enrichment/one_shot_spec.rb | 7 +- .../github/enrichment/summary_spec.rb | 77 ++++++- spec/services/github/enrichment/tally_spec.rb | 21 +- .../services/github/enrichment_runner_spec.rb | 42 ++-- .../github/ingestion/page_writer_spec.rb | 79 ++----- spec/services/github/ingestion_runner_spec.rb | 22 +- spec/services/github/status/snapshot_spec.rb | 29 +-- .../shared_examples/enrichable_entity.rb | 95 ++------ .../support/shared_examples/enrichment_job.rb | 4 + 76 files changed, 1458 insertions(+), 1279 deletions(-) delete mode 100644 app/services/github/enrichment/age_out.rb create mode 100644 app/services/github/enrichment/backlog_metrics.rb create mode 100644 db/migrate/20260802000000_remove_skipped_budget_from_enrichment.rb create mode 100644 spec/db/remove_skipped_budget_from_enrichment_spec.rb delete mode 100644 spec/services/github/enrichment/age_out_spec.rb create mode 100644 spec/services/github/enrichment/backlog_metrics_spec.rb diff --git a/.env.example b/.env.example index 2dda257..10949b1 100644 --- a/.env.example +++ b/.env.example @@ -112,31 +112,25 @@ GITHUB_MODE=live # zero indefinitely. # # It is a guarantee, not a cap. A class may borrow the other's unused capacity when -# the other class has no *currently eligible* candidate — not merely no rows — so a +# the other class has no *currently claimable* backlog candidate — not merely no rows — so a # quiet hour for repositories is spent on actors rather than wasted. Both ends of # the range are legal: 0.0 means "repositories first, actors during the quiet # periods". Below 0 or above 1 is rejected at startup, because either would put one # guarantee above the class allowance. # ACTOR_ENRICHMENT_SHARE=0.50 -# How long an entity stays worth enriching, and how long an enriched one stays -# fresh (§10): -# -# ENRICHMENT_ELIGIBILITY_WINDOW_SECONDS - a candidate whose most recent distinct -# push event is older than this transitions to skipped_budget. This is what -# bounds the backlog: at ~2,000 candidates an hour against 40 available -# requests, without it the pending queue grows forever. A skip is not -# permanent — a genuinely new push event referencing the entity reactivates it, -# while a duplicate replay never does. -# ACTOR_REFRESH_TTL_SECONDS / REPOSITORY_REFRESH_TTL_SECONDS - how long an -# enriched entity is reused before it is re-fetched. Never-enriched candidates -# always take priority over refreshes, so raising these buys coverage of new -# entities rather than freshness of old ones. -# -# All three must be greater than zero. A zero window skips every candidate on -# sight; a zero TTL turns the freshness cache off entirely. "Never refresh" is a -# large number, not zero. -# ENRICHMENT_ELIGIBILITY_WINDOW_SECONDS=3600 +# How long before an enriched entity becomes refresh-eligible (§10): +# +# ACTOR_REFRESH_TTL_SECONDS / REPOSITORY_REFRESH_TTL_SECONDS - the minimum time an +# enriched entity is reused before it may be re-fetched. Never-enriched entity rows +# are a durable FIFO backlog: quota exhaustion defers them to a later window and +# never terminates them. A selection that observes never-enriched work does not choose +# a refresh, so raising these values affects freshness only after the durable backlog +# drains. Ingestion can commit one new row between that read and a request debit; the +# runner's one-request cycle bounds the window and the next selection suppresses refresh. +# +# Both must be greater than zero. A zero TTL turns the freshness cache off entirely. +# "Never refresh" is a large number, not zero. # ACTOR_REFRESH_TTL_SECONDS=86400 # REPOSITORY_REFRESH_TTL_SECONDS=86400 @@ -148,8 +142,7 @@ GITHUB_MODE=live # # The window is measured on push_events.created_at, the instant this application # persisted the row, not on GitHub's occurred_at. Coverage grades this -# application's enrichment pipeline, and that pipeline's own eligibility rule is -# already COALESCE(last_seen_at, created_at) — one clock, not two. It also keeps +# application's enrichment pipeline and uses one local clock throughout. It also keeps # the offline reviewer path meaningful: the fixture corpus pins its event # timestamps to a fixed date, so an occurred_at basis would report null coverage # to anyone running the walkthrough after that date. diff --git a/CLAUDE.md b/CLAUDE.md index d85d66a..548ced6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,8 @@ plan wins. 1. Read `IMPLEMENTATION_PLAN.md` (repository root). It is the frozen execution plan; its pre-implementation revision history lives in Git and in its - Appendices A–D, and Appendix E records how the build diverged from it. + Appendices A–D, Appendix E records how the build diverged from it, and Appendix F + supersedes the enrichment load-shedding policy with a durable backlog. 2. Read `docs/DESIGN_BRIEF.md` and the ADRs under `docs/adr/`. 3. Do not change architectural direction, add infrastructure, or add dependencies without first updating the plan and stating the tradeoff. @@ -55,7 +56,7 @@ acquire `SourceLock` — they take only the request gate. Repeated observation and job execution are expected. State only the two proved ingestion invariants: a duplicate GitHub event ID cannot create another `push_events` row, and that -duplicate cannot register entity activity or reactivate a `skipped_budget` entity. +duplicate cannot register entity activity. Executions, ingestion runs, quarantine occurrence counts, budget debits, and logs may repeat or change. Recovery before commit is conditional on the event remaining in a later sliding-feed response. Never claim or code against exactly-once execution or universal @@ -64,9 +65,9 @@ idempotency of persisted state. ### Duplicate-event invariants (plan §7) - `push_events` inserts use `ON CONFLICT (github_event_id) DO NOTHING RETURNING id`. -- Entity activity fields (`last_seen_at`, `latest_event_at`, reactivation) update - only when `RETURNING` produced a row. Duplicate replays may refresh identity - fields but must never reactivate a `skipped_budget` entity. +- Entity activity fields (`last_seen_at`, `latest_event_at`) update only when + `RETURNING` produced a row. Duplicate replays may refresh identity fields but must + never register new entity activity. - Quarantine identity is `payload_fingerprint` alone: SHA-256 of compact UTF-8 JSON with recursively sorted object keys. One algorithm, no alternates. @@ -76,6 +77,14 @@ Enrichment fetches only validated URLs: HTTPS, host exactly `api.github.com`, no userinfo, no non-default port, no IP literals, bounded re-validated redirects. Fixture mode fails closed — never a live fallback. +### Durable enrichment backlog (plan §10, Appendix F) + +Never-enriched entity rows remain actionable across quota windows. Select them FIFO by +`created_at ASC, id ASC`; quota or fairness denial defers rather than terminates. The +default hourly split is 12 polling requests, 40 backlog-enrichment requests, and 8 safety +reserve requests, with 20/20 actor/repository guarantees and borrowing. Do not schedule a +refresh while either class has never-enriched work. + ## Database changes Schema changes go through migrations with intentional indexes, constraints where diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md index 8d596a1..3ec4412 100644 --- a/IMPLEMENTATION_PLAN.md +++ b/IMPLEMENTATION_PLAN.md @@ -1,6 +1,6 @@ # GitHub Push Event Ingestion Service — Implementation Plan -> **This plan was finalized through four pre-implementation review rounds**: an adversarial multi-lens design review with live probes of the unauthenticated GitHub API (**Appendix A**), an independent validation pass against official GitHub, PostgreSQL, and Rails/Solid Queue documentation (**Appendix B**), an implementation-readiness re-check that corrected locking, scheduling, Compose, and PR-ordering defects (**Appendix C**), and a freeze-readiness pass that corrected lock scoping, class-level blocking, bootstrap, restart, and reactivation semantics (**Appendix D**). The initial plan (V1) and the full revision trail are preserved in Git history; the appendices record what changed and why. The one section added during revision is numbered **2A** to keep the original numbering stable. +> **This plan was finalized through four pre-implementation review rounds**: an adversarial multi-lens design review with live probes of the unauthenticated GitHub API (**Appendix A**), an independent validation pass against official GitHub, PostgreSQL, and Rails/Solid Queue documentation (**Appendix B**), an implementation-readiness re-check that corrected locking, scheduling, Compose, and PR-ordering defects (**Appendix C**), and a freeze-readiness pass that corrected lock scoping, class-level blocking, bootstrap, restart, and entity-activity semantics (**Appendix D**). The initial plan (V1) and the full revision trail are preserved in Git history; the appendices record what changed and why. Appendix F supersedes the original enrichment load-shedding policy with the durable-backlog design adopted on 2026-08-02. The one section added during revision is numbered **2A** to keep the original numbering stable. > > File locations: this plan lives at the repository root. `DESIGN_BRIEF.md` and the ADRs live under `docs/`. @@ -32,7 +32,7 @@ The delivered implementation will satisfy the required public-events source whil - `PushEvent` filtering and processing - Raw event payload retention (semantic retention via `jsonb` — see Section 7) - Structured push-event fields -- Actor and repository enrichment (**budget-bounded, best-effort sampling with per-class fairness** — see Section 10) +- Actor and repository enrichment (**durable, quota-paced FIFO backlog with per-class fairness** — see Section 10) - Duplicate-safe `push_events` persistence - Pagination via the `Link` response header - ETag and `304 Not Modified` handling (bandwidth/correctness measure, scoped to the canonical first-page request — see Sections 9–10) @@ -64,7 +64,7 @@ The delivered implementation will satisfy the required public-events source whil - GitHub authentication - Private repository ingestion - Complete event capture — not guaranteed by the bounded, delayed Events polling API (see Section 10) -- Complete enrichment coverage — not sustainable under the observed global-feed demand and the unauthenticated 60-request hourly budget (see Section 10) +- Bounded-time enrichment completion — sustained unique-entity arrivals can exceed the unauthenticated backlog service rate (see Section 10) - Event processors beyond `PushEvent` - Object storage (Extension C — deliberately not attempted; rationale in the design brief) - Production cloud deployment @@ -204,9 +204,9 @@ Child issues: 4. Fetch repository data from payload-provided URLs (validated — Section 10) 5. Add freshness-based durable caching (entity `fetched_at` + refresh TTLs) 6. Prevent duplicate concurrent enrichment (keyed by entity row) -7. Enforce the hourly enrichment allowance with **per-class fairness shares and borrowing**, the `skipped_budget` state, and the distinct-event reactivation rule +7. Enforce the hourly enrichment allowance with **per-class fairness shares and borrowing**, durable FIFO pending work, and quota deferral without termination 8. Add pending-work reconciliation (entity-scoped scan) -9. Test failed, repeated, skipped, reactivated, replay-non-reactivated, and starved-class enrichment +9. Test failed, repeated, quota-deferred, FIFO, durable-backlog, refresh-priority, and starved-class enrichment ### Story 4 — Operability and Observability @@ -216,7 +216,8 @@ Child issues: 2. Add ingestion-run correlation IDs (`run_id` UUID) 3. Add GitHub event IDs and job IDs to logs 4. Add `/health/live` and `/health/ready` endpoints (no GitHub calls, no budget consumption) -5. Add ingestion status endpoint (per-class budget state, defined coverage formulas, pending/skipped counts) +5. Add ingestion status endpoint (per-class budget state, defined coverage formulas, + backlog size and oldest pending timestamp/age; no unsupported drain ETA) 6. Add malformed-payload quarantine handling 7. Add retry and failure logging 8. Add container health checks @@ -248,7 +249,7 @@ Child issues: 5. Preserve pending work in business tables 6. Reconcile work not enqueued before a crash 7. Add crash-safe source ownership (session advisory lock; verify release on session death) -8. Bound the enrichment backlog via the eligibility window and `skipped_budget` state (no unbounded growth) +8. Preserve never-enriched entity rows as durable backlog work across quota windows; select FIFO oldest-first and report unbounded growth honestly 9. Add Docker restart policies; test API-stop and main-process-crash paths separately 10. Document processing guarantees @@ -263,13 +264,13 @@ Child issues: 1. Add deterministic GitHub fixtures (static JSON corpus + fixture source + fixture transport) 2. Test `PushEvent` filtering and tolerant parsing 3. Test raw and structured persistence -4. Test duplicate ingestion (including replay-does-not-reactivate) +4. Test duplicate ingestion (including replay-does-not-register-activity) 5. Test pagination stopping conditions (cap / allowance / no-next-link / empty page) 6. Test ETag scoping and `304` behavior (including quota accounting) 7. Test budget-ledger reservation, the allowance formula, fairness rounding/borrowing, per-window bootstrap, and exhaustion (global vs class blocking) 8. Test transient retries 9. Test the canonical fingerprint algorithm and quarantine occurrence counting -10. Test enrichment caching, terminal skip, and distinct-event reactivation +10. Test enrichment caching, durable quota deferral, FIFO selection, and refresh suppression while backlog exists 11. Test pending-work recovery and advisory-lock release on session death 12. Add Docker-based end-to-end verification (fixture mode, fail-closed) and container-kill recovery checks @@ -541,15 +542,14 @@ Malformed-event taxonomy: - `avatar_url` - `raw_payload` (full document after enrichment; `NULL` on stubs) - **Enrichment state machine (entity-level, V2):** - - `enrichment_status` — `pending | complete | retryable_failure | permanent_failure | skipped_budget` + - `enrichment_status` — `pending | complete | retryable_failure | permanent_failure` - `enrichment_attempts` - `next_retry_at` - `last_error` - `fetched_at` - `first_seen_at` — first time a **distinct persisted** push referenced this entity - - `last_seen_at` — most recent local observation of a **distinct persisted** push; drives the newest-first policy + - `last_seen_at` — most recent local observation of a **distinct persisted** push - `latest_event_at` — greatest GitHub event `created_at` among distinct persisted pushes - - `skipped_at` - timestamps Envelope-to-stub field mapping (explicit — the envelope and the enriched document are different shapes): @@ -566,22 +566,28 @@ actor.avatar_url → avatar_url 1. Upsert entity identity stubs — envelope values may refresh identity fields (login, display_login, API URL, avatar URL) on any observation, including duplicates; an envelope upsert never clears a previously stored enrichment payload or `name`. 2. `INSERT push_events … ON CONFLICT DO NOTHING RETURNING id`. -3. **Only when RETURNING produces a row** (a genuinely new event): update `last_seen_at` and `latest_event_at`, set `first_seen_at` if unset, and apply `skipped_budget` reactivation. -4. A duplicate event replay may refresh harmless identity fields but **can never reactivate enrichment** — otherwise a re-polled window would resurrect skipped entities with no new activity. +3. **Only when RETURNING produces a row** (a genuinely new event): update `last_seen_at` and `latest_event_at`, and set `first_seen_at` if unset. +4. A duplicate event replay may refresh harmless identity fields but **can never register new entity activity**. -A `complete` enrichment is not reset to `pending` by any duplicate; it returns to `pending` only when missing, explicitly stale (past its refresh TTL), or reactivated after a budget skip. +A `complete` enrichment is not reset to `pending` by any duplicate. Staleness is a derived +refresh predicate over a complete row, not a destructive status transition. -**Reactivation rule:** `skipped_budget` is terminal for the entity’s current eligibility window, not forever. A **newly persisted** push event referencing the entity updates its activity fields and may transition it back to `pending` when its enrichment is missing or stale. This also handles delayed-but-new events correctly: even with an old `created_at` (documented 30s–6h latency), a distinct event ID proves new activity. Partial enrichment is expected by design; pending work is bounded — candidates that age beyond the configured eligibility window transition to `skipped_budget`. +**Durable backlog rule:** a never-enriched entity remains `pending` or +`retryable_failure` until a real enrichment response produces a success or +entity-specific terminal failure. Quota exhaustion and fairness-share denial leave the row +actionable for a later window. Candidate selection is FIFO by immutable, non-null +`created_at ASC, id ASC`; `first_seen_at` can be null, so it is activity metadata rather +than the queue key. Sustained new arrivals therefore cannot starve old work. Partial index matching the reconciler’s exact predicate: ```sql -(next_retry_at, last_seen_at) WHERE enrichment_status IN ('pending', 'retryable_failure') +(created_at, id) WHERE enrichment_status IN ('pending', 'retryable_failure') ``` ### `github_repositories` -- `id`, `github_id` (`bigint`, unique), `name`, `full_name`, `api_url`, `description`, `language`, `owner_github_id`, `raw_payload`, plus the identical enrichment state machine, merge rules, distinct-event activity gating, reactivation rule, partial index, and timestamps. +- `id`, `github_id` (`bigint`, unique), `name`, `full_name`, `api_url`, `description`, `language`, `owner_github_id`, `raw_payload`, plus the identical enrichment state machine, merge rules, distinct-event activity gating, durable-backlog rule, partial index, and timestamps. Envelope-to-stub field mapping — the envelope’s `repo.name` is the qualified `owner/repository` form; it is **not** silently equated with the enriched `name`: @@ -607,7 +613,7 @@ A GitHub event is considered accepted only after its `push_events` row is commit 5. Normalize required attributes tolerantly; route failures to `quarantined_events`. 6. Upsert stub actor and repository rows (identity fields only). 7. Insert the raw and structured event row with conflict skipping (`RETURNING id`). -8. For rows actually inserted: apply entity activity updates and reactivation (Section 7). +8. For rows actually inserted: apply entity activity updates (Section 7). 9. Commit the PostgreSQL transaction (short-lived — the advisory lock, not the transaction, spans the HTTP work). 10. Enqueue enrichment after commit — Solid Queue lives in its own database, so same-transaction enqueue is not available; the committed entity state is the durable record of pending work (outbox-style recovery, per Section 2A). 11. Reconcile entities whose enrichment was not scheduled or completed. @@ -638,7 +644,7 @@ prevent duplicate entity rows, but execution and operational side effects may re Repeated observation and job delivery are expected. The system enforces two narrower invariants: a duplicate GitHub event ID cannot create another `push_events` row, and that -duplicate cannot register entity activity or reactivate a `skipped_budget` entity. A +duplicate cannot register entity activity. A duplicate may refresh permitted identity fields. Executions, `ingestion_runs`, quarantine occurrence counts, budget debits, and logs may repeat or change. The system does not claim exactly-once execution or universal idempotency of persisted state. @@ -798,10 +804,19 @@ Consequence, stated honestly: one observed live page of `/events` held ~92–95 ```text 89 actors + 92 repositories = 181 entity requests/page (cold) 181 × 12 polls/hour ≈ 2,172 requests/hour of cold demand -40 available enrichments/hour ≈ 1.8% theoretical cold coverage +40 available enrichments/hour ≈ 1.8% same-hour service ratio under the all-cold assumption ``` -**Enrichment is best-effort sampling by design**, and it must be **fair across classes**: repository candidates alone exceed the entire hourly allowance, so a naive repo-first policy would starve actor enrichment to zero indefinitely — violating Story 3, which requires both. Fairness policy with explicit rounding: +This is a cold-demand pressure scenario, not a measured arrival rate: it assumes every +poll contains entirely new identities, while entity rows deduplicate repeated actors and +repositories across events and overlapping pages. The actual unique arrival rate must be +measured against the 40-attempt service rate. + +**Enrichment is a durable, quota-paced backlog**, and it must be **fair across classes**: +repository candidates alone exceed the entire hourly allowance, so a naive repo-first +policy would starve actor enrichment indefinitely — violating Story 3, which requires both. +The default ledger reserves 12 attempts for polling, 40 for draining never-enriched work, +and 8 as a safety reserve. Fairness policy with explicit rounding: ```text actor_guarantee = floor(enrichment_allowance × ACTOR_ENRICHMENT_SHARE) @@ -810,15 +825,22 @@ repository_guarantee = enrichment_allowance − actor_guarantee Defaults: ACTOR_ENRICHMENT_SHARE = 0.50 → 20 actor / 20 repository Borrowing: a class may borrow the other’s unused capacity only when the -other class has no CURRENTLY ELIGIBLE candidate (not merely no rows). +other class has no CURRENTLY CLAIMABLE backlog candidate (not merely no rows). ``` -Within each class, never-enriched `pending` candidates always precede TTL-stale refreshes — a refresh spends budget only when no pending candidate is currently eligible. Among pending candidates the service enriches newest-first (`last_seen_at`); candidates that age beyond the eligibility window transition to `skipped_budget`, and entities referenced by newly persisted events reactivate (Section 7). The backlog is therefore bounded, `skipped_budget` is a normal documented outcome, and `/status` reports per-class usage and coverage so an operator sees the sampling rate instead of a mysteriously growing queue. +Within each class, never-enriched candidates are served FIFO by `created_at ASC, id ASC`. +Their entity rows remain durable across quota exhaustion, +window rollover, process restart, and lost enqueue hints. A denied reservation defers the +candidate; it never terminates it. A TTL-stale refresh receives no request while either +class has any never-enriched backlog work. If unique arrivals continuously exceed 40 +attempts per hour, the backlog can grow without a bounded completion estimate; `/status` +therefore reports backlog size, oldest pending timestamp/age, and reserved allowance usage. +It does not report a drain ETA because no durable outcome history exists from which to +derive an honest service rate. Timing configuration (pinned defaults; tunable): ```text -ENRICHMENT_ELIGIBILITY_WINDOW_SECONDS = 3600 ACTOR_REFRESH_TTL_SECONDS = 86400 REPOSITORY_REFRESH_TTL_SECONDS = 86400 ENRICHMENT_COVERAGE_WINDOW_SECONDS = 86400 @@ -895,7 +917,7 @@ Never disable the event source because one enrichment target disappeared. 1. Polling for new events (from `poll_attempt_allowance`) 2. Actor and repository enrichment (from `enrichment_allowance`, under the fairness guarantees — neither class can starve the other) -3. Refreshing stale enrichment (within each class’s share, and only when no never-enriched pending candidate is eligible) +3. Refreshing stale enrichment (within each class’s share, and only when neither entity class has any never-enriched backlog work) Polling receives priority because raw-event capture is more time-sensitive than enrichment — and the enrichment slice is guaranteed by its own allowance rather than starved by priority alone. @@ -905,7 +927,7 @@ Polling receives priority because raw-event capture is more time-sensitive than Structured JSON to stdout/stderr, with a `LOG_LEVEL` env (default `info`). -- **INFO**: ingestion run started/completed with summary counts (persisted, duplicates, quarantined, ignored non-push), enrichment completed/failed/skipped/reactivated, retry scheduled, budget state transitions (window initialized, `global_blocked_until` set/cleared, class exhaustion), source lock acquired/busy, reconciliation summaries +- **INFO**: ingestion run started/completed with summary counts (persisted, duplicates, quarantined, ignored non-push), enrichment completed/failed/deferred, retry scheduled, budget state transitions (window initialized, `global_blocked_until` set/cleared, class exhaustion), source lock acquired/busy, reconciliation summaries - **DEBUG**: per-request and per-page lines (GitHub request/response, page processed, `304` received) Rails and ActiveJob framework logging is routed through the same JSON formatter so `docker compose logs -f` stays one coherent stream — the INFO stream is sized so the events Story 4 asks reviewers to see are not buried. @@ -933,7 +955,8 @@ events_with_both_entities_enriched_pct = ÷ all distinct persisted push events in the window ``` - - `pending_actor_count`, `pending_repository_count`, `skipped_actor_count`, `skipped_repository_count` + - per-class backlog size and oldest pending timestamp/age, alongside reserved allowance + usage; no drain ETA without durable outcome history - `GET /api/push_events` - `GET /api/push_events/:id` @@ -956,10 +979,11 @@ Testing focuses on correctness boundaries rather than exhaustive framework behav - Retry and error-context classification (source vs entity) - Ledger accounting: class reservation, the allowance formula and startup validation, `304` debits, failure-stays-spent, monotonic reconciliation, per-window bootstrap and counter reset, global vs class blocking derivation - `effective_poll_time` / `effective_enrichment_time` (components independent; `global_blocked_until` only for global conditions; `--force` bypasses cadence + ETag only) -- Fairness rounding (floor/remainder) and eligibility-aware borrowing +- Fairness rounding (floor/remainder) and claimability-aware borrowing - Enrichment URL policy (reject non-HTTPS, wrong host, userinfo, ports, IP literals, unbounded redirects) - Pagination stop logic (`Link`-header driven; cap / allowance / no-next / empty) -- Enrichment state machine transitions, including distinct-event reactivation and replay-non-reactivation +- Enrichment state machine transitions, including durable quota deferral, FIFO ordering, + and duplicate replay without new activity ### Persistence tests @@ -974,12 +998,15 @@ Testing focuses on correctness boundaries rather than exhaustive framework behav - Public-event response ingestion - Non-push events ignored and counted -- Duplicate poll results (fixture replay) — duplicates skipped **and no entity reactivation occurs** +- Duplicate poll results (fixture replay) — duplicates skipped **and no new entity activity occurs** - Multiple-page ingestion via `Link` headers; full-page processing with duplicates absorbed by uniqueness (no known-event stop) - Actor and repository enrichment via stub → `complete` transitions - Reuse of fresh enriched records; TTL-driven staleness -- Enrichment allowance exhaustion → deferred → `skipped_budget` → reactivation only via a genuinely new event -- Class fairness: repository flood cannot starve actors (and vice versa); borrowing only when the other class has no eligible candidate +- Enrichment allowance exhaustion defers durable backlog work across window rollover without + changing it to a terminal state +- FIFO selection by `created_at ASC, id ASC` under sustained arrivals +- Refresh suppression while any never-enriched actor or repository work remains +- Class fairness: repository flood cannot starve actors (and vice versa); borrowing only when the other class has no claimable backlog candidate - Poll allowance protected from enrichment demand — and vice versa (class-blocking isolation: one class exhausted, the other proceeds) - Rate-limit exhaustion (`403` + headers → `global_blocked_until`); routine `reset_at` never defers; secondary limit blocks globally including enrichment - Per-window bootstrap: new window → counters reset → enrichment ineligible until the first poll initializes it @@ -1024,7 +1051,11 @@ Processor registry; tolerant `PushEvent` processor; quarantine taxonomy + canoni Poll-attempt allowance enforcement; `Link`-header pagination with budget-bounded stops (no known-event stop, ETag scoped to page 1); corrected `304` debit; `effective_poll_time` with independent components; `global_blocked_until` vs derived class blocking; secondary-limit global handling; `Retry-After` handling; persisted poll state; **committed dated live-probe transcript for the 304 finding (required validation gate)** ### PR 7 — Enrichment budget and fairness -Actor/repository enrichment against the entity state machine; fairness guarantees (floor/remainder rounding) with eligibility-aware borrowing; newest-first eligibility; `skipped_budget` + distinct-event reactivation; freshness cache + refresh TTLs; error-context classification (entity vs source); `effective_enrichment_time` +Actor/repository enrichment against the entity state machine; fairness guarantees +(floor/remainder rounding) with claimability-aware borrowing; durable FIFO selection by +`created_at ASC, id ASC`; quota deferral without termination; freshness cache with refreshes +suppressed while never-enriched work remains; refresh TTLs; error-context classification +(entity vs source); `effective_enrichment_time` ### PR 8 — Background processing and recovery Solid Queue setup (own database in the same Postgres container); worker container; recurring polling task; enrichment jobs; post-commit enqueue; entity-scoped reconciler; recovery tests (including advisory-lock release on session death) @@ -1056,7 +1087,9 @@ File locations: `IMPLEMENTATION_PLAN.md` at the repository root; `DESIGN_BRIEF.m Must include: -- Pointer to `IMPLEMENTATION_PLAN.md`, noting its revision history lives in Git and Appendices A–D +- Pointer to `IMPLEMENTATION_PLAN.md`, noting pre-implementation history in Git and + Appendices A–D, execution deltas in Appendix E, and the durable-backlog correction in + Appendix F - Problem overview - Architecture summary - Requirements @@ -1073,7 +1106,7 @@ Must include: - Rate-limit behavior: the allowance formula, the budget table, global-vs-class blocking, and per-window bootstrap - Separate API-stop and main-process-crash verification steps (Section 15) - Reset instructions -- Known limitations (sampling-based enrichment coverage; no guaranteed complete capture; shared-IP budget interference) +- Known limitations (unbounded enrichment drain time under sustained arrivals; no guaranteed complete capture; shared-IP budget interference) Required reviewer commands: @@ -1093,16 +1126,23 @@ Keep within one to two pages — the brief is the reviewer’s primary architect - Data model - Durability boundary - **The request-budget formula and table, and the unauthenticated `304` finding** — worded precisely: the endpoint documentation contains a general statement that `304` responses do not affect the rate limit, while the REST best-practices documentation limits that exemption to correctly authorized requests; dated unauthenticated probes showed `x-ratelimit-used` increasing across a `304`; this implementation therefore budgets unauthenticated conditional requests as one request -- **Enrichment as bounded best-effort sampling with per-class fairness; eligibility windows, `skipped_budget`, and distinct-event reactivation as the answer to unbounded growth** +- **Enrichment as a durable FIFO backlog with per-class fairness; quota exhaustion defers + without terminating, refresh waits for the never-enriched pool to empty, and unbounded + growth is reported rather than hidden** - Duplicate-safe event persistence and restart recovery (advisory-lock ownership; outbox-style recovery; Docker restart policies) - Enrichment strategy and the SSRF boundary - Tradeoffs and assumptions (including `jsonb` semantic retention) - Intentional omissions (Extension C; authentication; complete capture) -- Future scaling path (a larger authenticated allowance could materially increase feasible coverage without guaranteeing capture or enrichment; API-version upgrade to `2026-03-10` after payload re-verification) +- Future scaling path (a larger authenticated allowance could materially increase backlog + throughput without guaranteeing upstream capture or a bounded drain time; API-version + upgrade to `2026-03-10` after payload re-verification) ### Plan history -The plan’s pre-implementation revision history is preserved in Git history and summarized in Appendices A–D — the review-driven revision rounds are themselves submission-worthy evidence of process. At completion, add a short execution summary describing what changed from this plan during the build and why. +The plan’s pre-implementation revision history is preserved in Git history and summarized +in Appendices A–D — the review-driven revision rounds are themselves submission-worthy +evidence of process. Appendix E records execution deltas and Appendix F the durable-backlog +correction. ### Architecture Decision Records (`docs/adr/`) @@ -1112,7 +1152,8 @@ Short ADRs for: - Session advisory locks for source ownership and the global request gate (vs `FOR UPDATE` row claims; lock-order invariant) - Repeated execution with duplicate-safe event writes and distinct-event activity gating - Event-source adapter and transport seams, each with a shipped fixture implementation -- Class-aware budget ledger, allowance formula, global-vs-class blocking, and the enrichment fairness/sampling policy +- Class-aware budget ledger, allowance formula, global-vs-class blocking, and the durable + enrichment backlog/fairness policy - `jsonb` semantic retention (not byte-exact) - Pinned API version `2022-11-28` (evidence gathered under it) with the `2026-03-10` upgrade path - Why Kafka was not selected @@ -1127,7 +1168,10 @@ The README must provide exact steps to: 4. Follow application and worker logs. 5. Query persisted push events. 6. Inspect PostgreSQL record counts. -7. Run the fixture replay scenario and confirm `duplicates_skipped > 0` in the summary — and that no skipped entity was reactivated by the replay. (Live re-runs are not relied upon to demonstrate dedup: probe-dated observations showed little or no overlap between consecutive live polls.) +7. Run the fixture replay scenario and confirm `duplicates_skipped > 0` in the summary — + and that entity activity timestamps do not move. (Live re-runs are not relied upon to + demonstrate dedup: probe-dated observations showed little or no overlap between + consecutive live polls.) 8. **Verify operator-stop semantics and restart-policy crash recovery as separate paths:** ```bash @@ -1156,7 +1200,7 @@ evidence for the other. - Required fields are structured, typed, and `NOT NULL`; unknown payload fields tolerated; 40- and 64-char SHAs accepted - Raw payload is retained (semantic retention, documented) - **Both actor and repository enrichment demonstrably occur** within their fairness guarantees -- Duplicate event IDs cannot create another `push_events` row, and duplicate replays never reactivate skipped entities +- Duplicate event IDs cannot create another `push_events` row, and duplicate replays never register new entity activity - `Link`-header pagination is handled; every fetched page fully processed - Rate-limit behavior is demonstrated: `304` quota accounting, class-aware ledger enforcement, global-vs-class blocking, per-window bootstrap, scheduling rules - Malformed data is quarantined durably per the taxonomy (canonical fingerprints, occurrence-counted) and does not terminate the batch @@ -1172,14 +1216,19 @@ evidence for the other. - Advisory locks provably release on session death (tested) - The covered enrichment redelivery cannot create another entity row - Reconciliation recovers missing enrichment scheduling -- The enrichment backlog is bounded (eligibility window + `skipped_budget` + distinct-event reactivation) +- Never-enriched entity rows remain durable across quota exhaustion and window rollover; + FIFO selection prevents newer arrivals from starving older work +- A selection that observes either class has never-enriched backlog work does not choose a + refresh; a concurrent insert after that read can cross at most one one-request runner cycle ### Operability - Logs are readable through `docker compose logs -f` at the default level - Correlation fields (`run_id`, job ID) are present - `/health/live` and `/health/ready` are meaningful and never consume budget -- `/status` reports window status, poll state, per-class ledger state, pending/skipped counts, and coverage percentages computed by the defined formulas — without initiating GitHub requests +- `/status` reports window status, poll state, per-class ledger state, backlog size, oldest + pending timestamp/age, reserved allowance usage, and coverage percentages computed by the + defined formulas — without initiating GitHub requests or fabricating a drain ETA - Retry behavior is visible - Failures contain actionable context @@ -1216,7 +1265,8 @@ The target is a system that is: - Complete enough to trust - Extensible without being speculative - Durable across normal container and process failures -- Honest about the limitations of the GitHub polling source — including the arithmetic that makes enrichment a bounded sample +- Honest about the limitations of the GitHub polling source and the arithmetic that can + make the durable enrichment backlog grow without a bounded drain time --- @@ -1230,7 +1280,7 @@ Each change came out of a multi-lens adversarial review with independent verific |---|---|---| | 1 | **304 handling corrected: unauthenticated 304s consume quota.** V1’s 304 branch scheduled the next poll from `X-Poll-Interval`, which only makes sense if 304s were free. | Two independent live probes: conditional `If-None-Match` request to `/events` returned HTTP 304 with `x-ratelimit-used` incremented (one transcript: used 4→5, remaining 56→55). Best-practices doc scopes the 304 exemption to requests “correctly authorized with an Authorization header”; the events page carries a broader unqualified statement | | 2 | **Request-budget arithmetic added, cadence derived from budget.** V1 had mechanisms but no numbers: polling at `X-Poll-Interval` (60s) = 60 req/hr = 100% of the budget at 1 page (300% at 3 pages), starving required Story 3 enrichment. | Rate-limits doc: 60 req/hr unauthenticated, IP-keyed; live headers: `x-ratelimit-limit: 60`, `x-poll-interval: 60` on both 200 and 304, `x-ratelimit-resource: core` shared across `/events`, `/users/*`, `/repos/*`; `Link` header `rel="last"` page=3 at `per_page=100` | -| 3 | **Enrichment declared bounded best-effort sampling** with a budget-skip state (refined by Appendices B–D). | One live page: ~92–95 PushEvents, ~89 distinct actors, ~92 distinct repos ≈ 181 cold entity requests/page ≈ 2,172/hr at default cadence vs 40/hr supply ≈ 1.8% theoretical cold coverage | +| 3 | The initial review introduced terminal budget load shedding; Appendix F supersedes it with durable FIFO backlog work. | One live page: ~92–95 PushEvents, ~89 distinct actors, ~92 distinct repos ≈ 181 cold entity requests/page, or ≈2,172/hr only if every poll contains new identities — a pressure scenario, not a measured deduplicated arrival rate and not justification for discarding work | | 4 | **Enrichment state moved from `push_events` to entity tables, with stub upserts in the ingest transaction.** | Review verdict (upheld): shape defect on V1 Section 7’s seven per-event enrichment columns | | 5 | **Stack decisions pinned (Section 2A).** V1 named no versions, job backend, HTTP client, test tooling, recurring-poll mechanism, or compose topology. | Verified absence in V1 | | 6 | **Post-commit enqueue + reconciler kept** (critique refuted). | Solid Queue defaults to a separate queue database and documents `enqueue_after_transaction_commit` | @@ -1256,7 +1306,7 @@ A second, independent validation pass approved the direction and required these | 2 | Poll scheduling decomposed (cadence vs server floor vs deferrals) | Section 9 | | 3 | `--force` restricted to cadence + stored ETag | Section 9 | | 4 | Global live-request gate (serial outbound concurrency of one) | Sections 2A, 5, 10 | -| 5 | Enrichment state machine completed with reactivation rule and labeled coverage metrics | Sections 7, 10, 11 | +| 5 | Enrichment state machine completed with labeled coverage metrics (backlog behavior later superseded by Appendix F) | Sections 7, 10, 11 | | 6 | Stub upsert merge rules; reconciler partial index matches the real predicate | Section 7 | | 7 | Quarantine keyed by SHA-256 payload fingerprint; malformed-event taxonomy | Section 7 | | 8 | Fixture source vs transport resolved; fail-closed; VCR rationale reworded | Sections 2A, 5, 6, 12 | @@ -1277,7 +1327,7 @@ A third review round conditionally approved V2 and required eight substantive co | 2 | `reset_at` removed from routine scheduling; blocking timestamp added; scheduling components persisted separately | Sections 7, 9, 10 | | 3 | **Compose profiles** for `ingest`/`test` + one-shot `setup` service (unprofiled services all start on `up`; concurrent `db:prepare` raced) | Section 2A | | 4 | **PR order made dependency-consistent** (gate + ledger cores in PR 4, before budget-spending PRs) | Section 13 | -| 5 | **Enrichment fairness shares** with borrowing; per-class `/status`; pinned eligibility/TTL envs | Sections 7, 10, 11 | +| 5 | **Enrichment fairness shares** with borrowing; per-class `/status`; pinned refresh-TTL envs | Sections 7, 10, 11 | | 6 | **Stop-on-known-event removed for the live source; ETag scoped to page 1**; overlap claim softened to probe-dated observation | Sections 9, 12, 15 | | 7 | **One authoritative allowance formula** with startup validation; “request-attempt” naming; fresh-install bootstrap concept | Sections 7, 10 | | 8 | **Schema completed**: actor `display_login`/`name` + envelope mappings; repo `full_name` mapping; typed columns; canonical fingerprint + occurrence counting | Section 7 | @@ -1293,7 +1343,7 @@ A fourth review round validated the external facts (GitHub, Rails, Ruby, Solid Q | 2 | **Global vs class blocking split**: `global_blocked_until` stores only truly global conditions (primary exhaustion, reserve reached, secondary limits); class blocking derived from counters (`poll_used >= poll_allowance ? reset_at : nil`); separate `effective_enrichment_time`; **secondary limits block globally** (they can arise on enrichment requests, which have no source row) | One timestamp could not defer “only that class”: enrichment exhaustion would have stopped polling and vice versa — a direct contradiction in the C-round text | Sections 7, 9, 10 | | 3 | **Bootstrap = the first real poll of every window**, not an extra discovery request: window lifecycle (`uninitialized → active → globally_blocked`), counters reset per window, enrichment ineligible until initialized from authoritative headers | An extra quota-discovery request wastes budget; per-window (not just fresh-install) matters because IP co-tenants may spend immediately after each reset | Sections 7, 10 | | 4 | **Docker restart policies added** (`unless-stopped` for `db`/`web`/`worker`, `stop_grace_period: 30s` on worker, `no` for one-shots), with the recovery runbook later corrected by Appendix E's API-stop/process-crash distinction | Docker’s default restart policy is `no` — the durability story silently assumed restarts that would never happen | Sections 2A, 8, 15, 16 | -| 5 | **Entity activity gated on distinct events**: `last_seen_at`/`latest_event_at`/reactivation update only when `INSERT … RETURNING id` produces a row; duplicate replays may refresh identity fields but never reactivate | “Every observed event updates `last_seen_at`” let a replayed duplicate resurrect a `skipped_budget` entity with no new activity | Sections 5, 7, 8, 12 | +| 5 | **Entity activity gated on distinct events**: `last_seen_at`/`latest_event_at` update only when `INSERT … RETURNING id` produces a row; duplicate replays may refresh identity fields but never register activity | “Every observed event updates `last_seen_at`” let a replayed duplicate falsely look like new activity | Sections 5, 7, 8, 12 | | 6 | **SHA columns widened to `varchar(64)`** accepting 40- or 64-char hex | Git object names are 40 hex (SHA-1) or 64 hex (SHA-256); hard-coding 40 contradicted the tolerant-parser goal | Section 7 | | 7 | **Quarantine identity made unambiguous**: `payload_fingerprint` is the sole unique key (`github_event_id` indexed, not unique); one canonicalization definition (SHA-256 of compact UTF-8 JSON with recursively sorted keys) | Dual unique keys left an unhandled conflict path (same event ID, different malformed payload); “or equivalently normalized `jsonb`” specified two algorithms | Section 7 | | 8 | Precision edits: `ingest` depends on `setup`, `test` depends only on `db` and self-prepares; **Ruby switched to 3.4.10** (3.3 is security-maintenance-only — weaker greenfield signal; 3.4.10 verified current, released 2026-06-30); “Rails 8 bundles Solid Queue” reworded to “default Active Job backend in new Rails 8 applications”; `X-RateLimit-Resource` added to processed headers with a `core` verification; fairness rounding defined (floor/remainder); borrowing requires no *currently eligible* candidate; operational defaults pinned (HTTP timeouts, retries, redirects, lock wait); `/status` coverage formulas defined | Accuracy and reviewer experience | Sections 2A, 10, 11 | @@ -1313,7 +1363,7 @@ is the plan meeting a fact it could not have known in advance. | What the plan said | What was built | Why | Record | |---|---|---|---| -| Section 8 used a broad persisted-outcome equivalence and implied every uncommitted event would return on the next poll | The documented guarantee is limited to one `push_events` row per GitHub event ID and no activity/reactivation from duplicate observations; pre-commit recovery depends on the event remaining in a later sliding-feed response | Quarantine counters, run summaries, budget debits, executions, and logs may repeat, while the upstream window can advance past an uncommitted event. The old shorthand overstated both persistence and source-delivery guarantees; this is a wording correction, not an architecture change | ADR 0005; Section 8 amendment | +| Section 8 used a broad persisted-outcome equivalence and implied every uncommitted event would return on the next poll | The documented guarantee is limited to one `push_events` row per GitHub event ID and no new entity activity from duplicate observations; pre-commit recovery depends on the event remaining in a later sliding-feed response | Quarantine counters, run summaries, budget debits, executions, and logs may repeat, while the upstream window can advance past an uncommitted event. The old shorthand overstated both persistence and source-delivery guarantees; this is a wording correction, not an architecture change | ADR 0005; Section 8 amendment | | Section 16 gates on “plain `docker compose up --build` starts exactly `db`, `setup`, `web`, `worker`” | `web` and `worker` no longer declare a `build:`; `setup` builds the shared image and they wait on it, while the `tools` one-shots keep their own build plus `pull_policy: build` | **The clean-checkout verification found the gate was false.** Compose Bake — on by default in Docker Desktop — makes every service with a `build:` its own bake target, and targets exporting the same `image:` tag race. From a cold image the reviewer's first command failed with `image "github-push-ingestor-app:latest": already exists` and started **zero** containers. It reproduces only when the image is absent, so every prior run on a warm machine passed. This is the defect the deliverable exists to catch | `docker-compose.yml`, `spec/docker_compose_spec.rb` | | Section 15 step 8 originally treated `docker kill` as restart-policy verification | `script/verify_recovery.sh` performs **both** the documented API stop and a host-PID-namespace main-process crash, and reports both outcomes separately | `docker kill` is an API stop, and `restart: unless-stopped` skips a container the daemon recorded as manually stopped. Only the independently observed process-crash path exercises automatic restart; substituting that path without recording the distinction would hide the original defect | [`docs/evidence/2026-07-31-container-kill-recovery.md`](docs/evidence/2026-07-31-container-kill-recovery.md), README “Crash recovery verification” | | `ENABLED_LIVE_SOURCE_COUNT` is the allowance formula's source-count input | Demoted to a **fallback**. The formula counts enabled, in-service `event_sources` rows of the running mode at window initialization and rollover, and logs `budget.source_allocation_drift` when the two disagree | A configured count that drifts from the table silently mis-sizes every allowance. Boot validation still reads no database, so the refuse-to-boot check is unchanged | ADR 0009 | @@ -1328,3 +1378,35 @@ Two things worth stating after hardening: - **Guarantee wording now names only proved invariants.** Broad outcome shorthand was removed; any reference to exactly-once behavior or complete capture/enrichment is a negation or limitation. - **The 304 finding survived first-party re-verification.** PR 6's required gate re-ran the probe under `X-GitHub-Api-Version: 2022-11-28` and committed a dated transcript; `x-ratelimit-used` incremented across an unauthenticated `304`, exactly as the review-supplied evidence in Appendix A had reported. The budget arithmetic that rests on it did not have to change. + +## Appendix F — Durable enrichment backlog correction (2026-08-02) + +The original plan treated enrichment demand above the hourly allowance as grounds for +terminating old candidates. That conflated a rate limit with a business outcome. The entity +tables already provide durable, deduplicated work records, so quota scarcity should control +throughput rather than delete intent. + +This amendment supersedes every earlier backlog-discarding statement in the plan: + +- Never-enriched actor and repository rows remain `pending` or `retryable_failure` until a + real enrichment response produces success or an entity-specific terminal failure. +- The migration restores legacy budget-dropped rows to `pending` or `retryable_failure` + from their attempt history, removes the obsolete timestamp column and status constraint + value, and replaces the candidate indexes with partial `(created_at, id)` FIFO indexes. +- Quota, reserve, and fairness denials defer the row without changing its business state. +- The default hourly ledger reserves 12 requests for polling, 40 for the enrichment class, + and 8 for safety. Durable backlog has priority within the 40, which carry 20/20 + actor/repository guarantees with borrowing when the other class has no currently + claimable backlog candidate. +- Each class selects FIFO by immutable, non-null `created_at ASC, id ASC`. +- A selection that observes never-enriched work in either class does not choose refresh. + Ingestion can commit a row after that read; because one runner cycle issues at most one + request, at most one refresh crosses the boundary before the next selection suppresses it. +- `/status` reports backlog size, oldest pending timestamp/age, and reserved allowance + usage. It intentionally omits a drain ETA because no durable outcome history supports an + honest completion rate. + +The arithmetic in Appendix A still matters, but its conclusion changes: if unique arrivals +continue above the 40-attempt service rate, backlog size and oldest pending age can grow +without a bounded completion estimate. That is an operational fact to expose and capacity +to revisit, not permission to discard durable work. diff --git a/README.md b/README.md index 6ec0c86..be5c0df 100644 --- a/README.md +++ b/README.md @@ -48,10 +48,14 @@ What it does, running: - **Recovers** on its own: advisory locks die with their session, entity leases expire by arithmetic, and a 60-second reconciler rebuilds pending work from committed rows. -**Enrichment is bounded best-effort sampling, and says so.** Against roughly 2,172 cold -entity requests an hour of demand and 40 available, partial coverage is the design rather -than a shortfall — `skipped_budget` is a normal documented outcome, and `/status` publishes -the real sampling rate. See [Known limitations](#known-limitations). +**Enrichment is a durable, quota-paced backlog.** With the defaults, the hourly ledger +reserves 12 requests for polling, 40 for the enrichment class, and 8 as a safety reserve. +Durable backlog work has priority over refreshes within those 40 attempts, which carry +20/20 actor/repository guarantees with borrowing. Entity +rows remain actionable until a real enrichment response produces success or an +entity-specific terminal failure: quota exhaustion defers them to a later window, FIFO +oldest-first, and never converts delay into a terminal outcome. See [Known +limitations](#known-limitations) for what happens when arrivals outpace that service rate. **With the default `GITHUB_MODE=live`, `docker compose up` starts spending real unauthenticated quota** — twelve poll requests an hour at the default cadence, plus @@ -59,8 +63,8 @@ enrichment inside its allowance. That is the intended runtime behaviour (plan § `GITHUB_MODE=fixture docker compose up --build` runs the same flow entirely offline. Poll observations and job deliveries may repeat. The proven write invariant is narrower: -observing the same valid event again cannot add another `push_events` row or reactivate an -entity that was skipped for budget. Executions, ingestion-run rows, quarantine occurrence +observing the same valid event again cannot add another `push_events` row or register new +entity activity. Executions, ingestion-run rows, quarantine occurrence counters, budget use, and logs may repeat. This system does not claim exactly-once execution. See [Processing guarantees](#processing-guarantees). @@ -68,7 +72,7 @@ counters, budget use, and logs may repeat. This system does not claim exactly-on best place to start. The authoritative execution plan is [`IMPLEMENTATION_PLAN.md`](IMPLEMENTATION_PLAN.md) — its pre-implementation revision history lives in Git and in its Appendices A–D, and Appendix E records how the build diverged from -it. Delivery is tracked on the +it. Appendix F records the durable-enrichment-backlog correction. Delivery is tracked on the [GitHub Push Ingestor Delivery project](https://github.com/users/batbrainy/projects/1). ## Requirements @@ -244,13 +248,22 @@ output too. The removed one-shot container cannot be queried with `docker compos ```bash sleep 60 +activity_before="$(docker compose exec -T db psql -U postgres \ + -d github_push_ingestor_development -Atc \ + "SELECT md5(string_agg(row_to_json(entities)::text, ',' ORDER BY kind, github_id)) + FROM (SELECT 'actor' AS kind, github_id, first_seen_at, last_seen_at, latest_event_at FROM github_actors + UNION ALL + SELECT 'repository', github_id, first_seen_at, last_seen_at, latest_event_at FROM github_repositories) entities;")" fixture_replay_output="$(mktemp)" GITHUB_MODE=fixture docker compose run --rm ingest --force 2>&1 | tee "$fixture_replay_output" grep -E 'Duplicates skipped:[[:space:]]+4' "$fixture_replay_output" -if grep -q 'enrichment.reactivated' "$fixture_replay_output"; then - echo "duplicate replay reactivated an entity" >&2 - exit 1 -fi +activity_after="$(docker compose exec -T db psql -U postgres \ + -d github_push_ingestor_development -Atc \ + "SELECT md5(string_agg(row_to_json(entities)::text, ',' ORDER BY kind, github_id)) + FROM (SELECT 'actor' AS kind, github_id, first_seen_at, last_seen_at, latest_event_at FROM github_actors + UNION ALL + SELECT 'repository', github_id, first_seen_at, last_seen_at, latest_event_at FROM github_repositories) entities;")" +test "$activity_before" = "$activity_after" rm -f "$fixture_ingest_output" "$fixture_replay_output" ``` @@ -434,10 +447,9 @@ bypasses this application's cadence, never GitHub's floor. Once past it, nothing is created: the same page is absorbed as **4 duplicates**, the three quarantine rows stay three rows with their occurrence counts at 2, and no -entity's activity moves — and **no skipped entity is reactivated**, which is the -half of plan §7's merge rules a re-polled window would otherwise break. Re-running -ingestion cannot duplicate accepted event rows or reactivate a skipped entity from the -same event ID; run summaries, quarantine counters, budget use, and logs may still change. See +entity's activity moves. Re-running ingestion cannot duplicate accepted event rows or +register new entity activity from the same event ID; run summaries, quarantine counters, +budget use, and logs may still change. See [ADR 0005](docs/adr/0005-at-least-once-with-idempotent-writes.md). ### Enrichment, offline @@ -522,7 +534,7 @@ bypass, so allow a minute between successive ingestion scenarios. | Scenario | Command | What it shows | Cleanup | |---|---|---|---| | `default` | `GITHUB_MODE=fixture docker compose run --rm ingest` | 4 push events, 3 actors, 3 repositories, 3 quarantined | none | -| `default` (replay) | `… run --rm ingest --force` after 60s | `duplicates_skipped > 0`, occurrence counts climb, no skipped entity reactivated | none | +| `default` (replay) | `… run --rm ingest --force` after 60s | `duplicates_skipped > 0`, occurrence counts climb, entity activity does not move | none | | `default` (enrich) | `… run --rm enrich --limit 6` | both classes enrich within their fairness shares | none | | `paginated` | `GITHUB_FIXTURE_SCENARIO=paginated MAX_PAGES_PER_POLL=3 … ingest` | `Link`-driven walk over 3 pages | none | | `paginated_final_page` | `GITHUB_FIXTURE_SCENARIO=paginated_final_page … ingest` | the walk stops when no `next` link exists | none | @@ -553,7 +565,8 @@ curl -s http://localhost:3000/api/push_events/ | jq .data.raw_p Reports persisted state only: the poll schedule per event source (all five of §9's components, plus which one is binding), the per-class ledger state, §11's three -enrichment coverage percentages, and the per-status entity counts. +enrichment coverage percentages, the per-status entity counts, and durable-backlog +telemetry: per-class size, oldest pending timestamp/age, and reserved allowance usage. Three conventions in the response body are worth knowing before reading one: @@ -567,18 +580,21 @@ Three conventions in the response body are worth knowing before reading one: so you can check it. - **The coverage window is measured on `created_at`** — when *this application* persisted the event, not GitHub's `occurred_at`. Coverage grades this - application's enrichment pipeline, and that pipeline's own eligibility and - freshness rules already run on this clock. The basis is published as + application's enrichment pipeline and uses the same local clock as backlog and + freshness reporting. The basis is published as `coverage.basis` so the choice is visible rather than assumed. Widen `ENRICHMENT_COVERAGE_WINDOW_SECONDS` when reviewing a fixture corpus that has aged. - **`actor_requests.available` is a floor, not a ceiling.** §10 lets one class - borrow the other's unspent capacity when the other has no eligible candidate, so a + borrow the other's unspent capacity when the other has no claimable backlog candidate, so a class does not stop at zero available. The real ceiling is the `enrichment` pair beside it. `pending` in the entity counts means `enrichment_status = 'pending'` exactly. The -`candidates` figure beside it is pending **plus** `retryable_failure` — the "how much -work is left" number `bin/ingest` prints. Two questions, two names, deliberately. +`backlog_count` beside it is pending **plus** `retryable_failure` — the "how much work is +left" number the command summary prints. That work has no age cutoff. `/status` +deliberately does not publish a drain ETA: the schema has no durable outcome history from +which to calculate an honest service rate, and sustained arrivals can make the backlog grow +indefinitely. ### `GET /api/push_events` and `GET /api/push_events/:id` @@ -721,11 +737,10 @@ not claimed to be reconstructible. See | Column | Type | The rule it encodes | |---|---|---| | `github_id` | `bigint` **unique** | Identity. The stub upsert conflicts on this | -| `enrichment_status` | `text`, check-constrained | Five legal values; see below | +| `enrichment_status` | `text`, check-constrained | Four legal values; see below | | `first_seen_at`, `last_seen_at`, `latest_event_at` | `timestamp` | Activity. Updated **only** when a `push_events` insert actually returned a row | -| `next_retry_at` | `timestamp` | Double duty: the backoff instant *and* the enrichment lease. One column, one predicate, so the candidate query, the age-out sweep and the claim cannot drift apart | +| `next_retry_at` | `timestamp` | Double duty: the backoff instant *and* the enrichment lease. One column, one predicate, so candidate selection and claiming cannot drift apart | | `fetched_at` | `timestamp` | When enrichment last succeeded — the input to the refresh TTL | -| `skipped_at` | `timestamp` | When the eligibility window expired and the row became `skipped_budget` | | `raw_payload` | `jsonb` nullable | The enrichment response. Null until a fetch succeeds | **`event_sources`** — five independent scheduling columns, never one collapsed timestamp: @@ -751,15 +766,14 @@ limits are IP-scoped rather than window-scoped | Status | Entered when | Left when | Spends budget | |---|---|---|---| -| `pending` | A stub row is created by ingestion | Enrichment succeeds, fails, or ages out | Yes — it is the candidate pool | -| `complete` | An enrichment fetch succeeded | The refresh TTL expires, making it a refresh candidate | Only on refresh | +| `pending` | A stub row is created by ingestion | Enrichment succeeds or reaches a failure outcome | Yes, when allowance is available | +| `complete` | An enrichment fetch succeeded | Remains complete; after the TTL it may be refreshed in place once a selection observes the never-enriched backlog empty | Only on refresh | | `retryable_failure` | A `5xx`, timeout, or transport error | The backoff expires and a retry runs | Yes — it stays a candidate | -| `permanent_failure` | A `404` or other permanent `4xx` on the entity URL | Never, automatically | No | -| `skipped_budget` | The eligibility window expired before budget was available | **Only** a genuinely new push event reactivates it | No | +| `permanent_failure` | A `404` or other permanent `4xx` on the entity URL | Never, automatically | Yes for the HTTP outcome; no future retries | -`skipped_budget` is what bounds the backlog. A duplicate replay refreshes identity fields -but never reactivates a skipped entity — that is the half of §7's merge rules a re-polled -window would otherwise break. +`pending` and `retryable_failure` rows are the durable backlog. They are selected FIFO, +oldest first, and quota exhaustion only defers them to a later window. A duplicate replay +may refresh identity fields but does not register new activity or change backlog priority. ### Replay behavior by table @@ -768,7 +782,7 @@ window would otherwise break. | `push_events` | `INSERT … ON CONFLICT (github_event_id) DO NOTHING RETURNING id` | No-op; the accepted raw event is never mutated | | `github_actors`, `github_repositories` | `INSERT … ON CONFLICT (github_id) DO UPDATE` on identity fields only | Identity refreshed; enrichment payload untouched | | `quarantined_events` | `INSERT … ON CONFLICT (payload_fingerprint) DO UPDATE` | `occurrence_count` increments; the first classification is permanent | -| Entity activity fields | Gated on the `push_events` insert returning a row | No activity registered, no reactivation | +| Entity activity fields | Gated on the `push_events` insert returning a row | No activity registered | Transactions are **one per event**, not one per page, so a single malformed envelope can never discard the events persisted beside it. Quarantine writes stand outside any @@ -781,13 +795,14 @@ planner scans only rows that could possibly qualify: - `index_event_sources_on_poll_due` on `(source_type, next_poll_at)` `WHERE enabled AND status = 'idle'` — the tick's due-source scan. -- `index_github_{actors,repositories}_on_enrichment_candidates` on - `(next_retry_at, last_seen_at)` `WHERE enrichment_status IN ('pending','retryable_failure')` - — the candidate pool. +- `index_github_{actors,repositories}_on_enrichment_candidates` on the oldest-first keys + `(created_at, id)`, + `WHERE enrichment_status IN ('pending','retryable_failure')` — the durable FIFO + candidate pool. `created_at` is immutable and non-null; `first_seen_at` can be null. - `index_github_{actors,repositories}_on_enrichment_refresh` on `(fetched_at, next_retry_at)` `WHERE enrichment_status = 'complete'` — the TTL refresh pool. -[`db/schema.rb`](db/schema.rb) is authoritative (version `2026_07_31_120000`). For live +[`db/schema.rb`](db/schema.rb) is authoritative (version `2026_08_02_000000`). For live truth: ```bash @@ -831,8 +846,9 @@ environment or timestamp: ### Sample stream -Captured verbatim from a fixture-mode run against an empty database, so you can reproduce -the same shapes offline: +Representative, intentionally abridged excerpts from a fixture-mode run against an empty +database, with queue and backlog fields updated to the current routing. Fields omitted from +the excerpts remain present in the actual structured logs. You can reproduce them offline: ```bash GITHUB_MODE=fixture docker compose up --build -d @@ -853,7 +869,7 @@ Boot, then a poll that created four events and quarantined three: {"timestamp":"2026-07-31T15:50:49.924Z","level":"info","service":"github-push-ingestor","environment":"development","event":"budget.window_initialized","limit":60,"reserve":8,"poll_allowance":12,"enrichment_allowance":40,"actor_guarantee":20,"repository_guarantee":20,"rate_limit_resource":"core","rate_limit_limit":60,"rate_limit_remaining":59,"rate_limit_used":1,"rate_limit_reset_at":"2026-07-31T16:50:49Z","poll_used":1} {"timestamp":"2026-07-31T15:50:49.965Z","level":"info","service":"github-push-ingestor","environment":"development","event":"ingestion.event_quarantined","run_id":"099d562d-1261-488d-9003-cb0c443cdb55","github_event_id":"58000000006","event_type":"PushEvent","error_code":"invalid_field_format","error_message":"payload.head is \"not-a-valid-object-name\", not 40 or 64 hexadecimal characters","payload_fingerprint":"a8ad67ca97a4c48049f5fa447d5d88ae10c58c514e0129546e18b5ff22368020"} {"timestamp":"2026-07-31T15:50:49.973Z","level":"info","service":"github-push-ingestor","environment":"development","event":"ingestion.run_completed","run_id":"099d562d-1261-488d-9003-cb0c443cdb55","event_source_id":1,"duration_ms":100.7,"next_poll_at":"2026-07-31T15:55:49Z","consecutive_failures":0,"run_status":"completed","classification":"ok","stop_reason":"no_next_link","pages_fetched":1,"events_received":8,"push_events_seen":6,"events_created":4,"duplicates_skipped":0,"events_quarantined":3,"events_ignored":1,"events_failed":0} -{"timestamp":"2026-07-31T15:50:50.024Z","level":"info","service":"github-push-ingestor","environment":"development","event":"enrichment.dispatched","actor_enqueued":1,"repository_enqueued":1,"reason":"ingestion","actor_counts":{"pending":3},"repository_counts":{"pending":3},"actor_share_used":0,"repository_share_used":0,"actor_guarantee":20,"repository_guarantee":20,"enrichment_used":0,"enrichment_allowance":40,"window_status":"active","claimable_now":true} +{"timestamp":"2026-07-31T15:50:50.024Z","level":"info","service":"github-push-ingestor","environment":"development","event":"enrichment.dispatched","actor_enqueued":1,"repository_enqueued":1,"reason":"ingestion","actor_counts":{"pending":3},"repository_counts":{"pending":3},"actor_backlog_count":3,"repository_backlog_count":3,"actor_oldest_pending_age_seconds":0,"repository_oldest_pending_age_seconds":0,"actor_share_used":0,"repository_share_used":0,"actor_guarantee":20,"repository_guarantee":20,"enrichment_used":0,"enrichment_allowance":40,"window_status":"active","claimable_now":true} ``` A second run inside the cadence window makes no request at all, and names the component @@ -867,7 +883,7 @@ From the worker — a job, its enrichment outcome, and the deliberate `404` the so a dead target fails the *entity* rather than the source: ```text -{"timestamp":"2026-07-31T15:50:50.780Z","level":"info","service":"github-push-ingestor","environment":"development","event":"job.completed","job_id":"58a1e78d-f473-4440-aee0-fe0bbb22027f","job_class":"EnrichActorJob","queue":"default","attempt":1,"duration_ms":74.6,"entity_type":"actor","github_actor_id":7700421,"enrichment_outcome":"failed"} +{"timestamp":"2026-07-31T15:50:50.780Z","level":"info","service":"github-push-ingestor","environment":"development","event":"job.completed","job_id":"58a1e78d-f473-4440-aee0-fe0bbb22027f","job_class":"EnrichActorJob","queue":"enrichment","attempt":1,"duration_ms":74.6,"entity_type":"actor","github_actor_id":7700421,"enrichment_outcome":"failed"} {"timestamp":"2026-07-31T15:50:50.780Z","level":"info","service":"github-push-ingestor","environment":"development","enrichment_outcome":"failed","entity_type":"actor","github_id":7700421,"pool":"pending","classification":"not_found","entity_status":"permanent_failure","enrichment_attempt":1,"error_message":"GitHub returned 404 (not_found)","duration_ms":56.6,"event":"enrichment.failed"} {"timestamp":"2026-07-31T15:51:00.949Z","level":"info","service":"github-push-ingestor","environment":"development","enrichment_outcome":"enriched","entity_type":"actor","github_id":1024025,"pool":"pending","classification":"ok","entity_status":"complete","enrichment_attempt":1,"duration_ms":27.3,"event":"enrichment.completed"} ``` @@ -914,10 +930,9 @@ more request attempts per poll than the whole poll allowance. Enrichment adds `enrichment.completed` and `enrichment.failed`, each carrying the entity type, its GitHub id, the response classification, the resulting entity status -and the attempt number; `enrichment.aged_out`, one summary line per class per sweep -rather than one per row; `enrichment.reactivated` when a genuinely new push event -brings a `skipped_budget` entity back; `enrichment.lease_lost` at warning level when -an outcome arrived after another worker had claimed the row; and +and the attempt number; budget/class-exhaustion and reconciliation summaries show when +durable backlog work is waiting for a later quota window; `enrichment.lease_lost` appears +at warning level when an outcome arrived after another worker had claimed the row; and `enrichment.cycle_failed` at error level when an exception escapes a cycle entirely, carrying the lease it released and the error class. @@ -1019,7 +1034,7 @@ At limit 60, reserve 8, cadence 300s, one source — the only variable is page d | 4 | 48 | 4 | 2 | 2 | | 5 | 60 | — | — | **refuses to boot** | -Capture depth is bought with enrichment coverage, at a fixed exchange rate, and the +Capture depth is bought with backlog-drain capacity, at a fixed exchange rate, and the formula tells you the price before you pay it. ### Global blocks versus class exhaustion @@ -1102,9 +1117,8 @@ is [`.env.example`](.env.example). | `MAX_PAGES_PER_POLL` | `1` | How many `Link`-followed pages one poll may fetch, and an allowance-formula input. Raising it trades enrichment allowance for capture depth — see [the budget table](#the-budget-table) (plan §9, §10) | | `ENABLED_LIVE_SOURCE_COUNT` | `1` | Allowance-formula input: live sources sharing one per-IP budget. **A fallback rather than the authority** — at window initialization and rollover the formula counts the enabled, in-service `event_sources` rows of the running mode and uses that instead, falling back to this value only when there are none yet. A disagreement is logged as `budget.source_allocation_drift` (plan §10, [ADR 0009](docs/adr/0009-runtime-source-allocation-and-shared-ip-observability.md)) | | `RATE_LIMIT_RESERVE` | `8` | Requests per hour left deliberately unspent (plan §10) | -| `ACTOR_ENRICHMENT_SHARE` | `0.50` | How the enrichment allowance splits between actors and repositories: `floor(allowance x this)` guarantees actors, the remainder goes to repositories. A guarantee, not a cap — either class may borrow the other's unused capacity when the other has no *currently eligible* candidate. Both ends of `[0, 1]` are legal (plan §10) | -| `ENRICHMENT_ELIGIBILITY_WINDOW_SECONDS` | `3600` | How long a candidate stays worth enriching. Past it, the entity transitions to `skipped_budget` — which is what bounds the backlog — until a genuinely new push event reactivates it (plan §10, B8) | -| `ACTOR_REFRESH_TTL_SECONDS` | `86400` | How long an enriched actor is reused before it is re-fetched. Never-enriched candidates always take priority over refreshes (plan §10) | +| `ACTOR_ENRICHMENT_SHARE` | `0.50` | How the enrichment allowance splits between actors and repositories: `floor(allowance x this)` guarantees actors, the remainder goes to repositories. A guarantee, not a cap — either class may borrow the other's unused capacity when the other has no *currently claimable* backlog candidate. Both ends of `[0, 1]` are legal (plan §10) | +| `ACTOR_REFRESH_TTL_SECONDS` | `86400` | Minimum reuse time before an enriched actor may be re-fetched. At each selection decision, refreshes are suppressed when either class has never-enriched backlog work (plan §10); see limitation 7 for the bounded concurrent-insert window | | `REPOSITORY_REFRESH_TTL_SECONDS` | `86400` | The same, for repositories | | `ENRICHMENT_COVERAGE_WINDOW_SECONDS` | `86400` | How far back `GET /status` looks when computing §11's three coverage percentages, measured on `push_events.created_at`. The only knob here that changes what the system *reports* rather than what it *does* | @@ -1130,14 +1144,14 @@ IngestionRunner ──► SourceLock ──► PollSchedule (due? — five compo │ ├──► PageLoop ──► RequestExecutor ──► RequestGate EnrichmentRunner ────────────────┘ │ ▲ ──► BudgetLedger.reserve! - AgeOut → skipped_budget │ │ ──► UrlPolicy - Fairness → class, pool, borrow │ └── LinkHeader.next_url ──► Transport (Faraday | Fixture) + FIFO backlog → class, borrow │ │ ──► UrlPolicy + Refresh only when backlog empty │ └── LinkHeader.next_url ──► Transport (Faraday | Fixture) Claim → lease on next_retry_at │ │ (never a SourceLock, §8 step 1) │ ▼ │ PageWriter: one transaction per event │ stub upserts → INSERT … ON CONFLICT - │ DO NOTHING RETURNING id → activity - │ updates and reactivation only when + │ DO NOTHING RETURNING id → entity + │ activity updates only when │ a row returned ▼ RateLimitPolicy ──► BudgetLedger#block_globally! @@ -1197,25 +1211,24 @@ EnrichmentRunner ────────────────┘ │ actually happened, so a budget denial or a held gate can neither advance the cadence nor burn a healthy source's failure count. - **`Github::EnrichmentRunner`** owns one enrichment cycle and enriches at most one - entity: it ages overdue candidates into `skipped_budget` first — unconditionally, because - §12's sequence needs skipping to keep happening precisely while the budget is exhausted — - then asks fairness which class works next, leases that row, fetches through the same - chain, and writes one outcome. It never takes a source lock and opens no transaction + entity: it asks fairness which class works next, claims the oldest row in that class, + fetches through the same chain, and writes one outcome. A denied reservation leaves the + row durable for a later window. It never takes a source lock and opens no transaction across the request. - **`Github::Enrichment::Fairness`** applies §10's ladder: a never-enriched candidate in a - class still inside its guarantee, then the same borrowing when the other class has no - *currently eligible* candidate, then a TTL-stale refresh only when no pending candidate - is eligible anywhere. The refresh pool allocates by the same two steps — prefer a class - with room, borrow only from a class with nothing to refresh — so neither pool can starve a - class. It decides; the ledger enforces, so a wrong answer produces a refused reservation - rather than an overspend + class still inside its guarantee, then borrowing when the other class has no + *currently claimable* backlog candidate, then a TTL-stale refresh only when the selection + observes no never-enriched work in either class. Once refresh is allowed, it uses the same + two steps — prefer a class with room, then borrow only from a class with nothing to + refresh — so one refresh class cannot starve the other. Fairness decides; the ledger + enforces, so a wrong answer produces a refused reservation rather than an overspend ([ADR 0007](docs/adr/0007-enrichment-fairness-shares-and-borrowing.md), [ADR 0010](docs/adr/0010-secondary-limit-escalation-and-refresh-pool-fairness.md)). - **`Github::Enrichment::Claim`** prevents two workers enriching one entity by leasing the row — a conditional `UPDATE` that pushes `next_retry_at` forward. One column, one meaning: the same predicate excludes leases, backoffs and secondary-limit deferrals from - both candidate pools and from the age-out sweep, so the four queries cannot drift apart. - A crashed worker leaves nothing to clean up; the lease expires. + candidate selection, so the queries cannot drift apart. A crashed worker leaves nothing + to clean up; the lease expires. - **`Github::Enrichment::EntityState`** is §10's response behaviour resolved onto one entity row, and `PollState`'s twin. Its rules: an attempt is counted only for an outcome that says something about *this entity* — a rate limit is a fact about the IP — and a @@ -1293,7 +1306,7 @@ schedules one cycle per class as soon as its rows are committed and its advisory lock is released. That enqueue is a *hint*: the durable record of pending work is the entity rows themselves, so `ReconcilePendingEnrichmentsJob` sweeps them every 60 seconds and schedules a cycle for any class that has claimable work and is not -blocked by the ledger. If a crash loses an enrichment-dispatch hint, the committed eligible +blocked by the ledger. If a crash loses an enrichment-dispatch hint, the committed entity state remains discoverable on a later successful scheduled tick without a special cleanup job or queue inspection (plan §8, [ADR 0008](docs/adr/0008-post-commit-enqueue-and-entity-scoped-reconciliation.md)). @@ -1301,7 +1314,9 @@ cleanup job or queue inspection (plan §8, Each cycle enriches at most one entity, chosen by §10's fairness policy under a lease, so a backlog of ninety pending actors is one queued job rather than ninety. Steady state at the defaults: twelve polls an hour, and at most forty enrichment -requests an hour split between the two classes. +requests an hour split into 20/20 actor/repository guarantees with borrowing. Within each +class the oldest never-enriched entity is selected first; a selection considers refresh +work only after observing the entire never-enriched backlog empty. ## Processing guarantees @@ -1311,8 +1326,8 @@ before its commit is recoverable only while the event remains in a later GitHub pending enrichment can be reconstructed from committed entity state. The demonstrated invariant is deliberately specific: a duplicate observation of a valid -event cannot create another `push_events` row, and cannot reactivate an entity already in -`skipped_budget`. That does not make the surrounding execution singular. A retry may create +event cannot create another `push_events` row or register new entity activity. That does +not make the surrounding execution singular. A retry may create another ingestion-run row, increment a malformed payload's occurrence counter, spend another budget reservation, execute another job, or emit another log line, depending on its crash boundary. @@ -1323,8 +1338,9 @@ boundary. activity update are replay-safe; execution records, counters, budget use, and logs may repeat. - **Not complete upstream capture.** The feed is a sliding window with hours of latency; see [Known limitations](#known-limitations). -- **Not complete enrichment coverage.** Demand exceeds the hourly budget by roughly fifty - to one; see [Known limitations](#known-limitations). +- **Not bounded-time enrichment completion.** Durable work is not discarded, but sustained + arrivals can exceed the 40-request hourly service rate; see [Known + limitations](#known-limitations). ### The four crash cases @@ -1338,8 +1354,8 @@ boundary. ### The mechanisms - `INSERT … ON CONFLICT (github_event_id) DO NOTHING RETURNING id` — the deduplication gate. -- Activity updates gated on that `RETURNING` — so a replay refreshes identity but never - reactivates a `skipped_budget` entity. +- Activity updates gated on that `RETURNING` — so a replay may refresh identity but cannot + register new activity or change FIFO backlog age. - `payload_fingerprint` uniqueness on quarantine — one row per distinct malformed payload, occurrence-counted. - The entity lease is a `next_retry_at` timestamp that **expires by arithmetic**, so a @@ -1529,15 +1545,18 @@ newest ~100 events, and the feed moves considerably faster than that. **This ser samples the public feed rather than mirroring it.** The nine limitations that follow are consequences of that, of the 60-request hourly -ceiling, and of deliberate scope decisions. None is a gap to be closed later; each is a -stated boundary. +ceiling, and of deliberate scope decisions. Each is a stated operational boundary. -**1. Enrichment is sampled, not exhaustive.** One observed live page held ~92–95 +**1. Enrichment has no bounded completion time.** One observed live page held ~92–95 `PushEvent` records with ~89 distinct actors and ~92 distinct repositories — 181 cold -entity requests per page, ~2,172 an hour at twelve polls, against 40 available. That is -roughly 1.8% theoretical cold coverage. `skipped_budget` is a normal documented outcome, -and `/status` publishes the three coverage percentages with every denominator so the real -sampling rate is visible rather than inferred. +entity requests per page, or ~2,172 an hour if every poll contained entirely new entities, +against 40 available. That extrapolation is a pressure scenario, not the measured +deduplicated arrival rate: repeated actors, repositories, and overlapping pages collapse to +shared rows. Entity rows nevertheless remain durable FIFO backlog work; quota exhaustion +defers them to later windows and never terminates them. If measured unique arrivals remain +above 40 attempts per hour, backlog size and oldest pending age grow and no +finite drain estimate is honest. `/status` publishes backlog count, oldest pending age, +and the reserved allowance's usage; it does not fabricate an ETA from incomplete history. **2. There is no guarantee of complete upstream capture.** Pagination deepens a single poll within the budget; it does not backfill. Events that rolled out of the feed's window @@ -1558,14 +1577,21 @@ count, but a second *live* source is a documented seam rather than shipped behav key order, and duplicate keys are lost; array order is preserved because it is meaningful ([ADR 0001](docs/adr/0001-jsonb-semantic-retention.md)). -**6. No authentication.** A larger authenticated budget could materially increase feasible -coverage, but would not make upstream capture or enrichment complete. The unauthenticated +**6. No authentication.** A larger authenticated budget could drain the durable enrichment +backlog faster, but would not make upstream event capture complete. The unauthenticated constraint is deliberate, not an oversight. See the scaling path in [`docs/DESIGN_BRIEF.md`](docs/DESIGN_BRIEF.md). -**7. An enriched entity can be up to 24 hours stale.** `ACTOR_REFRESH_TTL_SECONDS` and -`REPOSITORY_REFRESH_TTL_SECONDS` default to 86400, and a refresh only spends budget when -no never-enriched candidate is eligible anywhere. +**7. Enriched entities can remain stale while backlog exists.** +`ACTOR_REFRESH_TTL_SECONDS` and `REPOSITORY_REFRESH_TTL_SECONDS` default to 86400, but a +selection that observes never-enriched work does not choose a refresh. Under sustained +backlog pressure, staleness can therefore exceed the TTL; the TTL is an earliest refresh +time, not a deadline. There is one bounded concurrency window: ingestion can commit a new +candidate after fairness has observed both tables empty but before the chosen refresh is +debited. Because a runner cycle issues at most one entity request, at most one refresh can +cross that boundary; the next selection sees the backlog and suppresses refreshes. The race +cannot discard or terminally mark backlog work, and the ledger still enforces the 40-request +enrichment cap. **8. Extension C (object storage) was deliberately not attempted.** A decision with a stated reason, not an omission — the remaining budget went to rate-limit correctness, @@ -1579,13 +1605,14 @@ verification](#crash-recovery-verification); Extension D (testing strategy) is [Deterministic fixture verification](#deterministic-fixture-verification) and the suite described there. -**9. Business tables grow without bound, by design.** `push_events`, `ingestion_runs`, -and `quarantined_events` are append-only — this service is the system of record, and -retention, pruning, and archival were deliberately not built. The 60-request hourly -ceiling bounds the worst case: twelve poll attempts an hour at up to ~100 events each -is at most ~1,200 rows an hour, fewer after duplicate skips. The only shipped pruning -is Solid Queue's finished-job cleanup in the queue database, which holds no business -data. +**9. Business tables and the enrichment backlog can grow without bound.** `push_events`, +`ingestion_runs`, and `quarantined_events` are append-only, and never-enriched actor and +repository rows remain actionable until attempted — this service is the system of record, +and retention, pruning, and archival were deliberately not built. Twelve poll attempts an +hour at up to ~100 events each can add roughly 1,200 event rows and as many as 2,400 entity +references an hour before deduplication, while only 40 enrichment attempts drain the +backlog. The only shipped pruning is Solid Queue's finished-job cleanup in the queue +database, which holds no business data. ## Development @@ -1599,7 +1626,7 @@ AI-assisted development guidance for this repository lives in - [`docs/SUBMISSION_CHECKLIST.md`](docs/SUBMISSION_CHECKLIST.md) — the §16 quality gates as a pre-flight checklist. - [`IMPLEMENTATION_PLAN.md`](IMPLEMENTATION_PLAN.md) — the execution plan; Appendix E - records how the build diverged from it. + records build-time divergence and Appendix F the durable-backlog correction. ## License diff --git a/app/jobs/enrich_actor_job.rb b/app/jobs/enrich_actor_job.rb index fc873b9..9b6f8dd 100644 --- a/app/jobs/enrich_actor_job.rb +++ b/app/jobs/enrich_actor_job.rb @@ -2,15 +2,20 @@ # # It takes no actor id, and that is the design rather than an omission: # Github::EnrichmentRunner enriches at most one entity per call and *chooses* it through -# §10's fairness policy and a FOR UPDATE SKIP LOCKED lease, newest-first. An id-addressed job -# would have to bypass that ordering to honour its argument, which is how a repository flood -# starves actors. So the job says "do one actor's worth of work" and the runner decides -# whose — which is also why a duplicate delivery is harmless: it is one more cycle, and it -# finds either different work or none. +# §10's fairness policy and a FOR UPDATE SKIP LOCKED lease, in durable FIFO order. An +# id-addressed job would have to bypass that ordering to honour its argument, which is how a +# repository flood starves actors. So the job says "do one actor's worth of work" and the +# runner decides whose — which is also why a duplicate delivery is harmless: it is one +# more cycle, and it finds either different work or none. # # It never takes a source lock (§8 step 1: "enrichment jobs skip this step — they take only # the request gate"), and Github::LockOrder enforces that structurally. class EnrichActorJob < ApplicationJob + # Enrichment is deliberately isolated from polling and reconciliation. A deep durable + # entity backlog may keep this queue busy for many rate-limit windows, but it must never + # delay the control tick that discovers more committed work or the poller that creates it. + queue_as :enrichment + def perform result = Github::EnrichmentRunner.new.call(entity_class: GithubActor) diff --git a/app/jobs/enrich_repository_job.rb b/app/jobs/enrich_repository_job.rb index 3f784b7..1ad3ff2 100644 --- a/app/jobs/enrich_repository_job.rb +++ b/app/jobs/enrich_repository_job.rb @@ -6,6 +6,10 @@ # It takes no repository id, for the reason EnrichActorJob's comment gives: the entity is # chosen by §10's fairness policy under a lease, not by the caller. class EnrichRepositoryJob < ApplicationJob + # Shares one bounded worker with actor enrichment. Entity rows, not queued job count, are + # the backlog; each delivery is only a wake-up that asks the runner to claim one row. + queue_as :enrichment + def perform result = Github::EnrichmentRunner.new.call(entity_class: GithubRepository) diff --git a/app/jobs/poll_event_source_job.rb b/app/jobs/poll_event_source_job.rb index a8e989f..c36ccec 100644 --- a/app/jobs/poll_event_source_job.rb +++ b/app/jobs/poll_event_source_job.rb @@ -22,6 +22,11 @@ # tick writes nothing at all, and the lock frees a killed session in milliseconds where a # semaphore's fixed duration cannot. See spec/recovery/multi_poller_spec.rb and ADR 0008. class PollEventSourceJob < ApplicationJob + # Polling is isolated from the long-lived enrichment backlog. It shares a worker only + # with the bounded control queue, whose recurring tick cannot create an entity-sized + # workload. + queue_as :polling + # Facts about the process rather than about one source: continuing to the next source # would only repeat them, and a tick that "completed" after boot-level breakage would be a # lie. The same line Github::Ingestion::PageWriter::FATAL_ERRORS and diff --git a/app/jobs/reconcile_pending_enrichments_job.rb b/app/jobs/reconcile_pending_enrichments_job.rb index 864e7e5..5b2446c 100644 --- a/app/jobs/reconcile_pending_enrichments_job.rb +++ b/app/jobs/reconcile_pending_enrichments_job.rb @@ -16,6 +16,12 @@ # derived class block is in force — so an exhausted window costs one indexed EXISTS per class # per minute rather than a queue full of cycles the ledger would refuse. class ReconcilePendingEnrichmentsJob < ApplicationJob + # The reconciler is control-plane work: two indexed existence checks and at most one + # wake-up per entity class. Its named, bounded queue shares the poll worker but remains + # isolated from the durable enrichment backlog, so recovery hints cannot sit behind an + # entity-sized workload. + queue_as :control + def perform @outcome = Github::Enrichment::Dispatch.call(reason: "reconcile") end diff --git a/app/models/concerns/enrichable.rb b/app/models/concerns/enrichable.rb index 2dadb48..9fe674a 100644 --- a/app/models/concerns/enrichable.rb +++ b/app/models/concerns/enrichable.rb @@ -1,11 +1,9 @@ # Actors and repositories share an identical entity-level enrichment state machine # (IMPLEMENTATION_PLAN.md §7). This concern carries the *data* half — the value set, the -# enum, and the scope whose WHERE clause matches the partial index — plus the one -# transition that belongs to the ingest path rather than to enrichment: §7 merge rule 3's -# skipped_budget reactivation. +# enum, and the scope whose WHERE clause matches the partial index. # -# Every other transition is a fetch outcome and lives in Github::Enrichment::EntityState, -# Github::Enrichment::AgeOut, or Github::Enrichment::Claim, all of which are constructed +# Every other transition is a fetch outcome and lives in Github::Enrichment::EntityState +# or Github::Enrichment::Claim, both of which are constructed # without an executor or a transport so a GitHub request cannot be issued from them. # # Two column conventions this state machine relies on, stated here because both invite @@ -18,7 +16,7 @@ # * next_retry_at means one thing everywhere: *this entity may not be attempted before # T*. It is simultaneously the failure backoff, a secondary-limit deferral, and the # in-flight claim lease. One meaning is what lets a single predicate exclude -# in-flight rows from the candidate pools and from the age-out sweep at once. +# in-flight rows from both candidate pools at once. module Enrichable extend ActiveSupport::Concern @@ -27,7 +25,6 @@ module Enrichable complete retryable_failure permanent_failure - skipped_budget ].freeze # Exactly the predicate of index_*_on_enrichment_candidates. @@ -47,11 +44,10 @@ module Enrichable class_methods do # The activity half of §7 merge rule 3. Its gate — calling this only when the # push_events insert actually returned a row, so a duplicate replay cannot register - # activity — belongs to the ingest transaction in PR 5, and it is the same gate - # .reactivate_skipped! sits behind. + # activity — belongs to the ingest transaction in PR 5. # - # Every timestamp is monotonic, so a delayed or out-of-order observation can - # never move one backwards and distort newest-first enrichment ordering. + # Every timestamp is monotonic, so a delayed or out-of-order observation can never + # move activity history backwards. FIFO enrichment itself uses immutable created_at. # PostgreSQL's GREATEST/LEAST ignore NULL arguments, returning NULL only when # every argument is NULL, so no COALESCE is required for the first write. def touch_activity!(github_id:, seen_at:, event_occurred_at:) @@ -65,38 +61,5 @@ def touch_activity!(github_id:, seen_at:, event_occurred_at:) where(github_id: github_id).update_all(assignments) end - - # The reactivation half of §7 merge rule 3, and §7's reactivation rule: "skipped_budget - # is terminal for the entity's current eligibility window, not forever. A **newly - # persisted** push event referencing the entity … may transition it back to pending." - # - # Rule 4 — "a duplicate event replay … can never reactivate enrichment" — is held by - # the *call site*, not by a check here: PR 5 already calls this only when - # PushEvent.insert_if_new returned a row. That is what makes the guarantee structural. - # - # A second statement rather than a CASE folded into .touch_activity!, for one reason - # worth the extra write: §11 lists "reactivated" among the INFO events, and a - # set-based UPDATE that also touched non-skipped rows could not report how many rows - # it actually reactivated. This one's row count is exactly that number. It matches at - # most one row, only on a genuinely new event, and almost always zero. - # - # No "missing or stale" sub-predicate is needed, because skipped_budget implies - # missing enrichment. That is derived rather than assumed: the only writer of the - # status is Github::Enrichment::AgeOut, whose WHERE is CANDIDATE_STATUSES, and no row - # in those two statuses has ever completed — so fetched_at and raw_payload are NULL on - # every one of them. A spec pins the invariant. - # - # It clears skipped_at and nothing else. enrichment_attempts and last_error are - # records of *fetches*, and an inbound envelope is not a fetch — writing - # last_error = NULL from a path that issued no request would assert something false. - # next_retry_at is left because it is provably not blocking: AgeOut never skips a row - # whose retry is in the future, so every skipped_budget row carries NULL or an instant - # already past, and clearing it would only destroy history. - # - # @return [Integer] rows reactivated: 1 or 0. - def reactivate_skipped!(github_id:, now:) - where(github_id: github_id, enrichment_status: "skipped_budget") - .update_all(enrichment_status: "pending", skipped_at: nil, updated_at: now) - end end end diff --git a/app/models/push_event.rb b/app/models/push_event.rb index f8445f3..b8bcee3 100644 --- a/app/models/push_event.rb +++ b/app/models/push_event.rb @@ -29,8 +29,7 @@ class PushEvent < ApplicationRecord # The duplicate-event insert gate the accepted-row guarantee rests on (§7, §8). Returns # the new row's id, or nil when the event was already persisted — and the caller uses - # exactly that distinction to decide whether entity activity may be updated, so a - # re-polled window cannot resurrect skipped entities. + # exactly that distinction to decide whether entity activity may be updated. # # The explicit validate! is load-bearing: insert bypasses Active Record # validations, so without it SHA_FORMAT would never run on the real write path. diff --git a/app/services/github/allowances.rb b/app/services/github/allowances.rb index 1a04186..ea4118e 100644 --- a/app/services/github/allowances.rb +++ b/app/services/github/allowances.rb @@ -88,9 +88,9 @@ def feasible? # What to actually store when the observed limit makes the configuration # infeasible — an IP co-tenant scenario, a proxy, or an unexpected tier. # - # Polling wins the clamp: §10 ranks polling first, and enrichment reaching zero is - # an already-modelled, documented outcome (skipped_budget), whereas polling - # stopping is a Story 1 failure. Runtime degrades; only boot refuses. + # Polling wins the clamp: §10 ranks polling first, and enrichment reaching zero + # defers the durable backlog until a later window, whereas polling stopping is a + # Story 1 failure. Runtime degrades; only boot refuses. # # **The floor of one is what makes "degrades" true rather than "stops".** An observed # limit at or below the reserve leaves nothing spendable, and the plain minimum gave diff --git a/app/services/github/configuration.rb b/app/services/github/configuration.rb index c36a364..e654450 100644 --- a/app/services/github/configuration.rb +++ b/app/services/github/configuration.rb @@ -36,7 +36,6 @@ class Configuration "MAX_REDIRECTS" => "2", "SOURCE_LOCK_WAIT_SECONDS" => "30", "ACTOR_ENRICHMENT_SHARE" => "0.50", - "ENRICHMENT_ELIGIBILITY_WINDOW_SECONDS" => "3600", "ACTOR_REFRESH_TTL_SECONDS" => "86400", "REPOSITORY_REFRESH_TTL_SECONDS" => "86400", "ENRICHMENT_COVERAGE_WINDOW_SECONDS" => "86400" @@ -46,13 +45,11 @@ class Configuration # divides, and a zero page count or source count silently derives a poll allowance # of nothing. # - # The three enrichment timings join the group for the same reason. A zero - # eligibility window puts every candidate past its window the instant it is created, - # so the sweep skips the entire backlog and enrichment can never run; a zero refresh - # TTL makes every enriched entity instantly stale, turning off the freshness cache - # §13 lists as a PR 7 capability. "Never refresh" is a large number, not zero. + # The two enrichment refresh timings join the group for the same reason. A zero + # refresh TTL makes every enriched entity instantly stale, turning off the freshness + # cache §13 lists as a PR 7 capability. "Never refresh" is a large number, not zero. # - # The coverage window is the fourth, and it fails the same way from the other end. It + # The coverage window is reporting-only, and it fails the same way from the other end. It # is the sole denominator of §11's three percentages, so a zero window puts every # denominator at zero and Github::Enrichment::Coverage reports null for all three, # permanently — §11's headline metric disabled by a number rather than by a decision. @@ -66,7 +63,6 @@ class Configuration http_open_timeout_seconds: "HTTP_OPEN_TIMEOUT_SECONDS", http_read_timeout_seconds: "HTTP_READ_TIMEOUT_SECONDS", source_lock_wait_seconds: "SOURCE_LOCK_WAIT_SECONDS", - enrichment_eligibility_window_seconds: "ENRICHMENT_ELIGIBILITY_WINDOW_SECONDS", actor_refresh_ttl_seconds: "ACTOR_REFRESH_TTL_SECONDS", repository_refresh_ttl_seconds: "REPOSITORY_REFRESH_TTL_SECONDS", enrichment_coverage_window_seconds: "ENRICHMENT_COVERAGE_WINDOW_SECONDS" @@ -93,8 +89,7 @@ class Configuration :enabled_live_source_count, :rate_limit_reserve, :http_open_timeout_seconds, :http_read_timeout_seconds, :max_http_retries, :max_redirects, :source_lock_wait_seconds, - :actor_enrichment_share, :enrichment_eligibility_window_seconds, - :actor_refresh_ttl_seconds, :repository_refresh_ttl_seconds, + :actor_enrichment_share, :actor_refresh_ttl_seconds, :repository_refresh_ttl_seconds, :enrichment_coverage_window_seconds def initialize(env = ENV) diff --git a/app/services/github/enrichment/age_out.rb b/app/services/github/enrichment/age_out.rb deleted file mode 100644 index b02e57d..0000000 --- a/app/services/github/enrichment/age_out.rb +++ /dev/null @@ -1,93 +0,0 @@ -module Github - module Enrichment - # B8: "Bound the enrichment backlog via the eligibility window and skipped_budget - # state (no unbounded growth)." §10: "candidates that age beyond the eligibility - # window transition to skipped_budget." - # - # This is the **only** writer of skipped_budget, which is what lets - # Enrichable#touch_activity!'s reactivation CASE be one line: every skipped row came - # through a WHERE of CANDIDATE_STATUSES, and no row in those two statuses has ever - # completed, so "skipped_budget implies missing enrichment" is a derived invariant - # rather than an assumption. - # - # Budget exhaustion does not cause a skip. §12's sequence is "exhaustion → deferred → - # skipped_budget → reactivation only via a genuinely new event": exhaustion produces a - # deferral that writes no entity state at all, and a row becomes skipped only once its - # own activity has aged past the window. That is why Github::EnrichmentRunner runs - # this *before* it asks whether it may spend — skipping has to keep happening while - # the budget is exhausted, which is precisely when boundedness matters. - # - # Holds no executor and no transport. - class AgeOut - # An unbounded UPDATE over a table taking on §10's ~2,000 candidates an hour would - # eventually hold hundreds of thousands of row locks in one statement. Bounded is - # honest, and the ORDER BY makes the residue always the *least* overdue, so progress - # is monotone and no particular row can be passed over forever by an unlucky plan. - BATCH_SIZE = 1_000 - - def initialize(configuration: Github.configuration, - selector: CandidateSelector.new(configuration: configuration)) - @configuration = configuration - @selector = selector - end - - attr_reader :configuration, :selector - - # @return [Hash{Symbol => Integer}] rows skipped per entity type - def call(now:, entity_types: EntityType.all) - entity_types.index_with { |entity_type| sweep(entity_type, now: now) } - end - - private - - def sweep(entity_type, now:) - skipped = ActiveRecord::Base.connection.exec_update( - sweep_sql(entity_type, now: now), "Github::Enrichment::AgeOut Sweep" - ) - log(entity_type, skipped: skipped, now: now) - skipped - end - - # enrichment_attempts, last_error and next_retry_at are absent from the SET list on - # purpose. A skip is not an attempt and knows nothing about failures — and leaving - # next_retry_at is what makes reactivation's "immediately due" property provable: - # the WHERE requires it to be NULL or already past, so no skipped_budget row can - # carry a future instant, and touch_activity! needs no clearing clause. - # - # updated_at *is* bumped, unlike Github::Enrichment::Claim's lease, under the same - # rule stated there: this changes the entity's observable state. - # - # The retry-due clause inside the subquery is what stops the sweep skipping an - # entity another worker is currently enriching — a leased row carries - # claim_time + lease > now — and FOR UPDATE SKIP LOCKED stops the sweep blocking - # behind that worker's own statement. - def sweep_sql(entity_type, now:) - candidates = selector.expired_scope(entity_type, now: now) - .select(:id) - .order(Arel.sql("COALESCE(last_seen_at, created_at) ASC")) - .limit(BATCH_SIZE) - .lock("FOR UPDATE SKIP LOCKED").to_sql - - ActiveRecord::Base.sanitize_sql_array([ <<~SQL.squish, now, now ]) - UPDATE #{entity_type.table_name} - SET enrichment_status = 'skipped_budget', - skipped_at = ?, - updated_at = GREATEST(updated_at, ?) - WHERE id IN (#{candidates}) - SQL - end - - # §11 puts "enrichment … skipped" at INFO, but one line per row would emit thousands - # in a single sweep. One summary per class, and nothing at all when the count is - # zero — the argument Github::BudgetLedger#log_class_exhausted already makes: a line - # that recurs on every quiet cycle buries the stream §11 deliberately sizes. - def log(entity_type, skipped:, now:) - return if skipped.zero? - - Rails.logger.info(event: "enrichment.aged_out", entity_type: entity_type.key, - skipped_count: skipped, batch_size: BATCH_SIZE, - eligible_since: selector.eligibility_floor(now).utc.iso8601) - end - end - end -end diff --git a/app/services/github/enrichment/backlog_metrics.rb b/app/services/github/enrichment/backlog_metrics.rb new file mode 100644 index 0000000..332a692 --- /dev/null +++ b/app/services/github/enrichment/backlog_metrics.rb @@ -0,0 +1,77 @@ +module Github + module Enrichment + # Read-only measurements of the durable, eventually processed enrichment backlog. + # + # Solid Queue contains only bounded wake-up hints. The rows in github_actors and + # github_repositories are the source of truth, so queue depth would under-report work by + # design. This projection counts the same candidate statuses as + # Enrichable.enrichment_candidates, including work deferred by retry backoff, and reports + # how long the oldest row has waited. + # + # No selector is used here: selector scopes answer "claimable now" and may exclude a + # deferred row. Backlog observability answers the different question "what work must the + # service eventually finish?" and therefore must include every backlog entity. + class BacklogMetrics < Data.define(:actor, :repository) + Entry = Data.define(:status_counts, :backlog_count, :oldest_pending_at, + :oldest_pending_age_seconds) + + STATUSES = Enrichable::ENRICHMENT_STATUSES + + def self.capture(now: Time.current) + new(actor: entry_for(GithubActor, now: now), + repository: entry_for(GithubRepository, now: now)) + end + + def self.entry_for(model, now:) + # CandidateSelector's FIFO order is created_at, the immutable instant the entity + # entered this backlog. Reporting the same clock keeps "oldest" aligned with the + # row the worker will actually choose next. + # + # All status counts, the candidate count, and its oldest row come from one aggregate + # statement. A worker may commit between statements, so separate count/minimum reads + # could otherwise publish a combination that never existed in the database. + values = Array(model.unscoped.pick(*aggregate_columns(model))) + status_counts = STATUSES.zip(values.shift(STATUSES.length)).to_h + .transform_values(&:to_i) + .reject { |_status, count| count.zero? } + backlog_count = values.shift.to_i + oldest = values.shift + + Entry.new(status_counts: status_counts, + backlog_count: backlog_count, + oldest_pending_at: oldest, + oldest_pending_age_seconds: age_seconds(oldest, now: now)) + end + private_class_method :entry_for + + def self.aggregate_columns(model) + connection = model.connection + status_column = connection.quote_column_name(:enrichment_status) + created_at_column = connection.quote_column_name(:created_at) + candidate_values = Enrichable::CANDIDATE_STATUSES.map do |status| + connection.quote(status) + end.join(", ") + candidate_filter = "#{status_column} IN (#{candidate_values})" + + STATUSES.map do |status| + quoted_status = connection.quote(status) + Arel.sql("COUNT(*) FILTER (WHERE #{status_column} = #{quoted_status})") + end + [ + Arel.sql("COUNT(*) FILTER (WHERE #{candidate_filter})"), + Arel.sql("MIN(#{created_at_column}) FILTER (WHERE #{candidate_filter})") + ] + end + private_class_method :aggregate_columns + + # A database timestamp a fraction ahead of the application clock can occur around a + # snapshot boundary. A negative backlog age is never useful, so clamp that harmless + # skew to zero while retaining whole-second precision for an operator-facing metric. + def self.age_seconds(timestamp, now:) + return nil if timestamp.nil? + + [ (now - timestamp).floor, 0 ].max + end + private_class_method :age_seconds + end + end +end diff --git a/app/services/github/enrichment/backoff.rb b/app/services/github/enrichment/backoff.rb index 5a5cd51..461c986 100644 --- a/app/services/github/enrichment/backoff.rb +++ b/app/services/github/enrichment/backoff.rb @@ -11,12 +11,10 @@ module Enrichment # observed on the live feed. Entities have no X-Poll-Interval. Sixty is chosen # here because it matches Github::RateLimitPolicy::MIN_BLOCK_SECONDS and sits well # below the ~90-second mean interval between enrichment requests at the default - # 40/hour allowance: long enough for a blip to clear, short enough that the - # backoff is never the reason an entity ages out. - # * MAX_SECONDS is one rate-limit window, which does transfer, and gains a second - # enrichment-specific justification: it equals the pinned - # ENRICHMENT_ELIGIBILITY_WINDOW_SECONDS, so a backoff can never outlast the window - # that would age the row into skipped_budget. The cap is never what strands a row. + # 40/hour allowance: long enough for a blip to clear without monopolising the + # durable backlog. + # * MAX_SECONDS is one rate-limit window. A repeatedly failing entity yields to + # other due backlog entries for at most an hour before it becomes eligible again. # * The counting unit differs. PollBackoff counts polls of one source; this counts # fetch attempts against one entity, since the last success. # diff --git a/app/services/github/enrichment/candidate_selector.rb b/app/services/github/enrichment/candidate_selector.rb index 31097b4..49bfa07 100644 --- a/app/services/github/enrichment/candidate_selector.rb +++ b/app/services/github/enrichment/candidate_selector.rb @@ -6,14 +6,11 @@ module Enrichment # Github::Ingestion::PageWriter uses — so a GitHub request cannot be issued from a # selection query, and every method here is a read. # - # **Two pools, not one status.** §7 line 572 says an entity "returns to pending when - # missing, explicitly stale, or reactivated after a budget skip", and §10 line 816 - # says never-enriched pending candidates "always precede TTL-stale refreshes". Those - # are only compatible if staleness is a *derived predicate over complete rows* rather - # than a stored transition: collapsing stale rows into pending would erase the very - # distinction the priority rule is stated over. So the only stored path back to - # pending is reactivation (Enrichable#touch_activity!), and a stale-but-enriched - # entity keeps reading `complete` — which is true, the payload is there. + # **Two pools, not one status.** Never-enriched candidates remain durable pending work, + # while staleness is a derived predicate over complete rows. Collapsing stale rows + # into pending would erase the distinction that keeps first-time enrichment ahead of + # refresh traffic. A stale-but-enriched entity therefore keeps reading `complete` — + # which is true, because its payload is still present. # # Two further consequences of that reading, both intended: a background writer that # mutated rows purely because time passed is the shape this codebase refuses @@ -23,21 +20,14 @@ module Enrichment class CandidateSelector POOLS = %i[ pending refresh ].freeze - # §10: "Among pending candidates the service enriches newest-first (last_seen_at)." - # - # The id tie-break is mandatory rather than cosmetic. PageWriter stamps one - # received_at for a whole page, so every entity on one page shares an identical - # last_seen_at, and without a second key the order is plan-dependent and the specs - # are non-deterministic. - # - # NULLS LAST puts a stub that has never been referenced by a distinct persisted - # event behind every candidate that has been. See #eligibility_floor for why the - # ordering key is strictly last_seen_at while the *bound* coalesces. - PENDING_ORDER = "last_seen_at DESC NULLS LAST, id DESC".freeze - - # Oldest-fetched first, which is the rule that terminates: a monotone queue cannot - # starve a complete row behind a hotter neighbour. §10's newest-first is scoped by - # its own words to "Among pending candidates", so the refresh pool needs its own. + # Durable FIFO. created_at is the instant the entity entered the backlog; unlike + # last_seen_at it is immutable when later events reference the same entity. The id + # tie-break makes candidates created in the same timestamp deterministic. + PENDING_ORDER = "created_at ASC, id ASC".freeze + + # Oldest-fetched first, which is the rule that terminates: a monotone refresh queue + # cannot starve a complete row behind a hotter neighbour. The first-time backlog + # has its own FIFO key, so the refresh pool needs its own ordering rule. REFRESH_ORDER = "fetched_at ASC, id ASC".freeze # When a complete row next becomes refreshable. See #earliest_refresh_at. @@ -60,26 +50,30 @@ def scope(entity_type, pool:, now:) end end - # The question §10's borrowing rule asks: "the other class has no CURRENTLY - # ELIGIBLE candidate (not merely no rows)." - # - # Over the pending pool only. §10's prioritization ladder ranks refreshing stale - # enrichment (3) below enriching never-seen entities (2) *globally*. If this counted - # refresh candidates, one class would decline to borrow the other's idle capacity - # because that other class had a stale refresh waiting — letting a refresh outrank a - # never-enriched candidate and inverting the ladder. The accepted consequence is - # that a class may spend its own guaranteed share on a refresh while the other class - # still has a pending backlog, which is exactly what a guarantee means. + # Whether this class has first-time backlog work that can be claimed now. Fairness + # uses this narrower predicate for borrowing: a class in backoff need not leave the + # other class's reserved attempts idle. Refresh suppression uses #pending_backlog? + # instead, because it asks whether first-time work exists at all. def pending_available?(entity_type, now:) pending_scope(entity_type, now: now).exists? end + # Backlog presence is deliberately broader than current claimability. A row in + # backoff or carrying an in-flight lease still represents first-time enrichment + # work, so refresh traffic must not consume the quota reserved for that backlog. + def pending_backlog?(entity_type) + entity_type.model.where(enrichment_status: Enrichable::CANDIDATE_STATUSES).exists? + end + # Whether either pool could hand out work for this class right now. Asked before any # "when next?" question, because the two are different questions and deriving one # from the other is what makes a report say "due now" while the command it sits next # to says there is nothing to enrich. def claimable?(entity_type, now:) - pending_available?(entity_type, now: now) || refresh_available?(entity_type, now: now) + return true if pending_available?(entity_type, now: now) + return false if EntityType.all.any? { |type| pending_backlog?(type) } + + refresh_available?(entity_type, now: now) end def refresh_available?(entity_type, now:) @@ -90,21 +84,22 @@ def refresh_available?(entity_type, now:) # caller that has already established neither has any now. # @return [Time, nil] nil when nothing will ever become claimable without new activity def earliest_claimable_at(entity_type, now:) - [ earliest_pending_at(entity_type, now: now), - earliest_refresh_at(entity_type, now: now) ].compact.min + if EntityType.all.any? { |type| pending_backlog?(type) } + earliest_pending_at(entity_type, now: now) + else + earliest_refresh_at(entity_type, now: now) + end end - # A pending candidate held back only by its own backoff or a secondary-limit - # deferral. One aged past the eligibility window is excluded: it will be swept into - # skipped_budget rather than enriched, so naming its retry instant would promise an - # enrichment that is never going to happen. + # A pending candidate held back only by its own backoff, lease, or secondary-limit + # deferral. Pending work has no age cutoff: this instant is a real promise that the + # row returns to the actionable backlog. # @return [Time, nil] def earliest_pending_at(entity_type, now:) entity_type.model .where(enrichment_status: Enrichable::CANDIDATE_STATUSES) .where.not(next_retry_at: nil) .where(next_retry_at: now..) - .where(eligible_since_clause, floor: eligibility_floor(now)) .minimum(:next_retry_at) end @@ -127,45 +122,14 @@ def earliest_refresh_at(entity_type, now:) entity_type.model.complete.where.not(fetched_at: nil).minimum(Arel.sql(expression)) end - # Candidates whose activity has aged past §10's eligibility window and are therefore - # Github::Enrichment::AgeOut's business. Shares every clause with #pending_scope - # except the direction of the window comparison, which is what makes the two - # exhaustive: a due candidate is in exactly one of them. - def expired_scope(entity_type, now:) - due(entity_type.model.where(enrichment_status: Enrichable::CANDIDATE_STATUSES), now) - .where("COALESCE(last_seen_at, created_at) <= :floor", floor: eligibility_floor(now)) - end - - def eligibility_floor(now) - now - configuration.enrichment_eligibility_window_seconds - end - def stale_before(entity_type, now) now - entity_type.refresh_ttl_seconds(configuration) end private - # §10's eligibility window, over COALESCE(last_seen_at, created_at) while - # PENDING_ORDER sorts on last_seen_at alone. The asymmetry is the whole trick and it - # is not arbitrary: - # - # * In the *bound*, created_at is safe — it can only ever shorten a row's life, - # never promote it — and it is what makes the predicate total. A stub can be - # created with a NULL last_seen_at (PageWriter upserts the stub, insert_if_new - # returns nil on a duplicate, and the transaction still commits), and - # `NULL > floor` is NULL, so without the coalesce such a row would be neither - # eligible nor ageable and would sit pending forever — violating B8. - # * In the *order*, created_at is unsafe. It means "we saw an envelope", which a - # duplicate replay also produces, while §10 pins the ordering key by name to - # last_seen_at — the only column that means proven distinct activity. Sorting on - # the coalesce would let a replay-created stub outrank a genuinely hot entity. - # - # created_at is NOT NULL on both tables and appears in no IDENTITY_MERGE SET list, - # so it is immutable after the first observation. def pending_scope(entity_type, now:) due(entity_type.model.where(enrichment_status: Enrichable::CANDIDATE_STATUSES), now) - .where(eligible_since_clause, floor: eligibility_floor(now)) .order(Arel.sql(PENDING_ORDER)) end @@ -178,15 +142,10 @@ def refresh_scope(entity_type, now:) # One predicate, spelled once. next_retry_at means "may not be attempted before T" # everywhere in this state machine — a failure backoff, a secondary-limit deferral, # and Github::Enrichment::Claim's in-flight lease all write it — so this single - # clause excludes all three from both pools and from the age-out sweep, and the four - # queries cannot drift apart. + # clause excludes all three from both claimable pools. def due(relation, now) relation.where("next_retry_at IS NULL OR next_retry_at <= :now", now: now) end - - def eligible_since_clause - "COALESCE(last_seen_at, created_at) > :floor" - end end end end diff --git a/app/services/github/enrichment/claim.rb b/app/services/github/enrichment/claim.rb index 752cbbe..3dff449 100644 --- a/app/services/github/enrichment/claim.rb +++ b/app/services/github/enrichment/claim.rb @@ -10,12 +10,10 @@ module Enrichment # simply expires. # # Leasing on next_retry_at rather than on a new column is the decision this class - # rests on, and its payoff is elsewhere: Github::Enrichment::CandidateSelector's two - # pools and Github::Enrichment::AgeOut's sweep all spell the same + # rests on, and its payoff is elsewhere: both candidate pools spell the same # "next_retry_at IS NULL OR next_retry_at <= now" clause, so one predicate excludes - # in-flight rows from all four queries at once. A separate leased_until column would - # need every one of them to carry a second condition, and the first that forgot would - # either skip an entity mid-flight or hand it to a second worker. + # in-flight rows consistently. A separate leased_until column would need every query + # to carry a second condition, and the first that forgot could hand work to two workers. # # Holds no executor and no transport, so a GitHub request cannot be issued from # inside a claim. @@ -115,7 +113,7 @@ def lease_seconds private # The candidate CTE is Github::Enrichment::CandidateSelector's own scope, so the - # eligibility window, the TTL, and §10's two ordering rules are defined in exactly + # FIFO order, the TTL, and the two pool rules are defined in exactly # one place and this statement cannot drift from the pool it claims out of. # # Three details carry the correctness: diff --git a/app/services/github/enrichment/coverage.rb b/app/services/github/enrichment/coverage.rb index 7a29955..851b9a7 100644 --- a/app/services/github/enrichment/coverage.rb +++ b/app/services/github/enrichment/coverage.rb @@ -14,9 +14,8 @@ module Enrichment # ## The window is measured on created_at, not occurred_at # # Coverage grades *this application's* enrichment pipeline, and that pipeline runs on - # this application's clock throughout: eligibility is - # COALESCE(last_seen_at, created_at) > floor (Github::Enrichment::CandidateSelector), - # staleness is fetched_at + TTL, and the budget refills on a wall-clock rate-limit + # this application's clock throughout: FIFO backlog insertion uses created_at, + # staleness uses fetched_at + TTL, and the budget refills on a wall-clock rate-limit # window. A denominator defined by GitHub's event clock would mix two clocks inside one # ratio. §11's own wording — "distinct **persisted** push events in the coverage # window" — reads the same way, and after downtime created_at is the basis that answers @@ -54,14 +53,12 @@ class Coverage < Data.define(:window_seconds, :window_start, :event_count, ACTOR_COMPLETE = "github_actors.enrichment_status = '#{COMPLETE}'".freeze REPOSITORY_COMPLETE = "github_repositories.enrichment_status = '#{COMPLETE}'".freeze - # `>` rather than `>=`, matching CandidateSelector's eligibility floor — one window - # convention in this codebase, not two. The table qualifier is mandatory rather than - # tidy: all three joined tables carry created_at, so an unqualified column is - # ambiguous and PostgreSQL rejects the statement. + # `>` gives this reporting window an unambiguous open lower bound. The table + # qualifier is mandatory rather than tidy: all three joined tables carry created_at, + # so an unqualified column is ambiguous and PostgreSQL rejects the statement. # # No upper bound. A future-dated row is clock skew, and excluding it would remove the - # same row from the numerator and the denominator together — the eligibility window - # has none either, for the same reason. + # same row from the numerator and the denominator together. WINDOW_CLAUSE = "push_events.created_at > :floor".freeze # §11's three formulas, as six counts taken in one pass. @@ -85,9 +82,8 @@ class Coverage < Data.define(:window_seconds, :window_start, :event_count, "COUNT(*) FILTER (WHERE #{ACTOR_COMPLETE} AND #{REPOSITORY_COMPLETE})" }.freeze - # Two decimals. §10 sizes the honest steady state at a low single-digit percentage, so - # the second decimal is the one that moves; a fourth would read as precision the - # sampling rate does not have. + # Two decimals are enough for an operational completion ratio; more would imply + # precision that the rolling window does not have. PRECISION = 2 # The basis, published so a consumer never has to guess which clock bounds the window. diff --git a/app/services/github/enrichment/dispatch.rb b/app/services/github/enrichment/dispatch.rb index 3ac0026..8dfca5a 100644 --- a/app/services/github/enrichment/dispatch.rb +++ b/app/services/github/enrichment/dispatch.rb @@ -17,7 +17,7 @@ module Enrichment # how fast jobs can be created. One live page carries ~90 distinct actors and ~90 # distinct repositories; enqueuing per created event would put ~2,400 argument-identical # cycles an hour on a queue that can spend 40 requests, and every surplus one would run - # the age-out sweep and the fairness reads to be told no. The reconciler's 60-second + # the fairness reads only to be told no. The reconciler's 60-second # cadence is what refills the pipeline instead — it is faster than the budget can be # spent, and it self-limits when the budget is gone. # @@ -44,8 +44,12 @@ def initialize(configuration: Github.configuration, clock: -> { Time.current }, # @return [Hash] the payload it logged, so a caller can assert on it. def call(reason:) now = @clock.call - schedule = class_schedule(now: now) - blocked = !schedule.due?(now: now) + budget = GithubApiBudget.find_by(id: GithubApiBudget::SINGLETON_ID) + schedule = class_schedule(budget, now: now) + window_block = window_blocked_by(budget, now: now) + schedule_blocked = !schedule.due?(now: now) + blocked = schedule_blocked || window_block.present? + blocked_by = schedule_blocked ? schedule.binding_component : window_block payload = EntityType.all.each_with_object({}) do |entity_type, counts| enqueue = !blocked && @selector.claimable?(entity_type, now: now) @@ -54,7 +58,8 @@ def call(reason:) counts[:"#{entity_type.key}_enqueued"] = enqueue ? 1 : 0 end - log(payload.merge(reason: reason, blocked_by: (schedule.binding_component if blocked)).compact, now: now) + log(payload.merge(reason: reason, blocked_by: (blocked_by if blocked)).compact, + budget: budget, now: now) end private @@ -70,12 +75,11 @@ def call(reason:) # relieved by borrowing, not a deferral, so refusing to enqueue on it would withhold # work the ledger would have granted. # - # find_by, never bootstrap!: a read path must not create the ledger row. A clean - # checkout has no row, every component is nil, and the schedule is due — which is - # right, because the first poll is what initializes the window. - def class_schedule(now:) - budget = GithubApiBudget.find_by(id: GithubApiBudget::SINGLETON_ID) - + # The caller obtains the row with find_by, never bootstrap!: a read path must not + # create it. A missing or existing-uninitialized ledger blocks dispatch because the + # request gate would deny enrichment until the first poll supplies authoritative + # rate-limit headers. + def class_schedule(budget, now:) EnrichmentSchedule.new( next_retry_at: nil, global_blocked_until: budget&.global_blocked_until, @@ -83,6 +87,11 @@ def class_schedule(now:) ) end + def window_blocked_by(budget, now:) + return :window_uninitialized if budget.nil? || budget.window_initialized_at.nil? + :window_elapsed if budget.reset_at.present? && now >= budget.reset_at + end + # §11 lists "reconciliation summaries" among the INFO events, and this is that line — # but only when it scheduled something. A tick that enqueued nothing is the ordinary # steady state of an exhausted window, and at 60-second cadence it would emit a line a @@ -91,10 +100,11 @@ def class_schedule(now:) # # The summary is PR 7's, unchanged: per-status counts per class, per-class share usage, # the window state, and when enrichment is next due. - def log(payload, now:) + def log(payload, budget:, now:) enqueued = payload.fetch(:actor_enqueued) + payload.fetch(:repository_enqueued) entry = { event: "enrichment.dispatched", **payload, - **Summary.capture(now: now, configuration: @configuration, selector: @selector).to_log } + **Summary.capture(now: now, configuration: @configuration, + selector: @selector, budget: budget).to_log } enqueued.positive? ? Rails.logger.info(entry) : Rails.logger.debug(entry) payload diff --git a/app/services/github/enrichment/entity_state.rb b/app/services/github/enrichment/entity_state.rb index 35bab58..f51c4dd 100644 --- a/app/services/github/enrichment/entity_state.rb +++ b/app/services/github/enrichment/entity_state.rb @@ -126,12 +126,11 @@ def from_document(lease, fetched, document, now:) # next_retry_at is cleared because the next event for this row is a *refresh*, gated # by fetched_at + the TTL rather than by a retry instant; leaving the lease in place # would delay it by ten minutes and conflate two meanings on one column. last_error - # and skipped_at are cleared for PollState#success's reason — a stale error or skip - # instant on a successful row is a permanent lie. + # is cleared because a stale error on a successful row is a permanent lie. def complete(lease, document, now:) write(lease, { enrichment_status: "complete", enrichment_attempts: 0, next_retry_at: nil, - last_error: nil, skipped_at: nil, fetched_at: now, updated_at: now + last_error: nil, fetched_at: now, updated_at: now }.merge(document.attributes), outcome: "enriched") end diff --git a/app/services/github/enrichment/fairness.rb b/app/services/github/enrichment/fairness.rb index eb5d0bd..8b4a458 100644 --- a/app/services/github/enrichment/fairness.rb +++ b/app/services/github/enrichment/fairness.rb @@ -13,9 +13,9 @@ module Enrichment # 1. a never-enriched pending candidate in a class still inside its guarantee, with # actor before repository only as a tie-break; # 2. the same, borrowing, when the other class has no currently eligible candidate; - # 3. a TTL-stale refresh, "only when no never-enriched pending candidate is - # eligible" — and §10 scopes that condition globally rather than per class, so a - # refresh waits behind the *other* class's pending backlog too. + # 3. a TTL-stale refresh only when no never-enriched backlog row exists. Backoff and + # in-flight leases delay first-time work; they never release its reserved quota + # to refresh traffic. # # The borrow fact is computed here and asserted to the ledger, and it is stale by # construction: a poll can persist a new candidate between this query and the debit. @@ -73,7 +73,7 @@ def choose(entity_class: nil, now: Time.current) eligible = EntityType.all.index_with { |type| selector.pending_available?(type, now: now) } pending_choice(budget, requested, eligible) || - refresh_choice(budget, requested, eligible, now: now) || + refresh_choice(budget, requested, now: now) || Choice.none(reason: "no_candidate") end @@ -112,10 +112,9 @@ def pending_choice(budget, requested, eligible) Choice.new(entity_type: borrower, pool: :pending, borrow: true, reason: "borrowed_pending") end - # §10: "a refresh spends budget only when no pending candidate is currently - # eligible". Read over every class, not only the requested one, so --class actor - # cannot promote an actor refresh above a repository still waiting to be enriched - # for the first time. + # Refresh is strictly subordinate to the durable first-time backlog. This checks + # rows rather than only currently-due candidates, so a backoff or lease cannot let + # refresh traffic consume quota reserved for eventual enrichment. # # Then §10:898's second constraint on the same line — refreshes run "within each # class's share" — which makes this the same two-step #pending_choice performs, over @@ -125,14 +124,8 @@ def pending_choice(budget, requested, eligible) # repository's untouched guarantee and eligible stale rows were never selected, which # is the class starvation the split exists to prevent, reproduced one pool down. # - # The eligibility map is rebuilt over the *refresh* pool rather than reusing - # `eligible`. CandidateSelector#pending_available? is deliberately pending-only — - # counting refreshes there would let a refresh outrank a never-enriched candidate and - # invert §10's ladder — but that hazard cannot arise here: this method is only - # reached when no class has a pending candidate at all, so there is nothing left for - # a refresh to outrank. - def refresh_choice(budget, requested, eligible, now:) - return nil if eligible.values.any? + def refresh_choice(budget, requested, now:) + return nil if EntityType.all.any? { |type| selector.pending_backlog?(type) } refreshable = EntityType.all.index_with { |type| selector.refresh_available?(type, now: now) } available = requested.select { |type| refreshable.fetch(type) } @@ -158,12 +151,10 @@ def room_within_guarantee?(budget, entity_type) share_used(budget, entity_type) < guarantee end - # §10's borrowing condition, verbatim: the other class has no CURRENTLY ELIGIBLE - # candidate, not merely no rows. - # - # Generic over whichever eligibility map it is handed, because the condition is the - # same question asked of whichever pool is being allocated: #pending_choice passes - # the pending map, #refresh_choice the refresh one. + # Borrow only when the other class has nothing claimable in the same pool. This is + # intentionally about due work rather than all rows: a class in backoff must not + # strand capacity the other class can spend. Refresh reaches this helper only after + # the global first-time backlog has been proven empty. def other_classes_quiet?(entity_type, eligible) eligible.except(entity_type).values.none? end diff --git a/app/services/github/enrichment/summary.rb b/app/services/github/enrichment/summary.rb index 1969fd1..30d1831 100644 --- a/app/services/github/enrichment/summary.rb +++ b/app/services/github/enrichment/summary.rb @@ -5,25 +5,33 @@ module Enrichment # # A second object rather than more members on that one, because the two answer # different questions and §13 splits them across two PRs: StateSummary is §9's - # proof-of-state for the *polling* command, and §11 assigns the coverage percentages to - # PR 10's /status. What lands here is the part PR 7's own outcome would otherwise be - # invisible without — the per-status counts and the per-class share usage §10 defines. + # proof-of-state for the *polling* command, and /status owns the coverage percentages. + # What lands here is the part an enrichment outcome would otherwise leave invisible: + # the durable per-class backlog, its oldest wait, and reserved allowance usage. # The percentages themselves are Github::Enrichment::Coverage, which needs # ENRICHMENT_COVERAGE_WINDOW_SECONDS and a join against push_events; both arrived with # PR 10, and /status renders the two objects side by side. # # **It never initiates a GitHub request**, structurally and for StateSummary's reason: - # no executor, no transport, no ledger — three read statements over Active Record + # no executor, no transport, no ledger — read statements over Active Record # models. Reading the ledger row with find_by rather than through # Github::BudgetLedger matters for the same reason PollSchedule gives: a read path must # not create the row. - class Summary < Data.define(:actor_counts, :repository_counts, :actor_share_used, + class Summary < Data.define(:actor_counts, :repository_counts, + :actor_backlog_count, + :repository_backlog_count, + :actor_oldest_pending_at, :repository_oldest_pending_at, + :actor_oldest_pending_age_seconds, + :repository_oldest_pending_age_seconds, + :actor_share_used, :repository_share_used, :actor_guarantee, :repository_guarantee, :enrichment_used, - :enrichment_allowance, :window_status, :claimable_now, + :enrichment_allowance, :window_status, :window_ready, + :work_waiting, :claimable_now, :next_enrichment_at) NO_LEDGER = "not yet initialized".freeze DUE_NOW = "due now".freeze + WAITING_FOR_WINDOW = "waiting for authoritative poll".freeze # next_enrichment_at is nil in two states that are not the same fact: something is # claimable *right now*, and nothing will ever become claimable without new ingest @@ -34,12 +42,6 @@ class Summary < Data.define(:actor_counts, :repository_counts, :actor_share_used # the other side. NOTHING_WAITING = "nothing waiting".freeze - # The three statuses an operator acts on. permanent_failure and retryable_failure are - # rolled into the pending/complete/skipped triple's remainder rather than printed - # separately: §11's line is "pending/skipped counts", and a five-column row would - # bury the two numbers that describe the sampling rate. - REPORTED_STATUSES = %w[ pending complete skipped_budget ].freeze - class << self # @param budget [GithubApiBudget, nil] the ledger row, when the caller already holds # it. Github::Status::Snapshot passes one so /status reads the singleton exactly @@ -50,30 +52,42 @@ class << self # create the row. def capture(now: Time.current, configuration: Github.configuration, selector: CandidateSelector.new(configuration: configuration), - budget: GithubApiBudget.find_by(id: GithubApiBudget::SINGLETON_ID)) + budget: GithubApiBudget.find_by(id: GithubApiBudget::SINGLETON_ID), + backlog: BacklogMetrics.capture(now: now)) guarantees = guarantees_for(budget, configuration) - claimable = claimable_now?(budget, selector, now: now) + backlog_waiting = backlog.actor.backlog_count.positive? || + backlog.repository.backlog_count.positive? + work_waiting = backlog_waiting || complete_rows?(backlog) + claimable = claimable_now?( + budget, selector, backlog_waiting: backlog_waiting, now: now + ) new( - actor_counts: counts(GithubActor), - repository_counts: counts(GithubRepository), + actor_counts: backlog.actor.status_counts, + repository_counts: backlog.repository.status_counts, + actor_backlog_count: backlog.actor.backlog_count, + repository_backlog_count: backlog.repository.backlog_count, + actor_oldest_pending_at: backlog.actor.oldest_pending_at, + repository_oldest_pending_at: backlog.repository.oldest_pending_at, + actor_oldest_pending_age_seconds: backlog.actor.oldest_pending_age_seconds, + repository_oldest_pending_age_seconds: backlog.repository.oldest_pending_age_seconds, actor_share_used: budget&.actor_share_used, repository_share_used: budget&.repository_share_used, actor_guarantee: guarantees[:actor], repository_guarantee: guarantees[:repository], enrichment_used: budget&.enrichment_used, enrichment_allowance: budget&.enrichment_allowance, window_status: budget&.window_status, + window_ready: !window_unavailable?(budget, now: now), + work_waiting: work_waiting, claimable_now: claimable, - next_enrichment_at: next_enrichment_at(budget, selector, claimable, now: now) + next_enrichment_at: next_enrichment_at( + budget, selector, claimable, backlog_waiting: backlog_waiting, now: now + ) ) end private - def counts(model) - model.group(:enrichment_status).count - end - def guarantees_for(budget, configuration) return { actor: nil, repository: nil } if budget.nil? @@ -92,22 +106,36 @@ def guarantees_for(budget, configuration) # complete row deferred by a failed refresh was invisible for the same reason; and # one candidate due now beside one deferred printed the deferred instant while work # was in fact claimable. - def next_enrichment_at(budget, selector, claimable, now:) + def next_enrichment_at(budget, selector, claimable, backlog_waiting:, now:) blocked = blocked_until(budget, now: now) return blocked if blocked return nil if claimable + return nil if window_unavailable?(budget, now: now) + + # Fairness reserves refresh capacity for the durable first-time backlog. If all + # of that backlog is in backoff, a stale refresh is not the next legal action; + # the earliest pending retry is. + if backlog_waiting + return EntityType.all.filter_map do |type| + selector.earliest_pending_at(type, now: now) + end.min + end EntityType.all.filter_map { |type| selector.earliest_claimable_at(type, now: now) }.min end - # "A request would be issued if the runner ran right now." Both pools, and the - # ledger asked first: a global block or a spent class outranks every per-entity - # instant, and is the one case where the answer exists without reading an entity - # row at all. - def claimable_now?(budget, selector, now:) + # "A request would be issued if the runner ran right now." The ledger is asked + # first, then the same first-time-before-refresh rule as Fairness: a deferred + # backlog row suppresses otherwise-due refresh work. + def claimable_now?(budget, selector, backlog_waiting:, now:) return false if blocked_until(budget, now: now) + return false if window_unavailable?(budget, now: now) - EntityType.all.any? { |type| selector.claimable?(type, now: now) } + if backlog_waiting + EntityType.all.any? { |type| selector.pending_available?(type, now: now) } + else + EntityType.all.any? { |type| selector.refresh_available?(type, now: now) } + end end # nil unless the instant is genuinely still ahead: BudgetLedger derives blocking @@ -119,24 +147,50 @@ def blocked_until(budget, now:) blocked if blocked&.>(now) end + + # A missing row is unavailable too: the first attempted reservation would create + # it, then be denied until a real poll supplies authoritative rate-limit headers. + # An elapsed window is equivalent: reserve! rolls it to uninitialized before it can + # authorize enrichment, so only a poll can make the new window usable. + def window_unavailable?(budget, now:) + budget.nil? || budget.window_initialized_at.nil? || + (budget.reset_at.present? && now >= budget.reset_at) + end + + def complete_rows?(backlog) + [ backlog.actor, backlog.repository ].any? do |entry| + entry.status_counts.fetch("complete", 0).positive? + end + end end def to_s [ - # Kept under Report::LABEL_WIDTH so both values land in the same column as every - # other line in the report — a label that fills the width exactly gets no padding - # at all and its value runs straight into the colon. - Ingestion::Report.line("Actors pending/complete/skipped", status_line(actor_counts)), - Ingestion::Report.line("Repos pending/complete/skipped", status_line(repository_counts)), + Ingestion::Report.line("Actor backlog", Ingestion::Report.count(actor_backlog_count)), + Ingestion::Report.line("Repository backlog", + Ingestion::Report.count(repository_backlog_count)), + Ingestion::Report.line("Oldest actor pending", + oldest_pending(actor_oldest_pending_at, + actor_oldest_pending_age_seconds)), + Ingestion::Report.line("Oldest repository pending", + oldest_pending(repository_oldest_pending_at, + repository_oldest_pending_age_seconds)), Ingestion::Report.line("Actor requests used", share_line(actor_share_used, actor_guarantee)), Ingestion::Report.line("Repository requests used", share_line(repository_share_used, repository_guarantee)), - Ingestion::Report.line("Enrichment requests used", share_line(enrichment_used, enrichment_allowance)), - Ingestion::Report.line("Next enrichment due", next_enrichment) + Ingestion::Report.line("Enrichment backlog budget", + backlog_budget(enrichment_used, enrichment_allowance)), + Ingestion::Report.line("Next enrichment attempt", next_enrichment) ].join("\n") end def to_log { actor_counts: actor_counts, repository_counts: repository_counts, + actor_backlog_count: actor_backlog_count, + repository_backlog_count: repository_backlog_count, + actor_oldest_pending_at: Ingestion::Report.timestamp(actor_oldest_pending_at), + repository_oldest_pending_at: Ingestion::Report.timestamp(repository_oldest_pending_at), + actor_oldest_pending_age_seconds: actor_oldest_pending_age_seconds, + repository_oldest_pending_age_seconds: repository_oldest_pending_age_seconds, actor_share_used: actor_share_used, repository_share_used: repository_share_used, actor_guarantee: actor_guarantee, repository_guarantee: repository_guarantee, enrichment_used: enrichment_used, enrichment_allowance: enrichment_allowance, @@ -146,19 +200,26 @@ def to_log private - def status_line(counts) - REPORTED_STATUSES.map { |status| Ingestion::Report.count(counts.fetch(status, 0)) }.join(" / ") + def oldest_pending(timestamp, age_seconds) + return NOTHING_WAITING if timestamp.nil? + + "#{Ingestion::Report.timestamp(timestamp)} (#{Ingestion::Report.count(age_seconds)}s old)" end - # "of" rather than a slash, so this line cannot be misread as the status triple above - # it. NO_LEDGER for the same reason StateSummary spells its unknowns out: a fabricated - # zero on a fresh install is exactly the misleading guarantee §16 forbids. + # NO_LEDGER for the same reason StateSummary spells its unknowns out: a fabricated + # zero on a fresh install would claim a quota window had been observed when it had not. def share_line(used, allowance) return NO_LEDGER if used.nil? "#{Ingestion::Report.count(used)} of #{Ingestion::Report.count(allowance)}" end + def backlog_budget(used, allowance) + value = share_line(used, allowance) + + value == NO_LEDGER ? value : "#{value} used" + end + # Three answers, not two. A nil instant means "no deferral applies", which is true # both when a candidate is claimable this second and when the backlog is empty — # and an operator reads those two states completely differently. claimable_now is @@ -166,6 +227,8 @@ def share_line(used, allowance) # very next line of output was "nothing to enrich". def next_enrichment return DUE_NOW if claimable_now + return NOTHING_WAITING unless work_waiting + return WAITING_FOR_WINDOW unless window_ready return NOTHING_WAITING if next_enrichment_at.nil? Ingestion::Report.timestamp(next_enrichment_at) diff --git a/app/services/github/enrichment/tally.rb b/app/services/github/enrichment/tally.rb index 15fdc70..ebefaea 100644 --- a/app/services/github/enrichment/tally.rb +++ b/app/services/github/enrichment/tally.rb @@ -5,8 +5,7 @@ module Enrichment # Immutable, like Github::Ingestion::Tally: #record returns a new value rather than # mutating, so a partially accumulated count can never be observed and a caller cannot # hold a stale reference that later changes underneath it. - class Tally < Data.define(:cycles, :enriched, :failed, :deferred, :idle, :lease_lost, - :aged_out) + class Tally < Data.define(:cycles, :enriched, :failed, :deferred, :idle, :lease_lost) # Github::EnrichmentRunner::Result::STATUSES, as counters. Keyed by the same strings # so a new status cannot be silently uncounted — #record fetches and raises. COUNTERS = { @@ -15,15 +14,14 @@ class Tally < Data.define(:cycles, :enriched, :failed, :deferred, :idle, :lease_ }.freeze def self.empty - new(cycles: 0, enriched: 0, failed: 0, deferred: 0, idle: 0, lease_lost: 0, aged_out: 0) + new(cycles: 0, enriched: 0, failed: 0, deferred: 0, idle: 0, lease_lost: 0) end # @param result [Github::EnrichmentRunner::Result] def record(result) counter = COUNTERS.fetch(result.status) { raise ArgumentError, "unknown status #{result.status.inspect}" } - with(cycles: cycles + 1, aged_out: aged_out + result.aged_out, - counter => public_send(counter) + 1) + with(cycles: cycles + 1, counter => public_send(counter) + 1) end def to_log = to_h @@ -34,8 +32,7 @@ def to_s Ingestion::Report.line("Entities enriched", Ingestion::Report.count(enriched)), Ingestion::Report.line("Entities failed", Ingestion::Report.count(failed)), Ingestion::Report.line("Cycles deferred", Ingestion::Report.count(deferred)), - Ingestion::Report.line("Cycles with nothing eligible", Ingestion::Report.count(idle)), - Ingestion::Report.line("Candidates skipped (budget)", Ingestion::Report.count(aged_out)) + Ingestion::Report.line("Cycles with nothing eligible", Ingestion::Report.count(idle)) ].join("\n") end end diff --git a/app/services/github/enrichment_runner.rb b/app/services/github/enrichment_runner.rb index f4a0e2c..4814508 100644 --- a/app/services/github/enrichment_runner.rb +++ b/app/services/github/enrichment_runner.rb @@ -1,19 +1,11 @@ module Github - # One enrichment cycle (IMPLEMENTATION_PLAN.md §13's PR 7), in the order §10 and §12 - # require: + # One enrichment cycle (IMPLEMENTATION_PLAN.md §13's PR 7): # - # 1. age candidates past their eligibility window into skipped_budget — for both - # classes, on every invocation, before anything else - # 2. ask §10's fairness policy which class works next, from which pool, and whether it + # 1. ask §10's fairness policy which class works next, from which pool, and whether it # may borrow - # 3. lease that entity row, so a second worker cannot take it - # 4. fetch it through Github.executor — the one and only network call - # 5. record the global rate-limit consequence, then the entity outcome - # - # Step 1 comes first deliberately. §12's sequence is "exhaustion → deferred → - # skipped_budget → reactivation", which requires skipping to keep happening *while* the - # budget is exhausted — precisely when boundedness matters. Behind the fairness decision - # it would stop exactly then, and the backlog would grow without limit. + # 2. lease that entity row, so a second worker cannot take it + # 3. fetch it through Github.executor — the one and only network call + # 4. record the global rate-limit consequence, then the entity outcome # # **At most one entity per call.** §5 names EnrichActorJob and EnrichRepositoryJob, and one # entity is what each of them performs; batching is the caller's loop, which @@ -45,14 +37,10 @@ class EnrichmentRunner # lease_lost the outcome arrived after another worker had claimed the row class Result < Data.define(:status, :entity_type, :github_id, :pool, :borrow, :classification, :enrichment_status, :last_error, - :error_code, :deferral_reason, :next_retry_at, :aged_out, + :error_code, :deferral_reason, :next_retry_at, :duration_ms, :enrichment_attempt) STATUSES = %w[ enriched failed deferred idle lease_lost ].freeze - # §11 names the INFO events "enrichment completed/failed/skipped/reactivated", so the - # log vocabulary is the plan's rather than this object's. "skipped" is - # Github::Enrichment::AgeOut's line and "reactivated" is the ingest path's; the five - # here are the outcomes of one cycle. EVENTS = { "enriched" => "enrichment.completed", "failed" => "enrichment.failed", "deferred" => "enrichment.deferred", "idle" => "enrichment.idle", @@ -62,7 +50,7 @@ class Result < Data.define(:status, :entity_type, :github_id, :pool, :borrow, def initialize(status:, entity_type: nil, github_id: nil, pool: nil, borrow: false, classification: nil, enrichment_status: nil, last_error: nil, error_code: nil, deferral_reason: nil, next_retry_at: nil, - aged_out: 0, duration_ms: nil, enrichment_attempt: nil) + duration_ms: nil, enrichment_attempt: nil) raise ArgumentError, "unknown status #{status.inspect}" unless STATUSES.include?(status) super @@ -84,7 +72,7 @@ def to_log error_code: error_code, error_message: last_error, deferral_reason: deferral_reason, next_retry_at: next_retry_at&.utc&.iso8601, - aged_out: (aged_out if aged_out.positive?), duration_ms: duration_ms }.compact + duration_ms: duration_ms }.compact end end @@ -93,7 +81,7 @@ def initialize(executor: Github.executor, clock: -> { Time.current }, monotonic: MONOTONIC, rate_limit_policy: RateLimitPolicy.new, - selector: nil, fairness: nil, claim: nil, age_out: nil, entity_state: nil) + selector: nil, fairness: nil, claim: nil, entity_state: nil) @executor = executor @configuration = configuration @clock = clock @@ -102,7 +90,6 @@ def initialize(executor: Github.executor, @selector = selector || Enrichment::CandidateSelector.new(configuration: configuration) @fairness = fairness || Enrichment::Fairness.new(configuration: configuration, selector: @selector) @claim = claim || Enrichment::Claim.new(configuration: configuration, selector: @selector) - @age_out = age_out || Enrichment::AgeOut.new(configuration: configuration, selector: @selector) @entity_state = entity_state || Enrichment::EntityState.new end @@ -113,22 +100,21 @@ def initialize(executor: Github.executor, def call(entity_class: nil) now = @clock.call started = @monotonic.call - aged = @age_out.call(now: now).values.sum choice = @fairness.choose(entity_class: entity_class, now: now) - return idle(choice, aged: aged, started: started) unless choice.chosen? + return idle(choice, started: started) unless choice.chosen? lease = @claim.acquire(choice.entity_type, pool: choice.pool, now: now) # A lost race, or a row that moved between the query and the claim. Nothing is # wrong, and there is nothing to report about an entity we never held. - return idle(Enrichment::Fairness::Choice.none(reason: "no_candidate"), aged: aged, started: started) if lease.nil? + return idle(Enrichment::Fairness::Choice.none(reason: "no_candidate"), started: started) if lease.nil? - enrich(choice, lease, aged: aged, started: started) + enrich(choice, lease, started: started) end private - def enrich(choice, lease, aged:, started:) + def enrich(choice, lease, started:) fetched = @executor.call(request_for(choice, lease)) # Before the entity write: a rate limit is a fact about the IP, the global block has # to be recorded first, and the secondary-limit branch of the write matrix reads the @@ -140,7 +126,7 @@ def enrich(choice, lease, aged:, started:) decision: decision, now: @clock.call) @claim.release!(lease) if written.lease_held - complete(choice, lease, fetched, written, aged: aged, started: started) + complete(choice, lease, fetched, written, started: started) rescue Errors::FixtureMiss # §6 requires a corpus gap to be raised rather than laundered into a failed fetch. # The lease goes back untouched: an authoring bug is not an entity outcome and must @@ -187,7 +173,7 @@ def request_for(choice, lease) ) end - def complete(choice, lease, fetched, written, aged:, started:) + def complete(choice, lease, fetched, written, started:) result = Result.new( status: written.outcome, entity_type: choice.entity_type.key, github_id: lease.github_id, pool: choice.pool, borrow: choice.borrow, @@ -199,7 +185,7 @@ def complete(choice, lease, fetched, written, aged:, started:) enrichment_attempt: lease.enrichment_attempts + 1, last_error: written.last_error, error_code: written.error_code, deferral_reason: (fetched.classification.to_s if written.deferred?), - next_retry_at: written.next_retry_at, aged_out: aged, + next_retry_at: written.next_retry_at, duration_ms: elapsed_ms(started) ) @@ -207,7 +193,7 @@ def complete(choice, lease, fetched, written, aged:, started:) result end - def idle(choice, aged:, started:) + def idle(choice, started:) # A deferral the ledger would have refused is reported as such; genuinely having # nothing to do is idle. IngestionRunner draws the same line between "not due" and # "deferred", and for the same reason: they are different facts and an operator acts @@ -215,14 +201,13 @@ def idle(choice, aged:, started:) deferred = choice.reason != "no_candidate" result = Result.new(status: deferred ? "deferred" : "idle", - deferral_reason: choice.reason, aged_out: aged, + deferral_reason: choice.reason, duration_ms: elapsed_ms(started)) log(result) result end - # §11 lists "enrichment completed/failed/skipped/reactivated, retry scheduled" among - # the INFO events. A deferral or an idle cycle is DEBUG: under PR 8's recurring task + # A completed attempt is INFO. A deferral or an idle cycle is DEBUG: under the recurring task # an exhausted window would otherwise emit a line a minute for the rest of the hour, # which is the volume argument Github::BudgetLedger#log_class_exhausted already makes. def log(result) diff --git a/app/services/github/events/quarantine_reasons.rb b/app/services/github/events/quarantine_reasons.rb index 589afa0..17ee75d 100644 --- a/app/services/github/events/quarantine_reasons.rb +++ b/app/services/github/events/quarantine_reasons.rb @@ -59,7 +59,7 @@ module QuarantineReasons INVALID_REPOSITORY_REFERENCE = "invalid_repository_reference".freeze # `created_at` absent, blank, or not parseable as ISO 8601. push_events.occurred_at - # is NOT NULL and drives the newest-first enrichment ordering. + # is NOT NULL and preserves the event's upstream time. INVALID_OCCURRED_AT = "invalid_occurred_at".freeze # `payload` is not an object, or one of §7's five documented required fields — diff --git a/app/services/github/ingestion/page_writer.rb b/app/services/github/ingestion/page_writer.rb index 10e14bd..0b491c7 100644 --- a/app/services/github/ingestion/page_writer.rb +++ b/app/services/github/ingestion/page_writer.rb @@ -119,14 +119,11 @@ def persist(outcome, run_id:, received_at:) # §7 merge rule 3 and Appendix D item 5: activity updates happen **only** when # RETURNING produced a row. On a duplicate the transaction still commits, carrying - # rule 1's identity refresh and nothing else, so rule 4 — "a duplicate event - # replay can never reactivate enrichment" — holds structurally rather than by a - # later check. PR 5 owns this gate; PR 7 put the skipped_budget reactivation - # behind it, which is why that guarantee needed no code of its own. + # rule 1's identity refresh and nothing else. Durable enrichment state remains + # unchanged by a duplicate observation. next nil if id.nil? touch_activity(outcome, received_at: received_at) - reactivate(outcome, run_id: run_id, received_at: received_at) id end @@ -145,26 +142,6 @@ def touch_activity(outcome, received_at:) ) end - # §7's reactivation rule, behind the same RETURNING gate as the activity update - # above — which is the whole of rule 4's guarantee. Actor before repository, keeping - # the ordering the stub upserts established so two concurrent pages touching the - # same pair cannot deadlock. - # - # §11 puts "reactivated" at INFO, and it belongs there rather than at DEBUG: an - # entity coming back from skipped_budget is the observable half of §10's bounded - # backlog, and the README's Phase B replay check greps for exactly this event to be - # absent on a replay. - def reactivate(outcome, run_id:, received_at:) - [ [ GithubActor, outcome.actor_attributes, :github_actor_id ], - [ GithubRepository, outcome.repository_attributes, :github_repository_id ] ].each do |model, attributes, log_key| - github_id = attributes.fetch(:github_id) - next if model.reactivate_skipped!(github_id: github_id, now: received_at).zero? - - Rails.logger.info(event: "enrichment.reactivated", run_id: run_id, log_key => github_id, - github_event_id: outcome.github_event_id) - end - end - def log_persisted(outcome, run_id:, created_id:) if created_id.nil? Rails.logger.debug(event: "ingestion.event_duplicate", run_id: run_id, **outcome.to_log) diff --git a/app/services/github/ingestion/state_summary.rb b/app/services/github/ingestion/state_summary.rb index efb816b..73b7510 100644 --- a/app/services/github/ingestion/state_summary.rb +++ b/app/services/github/ingestion/state_summary.rb @@ -29,8 +29,8 @@ module Ingestion # question. pending_actor_count here is the enrichment_candidates scope — pending *plus* # retryable_failure, which is what "still to enrich" means for the operator about to run # bin/enrich. /status reports the literal status under that name and publishes the scope - # beside it as `candidates`, because a JSON consumer has no §9 context to disambiguate - # from. + # beside it as `backlog_count`, because a JSON consumer has no §9 context to disambiguate + # from it. class StateSummary < Data.define( :latest_run_at, :latest_run_id, :push_event_count, :pending_actor_count, :pending_repository_count, @@ -54,9 +54,9 @@ def self.capture(now: Time.current) push_event_count: PushEvent.count, # The scope whose WHERE clause matches index_*_on_enrichment_candidates exactly, so # both counts are partial-index counts. It counts pending *and* retryable_failure, - # which is what "still to enrich" means; Github::Enrichment::Summary prints the - # per-status split that bin/enrich needs, and §11's coverage percentages arrive - # with PR 10's /status. + # which is what "still to enrich" means; Github::Enrichment::Summary prints this + # backlog with its oldest wait and reserved allowance use, while /status adds the + # literal per-status split and §11's coverage percentages. pending_actor_count: GithubActor.enrichment_candidates.count, pending_repository_count: GithubRepository.enrichment_candidates.count, budget_resource: budget&.resource, budget_remaining: budget&.remaining, diff --git a/app/services/github/status/snapshot.rb b/app/services/github/status/snapshot.rb index ecb61f7..701f634 100644 --- a/app/services/github/status/snapshot.rb +++ b/app/services/github/status/snapshot.rb @@ -95,37 +95,36 @@ def payload private - # §11's "pending_actor_count / pending_repository_count / skipped_actor_count / - # skipped_repository_count", and the reason all five statuses are published rather - # than those two. - # - # §11 lists pending_* beside skipped_*, and skipped_budget is a value of - # Enrichable::ENRICHMENT_STATUSES — so its sibling is the status value too, and - # `pending` here means enrichment_status = 'pending' exactly. - # Github::Ingestion::StateSummary uses the same *name* for a different number: the - # enrichment_candidates scope, which is pending **plus** retryable_failure and is - # what "still to enrich" means when the question is how much work is left. Both are - # right for their own question, and publishing one of them under a name the other - # also uses is how two numbers silently become one. So this block names both: - # every status by its own name, and the scope as `candidates`. + # Each entity class exposes the durable backlog separately from its raw status + # counts. A backlog row may be temporarily deferred by retry backoff, so this number + # intentionally differs from claimable_now. Queue depth is not published: + # jobs are bounded wake-up hints and entity rows are the source of truth. def enrichment_payload - { actors: entity_counts(enrichment.actor_counts), - repositories: entity_counts(enrichment.repository_counts), + { actors: entity_counts(enrichment.actor_counts, + backlog_count: enrichment.actor_backlog_count, + oldest_pending_at: enrichment.actor_oldest_pending_at, + oldest_pending_age_seconds: + enrichment.actor_oldest_pending_age_seconds), + repositories: entity_counts( + enrichment.repository_counts, + backlog_count: enrichment.repository_backlog_count, + oldest_pending_at: enrichment.repository_oldest_pending_at, + oldest_pending_age_seconds: enrichment.repository_oldest_pending_age_seconds + ), claimable_now: enrichment.claimable_now, next_enrichment_at: Ingestion::Report.timestamp(enrichment.next_enrichment_at) } end - # fetch(status, 0) because GROUP BY returns no key for a status with no rows, and an - # absent key here would be the missing-key shape the payload rule forbids. These + # fetch(status, 0) because GROUP BY returns no key for a status with no rows. These # zeros are counted, not fabricated: the table was read and held nothing. - def entity_counts(counts) - Enrichable::ENRICHMENT_STATUSES.index_with { |status| counts.fetch(status, 0) } - .symbolize_keys - .merge(candidates: candidates(counts)) - end - - def candidates(counts) - Enrichable::CANDIDATE_STATUSES.sum { |status| counts.fetch(status, 0) } + def entity_counts(counts, backlog_count:, oldest_pending_at:, + oldest_pending_age_seconds:) + Enrichable::ENRICHMENT_STATUSES + .index_with { |status| counts.fetch(status, 0) } + .symbolize_keys + .merge(backlog_count: backlog_count, + oldest_pending_at: Ingestion::Report.timestamp(oldest_pending_at), + oldest_pending_age_seconds: oldest_pending_age_seconds) end end end diff --git a/config/queue.yml b/config/queue.yml index ee30fdd..52e0e1c 100644 --- a/config/queue.yml +++ b/config/queue.yml @@ -1,12 +1,13 @@ -# Solid Queue's process topology (IMPLEMENTATION_PLAN.md §2A). One supervisor per `worker` -# container: a dispatcher, a scheduler for config/recurring.yml, and one worker process. +# Solid Queue's process topology. One supervisor per `worker` container: a dispatcher, a +# scheduler for config/recurring.yml, and two worker processes. # -# Two threads, not the generator's three, and no JOB_CONCURRENCY knob. §5's global request -# gate makes outbound concurrency exactly one application-wide, so a third thread could only -# queue behind the gate — and while it waited it would hold a primary-database connection -# for up to Github::RequestGate::WAIT_SECONDS (45) out of the RAILS_MAX_THREADS (5) that -# config/database.yml grants per database. Two keeps a reconciler tick from sitting behind a -# poll that is mid-fetch, which is the only concurrency this system actually needs. +# The entity tables are the durable enrichment backlog. Queue deliveries are bounded wake-up +# hints, not one job per entity, but that backlog can remain non-empty for many quota windows. +# A dedicated single-thread enrichment worker keeps that long-lived workload isolated. The +# other worker polls only the bounded polling and control queues; polling is listed first so +# source acquisition stays prompt, while the one-per-minute job cadence keeps control work +# from accumulating behind it. Both request paths still serialize outbound traffic through +# Github::RequestGate, and two processes avoid opening a third primary-database pool. # # polling_interval stays at 1s. Nothing here is latency-sensitive: the poll cadence is # POLL_INTERVAL_SECONDS (300), enrichment is capped by the hourly allowance (40 at the @@ -16,8 +17,12 @@ default: &default - polling_interval: 1 batch_size: 500 workers: - - queues: "*" - threads: 2 + - queues: polling,control + threads: 1 + processes: 1 + polling_interval: 1 + - queues: enrichment + threads: 1 processes: 1 polling_interval: 1 diff --git a/config/recurring.yml b/config/recurring.yml index 71fb780..280b9f8 100644 --- a/config/recurring.yml +++ b/config/recurring.yml @@ -26,12 +26,15 @@ default: &default poll_event_sources: class: PollEventSourceJob + queue: polling schedule: every 60 seconds reconcile_pending_enrichments: class: ReconcilePendingEnrichmentsJob + queue: control schedule: every 60 seconds clear_solid_queue_finished_jobs: command: "SolidQueue::Job.clear_finished_in_batches(sleep_between_batches: 0.3)" + queue: control schedule: every hour at minute 12 development: diff --git a/db/migrate/20260802000000_remove_skipped_budget_from_enrichment.rb b/db/migrate/20260802000000_remove_skipped_budget_from_enrichment.rb new file mode 100644 index 0000000..095f8fc --- /dev/null +++ b/db/migrate/20260802000000_remove_skipped_budget_from_enrichment.rb @@ -0,0 +1,60 @@ +class RemoveSkippedBudgetFromEnrichment < ActiveRecord::Migration[8.1] + TABLES = %i[github_actors github_repositories].freeze + STATUSES = %w[pending complete retryable_failure permanent_failure].freeze + + def up + TABLES.each do |table| + restore_skipped_rows(table) + replace_status_constraint(table, STATUSES) + replace_candidate_index(table, %i[created_at id]) + remove_column table, :skipped_at, :datetime + end + end + + def down + TABLES.each do |table| + add_column table, :skipped_at, :datetime + replace_status_constraint(table, STATUSES + [ "skipped_budget" ]) + replace_candidate_index(table, %i[next_retry_at last_seen_at]) + end + end + + private + + # AgeOut previously erased whether a skipped row had been pending for its first attempt + # or waiting after a retryable failure. Attempt history is the remaining durable signal, + # so untouched rows return to pending and attempted rows return to retryable_failure. + # Either way the row is immediately eligible unless its preserved retry instant is + # already earlier. Quota delay can no longer turn either state into a terminal one. + def restore_skipped_rows(table) + execute <<~SQL.squish + UPDATE #{table} + SET enrichment_status = CASE + WHEN enrichment_attempts > 0 OR last_error IS NOT NULL + THEN 'retryable_failure' + ELSE 'pending' + END, + next_retry_at = CASE + WHEN next_retry_at > CURRENT_TIMESTAMP THEN CURRENT_TIMESTAMP + ELSE next_retry_at + END, + updated_at = GREATEST(updated_at, CURRENT_TIMESTAMP) + WHERE enrichment_status = 'skipped_budget' + SQL + end + + def replace_status_constraint(table, statuses) + name = "#{table}_enrichment_status_check" + remove_check_constraint table, name: name + quoted = statuses.map { |status| connection.quote(status) }.join(", ") + add_check_constraint table, "enrichment_status IN (#{quoted})", name: name + end + + def replace_candidate_index(table, columns) + name = "index_#{table}_on_enrichment_candidates" + remove_index table, name: name + add_index table, columns, + where: "enrichment_status IN ('pending', 'retryable_failure')", + name: name + end +end diff --git a/db/schema.rb b/db/schema.rb index d924955..72c5db2 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.1].define(version: 2026_07_31_120000) do +ActiveRecord::Schema[8.1].define(version: 2026_08_02_000000) do # These are extensions that must be enabled in order to support this database enable_extension "pg_catalog.plpgsql" @@ -52,13 +52,12 @@ t.text "name" t.datetime "next_retry_at" t.jsonb "raw_payload" - t.datetime "skipped_at" t.datetime "updated_at", null: false + t.index ["created_at", "id"], name: "index_github_actors_on_enrichment_candidates", where: "(enrichment_status = ANY (ARRAY['pending'::text, 'retryable_failure'::text]))" t.index ["fetched_at", "next_retry_at"], name: "index_github_actors_on_enrichment_refresh", where: "(enrichment_status = 'complete'::text)" t.index ["github_id"], name: "index_github_actors_on_github_id", unique: true - t.index ["next_retry_at", "last_seen_at"], name: "index_github_actors_on_enrichment_candidates", where: "(enrichment_status = ANY (ARRAY['pending'::text, 'retryable_failure'::text]))" t.check_constraint "enrichment_attempts >= 0", name: "github_actors_enrichment_attempts_nonnegative" - t.check_constraint "enrichment_status = ANY (ARRAY['pending'::text, 'complete'::text, 'retryable_failure'::text, 'permanent_failure'::text, 'skipped_budget'::text])", name: "github_actors_enrichment_status_check" + t.check_constraint "enrichment_status = ANY (ARRAY['pending'::text, 'complete'::text, 'retryable_failure'::text, 'permanent_failure'::text])", name: "github_actors_enrichment_status_check" end create_table "github_api_budget", id: :integer, default: 1, force: :cascade do |t| @@ -105,13 +104,12 @@ t.datetime "next_retry_at" t.bigint "owner_github_id" t.jsonb "raw_payload" - t.datetime "skipped_at" t.datetime "updated_at", null: false + t.index ["created_at", "id"], name: "index_github_repositories_on_enrichment_candidates", where: "(enrichment_status = ANY (ARRAY['pending'::text, 'retryable_failure'::text]))" t.index ["fetched_at", "next_retry_at"], name: "index_github_repositories_on_enrichment_refresh", where: "(enrichment_status = 'complete'::text)" t.index ["github_id"], name: "index_github_repositories_on_github_id", unique: true - t.index ["next_retry_at", "last_seen_at"], name: "index_github_repositories_on_enrichment_candidates", where: "(enrichment_status = ANY (ARRAY['pending'::text, 'retryable_failure'::text]))" t.check_constraint "enrichment_attempts >= 0", name: "github_repositories_enrichment_attempts_nonnegative" - t.check_constraint "enrichment_status = ANY (ARRAY['pending'::text, 'complete'::text, 'retryable_failure'::text, 'permanent_failure'::text, 'skipped_budget'::text])", name: "github_repositories_enrichment_status_check" + t.check_constraint "enrichment_status = ANY (ARRAY['pending'::text, 'complete'::text, 'retryable_failure'::text, 'permanent_failure'::text])", name: "github_repositories_enrichment_status_check" end create_table "ingestion_runs", force: :cascade do |t| diff --git a/docker-compose.yml b/docker-compose.yml index 31fc57d..cade1f3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,11 +41,9 @@ x-app-env: &app_env RATE_LIMIT_RESERVE: ${RATE_LIMIT_RESERVE:-8} # §10's enrichment policy, in the shared anchor for a sharper version of the same # reason. ACTOR_ENRICHMENT_SHARE decides how the ledger splits the enrichment allowance - # under its row lock, and the three timings decide which candidates are *currently - # eligible* — which is the input to the borrow decision the ledger then acts on. Two + # under its row lock, and the refresh TTLs decide when completed rows become stale. Two # processes reading different values would enforce different policies against one row. ACTOR_ENRICHMENT_SHARE: ${ACTOR_ENRICHMENT_SHARE:-0.50} - ENRICHMENT_ELIGIBILITY_WINDOW_SECONDS: ${ENRICHMENT_ELIGIBILITY_WINDOW_SECONDS:-3600} ACTOR_REFRESH_TTL_SECONDS: ${ACTOR_REFRESH_TTL_SECONDS:-86400} REPOSITORY_REFRESH_TTL_SECONDS: ${REPOSITORY_REFRESH_TTL_SECONDS:-86400} # §11's coverage window, forwarded for the same reason the four allowance inputs are — diff --git a/docs/DESIGN_BRIEF.md b/docs/DESIGN_BRIEF.md index af214de..9902a86 100644 --- a/docs/DESIGN_BRIEF.md +++ b/docs/DESIGN_BRIEF.md @@ -1,6 +1,6 @@ # Design brief — github-push-ingestor -This Rails 8.1 API service samples GitHub's public Events API, stores `PushEvent` +This Rails 8.1 API service polls GitHub's public Events API, stores `PushEvent` records and their raw payloads in PostgreSQL, and enriches referenced actors and repositories without a token. The README is the runbook; the [ADRs](adr/) and [`IMPLEMENTATION_PLAN.md`](../IMPLEMENTATION_PLAN.md) hold the detailed arguments and @@ -12,8 +12,9 @@ The assignment asks for durable ingestion, enrichment, and restart safety. The d constraint is the source: `/events` is a sliding window with documented delivery latency, and unauthenticated callers share 60 requests per hour per outbound IP — an event can leave the window unobserved, and enrichment demand can exceed the remaining budget by -orders of magnitude. The product is therefore an observable, bounded sampler, not a -mirror, with no guarantee of complete upstream capture or complete enrichment. +orders of magnitude. Event capture is therefore an observable, bounded sample rather than +a mirror. Enrichment is different: every never-enriched entity remains durable work, even +when the backlog cannot drain within a bounded time. ## Architecture, data, and durability @@ -49,8 +50,8 @@ SHA-256, since malformed data may lack a usable event ID. Acceptance occurs when a `push_events` row commits. Inserts use `ON CONFLICT (github_event_id) DO NOTHING RETURNING id`. A duplicate observation cannot -create a second event row, and entity activity or `skipped_budget` reactivation occurs only -when `RETURNING` yields a new row. A replay may still refresh permitted identity fields. +create a second event row, and entity activity is updated only when `RETURNING` yields a +new row. A replay may still refresh permitted identity fields. These invariants are deliberately narrow: executions, runs, quarantine counts, budget debits, and logs can repeat. @@ -87,19 +88,22 @@ unauthenticated [probe](evidence/2026-07-30-unauthenticated-304-quota-probe.md) request: ETags save bandwidth, not budget. The asymmetry decides it — a wrongly-free `304` can exhaust the shared window, while a wrongly-charged one costs only local opportunity. -## Bounded, fair, and safe enrichment +## Durable, fair, and safe enrichment -One observed page held roughly 180 distinct actors and repositories — cold lookups -competing for 40 hourly attempts. Candidates therefore age from `pending` to the -terminal `skipped_budget` state after an eligibility window, and only a newly inserted -push event can reactivate a skipped entity. This bounds actionable backlog, and -`/status` reports the sampled proportions. +One observed page held roughly 180 distinct entities — a cold-demand pressure scenario, +not a measured unique arrival rate, because identities deduplicate into shared rows. +Defaults reserve 12 attempts for polls, 40 for the enrichment class, and 8 for safety; +durable FIFO backlog has priority over refresh within the 40. Quota exhaustion only defers +rows. If unique arrivals exceed service, backlog size and age can grow without a bounded +completion time. -Each entity class receives a guaranteed share. A class may borrow beyond it only when the -other has no currently eligible candidate; the ledger independently refuses invalid -reservations. Never-enriched candidates precede stale refreshes, and selection plus -leases prevent concurrent duplicate work, so a repository-heavy feed cannot starve actor -enrichment ([ADR 0007](adr/0007-enrichment-fairness-shares-and-borrowing.md)). +`/status` exposes per-class count, oldest pending timestamp/age, and allowance usage; it +omits an ETA because there is no durable outcome history for an honest rate. The 40 attempts +carry 20/20 actor/repository guarantees with borrowing when the other class has no +claimable work. A selection that observes a never-enriched row suppresses refresh; one +concurrent insert can cross a one-request decision/debit window before the next selection +self-corrects. Selection and leases prevent duplicate work +([ADR 0007](adr/0007-enrichment-fairness-shares-and-borrowing.md)). Payload, pagination, and redirect URLs are attacker-influenceable, so the SSRF boundary allows only HTTPS URLs whose host is exactly `api.github.com` — no userinfo, non-default @@ -117,7 +121,7 @@ throughput and fan-out. Authentication, object storage, extra entity APIs, Kafka, and a frontend were omitted to keep the submission focused on correctness at the stated quota. The bottleneck is -upstream allowance, not local throughput: an authenticated budget would raise coverage -but not make capture or enrichment complete. Queue capacity and database contention -should be measured before any broker, and the pinned `2022-11-28` API version +upstream allowance, not local throughput: an authenticated budget would drain the durable +backlog faster but would not make upstream event capture complete. Queue capacity and +database contention should be measured before any broker, and the pinned `2022-11-28` API version revalidated before upgrade. diff --git a/docs/SUBMISSION_CHECKLIST.md b/docs/SUBMISSION_CHECKLIST.md index 9d8c443..1e950d1 100644 --- a/docs/SUBMISSION_CHECKLIST.md +++ b/docs/SUBMISSION_CHECKLIST.md @@ -112,18 +112,28 @@ globally named `github-push-ingestor_pgdata` volume. SELECT 'repository', enrichment_status, COUNT(*) FROM github_repositories GROUP BY 2;" ``` -- [ ] After the 60-second poll floor, capture `ingest --force`; require 4 duplicates and no - `enrichment.reactivated` line in that captured output: +- [ ] Before replay, record entity activity; after the 60-second poll floor, capture + `ingest --force`, require 4 duplicates, and prove the duplicate registered no new + entity activity: ```bash sleep 60 + activity_before="$(docker compose exec -T db psql -U postgres \ + -d github_push_ingestor_development -Atc \ + "SELECT md5(string_agg(row_to_json(entities)::text, ',' ORDER BY kind, github_id)) + FROM (SELECT 'actor' AS kind, github_id, first_seen_at, last_seen_at, latest_event_at FROM github_actors + UNION ALL + SELECT 'repository', github_id, first_seen_at, last_seen_at, latest_event_at FROM github_repositories) entities;")" fixture_replay_output="$(mktemp)" GITHUB_MODE=fixture docker compose run --rm ingest --force 2>&1 | tee "$fixture_replay_output" grep -E 'Duplicates skipped:[[:space:]]+4' "$fixture_replay_output" - if grep -q 'enrichment.reactivated' "$fixture_replay_output"; then - echo "duplicate replay reactivated an entity" >&2 - exit 1 - fi + activity_after="$(docker compose exec -T db psql -U postgres \ + -d github_push_ingestor_development -Atc \ + "SELECT md5(string_agg(row_to_json(entities)::text, ',' ORDER BY kind, github_id)) + FROM (SELECT 'actor' AS kind, github_id, first_seen_at, last_seen_at, latest_event_at FROM github_actors + UNION ALL + SELECT 'repository', github_id, first_seen_at, last_seen_at, latest_event_at FROM github_repositories) entities;")" + test "$activity_before" = "$activity_after" rm -f "$fixture_ingest_output" "$fixture_replay_output" ``` @@ -187,8 +197,8 @@ particular, read the erratum atop - [ ] **Both** actor and repository enrichment demonstrably occur within their fairness guarantees — `spec/services/github/enrichment/end_to_end_spec.rb`; fixture run gives `complete 2 / permanent_failure 1` per class -- [ ] A duplicate event ID cannot add another `push_events` row or reactivate a skipped entity - — fixture replay: 4 duplicates absorbed, no `enrichment.reactivated` +- [ ] A duplicate event ID cannot add another `push_events` row or register new entity + activity — fixture replay: 4 duplicates absorbed, activity hash unchanged - [ ] `Link`-header pagination is handled; every fetched page fully processed — `spec/services/github/ingestion/page_loop_spec.rb`; `paginated` scenario - [ ] Rate-limit behavior demonstrated: `304` quota accounting, class-aware ledger @@ -219,8 +229,13 @@ particular, read the erratum atop idempotency of persisted state - [ ] Reconciliation recovers missing enrichment scheduling — `spec/recovery/pending_enrichment_recovery_spec.rb` -- [ ] The enrichment backlog is bounded (eligibility window + `skipped_budget` + - distinct-event reactivation) — `spec/services/github/enrichment/age_out_spec.rb` +- [ ] Never-enriched actor and repository rows remain durable backlog work across quota + exhaustion and window rollover; candidates are selected FIFO oldest-first — + `spec/services/github/enrichment/candidate_selector_spec.rb`, + `spec/recovery/pending_enrichment_recovery_spec.rb` +- [ ] A selection that observes either entity class has never-enriched backlog work does + not choose a refresh; the documented concurrent-insert window is bounded to one + runner request — `spec/services/github/enrichment/fairness_spec.rb` --- @@ -232,9 +247,10 @@ particular, read the erratum atop trace is one hop - [ ] `/health/live` and `/health/ready` are meaningful and never consume budget — `spec/requests/health_spec.rb` -- [ ] `/status` reports window status, poll state, per-class ledger state, pending/skipped - counts, and coverage percentages by the defined formulas — without initiating GitHub - requests — `spec/requests/status_spec.rb`, `Github::Enrichment::Coverage` +- [ ] `/status` reports window status, poll state, per-class ledger state, backlog size, + oldest pending timestamp/age, reserved allowance usage, and coverage percentages by + the defined formulas — without initiating GitHub requests or fabricating a drain ETA + — `spec/requests/status_spec.rb`, `Github::Enrichment::Coverage` - [ ] During §1's empty-volume fixture phase, while the worker has never been started, hash the complete budget row and record all request counters. Call `/health/live`, `/health/ready`, and `/status` repeatedly, then require the hash and counters to be diff --git a/docs/adr/0005-at-least-once-with-idempotent-writes.md b/docs/adr/0005-at-least-once-with-idempotent-writes.md index e6047b3..0105777 100644 --- a/docs/adr/0005-at-least-once-with-idempotent-writes.md +++ b/docs/adr/0005-at-least-once-with-idempotent-writes.md @@ -2,7 +2,7 @@ Date: 2026-07-30 -Status: Accepted; guarantee wording clarified 2026-07-31 +Status: Accepted; guarantee wording clarified 2026-07-31 and 2026-08-02 ## Context @@ -26,8 +26,7 @@ Accept repeated execution and enforce two narrow ingestion invariants at the dat boundary: 1. A duplicate observation of a GitHub event cannot create a second `push_events` row. -2. A duplicate observation cannot register new entity activity or reactivate an entity in - `skipped_budget`. +2. A duplicate observation cannot register new entity activity. Those invariants are implemented by `ON CONFLICT (github_event_id) DO NOTHING RETURNING id` and by applying entity activity @@ -53,7 +52,7 @@ payload. What this buys: - Re-polling or manually re-running ingestion cannot duplicate an accepted event row or - falsely reactivate a skipped entity from the same event ID. + falsely register new entity activity from the same event ID. - Committed events survive process and container restarts without a separate dedup table or distributed transaction. - Committed entity state is sufficient for the reconciler to rediscover pending enrichment @@ -73,7 +72,7 @@ What it does not buy: precedence-ordered function of the payload. The suite tests the stated boundaries: fixture replay leaves four `push_events` rows, -increments the three quarantine occurrence counters, does not register duplicate entity -activity, and leaves a planted `skipped_budget` entity skipped. A separate recovery test +increments the three quarantine occurrence counters, and does not register duplicate entity +activity. A separate recovery test shows a delivered enrichment job may execute again without creating another entity row; that scenario does not widen the ingestion guarantees above. diff --git a/docs/adr/0007-enrichment-fairness-shares-and-borrowing.md b/docs/adr/0007-enrichment-fairness-shares-and-borrowing.md index fd553b5..5755616 100644 --- a/docs/adr/0007-enrichment-fairness-shares-and-borrowing.md +++ b/docs/adr/0007-enrichment-fairness-shares-and-borrowing.md @@ -2,15 +2,17 @@ Date: 2026-07-30 -Status: Accepted +Status: Accepted; durable-backlog policy amended 2026-08-02 ## Context §10's demand arithmetic settles the shape of enrichment before any code is written. One observed live page of `/events` held ~92–95 push events referencing ~89 distinct actors and ~92 distinct repositories: 181 cold entity requests per page, ~2,172 an hour at the default -cadence, against 40 available. Enrichment is bounded best-effort sampling, and the only -open question is *how* the 40 are allocated. +cadence if every poll contained new identities, against 40 available. This is a pressure +scenario rather than a measured deduplicated arrival rate. The 40 attempts are a reserved +service budget for a durable backlog, and the allocation must make progress across both +entity classes. Left to a simple queue it allocates badly. Repository candidates alone exceed the whole hourly allowance, so a repo-first policy — or any policy ordering purely by recency across @@ -22,7 +24,7 @@ actor_guarantee = floor(enrichment_allowance × ACTOR_ENRICHMENT_SHARE) repository_guarantee = enrichment_allowance − actor_guarantee Borrowing: a class may borrow the other's unused capacity only when the -other class has no CURRENTLY ELIGIBLE candidate (not merely no rows). +other class has no CURRENTLY CLAIMABLE backlog candidate (not merely no rows). ``` Two facts make this awkward to place. The guarantee is arithmetic over @@ -39,7 +41,7 @@ from inside the most contended row lock in the application would invert the lock outside the row lock would be advisory rather than enforcement. 2. **The borrow is a parameter, not a query.** `reserve!(request_class, now:, borrow:)` - takes the caller's assertion that the other class has no currently eligible candidate. + takes the caller's assertion that the other class has no currently claimable backlog candidate. `Github::Enrichment::Fairness` establishes it; the ledger enforces the arithmetic that follows from it. It defaults to `false`, so every existing caller and every careless one gets enforcement. @@ -70,7 +72,7 @@ from inside the most contended row lock in the application would invert the lock from `Github::EnrichmentSchedule`. §9's `effective_enrichment_time` names `enrichment_used >= enrichment_allowance`, the class cap, and admitting the share would have two consequences: there is no honest instant to name (a share is relieved either by - the window rolling *or* by the other class running out of eligible candidates, and the + the window rolling *or* by the other class running out of claimable backlog candidates, and the second has no timestamp), and it would make borrowing unreachable, because the schedule would answer "not due" before the runner ever computed a borrow. @@ -89,7 +91,7 @@ from inside the most contended row lock in the application would invert the lock - Fairness is real rather than advisory: a repository flood is refused at 20 requests with the actor guarantee untouched, and the refusal costs no quota. - Both classes demonstrably enrich within their guarantees (§16), and `bin/enrich` prints - the per-class usage so an operator sees the sampling rate rather than a growing queue. + per-class usage and backlog progress so an operator can see whether the queue is draining. - **The borrow fact is stale by construction.** A poll can persist a new candidate between the eligibility query and the debit. The exposure is bounded to one request — the runner enriches at most one entity per invocation — it self-corrects on the next call, and it can @@ -131,6 +133,9 @@ from inside the most contended row lock in the application would invert the lock ## Amendment (2026-07-31): the refresh pool follows the same two steps +The 2026-08-02 amendment below further restricts *when* this pool may run; the allocation +arithmetic here still applies after the durable never-enriched backlog is empty. + This ADR decided how the pending pool allocates and left the refresh pool's *selection order* unstated, and the shipped `#refresh_choice` did not follow it: it took the first refreshable class in `EntityType.all` order — always actor — and set `borrow` from whether @@ -149,3 +154,30 @@ about the *pending* borrow test and still holds. See [ADR 0010](0010-secondary-limit-escalation-and-refresh-pool-fairness.md) for the full context, the rejected "refreshes never borrow" reading of §10:898, and the `borrowed_refresh` choice reason. + +## Amendment (2026-08-02): pending work is durable FIFO backlog + +Quota scarcity delays enrichment; it does not make a never-enriched entity expendable. +Entity rows remain the durable work record until enrichment reaches a success or a genuine +entity-specific failure outcome. Exhausting the hourly allowance merely defers the row to a +later rate-limit window. The pending pool is ordered by immutable, non-null +`created_at ASC, id ASC`; `first_seen_at` can be null and is not a safe queue key. Sustained +arrivals therefore cannot keep older work from being attempted. + +The default hourly ledger reserves 12 attempts for polling, 40 for the enrichment class, +and 8 as a safety reserve. Within the 40, durable backlog has priority; actors and +repositories receive 20-attempt guarantees and may borrow only when the other class has no +currently claimable backlog candidate. Refresh work is lower priority than the entire +never-enriched pool: a selection that observes either class has never-enriched work does not +choose a refresh. Once that pool is observed empty, refreshes use the same guarantee and +borrowing arithmetic described above. + +That observation and the later debit are not one database snapshot. Ingestion can commit a +new candidate between them, so one already-selected refresh can cross the boundary. The +exposure is at most one request because a runner invocation handles one entity, and the next +selection self-corrects. This cannot erase backlog state or exceed the enrichment cap. + +This policy deliberately does not promise a bounded completion time. If unique entities +arrive faster than 40 attempts per hour can serve them, backlog size and oldest pending age +will grow. That pressure is reported directly; work is never converted into a terminal +budget outcome merely because the quota window ended. diff --git a/docs/adr/0008-post-commit-enqueue-and-entity-scoped-reconciliation.md b/docs/adr/0008-post-commit-enqueue-and-entity-scoped-reconciliation.md index 6d40bf5..2a6d696 100644 --- a/docs/adr/0008-post-commit-enqueue-and-entity-scoped-reconciliation.md +++ b/docs/adr/0008-post-commit-enqueue-and-entity-scoped-reconciliation.md @@ -2,7 +2,7 @@ Date: 2026-07-31 -Status: Accepted +Status: Accepted; durable-backlog ordering amended 2026-08-02 ## Context @@ -21,7 +21,8 @@ event rows per entity." A second fact shapes the design as much as the first: **enrichment jobs carry no entity id.** `Github::EnrichmentRunner` enriches one entity per call and chooses it itself, -through §10's fairness policy and a `FOR UPDATE SKIP LOCKED` lease, newest-first. That is +through §10's fairness policy and a `FOR UPDATE SKIP LOCKED` lease, FIFO by +`created_at ASC, id ASC`. That is not an accident to route around — an id-addressed job would have to bypass that ordering to honour its argument, which is precisely how a repository flood starves actors (ADR 0007). So an enqueue cannot mean "enrich this actor". It can only mean "there may be actor work". @@ -59,10 +60,11 @@ Positive: - The crash window has no special case. The system's recovery path and its steady-state path are the same code, which means the recovery path is exercised on every tick rather than only during an incident. -- Queue depth is bounded by the hourly allowance rather than by arrival rate. One live page +- Operational job-queue depth is bounded by dispatch rather than by entity arrival rate; + the durable business backlog is intentionally not bounded. One live page references ~181 distinct entities; enqueuing per created event would produce ~2,400 argument-identical cycles an hour against 40 spendable requests, and each surplus cycle - would run the age-out sweep and the fairness reads to be told no. + would run selection and fairness reads only to be told no. - The queue is never read to answer a question about business state, which keeps CLAUDE.md's source-of-truth rule intact: PostgreSQL business tables are the durable record, not the queue. @@ -141,5 +143,5 @@ each held a database connection. Deferred to PR 9, which owns multi-source alloc leans on) - ADR 0004 — the class-aware budget ledger (what bounds enrichment throughput) - ADR 0005 — repeated execution with duplicate-safe event writes (the narrow event-row and - skipped-reactivation guarantees under redelivery) + entity-activity guarantees under redelivery) - ADR 0007 — enrichment fairness shares and borrowing (why a job cannot carry an entity id) diff --git a/docs/adr/0010-secondary-limit-escalation-and-refresh-pool-fairness.md b/docs/adr/0010-secondary-limit-escalation-and-refresh-pool-fairness.md index 8c8a15a..afcecec 100644 --- a/docs/adr/0010-secondary-limit-escalation-and-refresh-pool-fairness.md +++ b/docs/adr/0010-secondary-limit-escalation-and-refresh-pool-fairness.md @@ -2,7 +2,7 @@ Date: 2026-07-31 -Status: Accepted +Status: Accepted; durable-backlog refresh priority amended 2026-08-02 ## Context @@ -79,19 +79,24 @@ request on a service that has never been throttled — is one statement that mat and takes no lock. Zero affected rows is the expected outcome, so it deliberately does not go through the `LedgerInvariantViolation` check `#debit!` applies to the same condition. -### The refresh pool allocates the same way the pending pool does +### The refresh pool runs only after the durable backlog is observed empty -`#refresh_choice` now mirrors `#pending_choice`: build an eligibility map, prefer a class -with `room_within_guarantee?`, and only then borrow — and only from a class that has nothing -to do. A borrowed refresh reports `borrowed_refresh`, mirroring `borrowed_pending`. +Before considering any refresh, fairness checks both entity tables for every +candidate-status row (`pending` or `retryable_failure`) without applying the due-time +predicate. A backed-off row and a row carrying an active lease are still durable +never-enriched work, so either blocks the entire refresh pool at that selection decision. +This gives backlog priority within the 40-request enrichment allowance. -The eligibility map is rebuilt over the refresh pool rather than reusing the pending one. -`CandidateSelector#pending_available?` stays pending-only, and its documented reason stands: -counting refreshes in the *pending* borrow test would let a refresh outrank a never-enriched -candidate and invert §10's prioritization ladder. That hazard cannot arise inside -`#refresh_choice`, which is only reached when no class has a pending candidate at all — so -there is nothing left for a refresh to outrank. Keeping two maps preserves the invariant -verbatim instead of weakening it. +Only after that durable backlog is empty does `#refresh_choice` mirror the per-class +allocation arithmetic: build a claimability map over stale refresh rows, prefer a class +with `room_within_guarantee?`, and only then borrow from a class that has nothing to +refresh. A borrowed refresh reports `borrowed_refresh`, mirroring `borrowed_pending`. + +The emptiness read and request debit are deliberately not serialized against ingestion. +A poll can commit a new candidate between them, allowing the one refresh already selected +to proceed. Since one runner invocation issues at most one entity request, the exposure is +bounded to one request; the next decision observes the row and closes the refresh pool. The +ledger cap remains authoritative and the durable row is never discarded. ## Consequences @@ -102,6 +107,9 @@ verbatim instead of weakening it. successful live request that matches no row in the normal case. - The refresh pool can no longer starve a class. Total enrichment spend is unchanged — the ledger's cap was always the bound, and this is a fairness fix rather than an overspend fix. +- A refresh TTL is an earliest eligible time rather than a completion deadline. Sustained + never-enriched backlog can postpone refresh indefinitely, which is preferable to spending + reserved backlog capacity on already-enriched rows. - `Choice::REASONS` gains `borrowed_refresh`. The only consumer of `Choice#reason` is `Github::EnrichmentRunner`'s deferral log, which reads it on the not-chosen path only. @@ -124,4 +132,5 @@ non-positive delta, which `#fallback_instant` already treats exactly as an absen "within each class's share". Rejected: a class whose guarantee rounds to zero (`ACTOR_ENRICHMENT_SHARE` at `0.0` or `1.0`) could then never refresh at all, and it would leave capacity idle whenever the other class has no stale rows — while §10:812's borrowing -rule is stated generally rather than scoped to the pending pool. +rule is stated generally rather than scoped to the pending pool. Borrowing remains valid +after the durable never-enriched backlog is empty. diff --git a/docs/adr/0012-solid-queue-over-kafka.md b/docs/adr/0012-solid-queue-over-kafka.md index c28120d..1fe12e9 100644 --- a/docs/adr/0012-solid-queue-over-kafka.md +++ b/docs/adr/0012-solid-queue-over-kafka.md @@ -21,8 +21,8 @@ The arithmetic: envelopes an hour** before filtering, still bounded by upstream quota rather than by anything downstream. - Enrichment is bounded by the same 60-request ceiling — around 40 requests an hour after - the poll allowance and reserve, which §10 states plainly is a *sample* of demand rather - than coverage of it. + the poll allowance and reserve. Those attempts are the service rate for a durable FIFO + entity backlog; demand beyond that rate is deferred across windows rather than dropped. Nothing in that profile is throughput-constrained. The bottleneck is a third party's rate limit, and no amount of broker capacity moves it. diff --git a/docs/evidence/2026-07-31-clean-checkout-verification.md b/docs/evidence/2026-07-31-clean-checkout-verification.md index 882f24f..e25b188 100644 --- a/docs/evidence/2026-07-31-clean-checkout-verification.md +++ b/docs/evidence/2026-07-31-clean-checkout-verification.md @@ -2,7 +2,15 @@ Date: 2026-07-31 -Status: First-party observation +Status: Historical first-party observation; enrichment policy superseded + +> [!IMPORTANT] +> This transcript verifies revision `6eab84c` and preserves its then-current +> `skipped_budget`/`enrichment.reactivated` assertions as historical evidence. They are not +> current acceptance gates. The 2026-08-02 durable-backlog correction removes that terminal +> state, restores affected rows, and makes quota exhaustion a deferral; see +> [`IMPLEMENTATION_PLAN.md` Appendix F](../../IMPLEMENTATION_PLAN.md) and +> [ADR 0007](../adr/0007-enrichment-fairness-shares-and-borrowing.md). Revision verified: `6eab84c6ce6b0660cc124b7b2d1c4012fc127222` (branch `issue-22-reviewer-documentation`; the post-merge run against the default branch is diff --git a/docs/evidence/2026-07-31-container-kill-recovery.md b/docs/evidence/2026-07-31-container-kill-recovery.md index 8a4129f..0035323 100644 --- a/docs/evidence/2026-07-31-container-kill-recovery.md +++ b/docs/evidence/2026-07-31-container-kill-recovery.md @@ -16,6 +16,12 @@ Status: Historical first-party observation; superseded for submission gating > [`2026-08-01-post-merge-verification.md`](2026-08-01-post-merge-verification.md) gate 1.18 > ran the corrected script against `88e2260c` — 45 checks passed, 0 failed. +> [!IMPORTANT] +> The raw transcript also predates the 2026-08-02 durable-backlog correction. Its +> `enrichment.aged_out` and skipped-count lines describe the old binary, not current policy. +> Current entity work is never terminally skipped for quota; see +> [`IMPLEMENTATION_PLAN.md` Appendix F](../../IMPLEMENTATION_PLAN.md). + ## Why this verification exists `IMPLEMENTATION_PLAN.md` §2A declares `restart: unless-stopped` on `db`, `web` and `worker`, diff --git a/docs/evidence/2026-08-01-post-merge-verification.md b/docs/evidence/2026-08-01-post-merge-verification.md index c1e9500..bc15fc8 100644 --- a/docs/evidence/2026-08-01-post-merge-verification.md +++ b/docs/evidence/2026-08-01-post-merge-verification.md @@ -5,6 +5,12 @@ Date: 2026-08-01 (UTC) Status: Verification of runtime behavior and repository-history gates at the default-branch SHA below, plus documentation-only gates against the PR #42 content identified separately +> [!IMPORTANT] +> This transcript predates the 2026-08-02 durable-backlog correction. Its +> `enrichment.reactivated` replay assertion verifies the old revision only and is no longer +> a current gate. Current quota exhaustion defers durable FIFO work instead of creating a +> terminal budget-skip state; see [`IMPLEMENTATION_PLAN.md` Appendix F](../../IMPLEMENTATION_PLAN.md). + ```text Runtime checkout: 88e2260c7f20fea06cdacb88d62132caaed1fc14 (default branch) Design-brief source: blob af214de7a2fc5c1ee3716ec7ff1ee2e73df21b06 diff --git a/spec/db/remove_skipped_budget_from_enrichment_spec.rb b/spec/db/remove_skipped_budget_from_enrichment_spec.rb new file mode 100644 index 0000000..05308bf --- /dev/null +++ b/spec/db/remove_skipped_budget_from_enrichment_spec.rb @@ -0,0 +1,102 @@ +require "rails_helper" +require Rails.root.join("db/migrate/20260802000000_remove_skipped_budget_from_enrichment").to_s + +RSpec.describe RemoveSkippedBudgetFromEnrichment, type: :migration do + self.use_transactional_tests = false + + ACTOR_IDS = [ 99_100_001, 99_100_002 ].freeze + REPOSITORY_IDS = [ 99_200_001, 99_200_002 ].freeze + + let(:connection) { ActiveRecord::Base.connection } + let(:migration) { described_class.new } + + around do |example| + previous_verbose = ActiveRecord::Migration.verbose + ActiveRecord::Migration.verbose = false + + begin + example.run + ensure + restore_current_schema! + GithubActor.where(github_id: ACTOR_IDS).delete_all + GithubRepository.where(github_id: REPOSITORY_IDS).delete_all + ActiveRecord::Migration.verbose = previous_verbose + end + end + + it "restores every discarded row to the durable FIFO and removes the old state" do + migration.migrate(:down) + reset_entity_schema_cache! + insert_legacy_rows! + + migration.migrate(:up) + reset_entity_schema_cache! + + expect(GithubActor.where(github_id: ACTOR_IDS).order(:github_id).pluck(:enrichment_status)) + .to eq(%w[pending retryable_failure]) + expect(GithubRepository.where(github_id: REPOSITORY_IDS).order(:github_id).pluck(:enrichment_status)) + .to eq(%w[pending retryable_failure]) + + attempted = GithubActor.find_by!(github_id: ACTOR_IDS.last) + expect(attempted).to have_attributes(enrichment_attempts: 2, last_error: "GitHub unavailable") + expect(attempted.next_retry_at).to be <= Time.current + + %w[github_actors github_repositories].each do |table| + expect(connection.column_exists?(table, :skipped_at)).to be(false) + + index = connection.indexes(table) + .find { _1.name == "index_#{table}_on_enrichment_candidates" } + expect(index.columns).to eq(%w[created_at id]) + expect(index.where).to include("pending", "retryable_failure") + end + + expect_violation(ActiveRecord::CheckViolation) do + GithubActor.where(github_id: ACTOR_IDS.first) + .update_all(enrichment_status: "skipped_budget") + end + end + + private + + def insert_legacy_rows! + now = connection.quote(Time.current - 2.days) + future = connection.quote(Time.current + 1.day) + + connection.execute(<<~SQL) + INSERT INTO github_actors + (github_id, login, enrichment_status, enrichment_attempts, next_retry_at, + last_error, created_at, updated_at, skipped_at) + VALUES + (#{ACTOR_IDS.first}, 'legacy-pending', 'skipped_budget', 0, #{future}, + NULL, #{now}, #{now}, #{now}), + (#{ACTOR_IDS.last}, 'legacy-retry', 'skipped_budget', 2, #{future}, + 'GitHub unavailable', #{now}, #{now}, #{now}) + SQL + + connection.execute(<<~SQL) + INSERT INTO github_repositories + (github_id, full_name, enrichment_status, enrichment_attempts, next_retry_at, + last_error, created_at, updated_at, skipped_at) + VALUES + (#{REPOSITORY_IDS.first}, 'legacy/pending', 'skipped_budget', 0, #{future}, + NULL, #{now}, #{now}, #{now}), + (#{REPOSITORY_IDS.last}, 'legacy/retry', 'skipped_budget', 2, #{future}, + 'GitHub unavailable', #{now}, #{now}, #{now}) + SQL + end + + def restore_current_schema! + if connection.column_exists?(:github_actors, :skipped_at) || + connection.column_exists?(:github_repositories, :skipped_at) + migration.migrate(:up) + end + + reset_entity_schema_cache! + end + + def reset_entity_schema_cache! + connection.schema_cache.clear! + GithubActor.reset_column_information + GithubRepository.reset_column_information + end +end diff --git a/spec/db/schema_spec.rb b/spec/db/schema_spec.rb index 57ec466..5897c45 100644 --- a/spec/db/schema_spec.rb +++ b/spec/db/schema_spec.rb @@ -85,11 +85,14 @@ it "each carry the same enrichment state columns" do enrichment_columns = %w[ enrichment_status enrichment_attempts next_retry_at last_error fetched_at - first_seen_at last_seen_at latest_event_at skipped_at + first_seen_at last_seen_at latest_event_at ] %w[github_actors github_repositories].each do |table| - expect(connection.columns(table).map(&:name)).to include(*enrichment_columns) + column_names = connection.columns(table).map(&:name) + + expect(column_names).to include(*enrichment_columns) + expect(column_names).not_to include("skipped_at") end end diff --git a/spec/docker_compose_spec.rb b/spec/docker_compose_spec.rb index f812018..88f83db 100644 --- a/spec/docker_compose_spec.rb +++ b/spec/docker_compose_spec.rb @@ -68,15 +68,14 @@ def unprofiled end # ACTOR_ENRICHMENT_SHARE decides how the ledger splits the enrichment allowance under - # its row lock, and the three timings decide which candidates are *currently eligible* - # — which is the input to the borrow decision the ledger then acts on. Two processes - # reading different values would enforce different policies against one row. - it "forwards §10's fairness share and enrichment timings, so one ledger sees one policy" do + # its row lock, and the two refresh timings decide when complete rows are eligible for + # another fetch. Two processes reading different values would enforce different + # policies against one row. + it "forwards the fairness share and refresh timings, so one ledger sees one policy" do environment = ingest.fetch("environment") expect(environment).to include( "ACTOR_ENRICHMENT_SHARE" => "${ACTOR_ENRICHMENT_SHARE:-0.50}", - "ENRICHMENT_ELIGIBILITY_WINDOW_SECONDS" => "${ENRICHMENT_ELIGIBILITY_WINDOW_SECONDS:-3600}", "ACTOR_REFRESH_TTL_SECONDS" => "${ACTOR_REFRESH_TTL_SECONDS:-86400}", "REPOSITORY_REFRESH_TTL_SECONDS" => "${REPOSITORY_REFRESH_TTL_SECONDS:-86400}" ) diff --git a/spec/jobs/poll_event_source_job_spec.rb b/spec/jobs/poll_event_source_job_spec.rb index b29a1d8..70c4dfe 100644 --- a/spec/jobs/poll_event_source_job_spec.rb +++ b/spec/jobs/poll_event_source_job_spec.rb @@ -15,6 +15,10 @@ before { allow(Github::IngestionRunner).to receive(:new).and_return(runner) } + it "runs on the polling queue, isolated from backlog work" do + expect(described_class.new.queue_name).to eq("polling") + end + def poll!(job = described_class.new) job.perform_now job diff --git a/spec/jobs/reconcile_pending_enrichments_job_spec.rb b/spec/jobs/reconcile_pending_enrichments_job_spec.rb index 3a2803b..62b8eab 100644 --- a/spec/jobs/reconcile_pending_enrichments_job_spec.rb +++ b/spec/jobs/reconcile_pending_enrichments_job_spec.rb @@ -7,7 +7,11 @@ # The recovery property it exists for — work committed before a crash but never enqueued — # is spec/recovery/pending_enrichment_recovery_spec.rb. RSpec.describe ReconcilePendingEnrichmentsJob do - before { active_budget_window(now: frozen_time) } + before { active_budget_window(now: Time.current) } + + it "runs on the bounded control queue rather than behind enrichment work" do + expect(described_class.new.queue_name).to eq("control") + end it "reconciles, and reports what it scheduled on the job's line" do create_actor(github_id: 583_231, last_seen_at: Time.current) diff --git a/spec/models/github_actor_spec.rb b/spec/models/github_actor_spec.rb index de6a28f..126f855 100644 --- a/spec/models/github_actor_spec.rb +++ b/spec/models/github_actor_spec.rb @@ -64,17 +64,13 @@ expect(actor.enrichment_status).to eq("complete") end - # Plan §7: a duplicate event replay may refresh harmless identity fields but can - # never reactivate enrichment, or a re-polled window would resurrect skipped - # entities with no new activity. - it "leaves a budget-skipped entity skipped, with its failure state intact" do + it "refreshes identity without clearing a retryable entity's failure state" do described_class.upsert_stub!(github_id: 4242, login: "octocat", now: frozen_time) described_class.where(github_id: 4242).update_all( - enrichment_status: "skipped_budget", - skipped_at: frozen_time, + enrichment_status: "retryable_failure", enrichment_attempts: 3, next_retry_at: frozen_time + 3600, - last_error: "enrichment allowance exhausted" + last_error: "GitHub unavailable" ) described_class.upsert_stub!(github_id: 4242, login: "octocat-renamed", @@ -82,11 +78,10 @@ actor = described_class.find_by(github_id: 4242) expect(actor.login).to eq("octocat-renamed") - expect(actor.enrichment_status).to eq("skipped_budget") - expect(actor.skipped_at).to eq(frozen_time) + expect(actor.enrichment_status).to eq("retryable_failure") expect(actor.enrichment_attempts).to eq(3) expect(actor.next_retry_at).to eq(frozen_time + 3600) - expect(actor.last_error).to eq("enrichment allowance exhausted") + expect(actor.last_error).to eq("GitHub unavailable") end # Sources commit independently and events arrive late, so an out-of-order envelope diff --git a/spec/models/github_repository_spec.rb b/spec/models/github_repository_spec.rb index 6520cba..81a0c84 100644 --- a/spec/models/github_repository_spec.rb +++ b/spec/models/github_repository_spec.rb @@ -116,15 +116,14 @@ expect(repository.enrichment_status).to eq("complete") end - it "leaves a budget-skipped entity skipped, with its failure state intact" do + it "refreshes identity without clearing a retryable entity's failure state" do described_class.upsert_stub!(github_id: 8484, full_name: "octocat/hello-world", now: frozen_time) described_class.where(github_id: 8484).update_all( - enrichment_status: "skipped_budget", - skipped_at: frozen_time, + enrichment_status: "retryable_failure", enrichment_attempts: 2, next_retry_at: frozen_time + 3600, - last_error: "enrichment allowance exhausted" + last_error: "GitHub unavailable" ) described_class.upsert_stub!(github_id: 8484, full_name: "octocat/renamed", @@ -132,10 +131,10 @@ repository = described_class.find_by(github_id: 8484) expect(repository.full_name).to eq("octocat/renamed") - expect(repository.enrichment_status).to eq("skipped_budget") - expect(repository.skipped_at).to eq(frozen_time) + expect(repository.enrichment_status).to eq("retryable_failure") expect(repository.enrichment_attempts).to eq(2) - expect(repository.last_error).to eq("enrichment allowance exhausted") + expect(repository.next_retry_at).to eq(frozen_time + 3600) + expect(repository.last_error).to eq("GitHub unavailable") end end diff --git a/spec/queue/configuration_spec.rb b/spec/queue/configuration_spec.rb index 43a2ce3..4a45c0b 100644 --- a/spec/queue/configuration_spec.rb +++ b/spec/queue/configuration_spec.rb @@ -43,6 +43,13 @@ end end + it "routes every recurring task onto a queue with a configured consumer" do + expect(recurring.fetch("production").transform_values { _1.fetch("queue") }) + .to eq("poll_event_sources" => "polling", + "reconcile_pending_enrichments" => "control", + "clear_solid_queue_finished_jobs" => "control") + end + # Solid Queue's recurring uniqueness guarantee — the unique index on (task_key, run_at) — # holds "as long as you keep the jobs around", so preserve_finished_jobs stays at its # default and the installer's hourly cleanup is what bounds the table instead. @@ -61,16 +68,22 @@ end # A job enqueued into a queue no worker polls is a silent, total failure, and nothing else - # in the suite would catch it. Every job here uses the default queue, so one worker on "*" - # is the whole guarantee. + # in the suite would catch it. it "works every queue this application enqueues into" do - # Through an instance: Active Job's default queue name is a lambda until a job resolves - # it. queues = [ PollEventSourceJob, EnrichActorJob, EnrichRepositoryJob, - ReconcilePendingEnrichmentsJob ].map { _1.new.queue_name }.uniq + ReconcilePendingEnrichmentsJob ].map { _1.new.queue_name }.uniq.sort + configured = queue_config.dig("production", "workers") + .flat_map { _1.fetch("queues").split(",") }.uniq.sort + + expect(queues).to eq(%w[control enrichment polling]) + expect(configured).to eq(queues) + end + + it "isolates the durable enrichment backlog from polling and control work" do + workers = queue_config.dig("production", "workers").index_by { _1.fetch("queues") } - expect(queues).to eq([ "default" ]) - expect(queue_config.dig("production", "workers").map { _1["queues"] }).to all(eq("*")) + expect(workers.keys).to contain_exactly("polling,control", "enrichment") + expect(workers.values).to all(include("threads" => 1, "processes" => 1)) end # §5's request gate makes outbound concurrency exactly one application-wide, so extra diff --git a/spec/queue/solid_queue_integration_spec.rb b/spec/queue/solid_queue_integration_spec.rb index 6d2a30f..0e1df85 100644 --- a/spec/queue/solid_queue_integration_spec.rb +++ b/spec/queue/solid_queue_integration_spec.rb @@ -16,7 +16,7 @@ job = SolidQueue::Job.last expect(job.class_name).to eq("EnrichActorJob") - expect(job.queue_name).to eq("default") + expect(job.queue_name).to eq("enrichment") expect(SolidQueue::ReadyExecution.where(job_id: job.id)).to exist end @@ -70,8 +70,15 @@ expect(configuration).to be_valid, -> { configuration.errors.full_messages.join("; ") } end - it "configures the three processes the worker container runs" do - expect(configuration.configured_processes.map(&:kind)).to contain_exactly(:dispatcher, :worker, :scheduler) + it "configures a dispatcher, scheduler, and isolated control and enrichment workers" do + expect(configuration.configured_processes.map(&:kind)) + .to contain_exactly(:dispatcher, :worker, :worker, :scheduler) + + worker_queues = configuration.configured_processes + .select { |process| process.kind == :worker } + .map { |process| process.attributes.fetch(:queues) } + + expect(worker_queues).to contain_exactly("polling,control", "enrichment") end it "hands the scheduler this application's two ticks" do diff --git a/spec/recovery/concurrent_write_spec.rb b/spec/recovery/concurrent_write_spec.rb index 34552e7..5a48a7e 100644 --- a/spec/recovery/concurrent_write_spec.rb +++ b/spec/recovery/concurrent_write_spec.rb @@ -66,12 +66,12 @@ # looks. Raw SQL rather than Active Record, because the pooled connection is inside the # example's fixture transaction and anything written through it would be invisible to the # index this file is about. - def commit_actor!(status: "pending", skipped_at: nil) - second_session.exec_params(<<~SQL, [ ACTOR_ID, "octocat", status, stamp, skipped_at ]) + def commit_actor!(status: "pending", next_retry_at: nil, last_error: nil) + second_session.exec_params(<<~SQL, [ ACTOR_ID, "octocat", status, stamp, next_retry_at, last_error ]) INSERT INTO github_actors (github_id, login, display_login, api_url, enrichment_status, enrichment_attempts, - last_seen_at, first_seen_at, created_at, updated_at, skipped_at) - VALUES ($1, $2, $2, 'https://api.github.com/users/octocat', $3, 0, $4, $4, $4, $4, $5) + last_seen_at, first_seen_at, created_at, updated_at, next_retry_at, last_error) + VALUES ($1, $2, $2, 'https://api.github.com/users/octocat', $3, 0, $4, $4, $4, $4, $5, $6) SQL end @@ -121,21 +121,20 @@ def commit_push_event! end end - # §7's rule 4 — "duplicate replays may refresh identity fields but must never reactivate a - # skipped_budget entity" — where the duplicate is another session's commit rather than this - # session's own earlier write. - describe "a skipped entity whose event another poller committed first" do + describe "a retryable entity whose event another poller committed first" do before do - commit_actor!(status: "skipped_budget", skipped_at: stamp) + commit_actor!(status: "retryable_failure", next_retry_at: stamp, + last_error: "GitHub unavailable") commit_repository! commit_push_event! end - it "cannot be reactivated by losing the race" do + it "preserves its failure state when this poller loses the event-insert race" do writer.write([ well_formed_envelope ], run_id: SecureRandom.uuid) expect(GithubActor.find_by(github_id: ACTOR_ID)) - .to have_attributes(enrichment_status: "skipped_budget", skipped_at: frozen_time) + .to have_attributes(enrichment_status: "retryable_failure", + next_retry_at: frozen_time, last_error: "GitHub unavailable") end end diff --git a/spec/recovery/crash_window_spec.rb b/spec/recovery/crash_window_spec.rb index 8dfde96..a04958a 100644 --- a/spec/recovery/crash_window_spec.rb +++ b/spec/recovery/crash_window_spec.rb @@ -133,31 +133,35 @@ def replay_whole_page expect(GithubRepository.count).to eq(3) end - # ADR 0005's fourth mechanism, holding across a crash boundary rather than across a plain - # replay: the duplicated envelopes produce no RETURNING row, so the reactivation they - # would otherwise trigger never runs. - it "reactivates nothing the prefix already recorded" do + it "preserves retry state for entities in the duplicated prefix" do write_prefix GithubActor.where(github_id: IngestionHelpers::ACTOR_GITHUB_ID) - .update_all(enrichment_status: "skipped_budget", skipped_at: frozen_time) + .update_all(enrichment_status: "retryable_failure", + next_retry_at: frozen_time + 3600, + last_error: "GitHub unavailable") replay_whole_page expect(GithubActor.find_by(github_id: IngestionHelpers::ACTOR_GITHUB_ID)) - .to have_attributes(enrichment_status: "skipped_budget", skipped_at: frozen_time) + .to have_attributes(enrichment_status: "retryable_failure", + next_retry_at: frozen_time + 3600, + last_error: "GitHub unavailable") end - # The other half of the same rule, so the first is not passing merely because nothing - # reactivates anything: a genuinely new event for the same entity does. - it "still reactivates that entity for an event the crash had not yet seen" do + it "registers activity for an event the crash had not yet seen without clearing retry state" do write_prefix GithubActor.where(github_id: IngestionHelpers::ACTOR_GITHUB_ID) - .update_all(enrichment_status: "skipped_budget", skipped_at: frozen_time) + .update_all(enrichment_status: "retryable_failure", + next_retry_at: frozen_time + 3600, + last_error: "GitHub unavailable") - writer.write([ well_formed_envelope("id" => "58000009999") ], run_id: SecureRandom.uuid) + writer.write([ well_formed_envelope("id" => "58000009999") ], + run_id: SecureRandom.uuid) expect(GithubActor.find_by(github_id: IngestionHelpers::ACTOR_GITHUB_ID)) - .to have_attributes(enrichment_status: "pending", skipped_at: nil) + .to have_attributes(enrichment_status: "retryable_failure", + next_retry_at: frozen_time + 3600, + last_error: "GitHub unavailable") end # PageWriter#quarantine is deliberately one statement outside every transaction. A crash @@ -241,12 +245,8 @@ def replay_whole_page let(:actor_type) { Github::Enrichment::EntityType.fetch(:actor) } let!(:event_source) { fixture_event_source } - # Ingested at Time.current rather than at frozen_time, for the reason - # pending_enrichment_recovery_spec.rb states: Solid Queue constructs - # ReconcilePendingEnrichmentsJob, so there is no clock to inject into it and its sweep - # measures §10's eligibility window against the wall clock. Entities stamped in 2026-07-29 - # would be outside that window today, and the reconciler would correctly find nothing — - # which would make this example pass for a reason that has nothing to do with recovery. + # Ingested at Time.current because Solid Queue constructs the reconciliation job and its + # operational timestamps should match the worker clock used in this recovery scenario. let(:crashed_at) { Time.current } before { active_budget_window(now: crashed_at) } diff --git a/spec/recovery/duplicate_job_execution_spec.rb b/spec/recovery/duplicate_job_execution_spec.rb index a002eba..fc39622 100644 --- a/spec/recovery/duplicate_job_execution_spec.rb +++ b/spec/recovery/duplicate_job_execution_spec.rb @@ -2,7 +2,7 @@ # §12's "Enrichment job executed twice" exercises one redelivery case. The ingestion-wide # guarantees are narrower: a duplicate event ID cannot create another push_events row or -# reactivate a skipped entity. Executions, run summaries, quarantine counters, budget use, +# register new entity activity. Executions, run summaries, quarantine counters, budget use, # and logs can repeat. Here the second execution really happens, and this scenario asserts # only that an already-complete actor is left unchanged by the freshness check. # diff --git a/spec/recovery/pending_enrichment_recovery_spec.rb b/spec/recovery/pending_enrichment_recovery_spec.rb index 8f5deb9..97941bd 100644 --- a/spec/recovery/pending_enrichment_recovery_spec.rb +++ b/spec/recovery/pending_enrichment_recovery_spec.rb @@ -8,15 +8,14 @@ # what it leaves behind is exactly this state, and §2A's claim is that this state is # recoverable *because* the entity rows are the durable record of pending work. # -# The page is ingested at Time.current rather than at frozen_time, because the reconciler -# reads the clock the worker will actually be holding — Solid Queue constructs the job, so -# there is nothing to inject — and §10's eligibility window is measured against it. +# The page is ingested at Time.current because the reconciler reads the worker's clock and +# Solid Queue constructs the job, so there is no test clock to inject into that boundary. RSpec.describe "recovering enrichment work that was never enqueued", type: :integration do let(:transport) { fixture_transport } let(:ingested_at) { Time.current } before do - active_budget_window(now: frozen_time) + active_budget_window(now: ingested_at) fixture_runner(transport: transport, now: ingested_at).call(event_source: fixture_event_source) # This line is the crash: the four push events and their six stub entities are committed, @@ -37,6 +36,17 @@ .and have_enqueued_job(EnrichRepositoryJob).exactly(:once) end + it "rediscovers pending rows even when their durable insertion time is very old" do + GithubActor.update_all(created_at: ingested_at - 30.days, + last_seen_at: ingested_at - 30.days) + GithubRepository.update_all(created_at: ingested_at - 30.days, + last_seen_at: ingested_at - 30.days) + + expect { ReconcilePendingEnrichmentsJob.perform_now } + .to have_enqueued_job(EnrichActorJob).exactly(:once) + .and have_enqueued_job(EnrichRepositoryJob).exactly(:once) + end + # §8: "a small, entity-scoped set, not N event rows per entity." Three actors behind four # events are one cycle, not three and not four — the queue is not where the backlog lives. it "schedules one cycle per class, not one per pending entity or per event" do diff --git a/spec/services/github/configuration_spec.rb b/spec/services/github/configuration_spec.rb index 127a351..e0fcf1a 100644 --- a/spec/services/github/configuration_spec.rb +++ b/spec/services/github/configuration_spec.rb @@ -25,7 +25,6 @@ def configuration(**overrides) # §10's enrichment block. Rational("0.50") == 0.5, so the literal reads naturally # while the arithmetic stays exact. actor_enrichment_share: 0.5, - enrichment_eligibility_window_seconds: 3600, actor_refresh_ttl_seconds: 86_400, repository_refresh_ttl_seconds: 86_400, # §11's coverage window, pinned at 86400 by §10. It arrives with the rich /status @@ -41,7 +40,6 @@ def configuration(**overrides) it "reports through the coverage window without scheduling, reserving or deferring on it" do expect(configuration(ENRICHMENT_COVERAGE_WINDOW_SECONDS: "60")) .to have_attributes(enrichment_coverage_window_seconds: 60, - enrichment_eligibility_window_seconds: 3600, poll_interval_seconds: 300) end @@ -146,11 +144,6 @@ def configuration(**overrides) expect(configuration(ACTOR_ENRICHMENT_SHARE: "0.29").actor_enrichment_share).to eq(Rational(29, 100)) end - it "rejects a non-positive eligibility window, which would skip every candidate on sight" do - expect { configuration(ENRICHMENT_ELIGIBILITY_WINDOW_SECONDS: "0").validate! } - .to raise_error(Github::Errors::ConfigurationError, /ENRICHMENT_ELIGIBILITY_WINDOW_SECONDS/) - end - it "rejects a zero refresh TTL, which would turn the freshness cache off from a number alone" do expect { configuration(ACTOR_REFRESH_TTL_SECONDS: "0").validate! } .to raise_error(Github::Errors::ConfigurationError, /ACTOR_REFRESH_TTL_SECONDS/) diff --git a/spec/services/github/enrichment/age_out_spec.rb b/spec/services/github/enrichment/age_out_spec.rb deleted file mode 100644 index 8086b54..0000000 --- a/spec/services/github/enrichment/age_out_spec.rb +++ /dev/null @@ -1,149 +0,0 @@ -require "rails_helper" - -RSpec.describe Github::Enrichment::AgeOut do - subject(:age_out) { described_class.new(configuration: configuration_with) } - - let(:now) { frozen_time } - let(:aged) { now - 3601 } - let(:actor_type) { Github::Enrichment::EntityType.fetch(:actor) } - - def sweep = age_out.call(now: now) - - describe "the eligibility window (plan §10, B8)" do - it "skips a candidate whose activity aged past the window" do - actor = create_actor(github_id: 1, last_seen_at: aged) - - expect(sweep.fetch(actor_type)).to eq(1) - expect(actor.reload).to have_attributes(enrichment_status: "skipped_budget", skipped_at: now) - end - - it "keeps a candidate that is still inside the window" do - actor = create_actor(github_id: 1, last_seen_at: now - 3599) - - expect(sweep.fetch(actor_type)).to eq(0) - expect(actor.reload.enrichment_status).to eq("pending") - end - - it "skips a retryable failure too, which is the other half of the candidate set" do - create_actor(github_id: 1, last_seen_at: aged, enrichment_status: "retryable_failure") - - expect(sweep.fetch(actor_type)).to eq(1) - end - - it "never touches a complete or terminally failed row, which are not backlog" do - complete = create_actor(github_id: 1, last_seen_at: aged, enrichment_status: "complete", - fetched_at: aged) - permanent = create_actor(github_id: 2, last_seen_at: aged, enrichment_status: "permanent_failure") - - sweep - - expect(complete.reload.enrichment_status).to eq("complete") - expect(permanent.reload.enrichment_status).to eq("permanent_failure") - end - - # The totality argument behind B8: without COALESCE such a row is neither eligible - # (NULL > floor is NULL) nor ageable, and it would sit pending forever. - it "keeps a stub with no last_seen_at until one window after it was created" do - fresh = create_actor(github_id: 1, last_seen_at: nil) - old = create_actor(github_id: 2, last_seen_at: nil, created_at: aged) - - sweep - - expect(fresh.reload.enrichment_status).to eq("pending") - expect(old.reload.enrichment_status).to eq("skipped_budget") - end - - it "sweeps both classes on every call, so neither backlog can grow unattended" do - create_actor(github_id: 1, last_seen_at: aged) - create_repository(github_id: 2, last_seen_at: aged) - - expect(sweep.values.sum).to eq(2) - expect(GithubRepository.sole.enrichment_status).to eq("skipped_budget") - end - end - - describe "what it deliberately leaves alone" do - # The same due predicate the two candidate pools use — one clause, so a leased row, - # a backed-off row and a secondary-limit deferral are all excluded at once. - it "never skips an entity another worker is currently enriching" do - actor = create_actor(github_id: 1, last_seen_at: aged, next_retry_at: now + 600) - - expect(sweep.fetch(actor_type)).to eq(0) - expect(actor.reload.enrichment_status).to eq("pending") - end - - it "never skips an entity whose backoff has not expired" do - create_actor(github_id: 1, last_seen_at: aged, next_retry_at: now + 60) - - expect(sweep.fetch(actor_type)).to eq(0) - end - - # A skip is not an attempt and knows nothing about failures. - it "leaves the attempt count and the last error alone" do - actor = create_actor(github_id: 1, last_seen_at: aged, enrichment_attempts: 2, last_error: "boom") - - sweep - - expect(actor.reload).to have_attributes(enrichment_attempts: 2, last_error: "boom") - end - - # This is what makes reactivation's "immediately due" property provable: the WHERE - # requires next_retry_at to be NULL or already past, so no skipped_budget row can carry - # a future instant, and Enrichable#reactivate_skipped! needs no clearing clause. - it "leaves next_retry_at alone, so a reactivated entity is provably due at once" do - create_actor(github_id: 1, last_seen_at: aged, next_retry_at: now - 60) - - sweep - - expect(GithubActor.where(enrichment_status: "skipped_budget").where(next_retry_at: now..)).to be_empty - end - end - - describe "the bounded batch" do - # An unbounded UPDATE over §10's ~2,000 candidates an hour would eventually hold - # hundreds of thousands of row locks in one statement. The ordering makes the residue - # always the *least* overdue, so progress is monotone. - it "sweeps the most overdue candidates first when the batch is bounded" do - stub_const("#{described_class}::BATCH_SIZE", 1) - recent = create_actor(github_id: 1, last_seen_at: aged) - oldest = create_actor(github_id: 2, last_seen_at: now - 100_000) - - sweep - - expect(oldest.reload.enrichment_status).to eq("skipped_budget") - expect(recent.reload.enrichment_status).to eq("pending") - end - - it "finishes the backlog across successive calls" do - stub_const("#{described_class}::BATCH_SIZE", 1) - create_actor(github_id: 1, last_seen_at: aged) - create_actor(github_id: 2, last_seen_at: aged) - - 2.times { sweep } - - expect(GithubActor.where(enrichment_status: "skipped_budget").count).to eq(2) - end - end - - describe "logging" do - # §11 puts "skipped" at INFO, but one line per row would emit thousands in a single - # sweep — the volume argument BudgetLedger#log_class_exhausted already makes. - it "logs one summary per class rather than one line per row" do - allow(Rails.logger).to receive(:info) - 3.times { |index| create_actor(github_id: index, last_seen_at: aged) } - - sweep - - expect(Rails.logger).to have_received(:info) - .with(hash_including(event: "enrichment.aged_out", entity_type: :actor, skipped_count: 3)).once - end - - it "says nothing at all when a class had nothing to skip" do - allow(Rails.logger).to receive(:info) - - sweep - - expect(Rails.logger).not_to have_received(:info).with(hash_including(event: "enrichment.aged_out")) - end - end -end diff --git a/spec/services/github/enrichment/backlog_metrics_spec.rb b/spec/services/github/enrichment/backlog_metrics_spec.rb new file mode 100644 index 0000000..e743125 --- /dev/null +++ b/spec/services/github/enrichment/backlog_metrics_spec.rb @@ -0,0 +1,80 @@ +require "rails_helper" + +RSpec.describe Github::Enrichment::BacklogMetrics do + let(:now) { frozen_time } + + def capture = described_class.capture(now: now) + + it "counts pending and retryable rows even when their next attempt is deferred" do + create_actor(github_id: 1, enrichment_status: "pending") + create_actor(github_id: 2, enrichment_status: "retryable_failure", + next_retry_at: now + 3600) + create_actor(github_id: 3, enrichment_status: "complete", fetched_at: now) + create_actor(github_id: 4, enrichment_status: "permanent_failure") + + expect(capture.actor).to have_attributes( + status_counts: { + "pending" => 1, "complete" => 1, + "retryable_failure" => 1, "permanent_failure" => 1 + }, + backlog_count: 2 + ) + end + + it "uses created_at for the oldest wait, matching FIFO selection" do + create_actor(github_id: 1, created_at: now - 300, last_seen_at: now) + create_actor(github_id: 2, created_at: now - 900, last_seen_at: now - 10) + + expect(capture.actor).to have_attributes( + backlog_count: 2, + oldest_pending_at: now - 900, + oldest_pending_age_seconds: 900 + ) + end + + it "reports each entity class independently" do + create_actor(github_id: 1, created_at: now - 300) + create_repository(github_id: 2, created_at: now - 600) + + expect(capture.actor).to have_attributes(backlog_count: 1, + oldest_pending_at: now - 300) + expect(capture.repository).to have_attributes(backlog_count: 1, + oldest_pending_at: now - 600) + end + + it "reports nil oldest metrics for an empty backlog" do + entry = capture.actor + + expect(entry).to have_attributes(backlog_count: 0, oldest_pending_at: nil, + oldest_pending_age_seconds: nil) + end + + it "clamps harmless database-clock skew instead of reporting a negative age" do + create_actor(github_id: 1, created_at: now + 1) + + expect(capture.actor.oldest_pending_age_seconds).to eq(0) + end + + it "reads persisted state without writing or initiating a GitHub request" do + create_actor(github_id: 1) + transport = fixture_transport + allow(Github).to receive(:transport).and_return(transport) + + expect(write_statements { capture }).to be_empty + expect(transport.requests).to be_empty + end + + it "captures counts and the oldest row in one aggregate statement per entity class" do + create_actor(github_id: 1) + create_repository(github_id: 2) + + statements = capture_sql { capture } + actor_reads = statements.grep(/FROM "github_actors"/) + repository_reads = statements.grep(/FROM "github_repositories"/) + + expect(actor_reads.one?).to be(true) + expect(repository_reads.one?).to be(true) + expect(actor_reads.first).to include("COUNT(*) FILTER", "MIN(") + expect(repository_reads.first).to include("COUNT(*) FILTER", "MIN(") + end +end diff --git a/spec/services/github/enrichment/backoff_spec.rb b/spec/services/github/enrichment/backoff_spec.rb index e5a6fee..a09ad61 100644 --- a/spec/services/github/enrichment/backoff_spec.rb +++ b/spec/services/github/enrichment/backoff_spec.rb @@ -34,12 +34,8 @@ expect(full_jitter.delay_for(20)).to eq(described_class::MAX_SECONDS.to_f) end - # The cap equals the pinned ENRICHMENT_ELIGIBILITY_WINDOW_SECONDS, so a backoff can - # never outlast the window that would age the row into skipped_budget — the cap is - # never what strands a row. - it "caps at exactly the pinned eligibility window, so a backoff never outlives it" do - expect(described_class::MAX_SECONDS) - .to eq(Github::Configuration::DEFAULTS.fetch("ENRICHMENT_ELIGIBILITY_WINDOW_SECONDS").to_i) + it "caps at one rate-limit window so a repeatedly failing row periodically rejoins the FIFO" do + expect(described_class::MAX_SECONDS).to eq(3600) end # Additive only: subtracting could schedule a retry sooner than the floor, and the diff --git a/spec/services/github/enrichment/candidate_selector_spec.rb b/spec/services/github/enrichment/candidate_selector_spec.rb index 3d75c6f..f531034 100644 --- a/spec/services/github/enrichment/candidate_selector_spec.rb +++ b/spec/services/github/enrichment/candidate_selector_spec.rb @@ -8,7 +8,6 @@ let(:actor_type) { Github::Enrichment::EntityType.fetch(:actor) } let(:repository_type) { Github::Enrichment::EntityType.fetch(:repository) } - # Inside the pinned 3600-second eligibility window, and safely clear of its edge. def pending_actor(github_id:, last_seen_at: now - 60, **overrides) create_actor(github_id: github_id, last_seen_at: last_seen_at, **overrides) end @@ -22,22 +21,21 @@ def next_refresh(entity_type = actor_type) end describe "the pending pool" do - # §10: "Among pending candidates the service enriches newest-first (last_seen_at)." - it "enriches newest-first, because the freshest activity is the most worth sampling" do - pending_actor(github_id: 1, last_seen_at: now - 600) - newest = pending_actor(github_id: 2, last_seen_at: now - 10) + it "enriches oldest-created first, so every durable backlog row advances toward service" do + oldest = pending_actor(github_id: 1, created_at: now - 600) + pending_actor(github_id: 2, created_at: now - 10) - expect(next_pending).to eq(newest) + expect(next_pending).to eq(oldest) end - # PageWriter stamps one received_at for a whole page, so every entity on a page shares - # an identical last_seen_at. Without a second key the order is plan-dependent and every - # example below would be flaky rather than wrong. - it "breaks a tie deterministically, because one page gives every entity the same last_seen_at" do - pending_actor(github_id: 1, last_seen_at: now - 60) - tied = pending_actor(github_id: 2, last_seen_at: now - 60) + # PageWriter creates many stubs in one page with one timestamp. The ascending id tie + # break preserves insertion order rather than letting PostgreSQL choose a plan-dependent + # winner on every reconciliation tick. + it "breaks a created-at tie by ascending id" do + first = pending_actor(github_id: 1, created_at: now - 60) + pending_actor(github_id: 2, created_at: now - 60) - expect(next_pending).to eq(tied) + expect(next_pending).to eq(first) end it "offers a retryable failure alongside a pending row, which is what the index predicate says" do @@ -58,45 +56,33 @@ def next_refresh(entity_type = actor_type) expect(next_pending).to eq(due) end - it "excludes a candidate whose activity aged past the eligibility window" do - pending_actor(github_id: 1, last_seen_at: now - 3601) + it "keeps an old candidate claimable until it is eventually enriched" do + old = pending_actor(github_id: 1, last_seen_at: now - 100_000, + created_at: now - 100_000) - expect(next_pending).to be_nil + expect(next_pending).to eq(old) end it "excludes a terminal status, which no amount of budget would help" do pending_actor(github_id: 1, enrichment_status: "permanent_failure") - pending_actor(github_id: 2, enrichment_status: "skipped_budget") expect(next_pending).to be_nil end # A stub can be created with a NULL last_seen_at: PageWriter upserts the stub, the # push_events insert returns nil on a duplicate, and the transaction still commits. - # Without COALESCE such a row is neither eligible (NULL > floor is NULL) nor ageable, - # and it would sit pending forever — which is exactly what B8 forbids. - it "keeps a stub with no last_seen_at eligible for one window after it was created" do - stub = create_actor(github_id: 1, last_seen_at: nil) + # created_at is total and immutable, so the durable FIFO can still order this work. + it "keeps a stub with no last_seen_at claimable regardless of age" do + stub = create_actor(github_id: 1, last_seen_at: nil, created_at: now - 100_000) expect(next_pending).to eq(stub) end - it "ages a stub with no last_seen_at out one window after it was created" do - create_actor(github_id: 1, last_seen_at: nil, created_at: now - 3601) + it "orders solely by durable insertion time, not later activity" do + oldest = pending_actor(github_id: 1, created_at: now - 600, last_seen_at: now - 10) + pending_actor(github_id: 2, created_at: now - 300, last_seen_at: now - 200) - expect(next_pending).to be_nil - expect(selector.expired_scope(actor_type, now: now).count).to eq(1) - end - - # created_at is safe in the *bound*, where it can only shorten a row's life, and unsafe - # in the *order*, where it means "we saw an envelope" — which a duplicate replay also - # produces. §10 pins the ordering key by name to last_seen_at, the only column that - # means proven distinct activity. - it "orders a stub with no last_seen_at behind every candidate that has one" do - create_actor(github_id: 1, last_seen_at: nil) - proven = pending_actor(github_id: 2, last_seen_at: now - 3000) - - expect(next_pending).to eq(proven) + expect(next_pending).to eq(oldest) end it "keeps the two classes apart, so a repository backlog never appears as actor work" do @@ -125,9 +111,8 @@ def complete_actor(github_id:, fetched_at:, **overrides) expect(next_refresh).to eq(stale) end - # Oldest-fetched first is the rule that terminates: a monotone queue cannot starve a - # complete row behind a hotter neighbour. §10's newest-first is scoped by its own words - # to "Among pending candidates". + # Oldest-fetched first is the rule that terminates: a monotone refresh queue cannot + # starve a complete row behind a hotter neighbour. it "refreshes the most stale record first, so no complete row can be starved" do complete_actor(github_id: 1, fetched_at: now - 90_000) oldest = complete_actor(github_id: 2, fetched_at: now - 200_000) @@ -150,9 +135,7 @@ def complete_actor(github_id:, fetched_at:, **overrides) expect(next_refresh(repository_type)).to be_nil end - # A refresh candidate is not subject to the eligibility window: it has a document - # already, and §10's window bounds the *backlog* of never-enriched work. - it "offers a refresh whose activity has long since aged out, because it is not backlog" do + it "offers a refresh regardless of how long ago the entity was referenced" do stale = complete_actor(github_id: 1, fetched_at: now - 90_000, last_seen_at: now - 100_000) expect(next_refresh).to eq(stale) @@ -160,13 +143,11 @@ def complete_actor(github_id:, fetched_at:, **overrides) end describe "#pending_available?" do - # §10's borrowing condition, verbatim: "the other class has no CURRENTLY ELIGIBLE - # candidate (not merely no rows)". - it "answers false for a class whose only rows are ineligible, not merely for an empty table" do - pending_actor(github_id: 1, last_seen_at: now - 3601) + it "answers true for old durable work rather than treating age as ineligibility" do + pending_actor(github_id: 1, last_seen_at: now - 100_000, created_at: now - 100_000) expect(GithubActor.count).to eq(1) - expect(selector.pending_available?(actor_type, now: now)).to be(false) + expect(selector.pending_available?(actor_type, now: now)).to be(true) end it "answers true while one eligible candidate remains" do @@ -185,6 +166,30 @@ def complete_actor(github_id:, fetched_at:, **overrides) end end + describe "#pending_backlog?" do + it "sees never-enriched work even while its retry is deferred" do + pending_actor(github_id: 1, enrichment_status: "retryable_failure", + next_retry_at: now + 3600) + + expect(selector.pending_available?(actor_type, now: now)).to be(false) + expect(selector.pending_backlog?(actor_type)).to be(true) + end + + it "does not count complete or permanently failed rows as never-enriched backlog" do + create_actor(github_id: 1, enrichment_status: "complete", fetched_at: now) + create_actor(github_id: 2, enrichment_status: "permanent_failure") + + expect(selector.pending_backlog?(actor_type)).to be(false) + end + + it "keeps entity classes independent" do + create_repository(github_id: 1) + + expect(selector.pending_backlog?(actor_type)).to be(false) + expect(selector.pending_backlog?(repository_type)).to be(true) + end + end + describe "#claimable?" do it "is true while a pending candidate is eligible" do pending_actor(github_id: 1) @@ -227,12 +232,11 @@ def complete_actor(github_id:, fetched_at:, **overrides) expect(selector.earliest_pending_at(actor_type, now: now)).to be_nil end - # It will be swept into skipped_budget rather than enriched, so naming its retry - # instant would promise an enrichment that is never going to happen. - it "ignores a candidate that has aged out, which will never become claimable" do - pending_actor(github_id: 1, next_retry_at: now + 60, last_seen_at: now - 3601) + it "names the retry instant even for a very old candidate" do + pending_actor(github_id: 1, next_retry_at: now + 60, + last_seen_at: now - 100_000, created_at: now - 100_000) - expect(selector.earliest_pending_at(actor_type, now: now)).to be_nil + expect(selector.earliest_pending_at(actor_type, now: now)).to eq(now + 60) end end @@ -293,11 +297,11 @@ def complete_actor(github_id:, fetched_at:, **overrides) end describe "#earliest_claimable_at" do - it "takes whichever pool comes back first" do + it "keeps refresh timing subordinate to the deferred first-time backlog" do pending_actor(github_id: 1, next_retry_at: now + 300) create_actor(github_id: 2, enrichment_status: "complete", fetched_at: now - 86_340) - expect(selector.earliest_claimable_at(actor_type, now: now)).to eq(now + 60) + expect(selector.earliest_claimable_at(actor_type, now: now)).to eq(now + 300) end it "falls back to the refresh pool when nothing is pending at all" do @@ -311,24 +315,6 @@ def complete_actor(github_id:, fetched_at:, **overrides) end end - describe "#expired_scope" do - it "is exactly the complement of the pending pool among due candidates" do - inside = pending_actor(github_id: 1, last_seen_at: now - 3599) - outside = pending_actor(github_id: 2, last_seen_at: now - 3601) - - expect(selector.scope(actor_type, pool: :pending, now: now).to_a).to eq([ inside ]) - expect(selector.expired_scope(actor_type, now: now).to_a).to eq([ outside ]) - end - - # The same due predicate both pools use, which is what keeps a leased row out of the - # sweep without a second condition anywhere. - it "excludes a row another worker holds, because its lease sits in the future" do - pending_actor(github_id: 1, last_seen_at: now - 3601, next_retry_at: now + 600) - - expect(selector.expired_scope(actor_type, now: now)).to be_empty - end - end - it "refuses an unknown pool rather than silently selecting nothing" do expect { selector.scope(actor_type, pool: :everything, now: now) } .to raise_error(ArgumentError, /everything/) diff --git a/spec/services/github/enrichment/coverage_spec.rb b/spec/services/github/enrichment/coverage_spec.rb index 6ff554e..e0c5ea1 100644 --- a/spec/services/github/enrichment/coverage_spec.rb +++ b/spec/services/github/enrichment/coverage_spec.rb @@ -85,14 +85,14 @@ def event(id, actor:, repository:, created_at: now - 60, occurred_at: now - 60) events_with_both_entities_enriched_pct: 0.0) end - it "counts only complete, not the other four statuses" do - %w[pending retryable_failure permanent_failure skipped_budget].each_with_index do |status, n| + it "counts only complete, not the other three statuses" do + %w[pending retryable_failure permanent_failure].each_with_index do |status, n| other = create_actor(github_id: 3000 + n, login: "user#{n}") other.update!(enrichment_status: status) event("4000000010#{n}", actor: other, repository: repository) end - expect(capture).to have_attributes(actor_count: 4, complete_actor_count: 0, + expect(capture).to have_attributes(actor_count: 3, complete_actor_count: 0, actor_coverage_pct: 0.0) end diff --git a/spec/services/github/enrichment/dispatch_spec.rb b/spec/services/github/enrichment/dispatch_spec.rb index 6b75d84..88c3e36 100644 --- a/spec/services/github/enrichment/dispatch_spec.rb +++ b/spec/services/github/enrichment/dispatch_spec.rb @@ -116,15 +116,39 @@ def repository(**overrides) end end - # A clean checkout has no ledger row: nothing seeds it, and only a reservation inside the - # request gate creates one. Reading a schedule must not. + # A clean checkout has no ledger row: the first enrichment reservation would create an + # uninitialized row and be denied until a poll supplies authoritative headers. Dispatch + # avoids enqueueing that known-no-op while remaining read-only. describe "before the first window exists" do - it "enqueues without creating the ledger row" do + it "does not enqueue or create the ledger row" do GithubApiBudget.delete_all actor - expect { dispatch.call(reason: "ingestion") }.to have_enqueued_job(EnrichActorJob) + expect { dispatch.call(reason: "ingestion") }.not_to have_enqueued_job expect(GithubApiBudget.count).to eq(0) + expect(dispatch.call(reason: "reconcile")).to include(blocked_by: :window_uninitialized) + end + + it "does not enqueue when an uninitialized ledger row already exists" do + GithubApiBudget.delete_all + Github::BudgetLedger.new.bootstrap!(now: frozen_time) + actor + + expect { dispatch.call(reason: "reconcile") }.not_to have_enqueued_job + expect(dispatch.call(reason: "reconcile")).to include(blocked_by: :window_uninitialized) + end + + [ 0, 40 ].each do |used| + it "does not enqueue after an old window elapses with #{used} enrichment attempts used" do + active_budget_window(now: frozen_time - 3600, reset_at: frozen_time - 1, + enrichment_used: used, + actor_share_used: used / 2, + repository_share_used: used / 2) + actor + + expect { dispatch.call(reason: "reconcile") }.not_to have_enqueued_job + expect(dispatch.call(reason: "reconcile")).to include(blocked_by: :window_elapsed) + end end end diff --git a/spec/services/github/enrichment/end_to_end_spec.rb b/spec/services/github/enrichment/end_to_end_spec.rb index 3ccc99c..5bf78d1 100644 --- a/spec/services/github/enrichment/end_to_end_spec.rb +++ b/spec/services/github/enrichment/end_to_end_spec.rb @@ -116,9 +116,7 @@ def enrich!(cycles: 1, **arguments) end end - # §12's named sequence, in one example each: "Enrichment allowance exhaustion → deferred → - # skipped_budget → reactivation only via a genuinely new event." - describe "exhaustion, skip and reactivation" do + describe "durable backlog across quota windows" do let(:actor) { GithubActor.find_by(github_id: 583_231) } before { ingest! } @@ -131,35 +129,17 @@ def enrich!(cycles: 1, **arguments) expect(actor.reload.attributes).to eq(before) end - it "skips the entity once its activity ages past the eligibility window" do - later = now + 3601 - fixture_enrichment_runner(transport: transport, now: later).call - - expect(actor.reload).to have_attributes(enrichment_status: "skipped_budget", skipped_at: later) - end - - it "reactivates a skipped entity when a genuinely new event references it" do - GithubActor.where(github_id: 583_231) - .update_all(enrichment_status: "skipped_budget", skipped_at: now) - - Github::Ingestion::PageWriter.new(clock: -> { now + 60 }).write( - [ well_formed_envelope("id" => "58000000099", "payload" => { "push_id" => 27_500_000_099 }) ], - run_id: "reactivation" - ) - - expect(actor.reload).to have_attributes(enrichment_status: "pending", skipped_at: nil) - end - - # §7 rule 4, and the README's Phase B replay check: a duplicate replay must emit no - # enrichment.reactivated event. - it "never reactivates a skipped entity on a duplicate replay" do - GithubActor.where(github_id: 583_231) - .update_all(enrichment_status: "skipped_budget", skipped_at: now) + it "eventually enriches work that waited beyond one quota window" do + active_budget_window(now: now, enrichment_used: 40) + expect(runner.call.status).to eq("deferred") - Github::Ingestion::PageWriter.new(clock: -> { now + 60 }) - .write([ well_formed_envelope ], run_id: "replay") + next_window = now + 7200 + active_budget_window(now: next_window, poll_used: 0, enrichment_used: 0, + actor_share_used: 0, repository_share_used: 0) + result = fixture_enrichment_runner(transport: transport, now: next_window).call - expect(actor.reload).to have_attributes(enrichment_status: "skipped_budget", skipped_at: now) + expect(result).to have_attributes(status: "enriched", github_id: actor.github_id) + expect(actor.reload.enrichment_status).to eq("complete") end end diff --git a/spec/services/github/enrichment/entity_state_spec.rb b/spec/services/github/enrichment/entity_state_spec.rb index f9b35f6..2a9c125 100644 --- a/spec/services/github/enrichment/entity_state_spec.rb +++ b/spec/services/github/enrichment/entity_state_spec.rb @@ -66,14 +66,14 @@ def document_for(body) = Github::Enrichment::ActorDocument.parse(body, github_id expect(actor.reload.enrichment_attempts).to eq(0) end - # A stale error or skip instant on a successful row is a permanent lie — - # PollState#success's clearing argument, applied to the entity. - it "clears the failure and skip state, which would otherwise outlive the failure" do - actor.update!(last_error: "boom", skipped_at: now - 60) + # A stale error on a successful row is a permanent lie — PollState#success's clearing + # argument, applied to the entity. + it "clears failure state, which would otherwise outlive the failure" do + actor.update!(last_error: "boom") record(classification: :ok, status: 200, body: good_body, document: document_for(good_body)) - expect(actor.reload).to have_attributes(last_error: nil, skipped_at: nil) + expect(actor.reload.last_error).to be_nil end # The next event for this row is a *refresh*, gated by fetched_at plus the TTL rather @@ -259,8 +259,6 @@ def document_for(body) = Github::Enrichment::ActorDocument.parse(body, github_id expect(actor.reload.attributes).to eq(before) end - # §12's sequence is "exhaustion → deferred → skipped_budget": a denial writes nothing at - # all, and the row becomes skipped only later, when its activity ages out. it "leaves the row exactly as it found it on a budget denial" do before = actor.reload.attributes diff --git a/spec/services/github/enrichment/fairness_spec.rb b/spec/services/github/enrichment/fairness_spec.rb index fa7896d..1695db2 100644 --- a/spec/services/github/enrichment/fairness_spec.rb +++ b/spec/services/github/enrichment/fairness_spec.rb @@ -116,9 +116,9 @@ def pending_repository(github_id: 2, **overrides) borrow: false) end - it "borrows when the other class has rows that are merely ineligible, which is the plan's distinction" do + it "borrows when the other class's durable backlog is not currently due" do pending_actor - create_repository(github_id: 2, last_seen_at: now - 3601) + pending_repository(github_id: 2, next_retry_at: now + 300) expect(choose).to have_attributes(borrow: true, reason: "borrowed_pending") end @@ -148,9 +148,7 @@ def stale_actor(github_id: 1) last_seen_at: now - 60) end - # §10: "Within each class, never-enriched pending candidates always precede TTL-stale - # refreshes — a refresh spends budget only when no pending candidate is currently - # eligible." + # The durable first-time backlog always precedes TTL refreshes. it "prefers a pending candidate over a stale refresh" do stale_actor pending_repository @@ -174,6 +172,14 @@ def stale_actor(github_id: 1) expect(choose(entity_class: :actor)).to have_attributes(chosen?: false, reason: "no_candidate") end + it "does not promote a refresh while the only never-enriched row is backed off" do + stale_actor + pending_repository(enrichment_status: "retryable_failure", + next_retry_at: now + 3600) + + expect(choose).to have_attributes(chosen?: false, reason: "no_candidate") + end + # The other class has no refresh of its own to do, so nothing is starved by lending # its idle capacity — the same condition #pending_choice borrows under. it "borrows for a refresh once the class has spent its guarantee" do diff --git a/spec/services/github/enrichment/fairness_stress_spec.rb b/spec/services/github/enrichment/fairness_stress_spec.rb index c7d652e..23fb829 100644 --- a/spec/services/github/enrichment/fairness_stress_spec.rb +++ b/spec/services/github/enrichment/fairness_stress_spec.rb @@ -203,46 +203,30 @@ def flood_actors(count) end end - # Extension B's eighth bullet — "bound the enrichment backlog via the eligibility window and - # skipped_budget (no unbounded growth)" — at flood scale. spec/services/github/enrichment/ - # age_out_spec.rb tests the predicate and the batch bound on a handful of rows; the fact - # asserted here is the interaction: a class starved by *budget* is still bounded by *time*, - # so the two mechanisms compose rather than each assuming the other is doing the work. - describe "the backlog stays bounded under a flood" do - let(:age_out) { Github::Enrichment::AgeOut.new(configuration: configuration, selector: selector) } - let(:past_window) { frozen_time + configuration.enrichment_eligibility_window_seconds + 1 } + describe "the backlog remains durable under a flood" do + let(:next_window) { frozen_time + 7200 } before do flood_repositories(60) flood_actors(3) - drain! + active_budget_window(now: frozen_time, enrichment_used: 40) end - # Sixty-three rows, comfortably under AgeOut's batch of 1,000, so this measures the - # eligibility window and not the batch bound. - it "ages the surviving flood into skipped_budget rather than letting it grow" do - age_out.call(now: past_window) - - expect(GithubRepository.where(enrichment_status: "pending")).to be_empty - expect(GithubRepository.where(enrichment_status: "skipped_budget").count).to eq(60) + it "keeps every entity pending after the exhausted window has passed" do + expect(GithubRepository.where(enrichment_status: "pending").count).to eq(60) + expect(GithubActor.where(enrichment_status: "pending").count).to eq(3) end - it "charges the skipped rows no attempt, because nothing was ever tried on them" do - age_out.call(now: past_window) - - skipped = GithubRepository.where(enrichment_status: "skipped_budget") - - expect(skipped.pluck(:enrichment_attempts).uniq).to eq([ 0 ]) - expect(skipped.pluck(:skipped_at).compact.size).to eq(60) + it "charges no entity attempt when quota exhaustion prevented every request" do + expect(GithubRepository.distinct.pluck(:enrichment_attempts)).to eq([ 0 ]) + expect(GithubActor.distinct.pluck(:enrichment_attempts)).to eq([ 0 ]) end - it "leaves nothing eligible once the window has passed" do - age_out.call(now: past_window) + it "makes the old flood claimable when a later quota window opens" do + active_budget_window(now: next_window, poll_used: 0, enrichment_used: 0, + actor_share_used: 0, repository_share_used: 0) - expect(selector.pending_available?(Github::Enrichment::EntityType.fetch(:repository), now: past_window)) - .to be(false) - expect(selector.pending_available?(Github::Enrichment::EntityType.fetch(:actor), now: past_window)) - .to be(false) + expect(fairness.choose(now: next_window)).to be_chosen end end end diff --git a/spec/services/github/enrichment/one_shot_spec.rb b/spec/services/github/enrichment/one_shot_spec.rb index d03cd49..b1e4535 100644 --- a/spec/services/github/enrichment/one_shot_spec.rb +++ b/spec/services/github/enrichment/one_shot_spec.rb @@ -153,14 +153,15 @@ def returning(*results) it "prints the state blocks even when nothing was enriched" do one_shot.call - expect(output.string).to include("Nothing to enrich", "Actors pending/complete/skipped", - "Persisted push events") + expect(output.string).to include("Nothing to enrich", "Actor backlog", + "Enrichment backlog budget", "Persisted push events") end it "prints the enrichment counters for the invocation" do one_shot.call - expect(output.string).to include("Enrichment cycles", "Candidates skipped (budget)") + expect(output.string).to include("Enrichment cycles", "Cycles with nothing eligible") + expect(output.string).not_to include("Candidates skipped") end it "writes the whole block in one call, so a JSON log line cannot split it" do diff --git a/spec/services/github/enrichment/summary_spec.rb b/spec/services/github/enrichment/summary_spec.rb index 9ae1ec1..765ff56 100644 --- a/spec/services/github/enrichment/summary_spec.rb +++ b/spec/services/github/enrichment/summary_spec.rb @@ -4,15 +4,23 @@ let(:now) { frozen_time } describe ".capture" do - it "counts each class by status, which is what a sampling rate looks like" do + it "reports raw statuses and the durable backlog separately" do create_actor(github_id: 1, enrichment_status: "complete", fetched_at: now) - create_actor(github_id: 2, enrichment_status: "skipped_budget", skipped_at: now) - create_repository(github_id: 3) + create_actor(github_id: 2, enrichment_status: "retryable_failure", + next_retry_at: now + 3600, created_at: now - 600) + create_repository(github_id: 3, created_at: now - 300) summary = described_class.capture(now: now) - expect(summary.actor_counts).to eq("complete" => 1, "skipped_budget" => 1) + expect(summary.actor_counts).to eq("complete" => 1, "retryable_failure" => 1) expect(summary.repository_counts).to eq("pending" => 1) + expect(summary).to have_attributes( + actor_backlog_count: 1, repository_backlog_count: 1, + actor_oldest_pending_at: now - 600, + repository_oldest_pending_at: now - 300, + actor_oldest_pending_age_seconds: 600, + repository_oldest_pending_age_seconds: 300 + ) end it "reports the per-class share usage against the guarantees the ledger enforces" do @@ -45,6 +53,49 @@ it "does not create the ledger row it reads, which only a reservation may" do expect { described_class.capture(now: now) }.not_to change(GithubApiBudget, :count).from(0) end + + it "does not claim enrichment can run before a poll initializes the window" do + create_actor(github_id: 1) + + summary = described_class.capture(now: now) + + expect(summary).to have_attributes(claimable_now: false, next_enrichment_at: nil) + expect(summary.to_s).to include(described_class::WAITING_FOR_WINDOW) + end + + it "still says nothing is waiting on an empty clean checkout" do + summary = described_class.capture(now: now) + + expect(summary).to have_attributes(work_waiting: false, claimable_now: false, + next_enrichment_at: nil) + expect(summary.to_s).to include(described_class::NOTHING_WAITING) + expect(summary.to_s).not_to include(described_class::WAITING_FOR_WINDOW) + end + + it "treats an existing uninitialized ledger the same way" do + Github::BudgetLedger.new.bootstrap!(now: now) + create_actor(github_id: 1) + + expect(described_class.capture(now: now)) + .to have_attributes(window_status: "uninitialized", claimable_now: false, + next_enrichment_at: nil) + end + + [ 0, 40 ].each do |used| + it "waits for a poll after an old window elapses with #{used} enrichment attempts used" do + active_budget_window(now: now - 3600, reset_at: now - 1, + enrichment_used: used, + actor_share_used: used / 2, + repository_share_used: used / 2) + create_actor(github_id: 1) + + summary = described_class.capture(now: now) + + expect(summary).to have_attributes(window_ready: false, work_waiting: true, + claimable_now: false, next_enrichment_at: nil) + expect(summary.to_s).to include(described_class::WAITING_FOR_WINDOW) + end + end end describe "#next_enrichment_at" do @@ -99,6 +150,16 @@ expect(described_class.capture(now: now).next_enrichment_at).to be_nil end + it "does not report a refresh due while first-time backlog is backed off" do + active_budget_window(now: now) + create_actor(github_id: 1, enrichment_status: "complete", fetched_at: now - 90_000) + create_repository(github_id: 2, enrichment_status: "retryable_failure", + next_retry_at: now + 300) + + expect(described_class.capture(now: now)) + .to have_attributes(claimable_now: false, next_enrichment_at: now + 300) + end + it "takes the earliest across both classes" do active_budget_window(now: now) create_actor(github_id: 1, enrichment_status: "complete", fetched_at: now) @@ -137,12 +198,14 @@ end end - it "prints each class as pending, complete and skipped" do - create_actor(github_id: 1) + it "prints each class's backlog and oldest wait" do + create_actor(github_id: 1, created_at: now - 300) create_actor(github_id: 2, enrichment_status: "complete", fetched_at: now) active_budget_window(now: now) - expect(described_class.capture(now: now).to_s).to include("Actors pending/complete/skipped:", "1 / 1 / 0") + expect(described_class.capture(now: now).to_s).to include( + "Actor backlog:", "1", "Oldest actor pending:", "300s old" + ) end it "says due now when a candidate is actually claimable" do diff --git a/spec/services/github/enrichment/tally_spec.rb b/spec/services/github/enrichment/tally_spec.rb index e147d6e..c767aea 100644 --- a/spec/services/github/enrichment/tally_spec.rb +++ b/spec/services/github/enrichment/tally_spec.rb @@ -1,8 +1,8 @@ require "rails_helper" RSpec.describe Github::Enrichment::Tally do - def result(status:, aged_out: 0) - Github::EnrichmentRunner::Result.new(status: status, aged_out: aged_out) + def result(status:) + Github::EnrichmentRunner::Result.new(status: status) end it "starts at zero on every counter" do @@ -18,14 +18,6 @@ def result(status:, aged_out: 0) expect(tally).to have_attributes(cycles: 3, enriched: 1, failed: 1, deferred: 1, idle: 0) end - it "accumulates the candidates each cycle aged out" do - tally = described_class.empty - .record(result(status: "idle", aged_out: 4)) - .record(result(status: "idle", aged_out: 2)) - - expect(tally.aged_out).to eq(6) - end - # Immutable, like Github::Ingestion::Tally: a partially accumulated count can never be # observed, and a caller cannot hold a reference that later changes underneath it. it "returns a new value rather than mutating, so no caller sees a partial count" do @@ -40,13 +32,14 @@ def result(status:, aged_out: 0) end it "refuses an unknown status rather than dropping it" do - expect { described_class.empty.record(Struct.new(:status, :aged_out).new("invented", 0)) } + expect { described_class.empty.record(Struct.new(:status).new("invented")) } .to raise_error(ArgumentError, /invented/) end - it "prints the counters an operator reads the sampling rate from" do - rendered = described_class.empty.record(result(status: "enriched", aged_out: 1_234)).to_s + it "prints cycle outcomes without a discarded-work counter" do + rendered = described_class.empty.record(result(status: "enriched")).to_s - expect(rendered).to include("Entities enriched", "Candidates skipped (budget)", "1,234") + expect(rendered).to include("Entities enriched", "Cycles deferred", "Cycles with nothing eligible") + expect(rendered).not_to include("skipped") end end diff --git a/spec/services/github/enrichment_runner_spec.rb b/spec/services/github/enrichment_runner_spec.rb index e483c87..2a0a46e 100644 --- a/spec/services/github/enrichment_runner_spec.rb +++ b/spec/services/github/enrichment_runner_spec.rb @@ -62,33 +62,7 @@ def ghostuser(**overrides) end end - describe "the order of one cycle" do - before { active_budget_window(now: now) } - - # §12's sequence is "exhaustion → deferred → skipped_budget → reactivation", which - # requires skipping to keep happening *while* the budget is exhausted — precisely when - # boundedness matters. Behind the fairness decision it would stop exactly then. - it "ages out overdue candidates before it asks whether it may spend" do - aged = octocat(last_seen_at: now - 3601) - active_budget_window(now: now, enrichment_used: 40) - - result = runner.call - - expect(result).to have_attributes(status: "deferred", deferral_reason: "class_exhausted", aged_out: 1) - expect(aged.reload.enrichment_status).to eq("skipped_budget") - end - - it "sweeps on every cycle, including the ones that enrich something" do - octocat - create_actor(github_id: 999, last_seen_at: now - 3601) - - expect(runner.call.aged_out).to eq(1) - end - end - describe "deferrals" do - # §12 line 981's first step. The entity is untouched and stays in the pool; it becomes - # skipped_budget only later, through the sweep, when its own activity ages out. it "returns deferred without touching the entity when the class allowance is gone" do actor = octocat active_budget_window(now: now, enrichment_used: 40) @@ -120,6 +94,22 @@ def ghostuser(**overrides) expect { runner.call }.not_to change { current_budget.enrichment_used }.from(40) end + + it "keeps old work durable across quota windows and enriches it when capacity returns" do + actor = octocat(last_seen_at: now - 100_000, created_at: now - 100_000) + active_budget_window(now: now, enrichment_used: 40) + + expect(runner.call).to have_attributes(status: "deferred", deferral_reason: "class_exhausted") + expect(actor.reload.enrichment_status).to eq("pending") + + next_window = now + 3601 + active_budget_window(now: next_window, poll_used: 0, enrichment_used: 0, + actor_share_used: 0, repository_share_used: 0) + result = fixture_enrichment_runner(transport: transport, now: next_window).call + + expect(result).to have_attributes(status: "enriched", github_id: actor.github_id) + expect(actor.reload.enrichment_status).to eq("complete") + end end describe "the guarantees §5 and §8 make structurally" do diff --git a/spec/services/github/ingestion/page_writer_spec.rb b/spec/services/github/ingestion/page_writer_spec.rb index ee887fe..8020872 100644 --- a/spec/services/github/ingestion/page_writer_spec.rb +++ b/spec/services/github/ingestion/page_writer_spec.rb @@ -84,8 +84,8 @@ def write(envelopes, at: nil) expect(GithubActor.sole.latest_event_at).to eq(Time.utc(2026, 7, 29, 11, 59, 0)) end - # §12: "Duplicate poll results (fixture replay) — duplicates skipped and no entity - # reactivation occurs." Appendix D item 5 is the reason the gate exists at all. + # Duplicate observations are absorbed at the event row while harmless identity fields may + # still be refreshed. Activity remains gated on INSERT ... RETURNING. describe "a duplicate replay" do let(:later) { frozen_time + 60 } @@ -97,8 +97,9 @@ def write(envelopes, at: nil) # a wall-clock touch here would block the refresh and the example would pass for the # wrong reason. GithubActor.where(github_id: 583_231) - .update_all(login: "stale-login", enrichment_status: "skipped_budget", - skipped_at: frozen_time, enrichment_attempts: 3, + .update_all(login: "stale-login", enrichment_status: "retryable_failure", + enrichment_attempts: 3, next_retry_at: later + 3600, + last_error: "GitHub unavailable", updated_at: frozen_time) end @@ -133,91 +134,55 @@ def write(envelopes, at: nil) expect(GithubActor.sole.first_seen_at).to eq(frozen_time) end - # §7 merge rule 4, and the whole reason for the gate: "a re-polled window would - # resurrect skipped entities with no new activity". - it "cannot reactivate an entity that a budget skip had terminated" do + it "does not clear retry state when no new event was persisted" do write(well_formed_envelope, at: later) expect(GithubActor.sole).to have_attributes( - enrichment_status: "skipped_budget", skipped_at: frozen_time, enrichment_attempts: 3 + enrichment_status: "retryable_failure", enrichment_attempts: 3, + next_retry_at: later + 3600, last_error: "GitHub unavailable" ) end - # The structural assertion, not just its side effects: the gate is the call site, and - # both halves of merge rule 3 sit behind it. - it "never calls the activity update or the reactivation at all" do + # The structural assertion, not just its side effects: the activity gate is the call + # site, so a duplicate cannot look like new demand. + it "never calls the activity update" do expect(GithubActor).not_to receive(:touch_activity!) expect(GithubRepository).not_to receive(:touch_activity!) - expect(GithubActor).not_to receive(:reactivate_skipped!) - expect(GithubRepository).not_to receive(:reactivate_skipped!) write(well_formed_envelope, at: later) end end - # The other side of the same gate, and the pair is the assertion: §7's reactivation rule - # says "a **newly persisted** push event referencing the entity … may transition it back - # to pending", while rule 4 says a replay never may. The two examples above and below - # differ only in whether the event is genuinely new. - describe "a genuinely new event referencing a skipped entity" do + describe "a genuinely new event referencing a retryable entity" do let(:later) { frozen_time + 60 } before do write(well_formed_envelope) GithubActor.where(github_id: 583_231) - .update_all(enrichment_status: "skipped_budget", skipped_at: frozen_time, - enrichment_attempts: 3, updated_at: frozen_time) - GithubRepository.where(github_id: 1_296_269) - .update_all(enrichment_status: "skipped_budget", skipped_at: frozen_time, - updated_at: frozen_time) + .update_all(enrichment_status: "retryable_failure", + enrichment_attempts: 3, next_retry_at: later + 3600, + last_error: "GitHub unavailable", updated_at: frozen_time) end def distinct_event(at:) write(well_formed_envelope("id" => "58000000099", "payload" => { "push_id" => 27_500_000_099 }), at: at) end - it "reactivates both entities, because a distinct event id proves new activity" do + it "moves activity timestamps while preserving the entity's FIFO position" do + created_at = GithubActor.sole.created_at distinct_event(at: later) - expect(GithubActor.sole).to have_attributes(enrichment_status: "pending", skipped_at: nil) - expect(GithubRepository.sole).to have_attributes(enrichment_status: "pending", skipped_at: nil) + expect(GithubActor.sole).to have_attributes(last_seen_at: later, created_at: created_at) end - # §7's reactivation rule covers delayed-but-new events explicitly: "even with an old - # created_at (documented 30s-6h latency), a distinct event ID proves new activity". - it "moves the activity timestamps that drive newest-first ordering" do - distinct_event(at: later) - - expect(GithubActor.sole.last_seen_at).to eq(later) - end - - # An inbound envelope performed no fetch, so writing last_error = NULL or resetting the - # attempt count would assert something that did not happen. it "keeps the failure history, because no fetch occurred" do distinct_event(at: later) - expect(GithubActor.sole.enrichment_attempts).to eq(3) - end - - it "logs the reactivation at INFO, which §11 lists among the enrichment events" do - allow(Rails.logger).to receive(:info) - - distinct_event(at: later) - - expect(Rails.logger).to have_received(:info) - .with(hash_including(event: "enrichment.reactivated", github_actor_id: 583_231)) - end - - it "logs nothing when there was nothing to reactivate" do - GithubActor.update_all(enrichment_status: "pending", skipped_at: nil) - GithubRepository.update_all(enrichment_status: "pending", skipped_at: nil) - allow(Rails.logger).to receive(:info) - - distinct_event(at: later) - - expect(Rails.logger).not_to have_received(:info) - .with(hash_including(event: "enrichment.reactivated")) + expect(GithubActor.sole).to have_attributes( + enrichment_status: "retryable_failure", enrichment_attempts: 3, + next_retry_at: later + 3600, last_error: "GitHub unavailable" + ) end end diff --git a/spec/services/github/ingestion_runner_spec.rb b/spec/services/github/ingestion_runner_spec.rb index 1312c1d..c5b9576 100644 --- a/spec/services/github/ingestion_runner_spec.rb +++ b/spec/services/github/ingestion_runner_spec.rb @@ -84,9 +84,9 @@ def ingest(runner = fixture_runner, **options) end end - # §12: "Duplicate poll results (fixture replay) — duplicates skipped and no entity - # reactivation occurs." A second transport instance restarts the scripted sequence, which - # is a faithful model of a second one-shot process. + # A second transport instance restarts the scripted sequence, which is a faithful model + # of a second one-shot process. Duplicate event writes are absorbed without registering + # new entity activity. describe "replaying the same page" do # Past the 300-second cadence the first run wrote, so the replay is genuinely due # rather than deferred. §12's "duplicate poll results (fixture replay)" is about the @@ -100,8 +100,9 @@ def ingest(runner = fixture_runner, **options) # assignment on EXCLUDED.updated_at >= the stored value, so a wall-clock touch here # would block the refresh and the example would pass for the wrong reason. GithubActor.where(github_id: 583_231) - .update_all(login: "stale-login", enrichment_status: "skipped_budget", - skipped_at: frozen_time, updated_at: frozen_time) + .update_all(login: "stale-login", enrichment_status: "retryable_failure", + next_retry_at: later + 3600, last_error: "GitHub unavailable", + updated_at: frozen_time) end let!(:replay) { ingest(fixture_runner(transport: fixture_transport, now: later)) } @@ -117,10 +118,10 @@ def ingest(runner = fixture_runner, **options) expect(GithubActor.find_by(github_id: 583_231).login).to eq("octocat") end - # §7 merge rules 3 and 4, and Appendix D item 5's reason for the gate. - it "registers no new activity and cannot reactivate a budget-skipped entity" do + it "registers no new activity and preserves retry state" do expect(GithubActor.find_by(github_id: 583_231)).to have_attributes( - last_seen_at: frozen_time, enrichment_status: "skipped_budget", skipped_at: frozen_time + last_seen_at: frozen_time, enrichment_status: "retryable_failure", + next_retry_at: later + 3600, last_error: "GitHub unavailable" ) end @@ -459,9 +460,8 @@ def ingest(runner = fixture_runner, **options) .and have_enqueued_job(EnrichRepositoryJob).exactly(:once) end - # §7 merge rule 4's boundary, at the queue: a replay refreshes identity and reactivates - # nothing, so there is no new work to schedule. Anything still pending from the first run - # is ReconcilePendingEnrichmentsJob's business, not this run's. + # A replay refreshes identity but creates no event row, so there is no new work hint to + # schedule. Anything still pending from the first run is the reconciler's business. it "enqueues nothing for a replay that created no events" do ingest diff --git a/spec/services/github/status/snapshot_spec.rb b/spec/services/github/status/snapshot_spec.rb index edd82c1..e96ca14 100644 --- a/spec/services/github/status/snapshot_spec.rb +++ b/spec/services/github/status/snapshot_spec.rb @@ -23,24 +23,25 @@ def payload end describe "the enrichment counts (plan §11)" do - # The example that keeps two numbers from silently becoming one. §11's - # pending_actor_count sits beside skipped_actor_count, so it means the *status*; - # Github::Ingestion::StateSummary uses the same name for the enrichment_candidates - # scope, which is pending plus retryable_failure. Both are right for their own - # question. Publishing both under distinct names is what makes them checkable. - it "reports pending as the status and the candidate scope under its own name" do - create_actor(github_id: 1) - create_actor(github_id: 2, login: "two", enrichment_status: "retryable_failure") + it "reports raw statuses and the durable backlog under distinct names" do + create_actor(github_id: 1, created_at: now - 600) + create_actor(github_id: 2, login: "two", enrichment_status: "retryable_failure", + next_retry_at: now + 3600, created_at: now - 300) actors = payload.dig(:enrichment, :actors) - expect(actors).to include(pending: 1, retryable_failure: 1, candidates: 2) + expect(actors).to include( + pending: 1, retryable_failure: 1, backlog_count: 2, + oldest_pending_at: (now - 600).utc.iso8601, + oldest_pending_age_seconds: 600 + ) expect(GithubActor.enrichment_candidates.count).to eq(2) expect(Github::Ingestion::StateSummary.capture(now: now).pending_actor_count).to eq(2) end it "names every status including the ones with no rows" do - expected = Enrichable::ENRICHMENT_STATUSES.map(&:to_sym) + [ :candidates ] + expected = Enrichable::ENRICHMENT_STATUSES.map(&:to_sym) + + %i[backlog_count oldest_pending_at oldest_pending_age_seconds] expect(payload.dig(:enrichment, :actors).keys).to eq(expected) expect(payload.dig(:enrichment, :repositories).keys).to eq(expected) @@ -48,10 +49,12 @@ def payload it "counts each class separately, so one cannot mask the other" do create_actor(github_id: 1) - create_repository(github_id: 2, enrichment_status: "skipped_budget") + create_repository(github_id: 2, enrichment_status: "permanent_failure") - expect(payload.dig(:enrichment, :actors)).to include(pending: 1, skipped_budget: 0) - expect(payload.dig(:enrichment, :repositories)).to include(pending: 0, skipped_budget: 1) + expect(payload.dig(:enrichment, :actors)).to include(pending: 1, permanent_failure: 0, + backlog_count: 1) + expect(payload.dig(:enrichment, :repositories)).to include(pending: 0, permanent_failure: 1, + backlog_count: 0) end end diff --git a/spec/support/shared_examples/enrichable_entity.rb b/spec/support/shared_examples/enrichable_entity.rb index 7b6b3db..93ea9e6 100644 --- a/spec/support/shared_examples/enrichable_entity.rb +++ b/spec/support/shared_examples/enrichable_entity.rb @@ -1,10 +1,8 @@ # github_actors and github_repositories carry an identical enrichment state machine # (IMPLEMENTATION_PLAN.md §7), so its data-level guarantees are asserted once here. # -# Scope note: this covers the two transitions the *ingest* path owns — the activity -# effect and §7 merge rule 3's reactivation. Every other transition is a fetch outcome -# and belongs to Github::Enrichment::EntityState, Github::Enrichment::AgeOut, and -# Github::Enrichment::Claim, each with its own spec. +# Scope note: activity is the transition the ingest path owns. Fetch outcomes belong to +# Github::Enrichment::EntityState, and leasing belongs to Github::Enrichment::Claim. # # The host group must provide `valid_attributes`. RSpec.shared_examples "an enrichable entity" do @@ -38,6 +36,17 @@ end end + it "rejects the removed budget-skip status at both the model and database levels" do + record = described_class.create!(valid_attributes) + + expect(record.update(enrichment_status: "skipped_budget")).to be(false) + expect(record.errors[:enrichment_status]).to be_present + + expect_violation(ActiveRecord::CheckViolation) do + described_class.where(id: record.id).update_all(enrichment_status: "skipped_budget") + end + end + it "rejects a negative attempt count at the database level" do record = described_class.create!(valid_attributes) @@ -64,7 +73,7 @@ .find { |i| i.name == candidate_index_name } expect(index).not_to be_nil - expect(index.columns).to eq(%w[next_retry_at last_seen_at]) + expect(index.columns).to eq(%w[created_at id]) # Assert semantically, not textually: PostgreSQL rewrites `IN (...)` into # `= ANY (ARRAY[...])`, so the stored predicate never matches the migration's @@ -129,8 +138,8 @@ end # A delayed event carries an older created_at (documented 30s-6h latency), and - # sources commit independently. An older observation must never move newest-first - # enrichment ordering backwards. + # sources commit independently. Activity timestamps must remain monotone even though + # durable FIFO selection is based on the entity row's immutable created_at. it "never regresses an activity timestamp for an older observation" do described_class.touch_activity!(github_id: record.github_id, seen_at: frozen_time + 300, @@ -154,80 +163,8 @@ expect(record.reload.first_seen_at).to eq(frozen_time) end - - # The two halves of §7 merge rule 3 are separate statements, and this is the one that - # keeps them honest: an activity update is not a state transition. Reactivation is - # .reactivate_skipped!'s single job, and separating them is what lets that method's row - # count be exactly the number §11 asks to see logged. - it "does not change enrichment status, which reactivation is separately responsible for" do - described_class.where(id: record.id) - .update_all(enrichment_status: "skipped_budget", skipped_at: frozen_time) - - described_class.touch_activity!(github_id: record.github_id, - seen_at: frozen_time + 600, - event_occurred_at: frozen_time + 600) - - record.reload - expect(record.enrichment_status).to eq("skipped_budget") - expect(record.skipped_at).to eq(frozen_time) - end - end - - # §7's reactivation rule: "skipped_budget is terminal for the entity's current - # eligibility window, not forever." Rule 4 — a duplicate replay can never reactivate — is - # held by the *call site*, so it is asserted in page_writer_spec.rb rather than here. - describe ".reactivate_skipped!" do - let!(:record) { described_class.create!(valid_attributes) } - - def skip!(**overrides) - described_class.where(id: record.id) - .update_all({ enrichment_status: "skipped_budget", skipped_at: frozen_time }.merge(overrides)) - end - - it "returns a skipped entity to pending, because a distinct persisted event proves new activity" do - skip! - - expect(described_class.reactivate_skipped!(github_id: record.github_id, now: frozen_time + 600)).to eq(1) - expect(record.reload).to have_attributes(enrichment_status: "pending", skipped_at: nil) - end - - # §7 line 572: "An entity in the complete state is not reset to pending by a duplicate." - it "leaves every other status alone, so no enriched or terminally failed row is disturbed" do - (Enrichable::ENRICHMENT_STATUSES - [ "skipped_budget" ]).each do |status| - described_class.where(id: record.id).update_all(enrichment_status: status) - - expect(described_class.reactivate_skipped!(github_id: record.github_id, now: frozen_time)).to eq(0) - expect(record.reload.enrichment_status).to eq(status) - end - end - - # enrichment_attempts and last_error are records of *fetches*, and an inbound envelope - # is not a fetch. next_retry_at is left because Github::Enrichment::AgeOut never skips a - # row whose retry is in the future, so a reactivated row is provably due immediately. - it "keeps the failure history a reactivated entity carries, because no fetch just happened" do - skip!(enrichment_attempts: 3, last_error: "boom", next_retry_at: frozen_time - 60) - - described_class.reactivate_skipped!(github_id: record.github_id, now: frozen_time) - - expect(record.reload).to have_attributes(enrichment_attempts: 3, last_error: "boom", - next_retry_at: frozen_time - 60) - end - - # Two representations of "skipped" is two things that can disagree, which is the drift - # EventSource's own comment names. - it "never leaves a skipped instant on a row that is no longer skipped" do - skip! - described_class.reactivate_skipped!(github_id: record.github_id, now: frozen_time) - - expect(described_class.where.not(enrichment_status: "skipped_budget").where.not(skipped_at: nil)) - .to be_empty - end end - # The derived invariant Enrichable#reactivate_skipped! relies on to need no - # "missing or stale" sub-predicate: skipped_budget implies missing enrichment, because - # AgeOut only ever skips rows in CANDIDATE_STATUSES and no row in those two statuses has - # ever completed a fetch. describe "the candidate-status invariant" do it "never leaves a candidate carrying a fetched document" do Enrichable::CANDIDATE_STATUSES.each_with_index do |status, index| diff --git a/spec/support/shared_examples/enrichment_job.rb b/spec/support/shared_examples/enrichment_job.rb index 8000073..e9dc561 100644 --- a/spec/support/shared_examples/enrichment_job.rb +++ b/spec/support/shared_examples/enrichment_job.rb @@ -13,6 +13,10 @@ before { allow(Github::EnrichmentRunner).to receive(:new).and_return(runner) } + it "runs on the dedicated enrichment queue" do + expect(described_class.new.queue_name).to eq("enrichment") + end + it "runs exactly one cycle, narrowed to its own class" do expect(runner).to receive(:call).with(entity_class: entity_class).once.and_return(enriched) From cfff765dd24eff44409731ab21a54743cb4a6f01 Mon Sep 17 00:00:00 2001 From: Umang Date: Sun, 2 Aug 2026 11:15:50 -0500 Subject: [PATCH 02/12] Refresh README for durable backlog --- README.md | 94 +++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 63 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index be5c0df..ae9084e 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,30 @@ # github-push-ingestor +## New here? Start with the offline demo + +This program regularly checks GitHub's public activity feed, saves public code-push +events, and lets you inspect who pushed to which repository. The offline demo uses +prerecorded GitHub responses, makes no internet requests, and walks through the complete +collect → save → enrich → inspect flow in a few minutes. + +```bash +GITHUB_MODE=fixture docker compose up --build -d db setup web +GITHUB_MODE=fixture docker compose run --rm ingest +GITHUB_MODE=fixture docker compose run --rm enrich --limit 6 +curl -s http://localhost:3000/api/push_events +``` + +On a fresh project database, ingestion saves **4 valid push events**, quarantines **3 +malformed events**, and ignores **1 event that is not a push**. Enrichment then attempts all +**6 actor/repository backlog rows**: **4 complete successfully**, while **2 intentional +`404` outcomes** become permanent failures for a deleted user and repository. Nothing in +this walkthrough contacts GitHub or consumes real API quota. + +Already used this project and seeing different totals or a deferred poll? The database is +preserved between runs. Follow [Deterministic fixture verification](#deterministic-fixture-verification) +for the clean-reset walkthrough and expected output. When you are ready for the complete +automatic service, continue to [Quick start](#quick-start). + Fault-tolerant Rails service for ingesting, enriching, and persisting GitHub Push events. @@ -50,11 +75,11 @@ What it does, running: **Enrichment is a durable, quota-paced backlog.** With the defaults, the hourly ledger reserves 12 requests for polling, 40 for the enrichment class, and 8 as a safety reserve. -Durable backlog work has priority over refreshes within those 40 attempts, which carry -20/20 actor/repository guarantees with borrowing. Entity -rows remain actionable until a real enrichment response produces success or an -entity-specific terminal failure: quota exhaustion defers them to a later window, FIFO -oldest-first, and never converts delay into a terminal outcome. See [Known +Durable backlog work has priority over refreshes within those 40 attempts. Actors and +repositories receive 20/20 guarantees with borrowing. Entity rows remain actionable until +enrichment succeeds or an entity-specific terminal outcome is established; quota exhaustion +only defers them to a later window. Selection remains FIFO, oldest-first within each class, +and delay never becomes a terminal outcome. See [Known limitations](#known-limitations) for what happens when arrivals outpace that service rate. **With the default `GITHUB_MODE=live`, `docker compose up` starts spending real @@ -574,10 +599,11 @@ Three conventions in the response body are worth knowing before reading one: exist prints `null`. Where `null` would be ambiguous the disambiguating fact gets its own field — `ledger.present` separates "no ledger row yet" from "remaining is genuinely 0", `due_now` separates "no constraint applies" from "unknown", and - `claimable_now` separates "work is claimable this second" from "nothing is - waiting". An empty coverage window reports `null` percentages, not `0.0`: the - ratio is undefined, not zero, and every denominator is published beside its ratio - so you can check it. + `claimable_now` says whether work is eligible under persisted backlog and ledger state; + `backlog_count`, `next_enrichment_at`, and the ledger window/block fields distinguish + deferred work from an empty backlog. An empty coverage window reports `null` + percentages, not `0.0`: the ratio is undefined, not zero, and every denominator is + published beside its ratio so you can check it. - **The coverage window is measured on `created_at`** — when *this application* persisted the event, not GitHub's `occurred_at`. Coverage grades this application's enrichment pipeline and uses the same local clock as backlog and @@ -766,14 +792,15 @@ limits are IP-scoped rather than window-scoped | Status | Entered when | Left when | Spends budget | |---|---|---|---| -| `pending` | A stub row is created by ingestion | Enrichment succeeds or reaches a failure outcome | Yes, when allowance is available | -| `complete` | An enrichment fetch succeeded | Remains complete; after the TTL it may be refreshed in place once a selection observes the never-enriched backlog empty | Only on refresh | -| `retryable_failure` | A `5xx`, timeout, or transport error | The backoff expires and a retry runs | Yes — it stays a candidate | -| `permanent_failure` | A `404` or other permanent `4xx` on the entity URL | Never, automatically | Yes for the HTTP outcome; no future retries | +| `pending` | Ingestion creates a stub row | Success, a retryable outcome, or a permanent outcome | When a request is admitted; not for a URL-policy rejection | +| `complete` | An enrichment fetch succeeds | Stays complete while fresh; after TTL it may refresh once the backlog is observed empty. Transient refresh failures keep it complete; terminal outcomes may make it `permanent_failure` | When a refresh request is admitted | +| `retryable_failure` | A retryable entity or transport outcome occurs | Retried after backoff; leaves on success or a permanent outcome | When a request is admitted | +| `permanent_failure` | A permanent response, document, redirect/policy, or transport outcome occurs | Never, automatically | Every admitted outbound attempt; a pre-gate URL rejection spends none | -`pending` and `retryable_failure` rows are the durable backlog. They are selected FIFO, -oldest first, and quota exhaustion only defers them to a later window. A duplicate replay -may refresh identity fields but does not register new activity or change backlog priority. +`pending` and `retryable_failure` rows are the durable backlog. Within each entity class +they are selected FIFO, oldest first, and quota exhaustion only defers them to a later +window. A duplicate replay may refresh identity fields but does not register new activity +or change backlog priority. ### Replay behavior by table @@ -1145,7 +1172,7 @@ IngestionRunner ──► SourceLock ──► PollSchedule (due? — five compo ├──► PageLoop ──► RequestExecutor ──► RequestGate EnrichmentRunner ────────────────┘ │ ▲ ──► BudgetLedger.reserve! FIFO backlog → class, borrow │ │ ──► UrlPolicy - Refresh only when backlog empty │ └── LinkHeader.next_url ──► Transport (Faraday | Fixture) + Refresh after backlog observed empty │ └── LinkHeader.next_url ──► Transport (Faraday | Fixture) Claim → lease on next_retry_at │ │ (never a SourceLock, §8 step 1) │ ▼ │ PageWriter: one transaction per event @@ -1287,9 +1314,12 @@ pinned API version (0011), and Solid Queue over Kafka (0012). ## Continuous ingestion -The `worker` container runs one Solid Queue supervisor: a dispatcher, the worker -threads from [`config/queue.yml`](config/queue.yml), and a scheduler running -[`config/recurring.yml`](config/recurring.yml). Two tasks fire every 60 seconds. +The `worker` container runs one Solid Queue supervisor: a dispatcher, a scheduler running +[`config/recurring.yml`](config/recurring.yml), and two single-thread worker processes from +[`config/queue.yml`](config/queue.yml). One process serves `polling,control`; the other is +dedicated to `enrichment`, so polling and control work never queue behind the enrichment +workload. Outbound polling and enrichment attempts still serialize through `RequestGate`. +Two tasks fire every 60 seconds. **A tick is not a poll.** `PollEventSourceJob` selects the sources whose cached `next_poll_at` has arrived, and `Github::IngestionRunner` then re-reads each one @@ -1311,8 +1341,9 @@ entity state remains discoverable on a later successful scheduled tick without a cleanup job or queue inspection (plan §8, [ADR 0008](docs/adr/0008-post-commit-enqueue-and-entity-scoped-reconciliation.md)). -Each cycle enriches at most one entity, chosen by §10's fairness policy under a -lease, so a backlog of ninety pending actors is one queued job rather than ninety. +Each dispatch call enqueues at most one job per class regardless of backlog depth; entity +rows, not queued jobs, are the backlog. Each cycle enriches at most one entity, chosen by +§10's fairness policy under a lease. Steady state at the defaults: twelve polls an hour, and at most forty enrichment requests an hour split into 20/20 actor/repository guarantees with borrowing. Within each class the oldest never-enriched entity is selected first; a selection considers refresh @@ -1552,10 +1583,10 @@ ceiling, and of deliberate scope decisions. Each is a stated operational boundar entity requests per page, or ~2,172 an hour if every poll contained entirely new entities, against 40 available. That extrapolation is a pressure scenario, not the measured deduplicated arrival rate: repeated actors, repositories, and overlapping pages collapse to -shared rows. Entity rows nevertheless remain durable FIFO backlog work; quota exhaustion -defers them to later windows and never terminates them. If measured unique arrivals remain -above 40 attempts per hour, backlog size and oldest pending age grow and no -finite drain estimate is honest. `/status` publishes backlog count, oldest pending age, +shared rows. Entity rows nevertheless remain durable backlog work, FIFO within each entity +class; quota exhaustion defers them to later windows and never terminates them. If measured +unique arrivals remain above 40 attempts per hour, backlog size and oldest pending age grow +and no finite drain estimate is honest. `/status` publishes backlog count, oldest pending age, and the reserved allowance's usage; it does not fabricate an ETA from incomplete history. **2. There is no guarantee of complete upstream capture.** Pagination deepens a single @@ -1606,13 +1637,14 @@ verification](#crash-recovery-verification); Extension D (testing strategy) is described there. **9. Business tables and the enrichment backlog can grow without bound.** `push_events`, -`ingestion_runs`, and `quarantined_events` are append-only, and never-enriched actor and -repository rows remain actionable until attempted — this service is the system of record, +`ingestion_runs`, and `quarantined_events` are append-only, and actor and repository rows +remain actionable until enrichment succeeds or establishes an entity-specific terminal +outcome — this service is the system of record, and retention, pruning, and archival were deliberately not built. Twelve poll attempts an hour at up to ~100 events each can add roughly 1,200 event rows and as many as 2,400 entity -references an hour before deduplication, while only 40 enrichment attempts drain the -backlog. The only shipped pruning is Solid Queue's finished-job cleanup in the queue -database, which holds no business data. +references an hour before deduplication, while only 40 enrichment request attempts per hour +are available to work that backlog. The only shipped pruning is Solid Queue's finished-job +cleanup in the queue database, which holds no business data. ## Development From 435af2db38a2e3707bba7721d86b5c0b67422def Mon Sep 17 00:00:00 2001 From: Umang Date: Sun, 2 Aug 2026 11:40:21 -0500 Subject: [PATCH 03/12] Add live backlog proof and rollover coverage --- ...08-02-live-durable-backlog-and-capacity.md | 258 ++++++++++++++++++ .../enrichment/candidate_selector_spec.rb | 7 + .../enrichment/multi_window_backlog_spec.rb | 159 +++++++++++ 3 files changed, 424 insertions(+) create mode 100644 docs/evidence/2026-08-02-live-durable-backlog-and-capacity.md create mode 100644 spec/services/github/enrichment/multi_window_backlog_spec.rb diff --git a/docs/evidence/2026-08-02-live-durable-backlog-and-capacity.md b/docs/evidence/2026-08-02-live-durable-backlog-and-capacity.md new file mode 100644 index 0000000..58597dd --- /dev/null +++ b/docs/evidence/2026-08-02-live-durable-backlog-and-capacity.md @@ -0,0 +1,258 @@ +# Live durable-backlog and capacity verification + +```text +Probe date: 2026-08-02 (UTC) +Runtime SHA: cfff765dd24eff44409731ab21a54743cb4a6f01 +Run window: 2026-08-02T16:24:25Z → 2026-08-02T16:28:27Z +Docker: 28.3.0 +Docker Compose: 2.38.1-desktop.1 +API version: 2022-11-28 +Authorization: none sent +GitHub mode: live +Isolation: fresh Compose project and fresh PostgreSQL volume; no web or worker +``` + +## Questions + +This probe answers three separate questions: + +1. Does the changed application still reach GitHub and enrich real entities? +2. When its enrichment allowance is exhausted, does waiting work remain byte-for-byte + actionable rather than becoming a terminal budget-skip outcome? +3. Is the default unauthenticated capacity large enough to catch up with the live backlog? + +The first two answers are **yes**. The third answer is **no under the traffic observed in +this short sample**. + +## Method and safety boundary + +The probe used the current branch image and a fresh, explicitly named Compose project: + +```text +gpi-durable-live-proof-20260802 +``` + +Only `db`, the one-shot `setup`, `ingest`, and `enrich` services ran. The normal project +database was never attached, and no background worker could make an unplanned request. + +To reach an allowance boundary without wasting forty live requests, the isolated process +used this deliberately conservative test configuration: + +```text +GITHUB_MODE=live +POLL_INTERVAL_SECONDS=3600 +MAX_PAGES_PER_POLL=1 +RATE_LIMIT_RESERVE=58 +``` + +Against GitHub's observed limit of 60, the application derived one poll request, one +enrichment request, and a reserve of 58. This is not the production default; it exercises +the same ledger denial path with the smallest safe live request count. + +No response bodies, logins, repository names, API URLs, avatar URLs, request IDs, cookies, +or IP addresses are included below. Counts, local row IDs, timestamps, classifications, and +rate-limit fields are sufficient to establish the result. + +## Live poll created a real backlog + +The first one-shot poll reached `https://api.github.com/events`, and GitHub's response +headers initialized the ledger: + +```text +budget.window_initialized + limit=60 remaining=59 used=1 reserve=58 + poll_allowance=1 enrichment_allowance=1 + reset_at=2026-08-02T17:24:26Z + +ingestion.run_completed + pages_fetched=1 events_received=100 push_events_seen=99 + events_created=99 events_ignored=1 events_failed=0 + +actor pending=99 +repository pending=99 +``` + +One live page therefore created **198 cold entity rows** in this sample. + +## A live request exhausted the scaled allowance + +The next one-shot enrichment selected the oldest repository and received a real `200` +document from GitHub: + +```text +enrichment.completed + pool=pending classification=ok entity_status=complete + +actor backlog=99 +repository backlog=98 +enrichment_used=1 enrichment_allowance=1 +remaining=58 reserve=58 +``` + +Exactly 197 rows remained actionable after that successful request. + +## The denied cycle changed nothing + +Immediately before the next enrichment cycle, all actionable entity-state fields were +hashed in deterministic class/id order: + +```text +actionable rows: 197 +actionable fingerprint: f5e9892d16a89d3182474b4b9aa2db4d +oldest actor: pending, enrichment_attempts=0 +oldest repository: pending, enrichment_attempts=0 +``` + +The next real-mode command stopped at the ledger and reported: + +```text +Enrichment deferred — class_exhausted +Entities enriched: 0 +Entities failed: 0 +Cycles deferred: 1 +``` + +The same database queries afterward returned: + +```text +actionable rows: 197 +actionable fingerprint: f5e9892d16a89d3182474b4b9aa2db4d +oldest actor: pending, enrichment_attempts=0 +oldest repository: pending, enrichment_attempts=0 +enrichment_used: 1 of 1 +remaining: 58 +``` + +That equality is the central proof: quota denial made no entity-state write, consumed no +additional ledger attempt, and discarded no row. + +## Controlled capacity reopening resumed the same oldest row + +Waiting for a naturally elapsed GitHub hour was intentionally not represented as necessary +for this bounded probe. Instead, after the documented poll floor had passed, only the +isolated ledger's `reset_at` was moved one second into the past. A second real GitHub poll +then exercised the production rollover path and reinitialized the ledger from new +authoritative response headers: + +```text +budget.window_rolled +budget.window_initialized + limit=60 remaining=55 used=5 + reset_at=2026-08-02T17:18:11Z +``` + +The live response reported `used=5` while the deliberately reset local ledger recorded only +its new poll. Four requests were therefore absent from the new local counters: two were this +probe's known first poll and first enrichment, which the controlled local rollover erased, +and two show other activity from the same outbound IP during the natural GitHub window. The +deliberately extreme reserve of 58 therefore blocked enrichment, as designed. For the final +resumption check, the isolated row's reserve was lowered to 54, opening exactly one safe slot +while leaving the one-request enrichment allowance unchanged. + +Before that slot opened, repository row `id=2` was the oldest waiting repository: + +```text +id=2 status=pending enrichment_attempts=0 +created_at=2026-08-02T16:24:26.68435Z +``` + +The next enrichment used a real GitHub response and changed that same row to: + +```text +id=2 status=complete fetched_at=2026-08-02T16:28:27.667939Z +enrichment_used=1 of 1 remaining=54 reserve=54 +``` + +This is a **controlled rollover/capacity-boundary test**, not evidence that a natural GitHub +hour elapsed. It shows that an old row survives denial and is selected again when the +production ledger makes capacity available. The automated test described below covers a +clock-driven later window deterministically. + +At the end of the live run: + +```text +skipped_budget rows: 0 +skipped_at columns: 0 +``` + +## Capacity finding: durability does not imply catch-up + +The first live page added 198 entity lookups. At the default unauthenticated enrichment +allowance of 40 request attempts per window, that page alone requires: + +```text +198 / 40 = 4.95 full hourly allowances +``` + +In other words, it needs capacity from five quota windows. That is not a 4.95-hour wall-clock +lower bound: work may arrive near a reset and each window's capacity can be spent in a +burst. It is also the optimistic request count—no new events, no retries, no redirects, and +every request settling one entity. + +The controlled second poll ran about 193 seconds later and added another 94 actors and 96 +repositories: **190 new cold entity rows**. This is a short pressure sample, not a sustained +arrival-rate measurement. It nevertheless confirms the order of magnitude behind the +documented limitation. If default five-minute polls continue to receive pages with roughly +190 cold entities, twelve polls add about 2,280 entity lookups per hour while at most 40 +attempts work the backlog. Under the optimistic assumption that every attempt settles one +row, the durable entity backlog then grows by roughly 2,240 rows per hour; retries and +redirects make the completion side smaller. + +In plain terms: this correction turns dropped work into a real waiting list. It does not +make the service desk faster. The waiting list shrinks only during periods when new unique +work arrives more slowly than enrichment completes. + +GitHub currently documents 60 requests/hour for unauthenticated clients and 5,000 +requests/hour for authenticated users and GitHub App installations, with some installations +scaling higher: [REST API rate limits](https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api). +Authentication is therefore the right source of request capacity, but it is not the only +change required here. + +The current automatic dispatcher normally enqueues at most two entity cycles per minute +from reconciliation, plus at most two after each successful poll. At the default twelve +polls/hour, that is roughly 144 automatic cycles/hour. This is deliberately above the +current 40-request allowance, but far below both an authenticated allowance and the live +pressure sample. A catch-up design needs both: + +1. authenticated requests with a safely reserved poll/control budget; and +2. a self-refilling or bounded-batch enrichment pump that can use that larger allowance + without weakening FIFO, fairness, the request gate, or secondary-limit backoff. + +No finite API allowance can guarantee bounded backlog against arbitrary retries or an +unbounded arrival rate. The operationally honest target is a drain-rate SLO: alert whenever +the measured unique-arrival rate remains above the measured completion rate, and provision +enough authenticated capacity and worker throughput to restore a negative backlog slope. + +## Automated coverage matching this behavior + +Automated tests intentionally block all external network access; a deterministic test must +not depend on today's public event feed. The live transport proof above is therefore kept +as dated evidence, while the state-machine guarantee is exercised offline. + +The correction adds an integration scenario with 23 actors and 23 repositories—46 durable +rows, larger than the default 40-request enrichment allowance. It uses the real runner, +fairness selector, request gate, budget ledger, claim, parser, and entity-state writer, with +Faraday intercepted by WebMock: + +1. the first window completes exactly the oldest 20 actors and oldest 20 repositories; +2. call 41 is `class_exhausted` and sends no HTTP request; +3. all six remaining rows are still `pending`, with zero attempts, no retry timestamp, and + no error; +4. a later window is opened through another poll/header reconciliation; +5. the remaining oldest three actors and three repositories complete FIFO; and +6. all 46 rows finish `complete`. + +An additional selector example explicitly mirrors FIFO ordering for two repository rows. +Migration, constraint, retry, recovery, backlog-metric, status, worker-routing, and +concurrency specs continue to cover their respective boundaries. + +## What this probe does not show + +- A natural GitHub rate-limit window did not elapse; rollover was controlled and is labeled + as such above. +- Two polls on one host and one date do not establish a sustained arrival average. +- The one-request live allowance is a test configuration, not the production default. +- No authenticated request was made, so the documented authenticated limits were not + measured here. +- The probe proves durability and honest capacity accounting. It does not claim that the + present unauthenticated deployment can catch up; the live data shows the opposite. diff --git a/spec/services/github/enrichment/candidate_selector_spec.rb b/spec/services/github/enrichment/candidate_selector_spec.rb index f531034..31c181b 100644 --- a/spec/services/github/enrichment/candidate_selector_spec.rb +++ b/spec/services/github/enrichment/candidate_selector_spec.rb @@ -28,6 +28,13 @@ def next_refresh(entity_type = actor_type) expect(next_pending).to eq(oldest) end + it "applies the same oldest-created FIFO to repositories" do + oldest = create_repository(github_id: 1, created_at: now - 600) + create_repository(github_id: 2, created_at: now - 10) + + expect(next_pending(repository_type)).to eq(oldest) + end + # PageWriter creates many stubs in one page with one timestamp. The ascending id tie # break preserves insertion order rather than letting PostgreSQL choose a plan-dependent # winner on every reconciliation tick. diff --git a/spec/services/github/enrichment/multi_window_backlog_spec.rb b/spec/services/github/enrichment/multi_window_backlog_spec.rb new file mode 100644 index 0000000..6df121e --- /dev/null +++ b/spec/services/github/enrichment/multi_window_backlog_spec.rb @@ -0,0 +1,159 @@ +require "rails_helper" + +# A backlog larger than one default enrichment allowance, driven through the same runner, +# request gate, ledger, response classification, document parser and entity state writes as +# production. WebMock prevents external connections while the Faraday request stack is still +# exercised. +RSpec.describe "a durable enrichment backlog spanning quota windows", type: :integration do + let(:now) { frozen_time } + let(:actor_ids) { (10_001..10_023).to_a } + let(:repository_ids) { (20_001..20_023).to_a } + let(:configuration) do + configuration_with( + POLL_INTERVAL_SECONDS: "300", MAX_PAGES_PER_POLL: "1", + ENABLED_LIVE_SOURCE_COUNT: "1", RATE_LIMIT_RESERVE: "8", + ACTOR_ENRICHMENT_SHARE: "0.50" + ) + end + + it "drains forty FIFO candidates, preserves every remainder, and finishes next window" do + request_order = [] + stub_backlog_documents!(request_order) + create_backlog! + + current_time = now + clock = -> { current_time } + executor = live_stubbed_executor(clock: clock, configuration: configuration) + runner = live_stubbed_runner(executor: executor, clock: clock, configuration: configuration) + open_window!(executor: executor, at: current_time) + + first_window = Array.new(40) { runner.call } + + expect(first_window).to all(be_enriched) + expect(request_order).to eq( + actor_ids.first(20).map { [ :actor, _1 ] } + + repository_ids.first(20).map { [ :repository, _1 ] } + ) + expect(current_budget).to have_attributes( + poll_used: 1, enrichment_used: 40, actor_share_used: 20, repository_share_used: 20 + ) + + expect(runner.call).to have_attributes(status: "deferred", deferral_reason: "class_exhausted") + expect(request_order.length).to eq(40) + expect_remaining_backlog(actor_ids.last(3), repository_ids.last(3)) + + current_time = now + 7200 + open_window!(executor: executor, at: current_time) + + second_window = Array.new(6) { runner.call } + + expect(second_window).to all(be_enriched) + expect(request_order.last(6)).to eq( + actor_ids.last(3).map { [ :actor, _1 ] } + + repository_ids.last(3).map { [ :repository, _1 ] } + ) + expect(current_budget).to have_attributes( + poll_used: 1, enrichment_used: 6, actor_share_used: 3, repository_share_used: 3 + ) + expect(GithubActor.where(github_id: actor_ids).distinct.pluck(:enrichment_status)) + .to eq([ "complete" ]) + expect(GithubRepository.where(github_id: repository_ids).distinct.pluck(:enrichment_status)) + .to eq([ "complete" ]) + expect(runner.call).to have_attributes(status: "idle", deferral_reason: "no_candidate") + expect(request_order.length).to eq(46) + expect(WebMock).to have_requested(:get, "https://api.github.com/events?per_page=1").twice + end + + private + + def create_backlog! + actor_ids.each_with_index do |github_id, index| + create_actor( + github_id: github_id, login: "backlog-user-#{github_id}", + display_login: "backlog-user-#{github_id}", + api_url: "https://api.github.com/users/backlog-user-#{github_id}", + created_at: now - 1000 + index + ) + end + + repository_ids.each_with_index do |github_id, index| + create_repository( + github_id: github_id, full_name: "backlog/repo-#{github_id}", name: "repo-#{github_id}", + api_url: "https://api.github.com/repos/backlog/repo-#{github_id}", + created_at: now - 1000 + index + ) + end + end + + def stub_backlog_documents!(request_order) + stub_request( + :get, + %r{\Ahttps://api\.github\.com/(?:users/backlog-user-\d+|repos/backlog/repo-\d+)\z} + ).to_return do |request| + github_id = request.uri.path[/-(\d+)\z/, 1].to_i + actor = request.uri.path.start_with?("/users/") + request_order << [ actor ? :actor : :repository, github_id ] + + body = if actor + { "id" => github_id, "name" => "Backlog User #{github_id}" } + else + { "id" => github_id, "description" => "Backlog repository #{github_id}", + "language" => "Ruby", "owner" => { "id" => github_id + 100_000 } } + end + + { status: 200, headers: { "Content-Type" => "application/json" }, body: JSON.generate(body) } + end + end + + def live_stubbed_executor(clock:, configuration:) + Github::RequestExecutor.new( + transport: Github::Transports::Faraday.new, + ledger: ledger_for(configuration), mode: :live, + sleeper: ->(_seconds) { }, clock: clock + ) + end + + def live_stubbed_runner(executor:, clock:, configuration:) + selector = Github::Enrichment::CandidateSelector.new(configuration: configuration) + Github::EnrichmentRunner.new( + executor: executor, configuration: configuration, clock: clock, + monotonic: -> { 0.0 }, selector: selector + ) + end + + def open_window!(executor:, at:) + stub_request(:get, "https://api.github.com/events?per_page=1").to_return( + status: 200, body: "[]", + headers: { + "X-RateLimit-Resource" => "core", "X-RateLimit-Limit" => "60", + "X-RateLimit-Remaining" => "59", "X-RateLimit-Used" => "1", + "X-RateLimit-Reset" => (at + 3600).to_i.to_s + } + ) + + result = executor.call( + Github::Request.new(url: "https://api.github.com/events?per_page=1", request_class: :poll) + ) + expect(result.classification).to eq(:ok) + expect(current_budget).to have_attributes( + window_status: "active", poll_used: 1, enrichment_used: 0, + actor_share_used: 0, repository_share_used: 0, enrichment_allowance: 40 + ) + end + + def expect_remaining_backlog(expected_actor_ids, expected_repository_ids) + actor_rows = GithubActor.where(github_id: actor_ids, + enrichment_status: Enrichable::CANDIDATE_STATUSES) + .order(:created_at, :id) + repository_rows = GithubRepository.where( + github_id: repository_ids, enrichment_status: Enrichable::CANDIDATE_STATUSES + ).order(:created_at, :id) + + expect(actor_rows.pluck(:github_id)).to eq(expected_actor_ids) + expect(repository_rows.pluck(:github_id)).to eq(expected_repository_ids) + expect(actor_rows.pluck(:enrichment_status, :enrichment_attempts, :next_retry_at, :last_error).uniq) + .to eq([ [ "pending", 0, nil, nil ] ]) + expect(repository_rows.pluck(:enrichment_status, :enrichment_attempts, :next_retry_at, :last_error).uniq) + .to eq([ [ "pending", 0, nil, nil ] ]) + end +end From eb8c4545900cb4f9163e86332ecfbba202293948 Mon Sep 17 00:00:00 2001 From: Umang Date: Sun, 2 Aug 2026 11:48:22 -0500 Subject: [PATCH 04/12] Fix backlog metrics security scan --- .../github/enrichment/backlog_metrics.rb | 33 ++++++++++++------- .../github/enrichment/backlog_metrics_spec.rb | 19 +++++++++-- 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/app/services/github/enrichment/backlog_metrics.rb b/app/services/github/enrichment/backlog_metrics.rb index 332a692..50251b4 100644 --- a/app/services/github/enrichment/backlog_metrics.rb +++ b/app/services/github/enrichment/backlog_metrics.rb @@ -45,24 +45,35 @@ def self.entry_for(model, now:) private_class_method :entry_for def self.aggregate_columns(model) - connection = model.connection - status_column = connection.quote_column_name(:enrichment_status) - created_at_column = connection.quote_column_name(:created_at) - candidate_values = Enrichable::CANDIDATE_STATUSES.map do |status| - connection.quote(status) - end.join(", ") - candidate_filter = "#{status_column} IN (#{candidate_values})" + table = model.arel_table + status_column = table[:enrichment_status] + candidate_filter = status_column.in(Enrichable::CANDIDATE_STATUSES) STATUSES.map do |status| - quoted_status = connection.quote(status) - Arel.sql("COUNT(*) FILTER (WHERE #{status_column} = #{quoted_status})") + count_if(status_column.eq(status)) end + [ - Arel.sql("COUNT(*) FILTER (WHERE #{candidate_filter})"), - Arel.sql("MIN(#{created_at_column}) FILTER (WHERE #{candidate_filter})") + count_if(candidate_filter), + minimum_if(candidate_filter, table[:created_at]) ] end private_class_method :aggregate_columns + # Build conditional aggregates as Arel nodes instead of interpolating quoted SQL. + # COUNT ignores the implicit NULL for rows that do not match the CASE predicate. + def self.count_if(predicate) + conditional = Arel::Nodes::Case.new.when(predicate).then(1) + + Arel::Nodes::NamedFunction.new("COUNT", [ conditional ]) + end + private_class_method :count_if + + def self.minimum_if(predicate, value) + conditional = Arel::Nodes::Case.new.when(predicate).then(value) + + Arel::Nodes::NamedFunction.new("MIN", [ conditional ]) + end + private_class_method :minimum_if + # A database timestamp a fraction ahead of the application clock can occur around a # snapshot boundary. A negative backlog age is never useful, so clamp that harmless # skew to zero while retaining whole-second precision for an operator-facing metric. diff --git a/spec/services/github/enrichment/backlog_metrics_spec.rb b/spec/services/github/enrichment/backlog_metrics_spec.rb index e743125..9620811 100644 --- a/spec/services/github/enrichment/backlog_metrics_spec.rb +++ b/spec/services/github/enrichment/backlog_metrics_spec.rb @@ -32,6 +32,21 @@ def capture = described_class.capture(now: now) ) end + it "excludes older terminal rows from the oldest backlog wait" do + create_actor(github_id: 1, enrichment_status: "complete", + fetched_at: now, created_at: now - 1800) + create_actor(github_id: 2, enrichment_status: "permanent_failure", + created_at: now - 1200) + create_actor(github_id: 3, enrichment_status: "pending", + created_at: now - 300) + + expect(capture.actor).to have_attributes( + backlog_count: 1, + oldest_pending_at: now - 300, + oldest_pending_age_seconds: 300 + ) + end + it "reports each entity class independently" do create_actor(github_id: 1, created_at: now - 300) create_repository(github_id: 2, created_at: now - 600) @@ -74,7 +89,7 @@ def capture = described_class.capture(now: now) expect(actor_reads.one?).to be(true) expect(repository_reads.one?).to be(true) - expect(actor_reads.first).to include("COUNT(*) FILTER", "MIN(") - expect(repository_reads.first).to include("COUNT(*) FILTER", "MIN(") + expect(actor_reads.first).to include("COUNT(CASE WHEN", "MIN(CASE WHEN") + expect(repository_reads.first).to include("COUNT(CASE WHEN", "MIN(CASE WHEN") end end From a18382e525a9642d376fa05b9c081b552f6a2ad5 Mon Sep 17 00:00:00 2001 From: Umang Date: Sun, 2 Aug 2026 12:26:15 -0500 Subject: [PATCH 05/12] Correct no-token capacity guidance --- ...08-02-live-durable-backlog-and-capacity.md | 60 ++++++++++++------- 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/docs/evidence/2026-08-02-live-durable-backlog-and-capacity.md b/docs/evidence/2026-08-02-live-durable-backlog-and-capacity.md index 58597dd..9585904 100644 --- a/docs/evidence/2026-08-02-live-durable-backlog-and-capacity.md +++ b/docs/evidence/2026-08-02-live-durable-backlog-and-capacity.md @@ -202,26 +202,42 @@ In plain terms: this correction turns dropped work into a real waiting list. It make the service desk faster. The waiting list shrinks only during periods when new unique work arrives more slowly than enrichment completes. -GitHub currently documents 60 requests/hour for unauthenticated clients and 5,000 -requests/hour for authenticated users and GitHub App installations, with some installations -scaling higher: [REST API rate limits](https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api). -Authentication is therefore the right source of request capacity, but it is not the only -change required here. - -The current automatic dispatcher normally enqueues at most two entity cycles per minute -from reconciliation, plus at most two after each successful poll. At the default twelve -polls/hour, that is roughly 144 automatic cycles/hour. This is deliberately above the -current 40-request allowance, but far below both an authenticated allowance and the live -pressure sample. A catch-up design needs both: - -1. authenticated requests with a safely reserved poll/control budget; and -2. a self-refilling or bounded-batch enrichment pump that can use that larger allowance - without weakening FIFO, fairness, the request gate, or secondary-limit backoff. - -No finite API allowance can guarantee bounded backlog against arbitrary retries or an -unbounded arrival rate. The operationally honest target is a drain-rate SLO: alert whenever -the measured unique-arrival rate remains above the measured completion rate, and provision -enough authenticated capacity and worker throughput to restore a negative backlog slope. +GitHub authentication is explicitly outside this project's permitted scope. The service +must remain inside GitHub's documented 60-request/hour unauthenticated limit: +[REST API rate limits](https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api). +No scheduler, queue, or additional local worker can turn that fixed upstream allowance into +more requests. + +The current automatic dispatcher can already offer roughly 144 entity cycles/hour, which +is deliberately above the 40-request enrichment allowance. A faster or self-refilling pump +would therefore spend the same 40 requests earlier in the window and then stop; it would not +increase service capacity. + +Under the simultaneous requirements of five-minute polling, full per-entity API enrichment, +no discarded backlog work, and no authentication, bounded catch-up is not achievable when +unique arrivals remain above 40/hour. The valid no-token choices are explicit tradeoffs: + +1. keep the present polling cadence and retain every enrichment request durably, accepting + that backlog size and age can grow without bound under sustained pressure; or +2. apply polling backpressure while backlog exists, reassign unused polling capacity to + enrichment, and accept substantially less event capture. + +With the 8-request safety reserve and polling suspended, at most 52 unauthenticated requests +per hour could work enrichment. A backpressure implementation would also have to let one +enrichment request bootstrap each later quota window; the current ledger permits only a poll +to initialize a new window, and that poll would add more work. With that correction, even +the first observed page's 198 cold rows would consume about 3.8 full hourly enrichment +allowances, with no intervening polls, retries, or redirects. As above, allowance-hours are +not a wall-clock lower bound because work can arrive near a reset. A strict bounded-backlog +mode would therefore poll only after the previous page's backlog had drained, materially +worsening the public feed sampling rate. Redefining the required enrichment fields to use +only event-envelope data would be a third product-scope choice, not an implementation +optimization: actor names and repository descriptions and languages are absent from those +envelopes. + +The honest operational response is to expose arrival rate, completion rate, backlog slope, +and oldest age, then make the polling-versus-backlog priority explicit. There is no hidden +no-token implementation that preserves all four requirements and guarantees catch-up. ## Automated coverage matching this behavior @@ -252,7 +268,7 @@ concurrency specs continue to cover their respective boundaries. as such above. - Two polls on one host and one date do not establish a sustained arrival average. - The one-request live allowance is a test configuration, not the production default. -- No authenticated request was made, so the documented authenticated limits were not - measured here. +- No authenticated request was made or proposed; authentication is outside the permitted + project scope. - The probe proves durability and honest capacity accounting. It does not claim that the present unauthenticated deployment can catch up; the live data shows the opposite. From 1118220bf4fde405679b4bf8595ebe76b098f4ad Mon Sep 17 00:00:00 2001 From: Umang Date: Sun, 2 Aug 2026 15:39:12 -0500 Subject: [PATCH 06/12] Fix multi-queue worker declaration in Solid Queue config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SolidQueue::QueueSelector wraps its configured value in Array(), so `queues: polling,control` declared one queue literally named "polling,control" — a name no job is ever enqueued into. That worker registered, heartbeated, and polled for the life of the process while claiming nothing, so the always-on container never polled a source or ran a reconcile tick; only the single-name enrichment worker functioned. Found by watching a live run make no progress: recurring jobs accumulated in solid_queue_ready_executions with zero claimed executions. Both queue specs asserted the author's intent rather than the runtime's reading of it — one split the configured string itself, the other compared against the same joined string — so neither could fail on this. They now assert through SolidQueue::QueueSelector and Array(). Co-Authored-By: Claude Fable 5 --- config/queue.yml | 14 +++++++++++--- spec/queue/configuration_spec.rb | 21 +++++++++++++++------ spec/queue/solid_queue_integration_spec.rb | 11 +++++++---- 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/config/queue.yml b/config/queue.yml index 52e0e1c..38377b5 100644 --- a/config/queue.yml +++ b/config/queue.yml @@ -10,14 +10,22 @@ # Github::RequestGate, and two processes avoid opening a third primary-database pool. # # polling_interval stays at 1s. Nothing here is latency-sensitive: the poll cadence is -# POLL_INTERVAL_SECONDS (300), enrichment is capped by the hourly allowance (40 at the -# defaults), and both recurring tasks fire on a 60-second schedule. +# POLL_INTERVAL_SECONDS (300), enrichment is paced by the Search ledger and capped by the +# core detail-fallback allowance, and both recurring tasks fire on a 60-second schedule. +# +# Multi-queue workers are declared as a YAML **list**, never a comma-joined string. +# SolidQueue::QueueSelector takes `Array(queue_list)`, so `queues: polling,control` is one +# queue literally named "polling,control" and matches nothing — that worker claims no job +# for the life of the process, silently: it registers, heartbeats, and polls forever while +# ready executions accumulate. Found by watching a live run make no progress. default: &default dispatchers: - polling_interval: 1 batch_size: 500 workers: - - queues: polling,control + - queues: + - polling + - control threads: 1 processes: 1 polling_interval: 1 diff --git a/spec/queue/configuration_spec.rb b/spec/queue/configuration_spec.rb index 4a45c0b..3851a3f 100644 --- a/spec/queue/configuration_spec.rb +++ b/spec/queue/configuration_spec.rb @@ -69,20 +69,29 @@ # A job enqueued into a queue no worker polls is a silent, total failure, and nothing else # in the suite would catch it. + # + # Asserted through SolidQueue::QueueSelector rather than by splitting the configured + # value here, because that is precisely the bug this example exists to catch: a + # comma-joined string reads as three queue names to a human and as ONE queue literally + # named "polling,control" to Array(), which matches no execution ever. The worker still + # registers, heartbeats, and polls — it simply claims nothing, forever. Splitting the + # string in the spec proved the author's intent and nothing about the runtime. it "works every queue this application enqueues into" do - queues = [ PollEventSourceJob, EnrichActorJob, EnrichRepositoryJob, + queues = [ PollEventSourceJob, EnrichmentCycleJob, ReconcilePendingEnrichmentsJob ].map { _1.new.queue_name }.uniq.sort - configured = queue_config.dig("production", "workers") - .flat_map { _1.fetch("queues").split(",") }.uniq.sort + selected = queue_config.dig("production", "workers").flat_map do |worker| + SolidQueue::QueueSelector.new(worker.fetch("queues"), SolidQueue::ReadyExecution) + .send(:eligible_queues) + end.uniq.sort expect(queues).to eq(%w[control enrichment polling]) - expect(configured).to eq(queues) + expect(selected).to eq(queues) end it "isolates the durable enrichment backlog from polling and control work" do - workers = queue_config.dig("production", "workers").index_by { _1.fetch("queues") } + workers = queue_config.dig("production", "workers").index_by { Array(_1.fetch("queues")) } - expect(workers.keys).to contain_exactly("polling,control", "enrichment") + expect(workers.keys).to contain_exactly(%w[polling control], %w[enrichment]) expect(workers.values).to all(include("threads" => 1, "processes" => 1)) end diff --git a/spec/queue/solid_queue_integration_spec.rb b/spec/queue/solid_queue_integration_spec.rb index 0e1df85..2b07bf5 100644 --- a/spec/queue/solid_queue_integration_spec.rb +++ b/spec/queue/solid_queue_integration_spec.rb @@ -12,10 +12,10 @@ describe "enqueueing" do it "writes a job row and a ready execution" do - expect { EnrichActorJob.perform_later }.to change(SolidQueue::Job, :count).by(1) + expect { EnrichmentCycleJob.perform_later }.to change(SolidQueue::Job, :count).by(1) job = SolidQueue::Job.last - expect(job.class_name).to eq("EnrichActorJob") + expect(job.class_name).to eq("EnrichmentCycleJob") expect(job.queue_name).to eq("enrichment") expect(SolidQueue::ReadyExecution.where(job_id: job.id)).to exist end @@ -74,11 +74,14 @@ expect(configuration.configured_processes.map(&:kind)) .to contain_exactly(:dispatcher, :worker, :worker, :scheduler) + # The list form matters at runtime, not only in the file: SolidQueue::Worker + # wraps this value in Array(), so a comma-joined string would become one queue + # name no execution ever carries. worker_queues = configuration.configured_processes .select { |process| process.kind == :worker } - .map { |process| process.attributes.fetch(:queues) } + .map { |process| Array(process.attributes.fetch(:queues)) } - expect(worker_queues).to contain_exactly("polling,control", "enrichment") + expect(worker_queues).to contain_exactly(%w[polling control], [ "enrichment" ]) end it "hands the scheduler this application's two ticks" do From b7d6be4a9806709c6f0aaa7fd241aaf2f3fca9f6 Mon Sep 17 00:00:00 2001 From: Umang Date: Sun, 2 Aug 2026 15:39:35 -0500 Subject: [PATCH 07/12] Enrich in Search batches instead of one request per entity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #45. The durable backlog from #44 survives quota exhaustion but cannot drain it: one core request per entity gives 40 completions an hour against roughly 2,280 cold arrivals, and authentication is out of scope. Derivation-first staged enrichment replaces the per-entity normal path: - Every field computable from the stored event is derived on ingest, with no network call. Repository owner login comes from the event's own qualified name; both event-native identity fragments are preserved as append-only observations committed with the push event. - The normal path is an unauthenticated GitHub Search request carrying up to SEARCH_BATCH_SIZE repeated exact qualifiers (user: / repo:, never OR-joined, which answers 422). It spends a separate per-minute Search ledger — ceiling 10, reserve 2, paced — so Search pressure can neither consume nor block the polling allocation. - Returned items are applied only when the immutable GitHub id matches the claimed row. Missing, renamed, mismatched, and contract-invalid items go to the payload-URL detail fallback, which spends an explicit core allowance (CORE_DETAIL_FALLBACK_ALLOWANCE, 4/hour) and never the poll allocation. The core formula becomes 12 + 4 + 8 <= 60. - enrichment_status stays the business outcome; enrichment_stage carries the pipeline position through seven resting stages. Claims are explicit leases (lease_token, leased_until) rather than an overloaded next_retry_at, so a retry instant and a live claim can no longer be confused. Every projection is guarded by the lease, so a reclaimed row cannot be double-applied. - Quota, pacing, and reserve denials defer. The only terminal outcome is an entity-specific permanent fact, and it retains the event data, observations, reason, and timestamps. - Raw evidence is append-only: batch envelopes record the request, its counts, and its rate-limit headers; observations record every item. A refresh appends and repoints the projection rather than overwriting the only retained response. One EnrichmentCycleJob replaces the two per-class jobs: it loops until a ledger denies, waiting out pacing, because 6-second pacing is far finer than the 60-second tick and the queue has one thread. The retired per-entity path (EnrichmentRunner, Fairness, CandidateSelector, Claim, EntityState) is deleted rather than left as a second writer. Co-Authored-By: Claude Fable 5 --- app/jobs/application_job.rb | 2 +- app/jobs/enrich_actor_job.rb | 29 -- app/jobs/enrich_repository_job.rb | 19 - app/jobs/enrichment_cycle_job.rb | 26 ++ app/models/concerns/enrichable.rb | 46 ++- app/models/enrichment_batch.rb | 24 ++ app/models/enrichment_observation.rb | 20 ++ app/models/github_repository.rb | 6 + app/models/github_search_budget.rb | 26 ++ app/services/github/allowances.rb | 7 +- app/services/github/configuration.rb | 109 +++++- .../github/enrichment/actor_document.rb | 10 +- app/services/github/enrichment/admission.rb | 86 +++++ .../github/enrichment/backlog_metrics.rb | 92 +++-- app/services/github/enrichment/backoff.rb | 18 +- app/services/github/enrichment/batch_claim.rb | 162 +++++++++ .../github/enrichment/batch_quality.rb | 102 ++++++ .../github/enrichment/batch_runner.rb | 325 ++++++++++++++++++ .../github/enrichment/candidate_selector.rb | 151 -------- app/services/github/enrichment/claim.rb | 194 ----------- .../github/enrichment/cycle_runner.rb | 191 ++++++++++ .../github/enrichment/detail_claim.rb | 89 +++++ .../github/enrichment/detail_runner.rb | 206 +++++++++++ app/services/github/enrichment/dispatch.rb | 128 +++---- .../github/enrichment/entity_state.rb | 236 ------------- app/services/github/enrichment/entity_type.rb | 7 +- app/services/github/enrichment/fairness.rb | 167 --------- .../github/enrichment/observation_recorder.rb | 23 ++ app/services/github/enrichment/one_shot.rb | 215 ++++++++---- app/services/github/enrichment/parser.rb | 14 + .../github/enrichment/repository_document.rb | 39 ++- .../github/enrichment/search_query.rb | 34 ++ .../github/enrichment/search_response.rb | 28 ++ app/services/github/enrichment/summary.rb | 270 +++++++-------- app/services/github/enrichment/tally.rb | 64 +++- app/services/github/enrichment/throughput.rb | 117 +++++++ app/services/github/ingestion/page_writer.rb | 47 +++ app/services/github/ingestion/poll_state.rb | 4 +- app/services/github/request.rb | 14 +- app/services/github/request_executor.rb | 13 +- app/services/github/search_budget_ledger.rb | 186 ++++++++++ app/services/github/status/ledger_state.rb | 6 +- .../github/status/scheduler_settings.rb | 49 +++ .../github/status/search_ledger_state.rb | 64 ++++ app/services/github/status/snapshot.rb | 107 +++--- bin/enrich | 8 +- ...60802010000_add_staged_batch_enrichment.rb | 210 +++++++++++ db/schema.rb | 136 +++++++- 48 files changed, 2899 insertions(+), 1227 deletions(-) delete mode 100644 app/jobs/enrich_actor_job.rb delete mode 100644 app/jobs/enrich_repository_job.rb create mode 100644 app/jobs/enrichment_cycle_job.rb create mode 100644 app/models/enrichment_batch.rb create mode 100644 app/models/enrichment_observation.rb create mode 100644 app/models/github_search_budget.rb create mode 100644 app/services/github/enrichment/admission.rb create mode 100644 app/services/github/enrichment/batch_claim.rb create mode 100644 app/services/github/enrichment/batch_quality.rb create mode 100644 app/services/github/enrichment/batch_runner.rb delete mode 100644 app/services/github/enrichment/candidate_selector.rb delete mode 100644 app/services/github/enrichment/claim.rb create mode 100644 app/services/github/enrichment/cycle_runner.rb create mode 100644 app/services/github/enrichment/detail_claim.rb create mode 100644 app/services/github/enrichment/detail_runner.rb delete mode 100644 app/services/github/enrichment/entity_state.rb delete mode 100644 app/services/github/enrichment/fairness.rb create mode 100644 app/services/github/enrichment/observation_recorder.rb create mode 100644 app/services/github/enrichment/search_query.rb create mode 100644 app/services/github/enrichment/search_response.rb create mode 100644 app/services/github/enrichment/throughput.rb create mode 100644 app/services/github/search_budget_ledger.rb create mode 100644 app/services/github/status/scheduler_settings.rb create mode 100644 app/services/github/status/search_ledger_state.rb create mode 100644 db/migrate/20260802010000_add_staged_batch_enrichment.rb diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb index 6a182ca..84ab11d 100644 --- a/app/jobs/application_job.rb +++ b/app/jobs/application_job.rb @@ -8,7 +8,7 @@ # **No retry_on, deliberately, anywhere in this application.** Every job here can spend # GitHub request budget, and both retry ladders already exist and are durable: # Github::Ingestion::PollState writes consecutive_failures + retry_not_before_at for a -# source, and Github::Enrichment::EntityState writes next_retry_at for an entity. A second, +# source, and the enrichment runners write next_retry_at for an entity. A second, # uncoordinated Active Job ladder would re-poll a source whose backoff was just written and # spend the hourly allowance twice on the same failure. The 60-second recurring tick is the # retry — an escaped exception is a defect, so it fails the execution, job.failed says why, diff --git a/app/jobs/enrich_actor_job.rb b/app/jobs/enrich_actor_job.rb deleted file mode 100644 index 9b6f8dd..0000000 --- a/app/jobs/enrich_actor_job.rb +++ /dev/null @@ -1,29 +0,0 @@ -# One actor enrichment cycle (IMPLEMENTATION_PLAN.md §5, §8 step 10). -# -# It takes no actor id, and that is the design rather than an omission: -# Github::EnrichmentRunner enriches at most one entity per call and *chooses* it through -# §10's fairness policy and a FOR UPDATE SKIP LOCKED lease, in durable FIFO order. An -# id-addressed job would have to bypass that ordering to honour its argument, which is how a -# repository flood starves actors. So the job says "do one actor's worth of work" and the -# runner decides whose — which is also why a duplicate delivery is harmless: it is one -# more cycle, and it finds either different work or none. -# -# It never takes a source lock (§8 step 1: "enrichment jobs skip this step — they take only -# the request gate"), and Github::LockOrder enforces that structurally. -class EnrichActorJob < ApplicationJob - # Enrichment is deliberately isolated from polling and reconciliation. A deep durable - # entity backlog may keep this queue busy for many rate-limit windows, but it must never - # delay the control tick that discovers more committed work or the poller that creates it. - queue_as :enrichment - - def perform - result = Github::EnrichmentRunner.new.call(entity_class: GithubActor) - - # idle and deferred are ordinary outcomes, not errors: nothing was eligible, or the - # ledger refused. Github::EnrichmentRunner already logged the cycle; this joins it to the - # job. Errors::FixtureMiss and anything unexpected propagate — the runner released the - # lease before re-raising, and ApplicationJob turns it into job.failed. - @outcome = { entity_type: result.entity_type, github_actor_id: result.github_id, - enrichment_outcome: result.status }.compact - end -end diff --git a/app/jobs/enrich_repository_job.rb b/app/jobs/enrich_repository_job.rb deleted file mode 100644 index 1ad3ff2..0000000 --- a/app/jobs/enrich_repository_job.rb +++ /dev/null @@ -1,19 +0,0 @@ -# One repository enrichment cycle (IMPLEMENTATION_PLAN.md §5, §8 step 10). EnrichActorJob's -# twin, and separate rather than one parameterised class because §5 names both and a class -# per queue-visible unit of work is what makes `SELECT class_name, count(*) FROM -# solid_queue_jobs GROUP BY 1` answer a reviewer's question. -# -# It takes no repository id, for the reason EnrichActorJob's comment gives: the entity is -# chosen by §10's fairness policy under a lease, not by the caller. -class EnrichRepositoryJob < ApplicationJob - # Shares one bounded worker with actor enrichment. Entity rows, not queued job count, are - # the backlog; each delivery is only a wake-up that asks the runner to claim one row. - queue_as :enrichment - - def perform - result = Github::EnrichmentRunner.new.call(entity_class: GithubRepository) - - @outcome = { entity_type: result.entity_type, github_repository_id: result.github_id, - enrichment_outcome: result.status }.compact - end -end diff --git a/app/jobs/enrichment_cycle_job.rb b/app/jobs/enrichment_cycle_job.rb new file mode 100644 index 0000000..7b30a53 --- /dev/null +++ b/app/jobs/enrichment_cycle_job.rb @@ -0,0 +1,26 @@ +# One staged-enrichment cycle (IMPLEMENTATION_PLAN.md §5, Appendix G). +# +# It takes no arguments and no entity class: Github::Enrichment::CycleRunner works both +# lanes itself — Search batches while the search ledger grants, then detail fallbacks +# while the core allowance grants — choosing lanes through the weighted schedule and +# FOR UPDATE SKIP LOCKED claims in durable FIFO order. One looping job rather than a +# job per request, because pacing (6s) is far finer than the 60-second tick and the +# enrichment queue deliberately has one thread. +# +# Duplicate deliveries are harmless by construction: a surplus cycle's first admission +# check or claim finds pacing, exhaustion, or no claimable work, and the cycle exits in +# milliseconds having created no batch row and spent no budget. +# +# It never takes a source lock (§8 step 1: enrichment takes only the request gate), and +# Github::LockOrder enforces that structurally. +class EnrichmentCycleJob < ApplicationJob + # Enrichment is deliberately isolated from polling and reconciliation. A deep durable + # entity backlog may keep this queue busy for many rate-limit windows, but it must + # never delay the control tick that discovers more committed work or the poller that + # creates it. + queue_as :enrichment + + def perform + @outcome = Github::Enrichment::CycleRunner.new.call.to_log + end +end diff --git a/app/models/concerns/enrichable.rb b/app/models/concerns/enrichable.rb index 9fe674a..2fe62ec 100644 --- a/app/models/concerns/enrichable.rb +++ b/app/models/concerns/enrichable.rb @@ -2,11 +2,22 @@ # (IMPLEMENTATION_PLAN.md §7). This concern carries the *data* half — the value set, the # enum, and the scope whose WHERE clause matches the partial index. # -# Every other transition is a fetch outcome and lives in Github::Enrichment::EntityState -# or Github::Enrichment::Claim, both of which are constructed -# without an executor or a transport so a GitHub request cannot be issued from them. +# Every other transition is a fetch outcome and lives in the batch and detail runners, +# which reach the entity rows through Github::Enrichment::BatchClaim and +# Github::Enrichment::DetailClaim — neither claim holds an executor or a transport, so a +# GitHub request cannot be issued from them. # -# Two column conventions this state machine relies on, stated here because both invite +# Two columns carry the *business* outcome and the *pipeline position* separately, and the +# split is load-bearing (plan Appendix G): enrichment_status is what an operator reports +# on, enrichment_stage is where the row sits in the staged pipeline. Legal pairs: +# +# pending → batch_pending, batch_in_flight, detail_pending, +# detail_in_flight, retry_scheduled +# retryable_failure → retry_scheduled, batch_in_flight, detail_pending, detail_in_flight +# complete → contract_complete, and the in-flight/pending stages during a refresh +# permanent_failure → terminal +# +# Three column conventions this state machine relies on, stated here because each invites # the other reading: # # * enrichment_attempts counts attempts **since the last success**, not for the @@ -14,12 +25,27 @@ # Github::Ingestion::PollState's consecutive_failures. Its only two consumers — the # backoff exponent and the log line — both want that number. # * next_retry_at means one thing everywhere: *this entity may not be attempted before -# T*. It is simultaneously the failure backoff, a secondary-limit deferral, and the -# in-flight claim lease. One meaning is what lets a single predicate exclude -# in-flight rows from both candidate pools at once. +# T*. It is both the failure backoff and a secondary-limit deferral. It is no longer +# the claim lease — leases are explicit now (lease_token, leased_until), so a retry +# instant and a live claim can no longer be mistaken for each other. +# * detail_attempts counts only fallback fetches, so the bounded core allowance's +# ladder is independent of how many times the Search lane batched the row. module Enrichable extend ActiveSupport::Concern + # Resting pipeline positions only. Event-native persistence, local derivation, + # and batch application are instants (event_native_at / derived_at / + # batch_applied_at); a row never rests in them, so they are not stages. + ENRICHMENT_STAGES = %w[ + batch_pending + batch_in_flight + detail_pending + detail_in_flight + retry_scheduled + contract_complete + terminal + ].freeze + ENRICHMENT_STATUSES = %w[ pending complete @@ -32,6 +58,8 @@ module Enrichable included do enum :enrichment_status, ENRICHMENT_STATUSES.index_by(&:itself), validate: true + enum :enrichment_stage, ENRICHMENT_STAGES.index_by(&:itself), validate: true, + prefix: :stage # Guards upsert_stub!, which otherwise reaches PostgreSQL directly and turns a # malformed envelope into a NotNullViolation that aborts the ingest transaction @@ -39,6 +67,10 @@ module Enrichable validates :github_id, presence: true scope :enrichment_candidates, -> { where(enrichment_status: CANDIDATE_STATUSES) } + scope :durable_enrichment_backlog, -> { where.not(enrichment_stage: %w[contract_complete terminal]) } + + belongs_to :latest_observation, class_name: "EnrichmentObservation", optional: true + belongs_to :current_enrichment_batch, class_name: "EnrichmentBatch", optional: true end class_methods do diff --git a/app/models/enrichment_batch.rb b/app/models/enrichment_batch.rb new file mode 100644 index 0000000..f7bbe78 --- /dev/null +++ b/app/models/enrichment_batch.rb @@ -0,0 +1,24 @@ +class EnrichmentBatch < ApplicationRecord + REQUEST_KINDS = %w[search detail].freeze + ENTITY_KINDS = %w[actor repository].freeze + STATUSES = %w[in_flight succeeded failed deferred stale_lease].freeze + + has_many :enrichment_observations, dependent: :restrict_with_error + + # The column's default is the server-side gen_random_uuid(), which Active Record + # cannot evaluate before validation — so a create! that relied on it would fail the + # presence validation below rather than reach the INSERT. Assigned here so the + # correlation id exists client-side, where the observations written alongside this + # batch need it anyway. + before_validation { self.correlation_id ||= SecureRandom.uuid } + + validates :request_kind, inclusion: { in: REQUEST_KINDS } + validates :entity_kind, inclusion: { in: ENTITY_KINDS } + validates :status, inclusion: { in: STATUSES } + validates :correlation_id, presence: true, uniqueness: true + validates :started_at, presence: true + validates :requested_count, :returned_count, :valid_count, :missing_count, :invalid_count, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } + + scope :during, ->(range) { where(started_at: range) } +end diff --git a/app/models/enrichment_observation.rb b/app/models/enrichment_observation.rb new file mode 100644 index 0000000..c996cfb --- /dev/null +++ b/app/models/enrichment_observation.rb @@ -0,0 +1,20 @@ +class EnrichmentObservation < ApplicationRecord + SOURCES = %w[event search detail].freeze + ENTITY_KINDS = %w[actor repository].freeze + + belongs_to :enrichment_batch, optional: true + belongs_to :push_event, optional: true + + validates :entity_kind, inclusion: { in: ENTITY_KINDS } + validates :source, inclusion: { in: SOURCES } + validates :observed_at, :raw_payload, :payload_fingerprint, :validation_outcome, + presence: true + + scope :during, ->(range) { where(observed_at: range) } + + # Audit evidence is append-only. Batch envelopes are updated as their request finishes; + # individual observations never are. + def readonly? + persisted? + end +end diff --git a/app/models/github_repository.rb b/app/models/github_repository.rb index 4967fab..22854b4 100644 --- a/app/models/github_repository.rb +++ b/app/models/github_repository.rb @@ -28,6 +28,10 @@ class GithubRepository < ApplicationRecord CASE WHEN EXCLUDED.updated_at >= github_repositories.updated_at THEN EXCLUDED.name END, github_repositories.name), + owner_login = COALESCE( + CASE WHEN EXCLUDED.updated_at >= github_repositories.updated_at + THEN EXCLUDED.owner_login END, + github_repositories.owner_login), api_url = COALESCE( CASE WHEN EXCLUDED.updated_at >= github_repositories.updated_at THEN EXCLUDED.api_url END, @@ -39,6 +43,7 @@ class GithubRepository < ApplicationRecord # from aborting the ingest transaction before it can be quarantined. def self.upsert_stub!(github_id:, full_name:, name: nil, api_url: nil, now: Time.current) + owner_login = full_name.to_s.split("/", 2).first if full_name.to_s.include?("/") new(github_id: github_id, full_name: full_name, name: name, api_url: api_url).validate! @@ -47,6 +52,7 @@ def self.upsert_stub!(github_id:, full_name:, name: nil, api_url: nil, github_id: github_id, full_name: full_name, name: name, + owner_login: owner_login, api_url: api_url, created_at: now, updated_at: now diff --git a/app/models/github_search_budget.rb b/app/models/github_search_budget.rb new file mode 100644 index 0000000..781a2ec --- /dev/null +++ b/app/models/github_search_budget.rb @@ -0,0 +1,26 @@ +class GithubSearchBudget < ApplicationRecord + self.table_name = "github_search_budget" + + SINGLETON_ID = 1 + + validates :request_ceiling, numericality: { only_integer: true, greater_than: 0 } + validates :reserve, :used, :actor_used, :repository_used, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } + + def available + local = [ request_ceiling - reserve - used, 0 ].max + return local if remaining.nil? + + [ local, remaining - reserve ].min.clamp(0, local) + end + + def to_log + { + resource: resource, limit: limit, remaining: remaining, reset_at: reset_at&.utc&.iso8601, + observed_at: observed_at&.utc&.iso8601, request_ceiling: request_ceiling, + reserve: reserve, used: used, actor_used: actor_used, repository_used: repository_used, + available: available, blocked_until: blocked_until&.utc&.iso8601, + last_request_at: last_request_at&.utc&.iso8601 + } + end +end diff --git a/app/services/github/allowances.rb b/app/services/github/allowances.rb index ea4118e..af2413b 100644 --- a/app/services/github/allowances.rb +++ b/app/services/github/allowances.rb @@ -44,7 +44,7 @@ def derive(configuration:, limit:, live_source_count: configuration.enabled_live limit: limit, reserve: configuration.rate_limit_reserve, poll_allowance: poll, - enrichment_allowance: limit - configuration.rate_limit_reserve - poll, + enrichment_allowance: configuration.core_detail_fallback_allowance, actor_enrichment_share: configuration.actor_enrichment_share ) end @@ -82,7 +82,7 @@ def repository_guarantee = guarantees.fetch(:repository) # is about capacity for Story 3, and one attempt of capacity is one attempt of # capacity whichever class holds it — a zero guarantee is relieved by borrowing. def feasible? - enrichment_allowance >= 1 + poll_allowance + reserve + enrichment_allowance <= limit end # What to actually store when the observed limit makes the configuration @@ -105,7 +105,8 @@ def clamped spendable = [ limit - reserve, 0 ].max poll = [ [ poll_allowance, spendable ].min, 1 ].max - with(poll_allowance: poll, enrichment_allowance: [ spendable - poll, 0 ].max) + with(poll_allowance: poll, + enrichment_allowance: [ enrichment_allowance, spendable - poll ].min.clamp(0, enrichment_allowance)) end # The guarantees rather than the share: they are the numbers that actually bind, and diff --git a/app/services/github/configuration.rb b/app/services/github/configuration.rb index e654450..732aea3 100644 --- a/app/services/github/configuration.rb +++ b/app/services/github/configuration.rb @@ -30,6 +30,22 @@ class Configuration "MAX_PAGES_PER_POLL" => "1", "ENABLED_LIVE_SOURCE_COUNT" => "1", "RATE_LIMIT_RESERVE" => "8", + "CORE_DETAIL_FALLBACK_ALLOWANCE" => "4", + "SEARCH_REQUEST_CEILING" => "10", + "SEARCH_SAFETY_RESERVE" => "2", + "SEARCH_BATCH_SIZE" => "10", + "SEARCH_PACING_SECONDS" => "6", + "SEARCH_WORKER_CONCURRENCY" => "1", + "ENRICHMENT_CYCLE_BUDGET_SECONDS" => "55", + "ACTOR_ENRICHMENT_WEIGHT" => "1", + "REPOSITORY_ENRICHMENT_WEIGHT" => "1", + "DETAIL_FALLBACK_MAX_ATTEMPTS" => "3", + "ENRICHMENT_LEASE_SECONDS" => "600", + "ENRICHMENT_RETRY_BASE_SECONDS" => "60", + "ENRICHMENT_RETRY_MAX_SECONDS" => "3600", + "ENRICHMENT_METRICS_WINDOW_SECONDS" => "3600", + "CATCH_UP_MIN_SAMPLE_SECONDS" => "900", + "REFRESH_ACTIVE_WITHIN_SECONDS" => "604800", "HTTP_OPEN_TIMEOUT_SECONDS" => "5", "HTTP_READ_TIMEOUT_SECONDS" => "15", "MAX_HTTP_RETRIES" => "2", @@ -65,14 +81,32 @@ class Configuration source_lock_wait_seconds: "SOURCE_LOCK_WAIT_SECONDS", actor_refresh_ttl_seconds: "ACTOR_REFRESH_TTL_SECONDS", repository_refresh_ttl_seconds: "REPOSITORY_REFRESH_TTL_SECONDS", - enrichment_coverage_window_seconds: "ENRICHMENT_COVERAGE_WINDOW_SECONDS" + enrichment_coverage_window_seconds: "ENRICHMENT_COVERAGE_WINDOW_SECONDS", + search_request_ceiling: "SEARCH_REQUEST_CEILING", + search_batch_size: "SEARCH_BATCH_SIZE", + search_worker_concurrency: "SEARCH_WORKER_CONCURRENCY", + enrichment_cycle_budget_seconds: "ENRICHMENT_CYCLE_BUDGET_SECONDS", + actor_enrichment_weight: "ACTOR_ENRICHMENT_WEIGHT", + repository_enrichment_weight: "REPOSITORY_ENRICHMENT_WEIGHT", + detail_fallback_max_attempts: "DETAIL_FALLBACK_MAX_ATTEMPTS", + enrichment_lease_seconds: "ENRICHMENT_LEASE_SECONDS", + enrichment_retry_base_seconds: "ENRICHMENT_RETRY_BASE_SECONDS", + enrichment_retry_max_seconds: "ENRICHMENT_RETRY_MAX_SECONDS", + enrichment_metrics_window_seconds: "ENRICHMENT_METRICS_WINDOW_SECONDS", + catch_up_min_sample_seconds: "CATCH_UP_MIN_SAMPLE_SECONDS", + refresh_active_within_seconds: "REFRESH_ACTIVE_WITHIN_SECONDS" }.freeze - # Zero is meaningful for all three: no reserve, no retries, no redirects. + # Zero is meaningful for every member: no reserve, no retries, no redirects, no + # detail fallback, no search reserve — and no pacing, which the offline fixture + # walkthrough uses so a one-shot can run both lanes back to back. NON_NEGATIVE_INTEGERS = { rate_limit_reserve: "RATE_LIMIT_RESERVE", max_http_retries: "MAX_HTTP_RETRIES", - max_redirects: "MAX_REDIRECTS" + max_redirects: "MAX_REDIRECTS", + core_detail_fallback_allowance: "CORE_DETAIL_FALLBACK_ALLOWANCE", + search_safety_reserve: "SEARCH_SAFETY_RESERVE", + search_pacing_seconds: "SEARCH_PACING_SECONDS" }.freeze # §10's fairness share. A named group of one rather than a one-off line, because the @@ -90,7 +124,14 @@ class Configuration :http_open_timeout_seconds, :http_read_timeout_seconds, :max_http_retries, :max_redirects, :source_lock_wait_seconds, :actor_enrichment_share, :actor_refresh_ttl_seconds, :repository_refresh_ttl_seconds, - :enrichment_coverage_window_seconds + :enrichment_coverage_window_seconds, :core_detail_fallback_allowance, + :search_request_ceiling, :search_safety_reserve, :search_batch_size, + :search_pacing_seconds, :search_worker_concurrency, + :actor_enrichment_weight, :repository_enrichment_weight, + :detail_fallback_max_attempts, :enrichment_lease_seconds, + :enrichment_retry_base_seconds, :enrichment_retry_max_seconds, + :enrichment_metrics_window_seconds, :catch_up_min_sample_seconds, + :refresh_active_within_seconds, :enrichment_cycle_budget_seconds def initialize(env = ENV) @mode = read(env, "GITHUB_MODE").downcase @@ -176,6 +217,7 @@ def validate! end validate_fractions! + validate_staged_enrichment! validate_allowances! self end @@ -208,14 +250,63 @@ def validate_allowances! return if derived.feasible? raise Errors::ConfigurationError, <<~MESSAGE.squish - the polling requirement leaves no capacity for enrichment: - poll_allowance (#{derived.poll_allowance}) + RATE_LIMIT_RESERVE (#{derived.reserve}) - reaches the #{derived.limit}/hour limit, leaving #{derived.enrichment_allowance} - enrichment attempts. Raise POLL_INTERVAL_SECONDS, or lower MAX_PAGES_PER_POLL, - ENABLED_LIVE_SOURCE_COUNT, or RATE_LIMIT_RESERVE. + polling, the core detail-fallback allowance, and the safety reserve exceed the core limit: + poll_allowance (#{derived.poll_allowance}) + CORE_DETAIL_FALLBACK_ALLOWANCE + (#{derived.enrichment_allowance}) + RATE_LIMIT_RESERVE (#{derived.reserve}) exceeds + #{derived.limit}/hour. Raise POLL_INTERVAL_SECONDS, or lower MAX_PAGES_PER_POLL, + ENABLED_LIVE_SOURCE_COUNT, CORE_DETAIL_FALLBACK_ALLOWANCE, or RATE_LIMIT_RESERVE. MESSAGE end + def validate_staged_enrichment! + if search_safety_reserve >= search_request_ceiling + raise Errors::ConfigurationError, + "SEARCH_SAFETY_RESERVE must be below SEARCH_REQUEST_CEILING" + end + if search_batch_size > 10 + raise Errors::ConfigurationError, "SEARCH_BATCH_SIZE must not exceed 10" + end + if search_worker_concurrency != 1 + raise Errors::ConfigurationError, + "SEARCH_WORKER_CONCURRENCY must be 1 while the global request gate serializes outbound calls" + end + if enrichment_retry_base_seconds > enrichment_retry_max_seconds + raise Errors::ConfigurationError, + "ENRICHMENT_RETRY_BASE_SECONDS must not exceed ENRICHMENT_RETRY_MAX_SECONDS" + end + if enrichment_cycle_budget_seconds >= 60 + raise Errors::ConfigurationError, + "ENRICHMENT_CYCLE_BUDGET_SECONDS must be below the 60-second dispatch tick" + end + if search_pacing_seconds >= enrichment_cycle_budget_seconds + raise Errors::ConfigurationError, + "SEARCH_PACING_SECONDS must be below ENRICHMENT_CYCLE_BUDGET_SECONDS " \ + "or no cycle could ever wait out its own pacing" + end + if enrichment_lease_seconds <= worst_case_fetch_seconds + raise Errors::ConfigurationError, <<~MESSAGE.squish + ENRICHMENT_LEASE_SECONDS (#{enrichment_lease_seconds}) must exceed the worst-case + single fetch of #{worst_case_fetch_seconds} seconds — (MAX_HTTP_RETRIES + 1) x + (MAX_REDIRECTS + 1) attempts, each waiting up to the #{RequestGate::WAIT_SECONDS}s + gate plus HTTP_OPEN_TIMEOUT_SECONDS + HTTP_READ_TIMEOUT_SECONDS — or a live + worker's claim could be stolen mid-request + MESSAGE + end + end + + public + + # The longest one executor call can legally take while still holding its claim: + # every retry restarts the redirect chain, and every hop waits for the gate and + # both HTTP timeouts. The lease must outlive this or a slow-but-alive worker + # loses its rows to a reclaim. + def worst_case_fetch_seconds + (max_http_retries + 1) * (max_redirects + 1) * + (RequestGate::WAIT_SECONDS + http_open_timeout_seconds + http_read_timeout_seconds) + end + + private + def read(env, variable) value = env[variable] value.nil? || value.to_s.strip.empty? ? DEFAULTS.fetch(variable) : value.to_s.strip diff --git a/app/services/github/enrichment/actor_document.rb b/app/services/github/enrichment/actor_document.rb index d298b9d..2956464 100644 --- a/app/services/github/enrichment/actor_document.rb +++ b/app/services/github/enrichment/actor_document.rb @@ -1,7 +1,7 @@ module Github module Enrichment - # §7's actor mapping, which is one line long: "(enrichment populates name and - # raw_payload)". + # The useful-data actor contract intentionally excludes full-profile fields. Search + # and detail responses contribute account type plus the complete raw item. # # login, display_login, api_url and avatar_url are deliberately absent. They are # envelope-owned — GithubActor::IDENTITY_MERGE is their only writer, and §7 keeps the @@ -15,7 +15,11 @@ class << self private def attributes_from(document) - { name: optional_string(document["name"]) } + { account_type: document["type"] } + end + + def contract_error(document) + required_string(document["type"], "type") unless document["type"].is_a?(String) && document["type"].present? end end end diff --git a/app/services/github/enrichment/admission.rb b/app/services/github/enrichment/admission.rb new file mode 100644 index 0000000..d67d13d --- /dev/null +++ b/app/services/github/enrichment/admission.rb @@ -0,0 +1,86 @@ +module Github + module Enrichment + # Read-side admission pre-checks for the two enrichment lanes. Advisory only: the + # ledgers re-check everything under their row locks, and a verdict that passes here + # can still lose the race there. What the pre-check buys is churn control — a denied + # tick enqueues no cycle and creates no enrichment_batches row. + # + # Never calls bootstrap! and never writes: a read path must not create ledger rows. + class Admission + Verdict = Data.define(:reason, :retry_in_seconds) do + def granted? = reason.nil? + end + + GRANTED = Verdict.new(reason: nil, retry_in_seconds: nil) + + def initialize(configuration: Github.configuration) + @configuration = configuration + end + + attr_reader :configuration + + # Mirrors SearchBudgetLedger#denial_reason in the same order, plus window + # awareness: counters from an elapsed window are stale, and the ledger will roll + # them on its next reservation. A missing row is a grant — the search ledger + # self-bootstraps from configuration, unlike core. + def search(now: Time.current) + budget = GithubSearchBudget.find_by(id: GithubSearchBudget::SINGLETON_ID) + return GRANTED if budget.nil? + + if budget.blocked_until.present? && budget.blocked_until > now + return deny(:search_blocked, budget.blocked_until - now) + end + if budget.last_request_at.present? + resume_at = budget.last_request_at + configuration.search_pacing_seconds + return deny(:search_pacing, resume_at - now) if resume_at > now + end + return GRANTED if window_elapsed?(budget, now: now) + + if budget.remaining.present? && budget.remaining <= budget.reserve + return deny(:search_reserve_reached, until_reset(budget, now)) + end + if budget.used >= budget.request_ceiling - budget.reserve + return deny(:search_ceiling_exhausted, until_reset(budget, now)) + end + + GRANTED + end + + # The core detail-fallback lane keeps the core ledger's bootstrap discipline: no + # window, no enrichment. The same checks the retired per-entity Dispatch made. + def detail(now: Time.current) + budget = GithubApiBudget.find_by(id: GithubApiBudget::SINGLETON_ID) + return deny(:window_uninitialized, nil) if budget.nil? || budget.window_initialized_at.nil? + return deny(:window_elapsed, nil) if budget.reset_at.present? && now >= budget.reset_at + + if budget.global_blocked_until.present? && budget.global_blocked_until > now + return deny(:globally_blocked, budget.global_blocked_until - now) + end + blocked_until = budget.enrichment_class_blocked_until(now: now) + if blocked_until.present? && blocked_until > now + return deny(:class_exhausted, blocked_until - now) + end + + GRANTED + end + + private + + def deny(reason, retry_in) + Verdict.new(reason: reason, retry_in_seconds: retry_in&.to_f) + end + + def window_elapsed?(budget, now:) + return now >= budget.reset_at if budget.reset_at.present? + + budget.last_request_at.present? && + budget.last_request_at <= now - SearchBudgetLedger::SEARCH_WINDOW_SECONDS && + budget.used.positive? + end + + def until_reset(budget, now) + budget.reset_at.present? ? budget.reset_at - now : nil + end + end + end +end diff --git a/app/services/github/enrichment/backlog_metrics.rb b/app/services/github/enrichment/backlog_metrics.rb index 50251b4..1714182 100644 --- a/app/services/github/enrichment/backlog_metrics.rb +++ b/app/services/github/enrichment/backlog_metrics.rb @@ -6,55 +6,94 @@ module Enrichment # github_repositories are the source of truth, so queue depth would under-report work by # design. This projection counts the same candidate statuses as # Enrichable.enrichment_candidates, including work deferred by retry backoff, and reports - # how long the oldest row has waited. + # how long the oldest row has waited — plus, per issue #45, the staged-pipeline view: + # per-stage counts with each stage's oldest FIFO instant, the contract backlog, and the + # windowed arrival/completion/terminal counts the catch-up verdict is computed from. # - # No selector is used here: selector scopes answer "claimable now" and may exclude a + # No claim scope is used here: claim scopes answer "claimable now" and exclude a # deferred row. Backlog observability answers the different question "what work must the # service eventually finish?" and therefore must include every backlog entity. - class BacklogMetrics < Data.define(:actor, :repository) - Entry = Data.define(:status_counts, :backlog_count, :oldest_pending_at, - :oldest_pending_age_seconds) + # + # Everything for one entity table comes from ONE aggregate statement. A worker may + # commit between statements, so separate reads could publish a combination of numbers + # that never existed in the database at any instant. The statement is a single + # sequential scan however many conditional aggregates ride on it; revisit with partial + # indexes on contract_completed_at / terminal_at only if the tables reach millions of + # rows. + class BacklogMetrics < Data.define(:actor, :repository, :window_seconds) + Entry = Data.define(:status_counts, :stage_counts, :stage_oldest, + :backlog_count, :contract_backlog_count, + :oldest_pending_at, :oldest_pending_age_seconds, + :arrivals, :completions, :terminals, :earliest_created_at) STATUSES = Enrichable::ENRICHMENT_STATUSES + STAGES = Enrichable::ENRICHMENT_STAGES + + def self.capture(now: Time.current, configuration: Github.configuration) + window_seconds = configuration.enrichment_metrics_window_seconds + floor = now - window_seconds - def self.capture(now: Time.current) - new(actor: entry_for(GithubActor, now: now), - repository: entry_for(GithubRepository, now: now)) + new(actor: entry_for(GithubActor, now: now, floor: floor), + repository: entry_for(GithubRepository, now: now, floor: floor), + window_seconds: window_seconds) end - def self.entry_for(model, now:) - # CandidateSelector's FIFO order is created_at, the immutable instant the entity - # entered this backlog. Reporting the same clock keeps "oldest" aligned with the - # row the worker will actually choose next. - # - # All status counts, the candidate count, and its oldest row come from one aggregate - # statement. A worker may commit between statements, so separate count/minimum reads - # could otherwise publish a combination that never existed in the database. - values = Array(model.unscoped.pick(*aggregate_columns(model))) + def self.entry_for(model, now:, floor:) + # The claim's FIFO order is created_at, the immutable instant the entity entered + # this backlog. Reporting the same clock keeps "oldest" aligned with the row the + # worker will actually choose next. + values = Array(model.unscoped.pick(*aggregate_columns(model, floor: floor))) + status_counts = STATUSES.zip(values.shift(STATUSES.length)).to_h .transform_values(&:to_i) .reject { |_status, count| count.zero? } + # Stage counts keep their zeros: the payload publishes every stage so a consumer + # never has to distinguish "absent key" from "counted zero". + stage_counts = STAGES.zip(values.shift(STAGES.length)).to_h.transform_values(&:to_i) + stage_oldest = STAGES.zip(values.shift(STAGES.length)).to_h + backlog_count = values.shift.to_i oldest = values.shift + contract_backlog_count = values.shift.to_i + arrivals = values.shift.to_i + completions = values.shift.to_i + terminals = values.shift.to_i + earliest_created_at = values.shift - Entry.new(status_counts: status_counts, + Entry.new(status_counts: status_counts, stage_counts: stage_counts, + stage_oldest: stage_oldest, backlog_count: backlog_count, + contract_backlog_count: contract_backlog_count, oldest_pending_at: oldest, - oldest_pending_age_seconds: age_seconds(oldest, now: now)) + oldest_pending_age_seconds: age_seconds(oldest, now: now), + arrivals: arrivals, completions: completions, terminals: terminals, + earliest_created_at: earliest_created_at) end private_class_method :entry_for - def self.aggregate_columns(model) + def self.aggregate_columns(model, floor:) table = model.arel_table status_column = table[:enrichment_status] + stage_column = table[:enrichment_stage] candidate_filter = status_column.in(Enrichable::CANDIDATE_STATUSES) + # Contract debt: not yet at the useful-data contract or a terminal outcome, and + # not a completed row transiting a refresh — the same rule Appendix G states. + contract_filter = stage_column.not_in(%w[contract_complete terminal]) + .and(status_column.not_eq("complete")) + bound = ->(value) { Arel::Nodes.build_quoted(value) } - STATUSES.map do |status| - count_if(status_column.eq(status)) - end + [ - count_if(candidate_filter), - minimum_if(candidate_filter, table[:created_at]) - ] + STATUSES.map { |status| count_if(status_column.eq(status)) } + + STAGES.map { |stage| count_if(stage_column.eq(stage)) } + + STAGES.map { |stage| minimum_if(stage_column.eq(stage), table[:created_at]) } + + [ + count_if(candidate_filter), + minimum_if(candidate_filter, table[:created_at]), + count_if(contract_filter), + count_if(table[:created_at].gt(bound.call(floor))), + count_if(table[:contract_completed_at].gt(bound.call(floor))), + count_if(table[:terminal_at].gt(bound.call(floor))), + Arel::Nodes::NamedFunction.new("MIN", [ table[:created_at] ]) + ] end private_class_method :aggregate_columns @@ -82,7 +121,6 @@ def self.age_seconds(timestamp, now:) [ (now - timestamp).floor, 0 ].max end - private_class_method :age_seconds end end end diff --git a/app/services/github/enrichment/backoff.rb b/app/services/github/enrichment/backoff.rb index 461c986..ae2f383 100644 --- a/app/services/github/enrichment/backoff.rb +++ b/app/services/github/enrichment/backoff.rb @@ -26,25 +26,33 @@ module Enrichment # retry sooner than the floor, and the floor is the one property §10 states # numerically. class Backoff + # Issue #45 makes the retry ladder configurable; these remain as the documented + # defaults ENRICHMENT_RETRY_BASE_SECONDS / ENRICHMENT_RETRY_MAX_SECONDS start from. BASE_SECONDS = 60 MAX_SECONDS = 3600 JITTER_FRACTION = 0.25 # @param random [Random] injected so a spec asserts the schedule without sleeping. - def initialize(random: Random.new) + def initialize(random: Random.new, base_seconds: nil, max_seconds: nil, + configuration: nil) + configuration ||= Github.configuration if base_seconds.nil? || max_seconds.nil? + @base_seconds = base_seconds || configuration.enrichment_retry_base_seconds + @max_seconds = max_seconds || configuration.enrichment_retry_max_seconds @random = random end + attr_reader :base_seconds, :max_seconds + # @param attempts [Integer] the count *including* the attempt being scheduled for, - # so the first failure waits BASE_SECONDS rather than half of it. + # so the first failure waits base_seconds rather than half of it. # @return [Float] seconds def delay_for(attempts) exponent = [ attempts.to_i, 1 ].max - 1 - base = [ BASE_SECONDS * (2**exponent), MAX_SECONDS ].min + base = [ base_seconds * (2**exponent), max_seconds ].min - # Capped after jitter, so MAX_SECONDS is an honest bound rather than a bound plus + # Capped after jitter, so max_seconds is an honest bound rather than a bound plus # up to 25%. - [ base + (@random.rand * base * JITTER_FRACTION), MAX_SECONDS.to_f ].min + [ base + (@random.rand * base * JITTER_FRACTION), max_seconds.to_f ].min end # @return [Time] diff --git a/app/services/github/enrichment/batch_claim.rb b/app/services/github/enrichment/batch_claim.rb new file mode 100644 index 0000000..bfcb0a1 --- /dev/null +++ b/app/services/github/enrichment/batch_claim.rb @@ -0,0 +1,162 @@ +module Github + module Enrichment + # Claims a coalesced batch of stable entity rows. The entity table is the work record; + # one row per GitHub id means repeated event demand is absorbed before planning. + # + # Composition policy (plan Appendix G): never-enriched backlog fills the batch FIFO + # by created_at, id. TTL-stale refresh candidates may only top up slots the backlog + # could not fill, and only while the other class has no claimable backlog either — + # otherwise the spare capacity belongs to that class's backlog, not to refresh. + class BatchClaim + class Item < Data.define(:id, :github_id, :identifier, :api_url, :previous_stage, + :enrichment_status, :enrichment_attempts) + end + + class Lease < Data.define(:entity_type, :batch, :token, :leased_until, :items) + def to_log + { entity_type: entity_type.key, enrichment_batch_id: batch.id, + batch_correlation_id: batch.correlation_id, + requested_count: items.length } + end + end + + # The stages a backlog claim may take. batch_in_flight is here so an expired + # lease is reclaimable; the leased_until clause keeps live leases invisible. + BACKLOG_STAGES = %w[batch_pending retry_scheduled batch_in_flight].freeze + REFRESH_STAGES = %w[contract_complete batch_in_flight].freeze + + def initialize(configuration: Github.configuration) + @configuration = configuration + end + + attr_reader :configuration + + # Never-enriched (or batch-retrying) rows this claim could take right now. + def backlog_scope(entity_type, now: Time.current) + due(entity_type.model, now: now) + .where(enrichment_status: Enrichable::CANDIDATE_STATUSES) + .where(enrichment_stage: BACKLOG_STAGES) + end + + # TTL-stale completed rows eligible for a staged refresh: recently active, + # not backed off, not held by a live lease. + def refresh_scope(entity_type, now: Time.current) + due(entity_type.model, now: now) + .where(enrichment_status: "complete") + .where(enrichment_stage: REFRESH_STAGES) + .where(fetched_at: ..(now - entity_type.refresh_ttl_seconds(configuration))) + .where(last_seen_at: (now - configuration.refresh_active_within_seconds)..) + end + + def claimable_backlog?(entity_type, now: Time.current) + backlog_scope(entity_type, now: now).exists? + end + + # Would #acquire return a lease for this class right now? Backlog always + # qualifies; refresh qualifies only under the composition policy above. + def claimable?(entity_type, now: Time.current) + return true if claimable_backlog?(entity_type, now: now) + + refresh_scope(entity_type, now: now).exists? && + EntityType.all.none? { |type| claimable_backlog?(type, now: now) } + end + + def acquire(entity_type, now: Time.current) + lease = nil + + entity_type.model.transaction do + rows = backlog_scope(entity_type, now: now) + .order(:created_at, :id) + .limit(configuration.search_batch_size) + .lock("FOR UPDATE SKIP LOCKED").to_a + rows += refresh_top_up(entity_type, taken: rows.length, now: now) + next if rows.empty? + + reclaim_stale_batches(entity_type, rows, now: now) + + identifiers = rows.map { |row| identifier_for(entity_type, row) } + batch = EnrichmentBatch.create!( + request_kind: "search", entity_kind: entity_type.key.to_s, + requested_github_ids: rows.map(&:github_id), requested_identifiers: identifiers, + requested_count: rows.length, + request_url: SearchQuery.build(entity_type, identifiers, mode: configuration.mode), + started_at: now + ) + token = SecureRandom.uuid + leased_until = now + configuration.enrichment_lease_seconds + + entity_type.model.where(id: rows.map(&:id)).update_all( + enrichment_stage: "batch_in_flight", lease_token: token, leased_until: leased_until, + current_enrichment_batch_id: batch.id + ) + + items = rows.zip(identifiers).map do |row, identifier| + Item.new(id: row.id, github_id: row.github_id, identifier: identifier, + api_url: row.api_url, previous_stage: row.enrichment_stage, + enrichment_status: row.enrichment_status, + enrichment_attempts: row.enrichment_attempts) + end + lease = Lease.new(entity_type: entity_type, batch: batch, token: token, + leased_until: leased_until, items: items.freeze) + end + + lease + end + + def release!(lease, stage: nil, now: Time.current) + lease.items.each do |item| + restored = stage || (item.enrichment_status == "complete" ? "contract_complete" : "batch_pending") + lease.entity_type.model.where(id: item.id, lease_token: lease.token, + current_enrichment_batch_id: lease.batch.id).update_all( + enrichment_stage: restored, lease_token: nil, leased_until: nil, + current_enrichment_batch_id: nil, updated_at: now + ) + end + end + + private + + # The single due predicate: a live lease or a scheduled retry excludes a row + # from every claim; expiry re-admits it. + def due(model, now:) + model.where(leased_until: nil).or(model.where(leased_until: ..now)) + .merge(model.where(next_retry_at: nil).or(model.where(next_retry_at: ..now))) + end + + def refresh_top_up(entity_type, taken:, now:) + spare = configuration.search_batch_size - taken + return [] unless spare.positive? + return [] if EntityType.all.any? { |type| claimable_backlog?(type, now: now) } + + refresh_scope(entity_type, now: now) + .order(:fetched_at, :id) + .limit(spare) + .lock("FOR UPDATE SKIP LOCKED").to_a + end + + # A row still pointing at an in-flight batch was leased by a worker that never + # finished; its lease has expired or we could not have locked it. The batch row + # is finalized as evidence rather than deleted. + def reclaim_stale_batches(entity_type, rows, now:) + stale_batch_ids = rows.filter_map(&:current_enrichment_batch_id).uniq + return if stale_batch_ids.empty? + + reclaimed = EnrichmentBatch.where(id: stale_batch_ids, status: "in_flight") + .update_all(status: "stale_lease", completed_at: now, + updated_at: now) + return unless reclaimed.positive? + + Rails.logger.warn(event: "enrichment.stale_lease_reclaimed", + entity_kind: entity_type.key, + enrichment_batch_ids: stale_batch_ids, count: reclaimed) + end + + def identifier_for(entity_type, row) + value = entity_type.key == :actor ? row.login : row.full_name + raise ArgumentError, "#{entity_type.key} #{row.github_id} has no Search identifier" if value.blank? + + value + end + end + end +end diff --git a/app/services/github/enrichment/batch_quality.rb b/app/services/github/enrichment/batch_quality.rb new file mode 100644 index 0000000..914dda0 --- /dev/null +++ b/app/services/github/enrichment/batch_quality.rb @@ -0,0 +1,102 @@ +module Github + module Enrichment + # Issue #45's batch-quality metrics: enrichment_batches aggregated over the trailing + # metrics window, grouped by request_kind x entity_kind. One grouped statement; the + # four groups are always present in the payload with counted zeros — the table was + # read and held nothing. + class BatchQuality < Data.define(:window_seconds, :window_start, :groups) + Group = Data.define(:attempts, :in_flight, :succeeded, :failed, :deferred, + :stale_lease, :requested_items, :returned_items, :valid_items, + :missing_items, :invalid_items, :fill_ratio, + :incomplete_results_count) + + EMPTY_GROUP = Group.new( + attempts: 0, in_flight: 0, succeeded: 0, failed: 0, deferred: 0, stale_lease: 0, + requested_items: 0, returned_items: 0, valid_items: 0, missing_items: 0, + invalid_items: 0, fill_ratio: nil, incomplete_results_count: 0 + ) + + STATUS_COLUMNS = EnrichmentBatch::STATUSES.freeze + + def self.capture(now: Time.current, configuration: Github.configuration) + window_seconds = configuration.enrichment_metrics_window_seconds + floor = now - window_seconds + + rows = EnrichmentBatch.where(started_at: floor..) + .group(:request_kind, :entity_kind) + .pluck(:request_kind, :entity_kind, *aggregates) + + groups = EnrichmentBatch::REQUEST_KINDS.index_with do |kind| + EnrichmentBatch::ENTITY_KINDS.index_with { EMPTY_GROUP } + end + rows.each do |row| + kind, entity_kind, *values = row + groups[kind][entity_kind] = group_from(values) + end + + new(window_seconds: window_seconds, window_start: floor, groups: groups) + end + + def self.aggregates + table = EnrichmentBatch.arel_table + star = Arel.star + + [ Arel::Nodes::NamedFunction.new("COUNT", [ star ]) ] + + STATUS_COLUMNS.map { |status| count_filter(table[:status].eq(status)) } + + %i[requested_count returned_count valid_count missing_count invalid_count].map do |column| + Arel::Nodes::NamedFunction.new( + "COALESCE", [ Arel::Nodes::NamedFunction.new("SUM", [ table[column] ]), + Arel::Nodes.build_quoted(0) ] + ) + end + [ count_filter(table[:incomplete_results].eq(true)) ] + end + private_class_method :aggregates + + # COUNT(CASE WHEN ...) rather than FILTER, matching the BacklogMetrics idiom. + def self.count_filter(predicate) + conditional = Arel::Nodes::Case.new.when(predicate).then(1) + + Arel::Nodes::NamedFunction.new("COUNT", [ conditional ]) + end + private_class_method :count_filter + + def self.group_from(values) + attempts = values.shift.to_i + statuses = STATUS_COLUMNS.map { values.shift.to_i } + requested, returned, valid, missing, invalid = Array.new(5) { values.shift.to_i } + incomplete = values.shift.to_i + + Group.new( + attempts: attempts, + in_flight: statuses[0], succeeded: statuses[1], failed: statuses[2], + deferred: statuses[3], stale_lease: statuses[4], + requested_items: requested, returned_items: returned, valid_items: valid, + missing_items: missing, invalid_items: invalid, + # A ratio with a zero denominator is null, never 0.0; the denominator is + # published beside it so the division is checkable. + fill_ratio: requested.zero? ? nil : (returned.to_f / requested).round(3), + incomplete_results_count: incomplete + ) + end + private_class_method :group_from + + def payload + { + window_seconds: window_seconds, + window_start: Ingestion::Report.timestamp(window_start), + search: kind_payload("search"), + detail: kind_payload("detail") + } + end + + private + + def kind_payload(kind) + { + actors: groups.fetch(kind).fetch("actor").to_h, + repositories: groups.fetch(kind).fetch("repository").to_h + } + end + end + end +end diff --git a/app/services/github/enrichment/batch_runner.rb b/app/services/github/enrichment/batch_runner.rb new file mode 100644 index 0000000..2cc8260 --- /dev/null +++ b/app/services/github/enrichment/batch_runner.rb @@ -0,0 +1,325 @@ +module Github + module Enrichment + # Executes one Search request for up to SEARCH_BATCH_SIZE stable entity ids, preserves + # every returned item, and projects only items that validate against the claimed id. + # + # incomplete_results=true is an envelope fact about the *query* (it timed out), not + # about any returned item: an item that validated against its immutable GitHub id is + # applied regardless, and only items missing from the response fall back to the + # bounded core detail lane. The envelope flag is retained on the batch row. + class BatchRunner + # Bodies are evidence only when something went wrong; successful batches already + # retain every item verbatim as observations. Bounded so a hostile or broken + # response cannot grow the batch table without limit. + RESPONSE_BODY_LIMIT = 65_536 + + class Result < Data.define(:status, :entity_type, :batch_id, :requested_count, + :returned_count, :valid_count, :fallback_count, + :deferral_reason) + def attempted? = %w[completed failed].include?(status) + end + + def initialize(executor: Github.executor, configuration: Github.configuration, + claim: BatchClaim.new(configuration: configuration), + search_ledger: SearchBudgetLedger.new(configuration: configuration), + backoff: Backoff.new(configuration: configuration), + clock: -> { Time.current }) + @executor = executor + @configuration = configuration + @claim = claim + @search_ledger = search_ledger + @backoff = backoff + @clock = clock + end + + def call(entity_class:) + entity_type = EntityType.fetch(entity_class) + lease = @claim.acquire(entity_type, now: @clock.call) + return idle(entity_type) if lease.nil? + + fetched = @executor.call(request_for(lease)) + @search_ledger.block_from!(fetched, now: @clock.call) + finish(lease, fetched) + rescue StandardError => error + abandon(lease, error) if lease + raise + end + + private + + def idle(entity_type) + Result.new(status: "idle", entity_type: entity_type.key, batch_id: nil, + requested_count: 0, returned_count: 0, valid_count: 0, + fallback_count: 0, deferral_reason: nil) + end + + def request_for(lease) + Request.new( + url: lease.batch.request_url, + request_class: lease.entity_type.search_request_class, + origin: :application, + context: lease.to_log + ) + end + + def finish(lease, fetched) + snapshot = fetched.rate_limit(observed_at: @clock.call) + record_response_metadata(lease.batch, fetched, snapshot) + + return defer(lease, fetched) if fetched.deferred? || %i[rate_limited secondary_limited].include?(fetched.classification) + return unsearchable_batch(lease, fetched) if unsearchable?(fetched) + return retry_later(lease, fetched) unless fetched.ok? + + response = SearchResponse.parse(fetched.body) + return malformed_batch(lease, response) unless response.ok? + + apply_response(lease, response) + end + + def apply_response(lease, response) + now = @clock.call + counts = { valid: 0, missing: 0, invalid: 0, fallback: 0 } + consumed = Set.new + + ActiveRecord::Base.transaction do + by_id = response.items.each_with_index.each_with_object({}) do |(raw, index), result| + result[raw["id"]] ||= [ raw, index ] if raw.is_a?(Hash) && raw["id"].is_a?(Integer) + end + + lease.items.each do |item| + raw, index = by_id[item.github_id] + unless raw + raw, index = identifier_match(lease.entity_type, response.items, item.identifier) + end + + if raw.nil? + counts[:missing] += 1 + counts[:fallback] += 1 + admit_fallback(lease, item, "missing_search_result", now: now) + next + end + + consumed << index + disposition = validate_item(lease.entity_type, item, raw) + observation = ObservationRecorder.record!( + entity_type: lease.entity_type, entity_github_id: item.github_id, + source: :search, raw_payload: raw, observed_at: now, + validation_outcome: disposition.fetch(:outcome), batch: lease.batch, + requested_identifier: item.identifier + ) + + if disposition.fetch(:apply) + applied = apply_projection(lease, item, disposition.fetch(:document), observation, now: now) + counts[applied ? :valid : :invalid] += 1 + else + counts[:invalid] += 1 + counts[:fallback] += 1 + admit_fallback(lease, item, disposition.fetch(:outcome), now: now) + end + end + + response.items.each_with_index do |raw, index| + next if consumed.include?(index) + + actual_id = raw.is_a?(Hash) && raw["id"].is_a?(Integer) ? raw["id"] : nil + ObservationRecorder.record!( + entity_type: lease.entity_type, entity_github_id: actual_id, + source: :search, raw_payload: raw, observed_at: now, + validation_outcome: "unrequested_result", batch: lease.batch + ) + counts[:invalid] += 1 + end + + lease.batch.update!( + status: "succeeded", completed_at: now, total_count: response.total_count, + incomplete_results: response.incomplete_results, + returned_count: response.items.length, valid_count: counts[:valid], + missing_count: counts[:missing], invalid_count: counts[:invalid] + ) + end + + result = Result.new(status: "completed", entity_type: lease.entity_type.key, + batch_id: lease.batch.id, requested_count: lease.items.length, + returned_count: response.items.length, valid_count: counts[:valid], + fallback_count: counts[:fallback], deferral_reason: nil) + Rails.logger.info(event: "enrichment.batch_completed", **result.to_h, + incomplete_results: response.incomplete_results) + result + end + + def validate_item(entity_type, item, raw) + return { apply: false, outcome: "identity_mismatch" } unless raw.is_a?(Hash) && raw["id"] == item.github_id + if entity_type.key == :repository && raw["full_name"].to_s.casecmp(item.identifier.to_s) != 0 + return { apply: false, outcome: "renamed_repository" } + end + + document = entity_type.document.parse(raw, github_id: item.github_id) + return { apply: false, outcome: document.error_code || document.kind.to_s } unless document.ok? + + { apply: true, outcome: "applied", document: document } + end + + def identifier_match(entity_type, items, identifier) + field = entity_type.key == :actor ? "login" : "full_name" + items.each_with_index.find do |raw, _index| + raw.is_a?(Hash) && raw[field].to_s.casecmp(identifier.to_s).zero? + end + end + + def apply_projection(lease, item, document, observation, now:) + model = lease.entity_type.model + attributes = document.attributes.merge( + enrichment_status: "complete", enrichment_stage: "contract_complete", + enrichment_attempts: 0, next_retry_at: nil, last_error: nil, + fetched_at: now, batch_applied_at: now, + contract_completed_at: keep_first(model, :contract_completed_at, now), + latest_observation_id: observation.id, latest_observation_source: "search", + latest_observed_at: now, lease_token: nil, leased_until: nil, + current_enrichment_batch_id: nil, updated_at: now + ) + + model.where(id: item.id, lease_token: lease.token, + current_enrichment_batch_id: lease.batch.id).update_all(attributes) == 1 + end + + # COALESCE(column, bound-now): the first completion instant is the durable one — + # a refresh must not re-count an entity as newly completed (§45 throughput). + def keep_first(model, column, now) + Arel::Nodes::NamedFunction.new( + "COALESCE", [ model.arel_table[column], Arel::Nodes.build_quoted(now) ] + ) + end + + def admit_fallback(lease, item, reason, now:) + lease.entity_type.model.where(id: item.id, lease_token: lease.token, + current_enrichment_batch_id: lease.batch.id).update_all( + enrichment_stage: "detail_pending", detail_pending_at: now, + last_error: reason, lease_token: nil, leased_until: nil, + current_enrichment_batch_id: nil, updated_at: now + ) + Rails.logger.info(event: "enrichment.fallback_admitted", + entity_type: lease.entity_type.key, + lease.entity_type.log_key => item.github_id, + reason: reason, enrichment_batch_id: lease.batch.id) + end + + def record_response_metadata(batch, fetched, snapshot) + batch.update!( + response_status: fetched.status, + response_body: retained_body(fetched), + rate_limit_resource: snapshot.resource, rate_limit_limit: snapshot.limit, + rate_limit_remaining: snapshot.remaining, rate_limit_used: snapshot.used, + rate_limit_reset_at: snapshot.reset_at + ) + end + + # Successful bodies live on as per-item observations; only failure evidence is + # retained here, bounded. + def retained_body(fetched) + return nil if fetched.ok? + + fetched.body.to_s.truncate(RESPONSE_BODY_LIMIT) + end + + def defer(lease, fetched) + now = @clock.call + reason = fetched.classification.to_s + lease.batch.update!(status: "deferred", completed_at: now, + last_error: fetched.error&.message || reason) + @claim.release!(lease, now: now) + Rails.logger.info(event: "enrichment.batch_deferred", **lease.to_log, + deferral_reason: reason) + Result.new(status: "deferred", entity_type: lease.entity_type.key, + batch_id: lease.batch.id, requested_count: lease.items.length, + returned_count: 0, valid_count: 0, fallback_count: 0, + deferral_reason: reason) + end + + def retry_later(lease, fetched) + reason = fetched.error&.message || fetched.classification.to_s + fail_batch(lease, reason, response_status: fetched.status) + end + + def malformed_batch(lease, response) + fail_batch(lease, response.error_message, + deferral_reason: "malformed_search_response") + end + + # Observed live, not in the probe: Search answers 422 rather than an empty result + # set when *every* requested identifier is unsearchable — the state a renamed or + # deleted entity is in (facebook/react, which now redirects to react/react). A + # batch of ten mixes them in and simply omits them, so this only occurs once the + # remaining members are all in that state. + # + # It is an authoritative per-item answer, so the members are admitted to the + # detail lane exactly as an omitted item would be. Retrying the search instead + # would reproduce the same 422 forever, and the stored payload URL is what + # actually resolves a rename. + UNSEARCHABLE_STATUS = 422 + UNSEARCHABLE_SIGNAL = "cannot be searched".freeze + + def unsearchable?(fetched) + fetched.status == UNSEARCHABLE_STATUS && + fetched.body.to_s.include?(UNSEARCHABLE_SIGNAL) + end + + def unsearchable_batch(lease, fetched) + now = @clock.call + lease.items.each do |item| + admit_fallback(lease, item, "unsearchable_identifier", now: now) + end + lease.batch.update!(status: "failed", completed_at: now, + last_error: "unsearchable_identifiers", + returned_count: 0, missing_count: lease.items.length) + Rails.logger.warn(event: "enrichment.batch_unsearchable", **lease.to_log, + response_status: fetched.status) + Result.new(status: "completed", entity_type: lease.entity_type.key, + batch_id: lease.batch.id, requested_count: lease.items.length, + returned_count: 0, valid_count: 0, + fallback_count: lease.items.length, deferral_reason: nil) + end + + def fail_batch(lease, reason, response_status: nil, deferral_reason: nil) + now = @clock.call + lease.batch.update!(status: "failed", completed_at: now, last_error: reason) + schedule_batch_retry(lease, now: now) + Rails.logger.warn(event: "enrichment.batch_failed", **lease.to_log, + reason: reason, response_status: response_status) + Result.new(status: "failed", entity_type: lease.entity_type.key, + batch_id: lease.batch.id, requested_count: lease.items.length, + returned_count: 0, valid_count: 0, fallback_count: 0, + deferral_reason: deferral_reason || reason) + end + + # A failed batch reschedules every member. Never-enriched members rest in + # retry_scheduled (the batch path owns that stage); already-complete refresh + # members return to contract_complete, their next_retry_at holding the next + # refresh claim off until the backoff clears. + def schedule_batch_retry(lease, now:) + lease.items.each do |item| + attempts = item.enrichment_attempts + 1 + complete = item.enrichment_status == "complete" + lease.entity_type.model.where(id: item.id, lease_token: lease.token, + current_enrichment_batch_id: lease.batch.id).update_all( + enrichment_status: complete ? "complete" : "retryable_failure", + enrichment_stage: complete ? "contract_complete" : "retry_scheduled", + enrichment_attempts: attempts, + next_retry_at: @backoff.retry_at(attempts, now: now), + retry_scheduled_at: now, + lease_token: nil, leased_until: nil, current_enrichment_batch_id: nil, + updated_at: now + ) + end + end + + # A crash between claim and outcome must not leave the batch row in_flight + # forever: finalize it as failed evidence, give the rows back, re-raise. + def abandon(lease, error) + now = @clock.call + lease.batch.update!(status: "failed", completed_at: now, + last_error: error.class.name) + @claim.release!(lease, now: now) + end + end + end +end diff --git a/app/services/github/enrichment/candidate_selector.rb b/app/services/github/enrichment/candidate_selector.rb deleted file mode 100644 index 49bfa07..0000000 --- a/app/services/github/enrichment/candidate_selector.rb +++ /dev/null @@ -1,151 +0,0 @@ -module Github - module Enrichment - # §10's two candidate pools and the predicate borrowing asks about. - # - # Deliberately holds no executor and no transport — the technique - # Github::Ingestion::PageWriter uses — so a GitHub request cannot be issued from a - # selection query, and every method here is a read. - # - # **Two pools, not one status.** Never-enriched candidates remain durable pending work, - # while staleness is a derived predicate over complete rows. Collapsing stale rows - # into pending would erase the distinction that keeps first-time enrichment ahead of - # refresh traffic. A stale-but-enriched entity therefore keeps reading `complete` — - # which is true, because its payload is still present. - # - # Two further consequences of that reading, both intended: a background writer that - # mutated rows purely because time passed is the shape this codebase refuses - # everywhere (Github::Ingestion::PollState, Github::BudgetLedger#log_block_cleared), - # and index_*_on_enrichment_candidates keeps its PR 3 predicate instead of silently - # absorbing every refresh row. - class CandidateSelector - POOLS = %i[ pending refresh ].freeze - - # Durable FIFO. created_at is the instant the entity entered the backlog; unlike - # last_seen_at it is immutable when later events reference the same entity. The id - # tie-break makes candidates created in the same timestamp deterministic. - PENDING_ORDER = "created_at ASC, id ASC".freeze - - # Oldest-fetched first, which is the rule that terminates: a monotone refresh queue - # cannot starve a complete row behind a hotter neighbour. The first-time backlog - # has its own FIFO key, so the refresh pool needs its own ordering rule. - REFRESH_ORDER = "fetched_at ASC, id ASC".freeze - - # When a complete row next becomes refreshable. See #earliest_refresh_at. - REFRESH_DUE_AT = "GREATEST(fetched_at + make_interval(secs => ?), next_retry_at)".freeze - - def initialize(configuration: Github.configuration) - @configuration = configuration - end - - attr_reader :configuration - - # @param entity_type [Github::Enrichment::EntityType] - # @param pool [Symbol] :pending or :refresh - # @return [ActiveRecord::Relation] - def scope(entity_type, pool:, now:) - case pool - when :pending then pending_scope(entity_type, now: now) - when :refresh then refresh_scope(entity_type, now: now) - else raise ArgumentError, "unknown pool #{pool.inspect}" - end - end - - # Whether this class has first-time backlog work that can be claimed now. Fairness - # uses this narrower predicate for borrowing: a class in backoff need not leave the - # other class's reserved attempts idle. Refresh suppression uses #pending_backlog? - # instead, because it asks whether first-time work exists at all. - def pending_available?(entity_type, now:) - pending_scope(entity_type, now: now).exists? - end - - # Backlog presence is deliberately broader than current claimability. A row in - # backoff or carrying an in-flight lease still represents first-time enrichment - # work, so refresh traffic must not consume the quota reserved for that backlog. - def pending_backlog?(entity_type) - entity_type.model.where(enrichment_status: Enrichable::CANDIDATE_STATUSES).exists? - end - - # Whether either pool could hand out work for this class right now. Asked before any - # "when next?" question, because the two are different questions and deriving one - # from the other is what makes a report say "due now" while the command it sits next - # to says there is nothing to enrich. - def claimable?(entity_type, now:) - return true if pending_available?(entity_type, now: now) - return false if EntityType.all.any? { |type| pending_backlog?(type) } - - refresh_available?(entity_type, now: now) - end - - def refresh_available?(entity_type, now:) - scope(entity_type, pool: :refresh, now: now).exists? - end - - # The soonest instant at which *either* pool will have work for this class, for a - # caller that has already established neither has any now. - # @return [Time, nil] nil when nothing will ever become claimable without new activity - def earliest_claimable_at(entity_type, now:) - if EntityType.all.any? { |type| pending_backlog?(type) } - earliest_pending_at(entity_type, now: now) - else - earliest_refresh_at(entity_type, now: now) - end - end - - # A pending candidate held back only by its own backoff, lease, or secondary-limit - # deferral. Pending work has no age cutoff: this instant is a real promise that the - # row returns to the actionable backlog. - # @return [Time, nil] - def earliest_pending_at(entity_type, now:) - entity_type.model - .where(enrichment_status: Enrichable::CANDIDATE_STATUSES) - .where.not(next_retry_at: nil) - .where(next_retry_at: now..) - .minimum(:next_retry_at) - end - - # When the freshness cache next lets go. A complete row's next legal fetch is the - # later of its TTL expiry and any retry instant a failed refresh left behind — so - # GREATEST, not the minimum of two independent columns, and the minimum is taken over - # that expression rather than over fetched_at alone: the oldest document may be the - # one carrying the longest backoff. - # - # GREATEST ignores NULL in PostgreSQL, which is the behaviour rather than the hazard - # here — a complete row with no retry instant is due at its TTL expiry, full stop. - # A NULL fetched_at is excluded for the same reason #refresh_scope excludes it: the - # refresh predicate could never match such a row, so it has no next refresh to name. - # @return [Time, nil] - def earliest_refresh_at(entity_type, now:) - expression = ActiveRecord::Base.sanitize_sql_array( - [ REFRESH_DUE_AT, entity_type.refresh_ttl_seconds(configuration) ] - ) - - entity_type.model.complete.where.not(fetched_at: nil).minimum(Arel.sql(expression)) - end - - def stale_before(entity_type, now) - now - entity_type.refresh_ttl_seconds(configuration) - end - - private - - def pending_scope(entity_type, now:) - due(entity_type.model.where(enrichment_status: Enrichable::CANDIDATE_STATUSES), now) - .order(Arel.sql(PENDING_ORDER)) - end - - def refresh_scope(entity_type, now:) - due(entity_type.model.complete, now) - .where(fetched_at: ..stale_before(entity_type, now)) - .order(Arel.sql(REFRESH_ORDER)) - end - - # One predicate, spelled once. next_retry_at means "may not be attempted before T" - # everywhere in this state machine — a failure backoff, a secondary-limit deferral, - # and Github::Enrichment::Claim's in-flight lease all write it — so this single - # clause excludes all three from both claimable pools. - def due(relation, now) - relation.where("next_retry_at IS NULL OR next_retry_at <= :now", now: now) - end - end - end -end diff --git a/app/services/github/enrichment/claim.rb b/app/services/github/enrichment/claim.rb deleted file mode 100644 index 3dff449..0000000 --- a/app/services/github/enrichment/claim.rb +++ /dev/null @@ -1,194 +0,0 @@ -module Github - module Enrichment - # S3.6: "Prevent duplicate concurrent enrichment (keyed by the entity row)." - # - # The fetch happens outside every transaction — §8 forbids one spanning network I/O, - # and Github::BudgetLedger#assert_committable! raises if one is open — so the claim - # cannot be a held lock. It is a **lease**: one conditional UPDATE pushes the entity's - # next_retry_at into the future, and every other reader of that column then treats the - # row as not attemptable. A crashed worker leaves nothing to clean up; the lease - # simply expires. - # - # Leasing on next_retry_at rather than on a new column is the decision this class - # rests on, and its payoff is elsewhere: both candidate pools spell the same - # "next_retry_at IS NULL OR next_retry_at <= now" clause, so one predicate excludes - # in-flight rows consistently. A separate leased_until column would need every query - # to carry a second condition, and the first that forgot could hand work to two workers. - # - # Holds no executor and no transport, so a GitHub request cannot be issued from - # inside a claim. - class Claim - # The candidate statuses each pool may claim from. The pending statement's guard - # excludes `complete`, so one statement provably cannot serve both pools. - POOL_STATUSES = { - pending: Enrichable::CANDIDATE_STATUSES, - refresh: %w[ complete ] - }.freeze - - RETURNED_COLUMNS = %w[ - id github_id api_url enrichment_status enrichment_attempts fetched_at last_seen_at - ].freeze - - # What one worker holds while it fetches. - class Lease < Data.define(:entity_type, :pool, :id, :github_id, :api_url, - :enrichment_status, :enrichment_attempts, :fetched_at, - :last_seen_at, :previous_next_retry_at, :leased_until) - def to_log - { entity_type: entity_type.key, entity_type.log_key => github_id, pool: pool, - entity_status: enrichment_status, enrichment_attempt: enrichment_attempts + 1 } - end - end - - def initialize(configuration: Github.configuration, - selector: CandidateSelector.new(configuration: configuration), - gate_wait_seconds: RequestGate::WAIT_SECONDS) - @configuration = configuration - @selector = selector - @gate_wait_seconds = gate_wait_seconds - end - - attr_reader :configuration, :selector - - # @param entity_type [Github::Enrichment::EntityType] - # @param pool [Symbol] :pending or :refresh - # @return [Lease, nil] nil when nothing was claimable — an empty pool, or a race - # another worker won. - def acquire(entity_type, pool:, now:) - statuses = POOL_STATUSES.fetch(pool) { raise ArgumentError, "unknown pool #{pool.inspect}" } - leased_until = now + lease_seconds - - row = execute(claim_sql(entity_type, pool: pool, statuses: statuses, now: now, - leased_until: leased_until), - "Github::Enrichment::Claim Acquire").first - return nil if row.nil? - - build_lease(entity_type, pool, row) - end - - # Undoes a claim that never became an attempt: a budget denial, a busy gate, or a - # corpus gap. It restores the *exact* prior instant and writes nothing else — no - # attempt count, no error, no status, not even updated_at — so a deferred cycle - # leaves the row byte-for-byte as it found it. §7's "failures stay spent" has a - # mirror here: deferrals leave no trace. - # - # Restoring rather than nulling is behaviourally identical (the claim guard proves - # the prior value was NULL or already past), but nulling would erase when the entity - # last backed off, from a code path that performed no fetch. - # - # The next_retry_at guard is load-bearing, not defensive: lease_seconds is the - # worst-case runtime by construction, so a lease expiring mid-flight is reachable, - # and an ungarded late release would clear another worker's fresh lease. - # @return [Boolean] whether this lease was still held - def release!(lease) - updated = ActiveRecord::Base.connection.exec_update( - ActiveRecord::Base.sanitize_sql_array([ <<~SQL.squish, lease.previous_next_retry_at, lease.id, lease.leased_until ]), - UPDATE #{lease.entity_type.table_name} SET next_retry_at = ? WHERE id = ? AND next_retry_at = ? - SQL - "Github::Enrichment::Claim Release" - ) - - updated == 1 - end - - # Derived rather than chosen, following RequestGate::WAIT_SECONDS' precedent — §2A - # pins the operational defaults and this follows from them. Every term is a real - # component's worst case: one gated attempt may wait the whole gate timeout plus - # both HTTP timeouts; each retry restarts redirect following from the original - # request (Github::RequestExecutor#follow_redirects); and the backoff sleeps between - # attempts still consume wall clock. - # - # At the pinned defaults: 3 attempts x 3 hops x (45 + 5 + 15) + 8.75 = 594 seconds. - def lease_seconds - attempts = 1 + configuration.max_http_retries - hops = 1 + configuration.max_redirects - per_request = @gate_wait_seconds + - configuration.http_open_timeout_seconds + - configuration.http_read_timeout_seconds - backoff = RetryPolicy::RETRY_BASE_DELAY_SECONDS * - ((2**attempts) - 1) * (1 + RetryPolicy::RETRY_JITTER_FRACTION) - - ((attempts * hops * per_request) + backoff).ceil - end - - private - - # The candidate CTE is Github::Enrichment::CandidateSelector's own scope, so the - # FIFO order, the TTL, and the two pool rules are defined in exactly - # one place and this statement cannot drift from the pool it claims out of. - # - # Three details carry the correctness: - # - # 1. FOR UPDATE SKIP LOCKED turns "correct" into "makes progress". Without it two - # workers pick the same row and one wastes a cycle; with it the second takes - # the next-best candidate. PR 8 runs several, so this matters. - # 2. The status and due guards are repeated on the *outer* UPDATE. Under READ - # COMMITTED a blocked UPDATE re-evaluates its quals against the newly committed - # row version, and `entities.id = candidate.id` alone is a constant that would - # pass — letting the second worker overwrite the first one's lease. Repeating - # the guards makes the re-check reject. The CTE's FOR UPDATE performs its own - # re-check too, so this is belt and braces; keeping it means the safety claim - # needs no argument about PostgreSQL internals. - # 3. RETURNING takes the *prior* instant from the CTE snapshot. PostgreSQL 16 has - # no RETURNING OLD.*, and entities.next_retry_at would hand back the lease we - # just wrote rather than the value a release has to restore. - # - # updated_at is deliberately absent from the SET list. GithubActor::IDENTITY_MERGE - # gates every identity refresh on `EXCLUDED.updated_at >= github_actors.updated_at`, - # so bumping it here would make a concurrently-processed page whose received_at - # predates the lease silently lose its refresh — to a write that may be released a - # second later. updated_at moves on writes that change the entity's observable - # state, and a lease is not observable state. - def claim_sql(entity_type, pool:, statuses:, now:, leased_until:) - table = entity_type.table_name - candidate = selector.scope(entity_type, pool: pool, now: now) - .select(:id, :next_retry_at).limit(1).lock("FOR UPDATE SKIP LOCKED").to_sql - returned = RETURNED_COLUMNS.map { |column| "entities.#{column}" }.join(", ") - - ActiveRecord::Base.sanitize_sql_array([ <<~SQL.squish, leased_until, statuses, now ]) - WITH candidate AS (#{candidate}) - UPDATE #{table} AS entities - SET next_retry_at = ? - FROM candidate - WHERE entities.id = candidate.id - AND entities.enrichment_status IN (?) - AND (entities.next_retry_at IS NULL OR entities.next_retry_at <= ?) - RETURNING #{returned}, entities.next_retry_at AS leased_until, - candidate.next_retry_at AS previous_next_retry_at - SQL - end - - def execute(sql, name) - ActiveRecord::Base.connection.exec_query(sql, name) - end - - # The two instants come back from RETURNING rather than from Ruby, so the lease this - # object holds is byte-identical to what PostgreSQL stored — which is what makes the - # `next_retry_at = ?` guards in #release! and in Github::Enrichment::EntityState - # exact rather than dependent on timestamp rounding. - def build_lease(entity_type, pool, row) - Lease.new( - entity_type: entity_type, - pool: pool, - id: row.fetch("id"), - github_id: row.fetch("github_id"), - api_url: row.fetch("api_url"), - enrichment_status: row.fetch("enrichment_status"), - enrichment_attempts: row.fetch("enrichment_attempts"), - fetched_at: timestamp(row.fetch("fetched_at")), - last_seen_at: timestamp(row.fetch("last_seen_at")), - previous_next_retry_at: timestamp(row.fetch("previous_next_retry_at")), - leased_until: timestamp(row.fetch("leased_until")) - ) - end - - # exec_query bypasses the attribute types a model would apply, so a timestamp comes - # back as whatever the adapter produced. - def timestamp(value) - return nil if value.nil? - return value if value.is_a?(Time) - - Time.zone.parse(value.to_s) - end - end - end -end diff --git a/app/services/github/enrichment/cycle_runner.rb b/app/services/github/enrichment/cycle_runner.rb new file mode 100644 index 0000000..a4a41f8 --- /dev/null +++ b/app/services/github/enrichment/cycle_runner.rb @@ -0,0 +1,191 @@ +module Github + module Enrichment + # One worker cycle over the staged pipeline: as many Search batches as the search + # ledger will grant (waiting out pacing when the wait fits), then as many detail + # fallbacks as the core allowance will grant, all inside one wall-clock budget that + # keeps a cycle shorter than the 60-second dispatch tick. + # + # This is the shape that reaches catch-up throughput: at the defaults — ceiling 10, + # reserve 2, pacing 6s, batches of 10 — one cycle can move ~80 entities/minute + # (~4,800/hour theoretical) where a job-per-request design at the tick cadence + # could never exceed ~1,200/hour. The pacing sleep happens here, on the dedicated + # single-thread enrichment worker, never inside a gate hold or a ledger lock. + class CycleRunner + # Two consecutive idle claims (a claimable? race that found nothing to lock) + # end the phase rather than spinning on it. + MAX_CONSECUTIVE_IDLE = 2 + + class Cycle < Data.define(:batches_attempted, :batches_completed, :batches_deferred, + :batches_failed, :items_requested, :items_valid, + :fallbacks_admitted, :details_attempted, :details_completed, + :details_terminal, :details_deferred, + :batch_stop_reason, :detail_stop_reason, :duration_ms) + def to_log = to_h.compact + end + + # Weighted round-robin over the two lanes, borrowing the slot when the scheduled + # lane has nothing claimable. For search batches the borrow is purely a + # scheduling fact — the search ledger has no per-lane caps and its actor_used / + # repository_used counters exist for observability. For detail requests the flag + # travels to the core ledger, which enforces the 2/2 share split under its row + # lock exactly as before. + class LaneSchedule + def initialize(actor_weight:, repository_weight:) + @rotation = ([ :actor ] * actor_weight) + ([ :repository ] * repository_weight) + @cursor = 0 + end + + # @param claimable [Proc] lane key -> Boolean + # @return [Array(Symbol, Boolean), nil] lane and whether the slot was borrowed + def next_claimable(claimable) + scheduled = @rotation[@cursor % @rotation.length] + @cursor += 1 + return [ scheduled, false ] if claimable.call(scheduled) + + other = scheduled == :actor ? :repository : :actor + return [ other, true ] if claimable.call(other) + + nil + end + end + + def initialize(configuration: Github.configuration, + batch_runner: BatchRunner.new(configuration: configuration), + detail_runner: DetailRunner.new(configuration: configuration), + admission: Admission.new(configuration: configuration), + batch_claim: BatchClaim.new(configuration: configuration), + detail_claim: DetailClaim.new(configuration: configuration), + clock: -> { Time.current }, + monotonic: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }, + sleeper: ->(seconds) { Kernel.sleep(seconds) }) + @configuration = configuration + @batch_runner = batch_runner + @detail_runner = detail_runner + @admission = admission + @batch_claim = batch_claim + @detail_claim = detail_claim + @clock = clock + @monotonic = monotonic + @sleeper = sleeper + end + + def call + started = @monotonic.call + @deadline = started + @configuration.enrichment_cycle_budget_seconds + @tally = Hash.new(0) + + batch_stop = batch_phase + detail_stop = detail_phase + + cycle = Cycle.new( + batches_attempted: @tally[:batches_attempted], + batches_completed: @tally[:batches_completed], + batches_deferred: @tally[:batches_deferred], + batches_failed: @tally[:batches_failed], + items_requested: @tally[:items_requested], + items_valid: @tally[:items_valid], + fallbacks_admitted: @tally[:fallbacks_admitted], + details_attempted: @tally[:details_attempted], + details_completed: @tally[:details_completed], + details_terminal: @tally[:details_terminal], + details_deferred: @tally[:details_deferred], + batch_stop_reason: batch_stop.to_s, + detail_stop_reason: detail_stop.to_s, + duration_ms: ((@monotonic.call - started) * 1000).round + ) + Rails.logger.info(event: "enrichment.cycle_completed", **cycle.to_log) + cycle + end + + private + + def batch_phase + lanes = LaneSchedule.new(actor_weight: @configuration.actor_enrichment_weight, + repository_weight: @configuration.repository_enrichment_weight) + idle_streak = 0 + + loop do + return :cycle_budget if @monotonic.call >= @deadline + + verdict = @admission.search(now: @clock.call) + unless verdict.granted? + next if waited_out_pacing?(verdict) + + return verdict.reason + end + + choice = lanes.next_claimable(->(lane) { @batch_claim.claimable?(EntityType.fetch(lane), now: @clock.call) }) + return :no_batch_work if choice.nil? + + result = @batch_runner.call(entity_class: choice.first) + case result.status + when "idle" + idle_streak += 1 + return :no_batch_work if idle_streak >= MAX_CONSECUTIVE_IDLE + when "deferred" + # The ledger's denial under its lock outranks the advisory pre-check. + tally_batch(result) + return result.deferral_reason + else + idle_streak = 0 + tally_batch(result) + end + end + end + + def detail_phase + lanes = LaneSchedule.new(actor_weight: @configuration.actor_enrichment_weight, + repository_weight: @configuration.repository_enrichment_weight) + idle_streak = 0 + + loop do + return :cycle_budget if @monotonic.call >= @deadline + + verdict = @admission.detail(now: @clock.call) + return verdict.reason unless verdict.granted? + + choice = lanes.next_claimable(->(lane) { @detail_claim.claimable?(EntityType.fetch(lane), now: @clock.call) }) + return :no_detail_work if choice.nil? + + lane, borrowed = choice + result = @detail_runner.call(entity_class: lane, borrow: borrowed) + case result.status + when "idle" + idle_streak += 1 + return :no_detail_work if idle_streak >= MAX_CONSECUTIVE_IDLE + when "deferred" + @tally[:details_attempted] += 1 + @tally[:details_deferred] += 1 + return result.reason + else + idle_streak = 0 + @tally[:details_attempted] += 1 + @tally[:details_completed] += 1 if result.status == "completed" + @tally[:details_terminal] += 1 if result.status == "terminal" + end + end + end + + # A pacing wait that fits before the deadline is slept through; one that does + # not ends the phase — the next tick's cycle resumes exactly where this left off. + def waited_out_pacing?(verdict) + return false unless verdict.reason == :search_pacing + return false if verdict.retry_in_seconds.nil? + return false if @monotonic.call + verdict.retry_in_seconds >= @deadline + + @sleeper.call(verdict.retry_in_seconds) + true + end + + def tally_batch(result) + @tally[:batches_attempted] += 1 + @tally[:batches_completed] += 1 if result.status == "completed" + @tally[:batches_deferred] += 1 if result.status == "deferred" + @tally[:batches_failed] += 1 if result.status == "failed" + @tally[:items_requested] += result.requested_count + @tally[:items_valid] += result.valid_count + @tally[:fallbacks_admitted] += result.fallback_count + end + end + end +end diff --git a/app/services/github/enrichment/detail_claim.rb b/app/services/github/enrichment/detail_claim.rb new file mode 100644 index 0000000..de882fc --- /dev/null +++ b/app/services/github/enrichment/detail_claim.rb @@ -0,0 +1,89 @@ +module Github + module Enrichment + # Claims one row from the bounded detail-fallback lane. Once a batch admits a row + # here it stays on the detail path until success or an entity-specific terminal + # outcome — the stage vocabulary keeps the two lanes' claims provably disjoint: + # the batch path owns batch_pending/retry_scheduled/batch_in_flight, this one owns + # detail_pending/detail_in_flight. + class DetailClaim + Item = Data.define(:id, :github_id, :identifier, :api_url, :enrichment_status, + :enrichment_attempts, :detail_attempts) + Lease = Data.define(:entity_type, :batch, :token, :leased_until, :item) + + CLAIMABLE_STAGES = %w[detail_pending detail_in_flight].freeze + + def initialize(configuration: Github.configuration) + @configuration = configuration + end + + attr_reader :configuration + + # FIFO by fallback admission: oldest detail_pending_at first. + def scope(entity_type, now: Time.current) + entity_type.model + .where(enrichment_stage: CLAIMABLE_STAGES) + .where.not(detail_pending_at: nil) + .where("next_retry_at IS NULL OR next_retry_at <= ?", now) + .where("leased_until IS NULL OR leased_until <= ?", now) + end + + def claimable?(entity_type, now: Time.current) + scope(entity_type, now: now).exists? + end + + def acquire(entity_type, now: Time.current) + lease = nil + entity_type.model.transaction do + row = scope(entity_type, now: now) + .order(:detail_pending_at, :id) + .lock("FOR UPDATE SKIP LOCKED").first + next if row.nil? + + reclaim_stale_batch(entity_type, row, now: now) + + identifier = entity_type.key == :actor ? row.login : row.full_name + batch = EnrichmentBatch.create!( + request_kind: "detail", entity_kind: entity_type.key.to_s, + requested_github_ids: [ row.github_id ], requested_identifiers: [ identifier ], + requested_count: 1, request_url: row.api_url, started_at: now + ) + token = SecureRandom.uuid + leased_until = now + @configuration.enrichment_lease_seconds + row.update_columns(enrichment_stage: "detail_in_flight", lease_token: token, + leased_until: leased_until, current_enrichment_batch_id: batch.id) + item = Item.new(id: row.id, github_id: row.github_id, identifier: identifier, + api_url: row.api_url, enrichment_status: row.enrichment_status, + enrichment_attempts: row.enrichment_attempts, + detail_attempts: row.detail_attempts) + lease = Lease.new(entity_type: entity_type, batch: batch, token: token, + leased_until: leased_until, item: item) + end + lease + end + + def release!(lease, now: Time.current) + lease.entity_type.model.where(id: lease.item.id, lease_token: lease.token, + current_enrichment_batch_id: lease.batch.id).update_all( + enrichment_stage: "detail_pending", lease_token: nil, leased_until: nil, + current_enrichment_batch_id: nil, updated_at: now + ) + end + + private + + def reclaim_stale_batch(entity_type, row, now:) + return if row.current_enrichment_batch_id.nil? + + reclaimed = EnrichmentBatch.where(id: row.current_enrichment_batch_id, status: "in_flight") + .update_all(status: "stale_lease", completed_at: now, + updated_at: now) + return unless reclaimed.positive? + + Rails.logger.warn(event: "enrichment.stale_lease_reclaimed", + entity_kind: entity_type.key, + enrichment_batch_ids: [ row.current_enrichment_batch_id ], + count: reclaimed) + end + end + end +end diff --git a/app/services/github/enrichment/detail_runner.rb b/app/services/github/enrichment/detail_runner.rb new file mode 100644 index 0000000..bfb0f4b --- /dev/null +++ b/app/services/github/enrichment/detail_runner.rb @@ -0,0 +1,206 @@ +module Github + module Enrichment + # The bounded exception path. It uses only the API URL retained from the event, and + # therefore cannot invent a login/name URL after a missing or renamed Search result. + # + # Outcomes (§10, restated by Appendix G): a confirmed 404/410 is the one immediate + # entity-specific terminal outcome; every other failure climbs the retry ladder and + # becomes terminal only after DETAIL_FALLBACK_MAX_ATTEMPTS. Quota denial defers and + # is never an attempt. + class DetailRunner + RESPONSE_BODY_LIMIT = BatchRunner::RESPONSE_BODY_LIMIT + + # Outcomes no retry can change, so the ladder is skipped entirely (§10's + # classification table, and the dispositions the retired per-entity path used): + # a deleted entity, a rejected request, and a URL this application's SSRF policy + # refuses or a redirect chain that exceeded its bound. The last is not + # hypothetical — a live actor login containing brackets yields an unparsable + # payload URL, and retrying it would spend the scarce core detail allowance three + # times to refuse the same stored string. + TERMINAL_CLASSIFICATIONS = %i[ not_found client_error permanent_error ].freeze + + Result = Data.define(:status, :entity_type, :github_id, :batch_id, :reason) + + def initialize(executor: Github.executor, configuration: Github.configuration, + claim: DetailClaim.new(configuration: configuration), + rate_limit_policy: RateLimitPolicy.new, + backoff: Backoff.new(configuration: configuration), + clock: -> { Time.current }) + @executor = executor + @configuration = configuration + @claim = claim + @rate_limit_policy = rate_limit_policy + @backoff = backoff + @clock = clock + end + + def call(entity_class:, borrow: false) + type = EntityType.fetch(entity_class) + lease = @claim.acquire(type, now: @clock.call) + return Result.new(status: "idle", entity_type: type.key, github_id: nil, + batch_id: nil, reason: nil) if lease.nil? + + fetched = @executor.call(Request.new( + url: lease.item.api_url, request_class: type.request_class, origin: :payload, + borrow: borrow, + context: { enrichment_batch_id: lease.batch.id, + batch_correlation_id: lease.batch.correlation_id, + type.log_key => lease.item.github_id } + )) + decision = @rate_limit_policy.apply!(fetched, now: @clock.call) + finish(lease, fetched, decision) + rescue StandardError => error + abandon(lease, error) if lease + raise + end + + private + + def finish(lease, fetched, _decision) + now = @clock.call + snapshot = fetched.rate_limit(observed_at: now) + lease.batch.update!( + response_status: fetched.status, + response_body: fetched.ok? ? nil : fetched.body.to_s.truncate(RESPONSE_BODY_LIMIT), + rate_limit_resource: snapshot.resource, rate_limit_limit: snapshot.limit, + rate_limit_remaining: snapshot.remaining, rate_limit_used: snapshot.used, + rate_limit_reset_at: snapshot.reset_at + ) + + if fetched.deferred? || %i[rate_limited secondary_limited].include?(fetched.classification) + lease.batch.update!(status: "deferred", completed_at: now, + last_error: fetched.error&.message || fetched.classification.to_s) + @claim.release!(lease, now: now) + Rails.logger.info(event: "enrichment.detail_deferred", + entity_type: lease.entity_type.key, + lease.entity_type.log_key => lease.item.github_id, + deferral_reason: fetched.classification.to_s) + return result(lease, "deferred", fetched.classification.to_s) + end + + if fetched.ok? + document = lease.entity_type.document.parse(fetched.body, github_id: lease.item.github_id) + raw = decode_raw(fetched.body) + observation = ObservationRecorder.record!( + entity_type: lease.entity_type, entity_github_id: lease.item.github_id, + source: :detail, raw_payload: raw, observed_at: now, + validation_outcome: document.ok? ? "applied" : (document.error_code || document.kind.to_s), + batch: lease.batch, requested_identifier: lease.item.identifier + ) + return apply_success(lease, document, observation, now: now) if document.ok? + + return retry_or_terminal(lease, document.error_message, now: now) + end + + reason = fetched.error&.message || fetched.classification.to_s + if TERMINAL_CLASSIFICATIONS.include?(fetched.classification) + return terminal(lease, terminal_reason(fetched, reason), now: now) + end + + retry_or_terminal(lease, reason, now: now) + end + + def apply_success(lease, document, observation, now:) + model = lease.entity_type.model + attributes = document.attributes.merge( + enrichment_status: "complete", enrichment_stage: "contract_complete", + enrichment_attempts: 0, detail_attempts: lease.item.detail_attempts + 1, + next_retry_at: nil, last_error: nil, fetched_at: now, batch_applied_at: now, + contract_completed_at: keep_first(model, :contract_completed_at, now), + latest_observation_id: observation.id, latest_observation_source: "detail", + latest_observed_at: now, lease_token: nil, leased_until: nil, + current_enrichment_batch_id: nil, updated_at: now + ) + applied = model.where(id: lease.item.id, lease_token: lease.token, + current_enrichment_batch_id: lease.batch.id) + .update_all(attributes) == 1 + lease.batch.update!(status: "succeeded", completed_at: now, returned_count: 1, + valid_count: applied ? 1 : 0, invalid_count: applied ? 0 : 1) + if applied + Rails.logger.info(event: "enrichment.detail_completed", + entity_type: lease.entity_type.key, + lease.entity_type.log_key => lease.item.github_id, + enrichment_batch_id: lease.batch.id, + detail_attempts: lease.item.detail_attempts + 1) + end + result(lease, applied ? "completed" : "lease_lost", applied ? nil : "lease_lost") + end + + def keep_first(model, column, now) + Arel::Nodes::NamedFunction.new( + "COALESCE", [ model.arel_table[column], Arel::Nodes.build_quoted(now) ] + ) + end + + # 404/410 says the entity is gone; the other terminal classifications name + # themselves through the executor's error message. + def terminal_reason(fetched, reason) + fetched.classification == :not_found ? "entity_gone_#{fetched.status}" : reason + end + + # An entity-specific, permanent fact is the only allowed terminal outcome: the + # event data, observations, reason, and timestamps all survive it. + def terminal(lease, reason, now:) + write_failure(lease, reason, terminal: true, now: now) + Rails.logger.warn(event: "enrichment.detail_terminal", + entity_type: lease.entity_type.key, + lease.entity_type.log_key => lease.item.github_id, + detail_attempts: lease.item.detail_attempts + 1, reason: reason) + result(lease, "terminal", reason) + end + + def retry_or_terminal(lease, message, now:) + attempts = lease.item.detail_attempts + 1 + return terminal(lease, message, now: now) if attempts >= @configuration.detail_fallback_max_attempts + + write_failure(lease, message, terminal: false, now: now) + Rails.logger.warn(event: "enrichment.detail_retry_scheduled", + entity_type: lease.entity_type.key, + lease.entity_type.log_key => lease.item.github_id, + detail_attempts: attempts, reason: message.to_s.truncate(200)) + result(lease, "retry_scheduled", message) + end + + # Retryable detail failures rest in detail_pending — never retry_scheduled, + # which the batch path owns. Re-batching a row whose search result already + # missed would spend search budget reproducing the same miss. + def write_failure(lease, message, terminal:, now:) + attempts = lease.item.detail_attempts + 1 + complete = lease.item.enrichment_status == "complete" + attributes = { + detail_attempts: attempts, enrichment_attempts: lease.item.enrichment_attempts + 1, + enrichment_status: terminal ? "permanent_failure" : + (complete ? "complete" : "retryable_failure"), + enrichment_stage: terminal ? "terminal" : "detail_pending", + next_retry_at: terminal ? nil : @backoff.retry_at(attempts, now: now), + retry_scheduled_at: terminal ? nil : now, + terminal_at: terminal ? now : nil, + last_error: message.to_s.truncate(1_000), lease_token: nil, leased_until: nil, + current_enrichment_batch_id: nil, updated_at: now + } + lease.entity_type.model.where(id: lease.item.id, lease_token: lease.token, + current_enrichment_batch_id: lease.batch.id).update_all(attributes) + lease.batch.update!(status: "failed", completed_at: now, last_error: message.to_s, + invalid_count: 1) + end + + def decode_raw(body) + body.is_a?(Hash) ? body : JSON.parse(body.to_s) + rescue JSON::ParserError + { "unparsed_body" => body.to_s } + end + + def abandon(lease, error) + now = @clock.call + lease.batch.update!(status: "failed", completed_at: now, + last_error: error.class.name) + @claim.release!(lease, now: now) + end + + def result(lease, status, reason) + Result.new(status: status, entity_type: lease.entity_type.key, + github_id: lease.item.github_id, batch_id: lease.batch.id, reason: reason) + end + end + end +end diff --git a/app/services/github/enrichment/dispatch.rb b/app/services/github/enrichment/dispatch.rb index 8dfca5a..85f1fba 100644 --- a/app/services/github/enrichment/dispatch.rb +++ b/app/services/github/enrichment/dispatch.rb @@ -2,111 +2,77 @@ module Github module Enrichment # §8 step 10's enqueue and step 11's reconciliation, as one rule with two callers. # - # Github::IngestionRunner calls it after a run whose events committed; PR 8's recurring - # ReconcilePendingEnrichmentsJob calls it every 60 seconds. Both ask the same question — - # "is there durable enrichment work this class could do right now?" — and the answer is - # read from the committed entity rows and the ledger, never from the queue. That is what - # makes the enqueue a *hint*: §2A's outbox-style recovery says "the committed entity - # state is the durable record of pending work". A process killed between the COMMIT and - # enqueue can lose that hint while leaving eligible pending entity state discoverable by - # a later successful reconciler tick. + # Github::IngestionRunner calls it after a run whose events committed; the recurring + # ReconcilePendingEnrichmentsJob calls it every 60 seconds. Both ask the same + # question — "is there staged enrichment work a cycle could do right now?" — and the + # answer is read from the committed entity rows and the two ledgers, never from the + # queue. That is what makes the enqueue a *hint*: §2A's outbox-style recovery says + # the committed entity state is the durable record of pending work. A process killed + # between COMMIT and enqueue loses the hint while leaving the work discoverable by a + # later reconciler tick. # - # **At most one job per class per call**, however deep the backlog. §5 gives each class - # one job and Github::EnrichmentRunner enriches at most one entity per call, so the - # queue depth that matters is set by §10's hourly allowance (40 at the defaults), not by - # how fast jobs can be created. One live page carries ~90 distinct actors and ~90 - # distinct repositories; enqueuing per created event would put ~2,400 argument-identical - # cycles an hour on a queue that can spend 40 requests, and every surplus one would run - # the fairness reads only to be told no. The reconciler's 60-second - # cadence is what refills the pipeline instead — it is faster than the budget can be - # spent, and it self-limits when the budget is gone. + # **At most one cycle per call**, however deep the backlog: EnrichmentCycleJob loops + # until a ledger denies, so queue depth is set by the budgets, not by how fast jobs + # can be created. This is also the churn gate — a tick with nothing admissible + # enqueues nothing, so an exhausted hour creates no cycle jobs and no batch rows. # # It takes no lock, opens no transaction, and makes no request. class Dispatch - # Job classes by name, constantized at the call, for EntityType's reason: a constant - # holding the class object would pin it across a development reload. - JOBS = { actor: "EnrichActorJob", repository: "EnrichRepositoryJob" }.freeze + # Constantized at the call, for EntityType's reason: a constant holding the class + # object would pin it across a development reload. + JOB = "EnrichmentCycleJob".freeze def self.call(reason:, **options) new(**options).call(reason: reason) end - def initialize(configuration: Github.configuration, clock: -> { Time.current }, selector: nil) + def initialize(configuration: Github.configuration, clock: -> { Time.current }, + admission: nil, batch_claim: nil, detail_claim: nil) @configuration = configuration @clock = clock - @selector = selector || CandidateSelector.new(configuration: configuration) + @admission = admission || Admission.new(configuration: configuration) + @batch_claim = batch_claim || BatchClaim.new(configuration: configuration) + @detail_claim = detail_claim || DetailClaim.new(configuration: configuration) end - # @param reason [String] what asked — "ingestion" or "reconcile". It is on every line - # because the two have different meanings when they disagree: an ingestion dispatch - # that enqueues nothing means the events created no new work, while a reconcile one - # that enqueues means something was committed and never scheduled. + # @param reason [String] what asked — "ingestion" or "reconcile". An ingestion + # dispatch that enqueues nothing means the events created no admissible work; a + # reconcile one that enqueues means something committed was never scheduled. # @return [Hash] the payload it logged, so a caller can assert on it. def call(reason:) now = @clock.call - budget = GithubApiBudget.find_by(id: GithubApiBudget::SINGLETON_ID) - schedule = class_schedule(budget, now: now) - window_block = window_blocked_by(budget, now: now) - schedule_blocked = !schedule.due?(now: now) - blocked = schedule_blocked || window_block.present? - blocked_by = schedule_blocked ? schedule.binding_component : window_block - payload = EntityType.all.each_with_object({}) do |entity_type, counts| - enqueue = !blocked && @selector.claimable?(entity_type, now: now) - JOBS.fetch(entity_type.key).constantize.perform_later if enqueue + search_verdict = @admission.search(now: now) + detail_verdict = @admission.detail(now: now) - counts[:"#{entity_type.key}_enqueued"] = enqueue ? 1 : 0 - end - - log(payload.merge(reason: reason, blocked_by: (blocked_by if blocked)).compact, - budget: budget, now: now) - end + # Pacing is admissible: the cycle can wait it out. Everything else is not. + search_admissible = search_verdict.granted? || search_verdict.reason == :search_pacing + batch_work = search_admissible && + EntityType.all.any? { |type| @batch_claim.claimable?(type, now: now) } + detail_work = detail_verdict.granted? && + EntityType.all.any? { |type| @detail_claim.claimable?(type, now: now) } - private + enqueue = batch_work || detail_work + JOB.constantize.perform_later if enqueue - # §9's effective_enrichment_time with the entity component omitted, because this object - # is not choosing an entity — Github::Enrichment::Claim does that, under a lease, after - # Github::Enrichment::Fairness has chosen a class. What it can answer cheaply is - # whether *any* enrichment is legal right now, and both of the remaining components are - # single reads of one row. - # - # The per-class share is deliberately absent, for the reason - # Github::EnrichmentSchedule's own comment gives: a share exhaustion is a denial - # relieved by borrowing, not a deferral, so refusing to enqueue on it would withhold - # work the ledger would have granted. - # - # The caller obtains the row with find_by, never bootstrap!: a read path must not - # create it. A missing or existing-uninitialized ledger blocks dispatch because the - # request gate would deny enrichment until the first poll supplies authoritative - # rate-limit headers. - def class_schedule(budget, now:) - EnrichmentSchedule.new( - next_retry_at: nil, - global_blocked_until: budget&.global_blocked_until, - enrichment_class_blocked_until: budget&.enrichment_class_blocked_until(now: now) - ) - end + blocked_by = unless enqueue + [ (search_verdict.reason || :no_batch_work), + (detail_verdict.reason || :no_detail_work) ] + end - def window_blocked_by(budget, now:) - return :window_uninitialized if budget.nil? || budget.window_initialized_at.nil? - :window_elapsed if budget.reset_at.present? && now >= budget.reset_at + log({ cycle_enqueued: enqueue ? 1 : 0, reason: reason, + blocked_by: blocked_by }.compact) end - # §11 lists "reconciliation summaries" among the INFO events, and this is that line — - # but only when it scheduled something. A tick that enqueued nothing is the ordinary - # steady state of an exhausted window, and at 60-second cadence it would emit a line a - # minute for the rest of the hour: the volume argument - # Github::BudgetLedger#log_class_exhausted and Github::EnrichmentRunner#log already make. - # - # The summary is PR 7's, unchanged: per-status counts per class, per-class share usage, - # the window state, and when enrichment is next due. - def log(payload, budget:, now:) - enqueued = payload.fetch(:actor_enqueued) + payload.fetch(:repository_enqueued) - entry = { event: "enrichment.dispatched", **payload, - **Summary.capture(now: now, configuration: @configuration, - selector: @selector, budget: budget).to_log } + private - enqueued.positive? ? Rails.logger.info(entry) : Rails.logger.debug(entry) + # INFO only when it scheduled something: a tick that enqueued nothing is the + # ordinary steady state of an exhausted window, and at 60-second cadence it would + # emit a line a minute for the rest of the hour. The rich per-stage summary lives + # on /status and the one-shot, not on this line. + def log(payload) + entry = { event: "enrichment.dispatched", **payload } + payload.fetch(:cycle_enqueued).positive? ? Rails.logger.info(entry) : Rails.logger.debug(entry) payload end end diff --git a/app/services/github/enrichment/entity_state.rb b/app/services/github/enrichment/entity_state.rb deleted file mode 100644 index f51c4dd..0000000 --- a/app/services/github/enrichment/entity_state.rb +++ /dev/null @@ -1,236 +0,0 @@ -module Github - module Enrichment - # §10's response behaviour, resolved onto one entity row — the entity-side twin of - # Github::Ingestion::PollState, and written to the same rule: **the state moves only - # when a fetch attempt actually happened, and only when its outcome is a fact about - # this entity.** - # - # Three rules govern the matrix below. - # - # 1. enrichment_attempts moves only for an outcome that says something about *this - # entity*. A primary or secondary rate limit is a fact about the IP; PollState - # makes exactly this call for the source ("GitHub answered … but nothing is wrong - # with the source"), and mirroring it keeps one rule across both state machines. - # Inflating an innocent entity's backoff for a condition it did not cause would, - # repeated, push it toward the hour-long cap. - # 2. A retryable outcome never downgrades a `complete` record. This is the one place - # the plan is silent, and the argument is concrete: a transient 500 on a *refresh* - # would otherwise flip a perfectly enriched entity to retryable_failure, dropping - # coverage for a network blip and jumping the row into the high-priority pending - # pool ahead of never-enriched candidates. The status conveys nothing that - # next_retry_at, last_error and enrichment_attempts do not already carry, and the - # row is still TTL-stale so it will be retried anyway. Terminal outcomes — 404, - # permanent, malformed — *do* overwrite `complete`, because they invalidate the - # stored document's premise. - # 3. enrichment_attempts counts attempts since the last success, so `complete` - # resets it to zero. PollState's consecutive_failures is the precedent. - # - # Holds no executor and no transport, so a GitHub request cannot be issued from a - # state write, and it runs after the fetch has returned — never inside a transaction - # that spans one. - class EntityState - MAX_ERROR_LENGTH = 1_000 - - # Dispatch is on the classification symbol and never on FetchResult#successful?, - # which answers true for :not_modified. A frozen Hash with no default, so an - # unenumerated classification raises rather than silently taking a branch: PR 4's - # :redirect never escapes Github::RequestExecutor#follow_redirects (it is either - # followed or converted into RedirectLimitExceeded → :permanent_error), so its - # presence here would be a claim about unreachable code. - DISPOSITIONS = { - ok: :document, - not_modified: :retryable, - not_found: :permanent, - client_error: :permanent, - server_error: :retryable, - transport_error: :retryable, - permanent_error: :permanent, - rate_limited: :defer, - secondary_limited: :secondary, - budget_denied: :defer, - gate_unavailable: :defer - }.freeze - - # What was written, so the runner's Result and its log line need no re-read. - # - # lease_held answers the one question the runner has left: whether it still owes a - # release. Every path that wrote next_retry_at has already replaced the lease, and a - # lost lease belongs to somebody else — only the do-nothing deferral leaves one - # outstanding. - class Written < Data.define(:outcome, :enrichment_status, :next_retry_at, :last_error, - :error_code, :lease_held) - OUTCOMES = %w[ enriched failed deferred lease_lost ].freeze - - def initialize(outcome:, enrichment_status: nil, next_retry_at: nil, - last_error: nil, error_code: nil, lease_held: false) - raise ArgumentError, "unknown outcome #{outcome.inspect}" unless OUTCOMES.include?(outcome) - - super - end - - # The deferral that wrote nothing at all, and therefore still owes a release. - def self.deferred(lease_held: true) - new(outcome: "deferred", lease_held: lease_held) - end - - def enriched? = outcome == "enriched" - def failed? = outcome == "failed" - def deferred? = outcome == "deferred" - def lease_lost? = outcome == "lease_lost" - - # Whether an outbound attempt was made *and* charged to this entity. The two - # deferral classifications spend nothing and record nothing. - def attempted? = enriched? || failed? - - def to_log - { enrichment_outcome: outcome, entity_status: enrichment_status, - next_retry_at: next_retry_at&.utc&.iso8601, error_code: error_code, - error_message: last_error }.compact - end - end - - def initialize(backoff: Backoff.new) - @backoff = backoff - end - - attr_reader :backoff - - # @param lease [Github::Enrichment::Claim::Lease] - # @param fetched [Github::FetchResult] - # @param document [Github::Enrichment::Document, nil] present only for a 200 - # @param decision [Github::RateLimitPolicy::Decision, nil] - # @return [Written] - def record!(lease:, fetched:, document: nil, decision: nil, now: Time.current) - case DISPOSITIONS.fetch(fetched.classification) { raise ArgumentError, unenumerated(fetched) } - when :document then from_document(lease, fetched, document, now: now) - when :permanent then permanent(lease, message_for(fetched), now: now) - when :retryable then retryable(lease, fetched, now: now) - when :secondary then secondary(lease, decision) - when :defer then Written.deferred - end - end - - private - - def from_document(lease, fetched, document, now:) - raise ArgumentError, "a 200 response was not parsed" if document.nil? - return complete(lease, document, now: now) if document.ok? - - # §10's third error-context row. Both remaining kinds are decided facts about the - # document rather than transport noise, so both are permanent — and neither - # touches the payload columns, so a bad refresh cannot delete a good document. - log_document_rejected(lease, fetched, document) - permanent(lease, document.error_message, now: now, error_code: document.error_code) - end - - # next_retry_at is cleared because the next event for this row is a *refresh*, gated - # by fetched_at + the TTL rather than by a retry instant; leaving the lease in place - # would delay it by ten minutes and conflate two meanings on one column. last_error - # is cleared because a stale error on a successful row is a permanent lie. - def complete(lease, document, now:) - write(lease, { - enrichment_status: "complete", enrichment_attempts: 0, next_retry_at: nil, - last_error: nil, fetched_at: now, updated_at: now - }.merge(document.attributes), outcome: "enriched") - end - - def permanent(lease, message, now:, error_code: nil) - write(lease, { - enrichment_status: "permanent_failure", - enrichment_attempts: lease.enrichment_attempts + 1, - next_retry_at: nil, last_error: truncate(message), updated_at: now - }, outcome: "failed", error_code: error_code) - end - - def retryable(lease, fetched, now:) - log_unexpected_not_modified(lease, fetched) if fetched.classification == :not_modified - attempts = lease.enrichment_attempts + 1 - - write(lease, { - # Rule 2. Read from the lease rather than re-queried: this worker holds the row, - # and nothing else may transition it while the lease stands. - enrichment_status: lease.enrichment_status == "complete" ? "complete" : "retryable_failure", - enrichment_attempts: attempts, - next_retry_at: backoff.retry_at(attempts, now: now), - last_error: truncate(message_for(fetched)), updated_at: now - }, outcome: "failed") - end - - # §10: on a secondary limit, "also update the request-specific source or entity retry - # state". Not redundant with the global block Github::RateLimitPolicy has already - # recorded — PollState#secondary_retry's reason transfers exactly: ROLL_WINDOW_SQL - # clears the global block at the window boundary while this component survives it, - # so a secondary limit that outlives a rollover still defers the entity that - # provoked it. - # - # No attempt is counted and no status moves: rule 1. updated_at is left alone for - # Github::Enrichment::Claim's reason — this is transient scheduling state, not - # observable entity state, and bumping it would cost a concurrent page its identity - # refresh. - def secondary(lease, decision) - retry_at = decision&.source_retry_at - return Written.deferred if retry_at.nil? - - write(lease, { next_retry_at: retry_at }, outcome: "deferred") - end - - # One guarded statement. The next_retry_at guard is the same one - # Github::Enrichment::Claim#release! carries and for the same reason: lease_seconds - # is the worst-case runtime by construction, so a lease expiring mid-flight is - # reachable, and writing an outcome onto a row another worker has since claimed - # would be the double-write the lease exists to prevent. Reporting it as lease_lost - # is the graceful degradation — the budget was spent and the document discarded, - # which is worth a WARN rather than a silent overwrite. - def write(lease, attributes, outcome:, error_code: nil) - updated = lease.entity_type.model - .where(id: lease.id, next_retry_at: lease.leased_until) - .update_all(attributes) - - return lease_lost(lease) if updated.zero? - - Written.new(outcome: outcome, enrichment_status: attributes[:enrichment_status], - next_retry_at: attributes[:next_retry_at], - last_error: attributes[:last_error], error_code: error_code, - lease_held: false) - end - - def lease_lost(lease) - Rails.logger.warn(event: "enrichment.lease_lost", **lease.to_log) - Written.new(outcome: "lease_lost", lease_held: false) - end - - # A 304 to a request that carried no validator. Enrichment has no entity ETag column - # — §7's column list has none, and §10's dated probe established that an - # unauthenticated 304 debits quota anyway, so a conditional enrichment request would - # cost the same and return no document. So this is an intermediary answering a - # conditional request we never made, or GitHub misbehaving: an assumption broke, and - # it is worth a line even though the row is handled as an ordinary retryable failure. - # - # fetched_at is deliberately not bumped on a refresh: we never told the server what - # we hold, so a 304 is not evidence that what we hold is current. - def log_unexpected_not_modified(lease, fetched) - Rails.logger.warn(event: "enrichment.unexpected_not_modified", **lease.to_log, - http_status: fetched.status) - end - - def log_document_rejected(lease, fetched, document) - Rails.logger.warn(event: "enrichment.document_rejected", **lease.to_log, - http_status: fetched.status, **document.to_log) - end - - def message_for(fetched) - return fetched.error.message if fetched.error - - "GitHub returned #{fetched.status} (#{fetched.classification})" - end - - def truncate(message) - message&.to_s&.truncate(MAX_ERROR_LENGTH) - end - - def unenumerated(fetched) - "no enrichment disposition for classification #{fetched.classification.inspect}" - end - end - end -end diff --git a/app/services/github/enrichment/entity_type.rb b/app/services/github/enrichment/entity_type.rb index d1c4e58..1e6e3e4 100644 --- a/app/services/github/enrichment/entity_type.rb +++ b/app/services/github/enrichment/entity_type.rb @@ -8,19 +8,22 @@ module Enrichment # A memoised class method rather than a constant assigned in the class body: a # constant would pin the model class objects across a development reload, which is the # same reason Github::EventSources::Base memoises its registry. - class EntityType < Data.define(:key, :model, :request_class, :document, :log_key) + class EntityType < Data.define(:key, :model, :request_class, :search_request_class, + :document, :log_key) class << self def all @all ||= [ new(key: :actor, model: GithubActor, request_class: :actor, + search_request_class: :actor_search, document: ActorDocument, log_key: :github_actor_id), new(key: :repository, model: GithubRepository, request_class: :repository, + search_request_class: :repository_search, document: RepositoryDocument, log_key: :github_repository_id) ].freeze end # @param key [Symbol, String, Class] a key, or the model class itself — which is - # what Github::EnrichmentRunner#call(entity_class:) is handed. + # what the runners' #call(entity_class:) is handed. # @return [EntityType] def fetch(key) resolve(key) || diff --git a/app/services/github/enrichment/fairness.rb b/app/services/github/enrichment/fairness.rb deleted file mode 100644 index 8b4a458..0000000 --- a/app/services/github/enrichment/fairness.rb +++ /dev/null @@ -1,167 +0,0 @@ -module Github - module Enrichment - # §10's fairness policy, on the *selection* side: which class works next, out of which - # pool, and whether it may spend past its guarantee. - # - # The division of labour mirrors the one between Github::RateLimitPolicy and - # Github::BudgetLedger. This decides; the ledger enforces. Nothing here can grant - # capacity — a wrong answer produces a refused reservation, never an overspend — - # because the ledger re-checks the same arithmetic under its row lock before debiting. - # - # §10's prioritization, applied in order: - # - # 1. a never-enriched pending candidate in a class still inside its guarantee, with - # actor before repository only as a tie-break; - # 2. the same, borrowing, when the other class has no currently eligible candidate; - # 3. a TTL-stale refresh only when no never-enriched backlog row exists. Backoff and - # in-flight leases delay first-time work; they never release its reserved quota - # to refresh traffic. - # - # The borrow fact is computed here and asserted to the ledger, and it is stale by - # construction: a poll can persist a new candidate between this query and the debit. - # The exposure is bounded to one request — Github::EnrichmentRunner enriches at most - # one entity per invocation — it self-corrects on the next call, and it can never - # exceed the class cap, which the ledger checks first. - class Fairness - # What to do next. `none` carries the reason, so a caller can report *why* nothing - # happened rather than only that nothing did. - class Choice < Data.define(:entity_type, :pool, :borrow, :reason) - REASONS = %w[ - pending borrowed_pending refresh borrowed_refresh - class_exhausted globally_blocked window_uninitialized no_candidate - ].freeze - - def self.none(reason:) - new(reason: reason) - end - - def initialize(reason:, entity_type: nil, pool: nil, borrow: false) - raise ArgumentError, "unknown reason #{reason.inspect}" unless REASONS.include?(reason) - - super - end - - def chosen? = !entity_type.nil? - - def to_log - { entity_type: entity_type&.key, pool: pool, borrow: (true if borrow), - choice_reason: reason }.compact - end - end - - def initialize(configuration: Github.configuration, - selector: CandidateSelector.new(configuration: configuration)) - @configuration = configuration - @selector = selector - end - - attr_reader :configuration, :selector - - # @param entity_class [Class, Symbol, nil] restricts the choice to one class; nil - # lets §10's fairness pick. PR 8's per-class jobs pass one and the one-shot's - # --class flag passes one, and neither bypasses any budget rule by doing so. - # @return [Choice] - def choose(entity_class: nil, now: Time.current) - budget = EnrichmentSchedule.current_budget - blocked = global_block(budget, now: now) - return blocked if blocked - - requested = entity_class.nil? ? EntityType.all : [ EntityType.fetch(entity_class) ] - # Asked once for both classes and then only read: the borrow condition is about - # the class this cycle did *not* pick, so re-querying per candidate would issue - # the same statement twice. - eligible = EntityType.all.index_with { |type| selector.pending_available?(type, now: now) } - - pending_choice(budget, requested, eligible) || - refresh_choice(budget, requested, now: now) || - Choice.none(reason: "no_candidate") - end - - private - - # The conditions that stop *all* enrichment, asked before any entity query so a - # blocked cycle costs one row read rather than five. :share_exhausted is deliberately - # not among them: it is a denial about one class rather than a stop, which is the - # same reason Github::EnrichmentSchedule excludes it. - def global_block(budget, now:) - return nil if budget.nil? - - if budget.global_blocked_until.present? && budget.global_blocked_until > now - Choice.none(reason: "globally_blocked") - elsif budget.window_initialized_at.nil? - # §7: enrichment is ineligible until the first real poll initializes the window - # from authoritative headers. Asked here as well as in the ledger so a fresh - # install reports the honest reason instead of taking the global request gate to - # be told the same thing. - Choice.none(reason: "window_uninitialized") - elsif budget.enrichment_used >= budget.enrichment_allowance - Choice.none(reason: "class_exhausted") - end - end - - def pending_choice(budget, requested, eligible) - available = requested.select { |type| eligible.fetch(type) } - return nil if available.empty? - - within = available.find { |type| room_within_guarantee?(budget, type) } - return Choice.new(entity_type: within, pool: :pending, borrow: false, reason: "pending") if within - - borrower = available.find { |type| other_classes_quiet?(type, eligible) } - return nil if borrower.nil? - - Choice.new(entity_type: borrower, pool: :pending, borrow: true, reason: "borrowed_pending") - end - - # Refresh is strictly subordinate to the durable first-time backlog. This checks - # rows rather than only currently-due candidates, so a backoff or lease cannot let - # refresh traffic consume quota reserved for eventual enrichment. - # - # Then §10:898's second constraint on the same line — refreshes run "within each - # class's share" — which makes this the same two-step #pending_choice performs, over - # a different pool: prefer a class that still has room, and only borrow when the - # other class genuinely has nothing to do. Picking the first refreshable class - # outright instead would hand actor every borrowed request in the window while - # repository's untouched guarantee and eligible stale rows were never selected, which - # is the class starvation the split exists to prevent, reproduced one pool down. - # - def refresh_choice(budget, requested, now:) - return nil if EntityType.all.any? { |type| selector.pending_backlog?(type) } - - refreshable = EntityType.all.index_with { |type| selector.refresh_available?(type, now: now) } - available = requested.select { |type| refreshable.fetch(type) } - return nil if available.empty? - - within = available.find { |type| room_within_guarantee?(budget, type) } - return Choice.new(entity_type: within, pool: :refresh, borrow: false, reason: "refresh") if within - - borrower = available.find { |type| other_classes_quiet?(type, refreshable) } - return nil if borrower.nil? - - Choice.new(entity_type: borrower, pool: :refresh, borrow: true, reason: "borrowed_refresh") - end - - # Derived from the *stored* enrichment_allowance, exactly as - # Github::BudgetLedger#share_cap is, so this predicate and the ledger's guard compute - # the same number from the same row. - def room_within_guarantee?(budget, entity_type) - return true if budget.nil? - - guarantee = Allowances.split(budget.enrichment_allowance, configuration.actor_enrichment_share) - .fetch(entity_type.request_class) - share_used(budget, entity_type) < guarantee - end - - # Borrow only when the other class has nothing claimable in the same pool. This is - # intentionally about due work rather than all rows: a class in backoff must not - # strand capacity the other class can spend. Refresh reaches this helper only after - # the global first-time backlog has been proven empty. - def other_classes_quiet?(entity_type, eligible) - eligible.except(entity_type).values.none? - end - - def share_used(budget, entity_type) - entity_type.request_class == :actor ? budget.actor_share_used : budget.repository_share_used - end - end - end -end diff --git a/app/services/github/enrichment/observation_recorder.rb b/app/services/github/enrichment/observation_recorder.rb new file mode 100644 index 0000000..74217bf --- /dev/null +++ b/app/services/github/enrichment/observation_recorder.rb @@ -0,0 +1,23 @@ +module Github + module Enrichment + class ObservationRecorder + def self.record!(entity_type:, entity_github_id:, source:, raw_payload:, observed_at:, + validation_outcome:, batch: nil, push_event: nil, + requested_identifier: nil, correlation_id: nil) + EnrichmentObservation.create!( + entity_kind: entity_type.key.to_s, + entity_github_id: entity_github_id, + source: source.to_s, + observed_at: observed_at, + raw_payload: raw_payload, + payload_fingerprint: Events::PayloadFingerprint.fingerprint(raw_payload), + enrichment_batch: batch, + push_event: push_event, + request_correlation_id: correlation_id || batch&.correlation_id, + requested_identifier: requested_identifier, + validation_outcome: validation_outcome + ) + end + end + end +end diff --git a/app/services/github/enrichment/one_shot.rb b/app/services/github/enrichment/one_shot.rb index 3855991..06de0b9 100644 --- a/app/services/github/enrichment/one_shot.rb +++ b/app/services/github/enrichment/one_shot.rb @@ -4,20 +4,25 @@ module Github module Enrichment - # The one-shot enrichment command — the operator surface for §13's PR 7, and the same - # contract Github::Ingestion::OneShot holds for polling. + # The one-shot enrichment command — the operator surface for the staged batch + # pipeline, holding the same contract Github::Ingestion::OneShot holds for polling. # # It returns its exit code and never calls exit; bin/enrich does, and that one line is # the only place in that file that ends a process. It owns the human text and the exit - # code, while Github::EnrichmentRunner owns the structured logs and every database + # code, while the batch and detail runners own the structured logs and every database # write — so neither can drift into the other's stream, which matters because §11 puts # both on stdout. The operator block is composed and written once, so a concurrent JSON # log line can never split it. # # **There is no source-lock wait and no Errors::SourceBusy path.** §8 step 1: # "Enrichment jobs skip this step — they take only the request gate." That absence is - # the guarantee made visible, and a spec asserts the runner is called without one. A - # held gate arrives as a FetchResult rather than an exception, and is a deferral. + # the guarantee made visible. A held gate arrives as a FetchResult rather than an + # exception, and is a deferral. + # + # **There is no pacing sleep here.** A pacing denial is reported and the command + # stops — the always-on cycle waits pacing out; a one-shot reports the truth and + # exits 0. The offline walkthrough sets SEARCH_PACING_SECONDS=0 to run both lanes + # back to back. class OneShot Result = Data.define(:outcome, :exit_code, :tally) @@ -27,34 +32,27 @@ class OneShot REFUSING_ERRORS = [ Errors::ConfigurationError, Errors::FixtureMiss, Errors::FixtureCorpusError ].freeze - # An entity that is provably gone is a *decided*, durable outcome — the command did - # exactly its job, and §10 is explicit that a 404 on one enrichment target is not a - # failure of anything else. Only an undecided attempt is a failed one. Without this - # line a reviewer running the deterministic fixture scenario would get exit 1 for - # ghostuser, which is correct behaviour reported as breakage. - DECIDED_STATUSES = %w[ complete permanent_failure ].freeze - - # Continuing past these would spin: nothing became eligible, or the next cycle would - # be refused identically. A failed entity is *not* among them — §16 requires that - # malformed data "does not terminate the batch", and the same reasoning applies to an - # entity that 404s halfway through a --limit. - STOPPING_STATUSES = %w[ idle deferred ].freeze + STAGES = %w[ batch detail ].freeze DEFAULT_LIMIT = 1 + # Two consecutive idle claims end a lane — the same race guard the cycle uses. + MAX_CONSECUTIVE_IDLE = CycleRunner::MAX_CONSECUTIVE_IDLE + USAGE = <<~TEXT.freeze Usage: bin/enrich [options] - Runs enrichment cycles against the persisted actor and repository backlog, then - prints what it did and the persisted state of the system. Each cycle enriches at - most one entity, chosen by the fairness policy in IMPLEMENTATION_PLAN.md §10. + Runs staged enrichment against the persisted actor and repository backlog, then + prints what it did and the persisted state of the system. Search batches run + first (up to SEARCH_BATCH_SIZE entities per request), then detail fallbacks from + the bounded core allowance. - --limit N Run up to N cycles (default #{DEFAULT_LIMIT}). Stops early when - nothing is eligible or the budget defers the next cycle. + --limit N Attempt up to N requests (default #{DEFAULT_LIMIT}). Stops early + when nothing is eligible or a ledger defers the next request. --class CLASS Restrict to actor or repository. It narrows selection and - bypasses nothing: the enrichment allowance, the per-class - fairness share, the reserve, the request gate and every - global block still bind. + bypasses nothing: both ledgers, pacing, the reserves, the + request gate and every global block still bind. + --stage STAGE Restrict to batch or detail. -h, --help Print this message. There is deliberately no --force. §9 licenses --force against the poll cadence and @@ -62,18 +60,25 @@ class OneShot one §10 requires to bind. Exit codes: - 0 enriched, decided, or deferred — nothing eligible, budget spent, gate held, - rate limited, globally blocked, or the entity is provably gone - 1 an attempt failed and is scheduled to be retried + 0 every attempted request was decided or deferred — applied, terminal, + nothing eligible, budget spent, pacing, gate held, or globally blocked + 1 at least one attempt failed and is scheduled to be retried 2 refused to run — bad option, bad configuration, or a corpus gap TEXT def initialize(argv: [], output: $stdout, error_output: $stderr, - runner: EnrichmentRunner.new) + configuration: Github.configuration, + batch_runner: nil, detail_runner: nil, admission: nil, + batch_claim: nil, detail_claim: nil) @argv = argv @output = output @error_output = error_output - @runner = runner + @configuration = configuration + @batch_runner = batch_runner || BatchRunner.new(configuration: configuration) + @detail_runner = detail_runner || DetailRunner.new(configuration: configuration) + @admission = admission || Admission.new(configuration: configuration) + @batch_claim = batch_claim || BatchClaim.new(configuration: configuration) + @detail_claim = detail_claim || DetailClaim.new(configuration: configuration) end # @return [Result] @@ -82,41 +87,129 @@ def call return Result.new(outcome: :usage_error, exit_code: REFUSED, tally: nil) if options.nil? return help if options.fetch(:help) - enrich(limit: options.fetch(:limit), entity_class: options.fetch(:entity_class)) + enrich(limit: options.fetch(:limit), entity_class: options.fetch(:entity_class), + stage: options.fetch(:stage)) end private - def enrich(limit:, entity_class:) - tally = Tally.empty - last = nil + def enrich(limit:, entity_class:, stage:) + @tally = Tally.empty + @headlines = [] + @limit = limit + lanes = lanes_for(entity_class) - limit.times do - last = @runner.call(entity_class: entity_class) - tally = tally.record(last) - break if STOPPING_STATUSES.include?(last.status) - end + batch_lane(lanes) if stage.nil? || stage == "batch" + detail_lane(lanes) if stage.nil? || stage == "detail" - report(tally) { headline(last) } - Result.new(outcome: last.status.to_sym, exit_code: exit_code_for(last), tally: tally) + report(@tally) { @headlines.last || "Nothing to enrich — no eligible candidate" } + undecided = @tally.batches_failed.positive? || @tally.details_retrying.positive? + Result.new(outcome: undecided ? :retry_scheduled : :decided, + exit_code: undecided ? FAILURE : SUCCESS, tally: @tally) rescue *REFUSING_ERRORS => error @error_output.puts("#{humanize(error)}: #{error.message}") report Result.new(outcome: :refused, exit_code: REFUSED, tally: nil) end - # Only an undecided attempt fails the command. A permanent_failure is a decided - # outcome; a retryable_failure is the one case where the work is genuinely unfinished - # and rerunning later is the right response. - def exit_code_for(result) - return SUCCESS unless result.failed? + def batch_lane(lanes) + schedule = CycleRunner::LaneSchedule.new( + actor_weight: @configuration.actor_enrichment_weight, + repository_weight: @configuration.repository_enrichment_weight + ) + idle_streak = 0 + + while @tally.requests < @limit + verdict = @admission.search + unless verdict.granted? + @headlines << "Search deferred — #{verdict.reason}" + return + end + + choice = schedule.next_claimable(->(lane) { + lanes.include?(lane) && @batch_claim.claimable?(EntityType.fetch(lane)) + }) + return if choice.nil? + + result = @batch_runner.call(entity_class: choice.first) + @tally = @tally.record_batch(result) + case result.status + when "idle" + idle_streak += 1 + return if idle_streak >= MAX_CONSECUTIVE_IDLE + when "deferred" + @headlines << "Search batch deferred — #{result.deferral_reason}" + return + else + idle_streak = 0 + @headlines << batch_headline(result) + end + end + end + + def detail_lane(lanes) + schedule = CycleRunner::LaneSchedule.new( + actor_weight: @configuration.actor_enrichment_weight, + repository_weight: @configuration.repository_enrichment_weight + ) + idle_streak = 0 + + while @tally.requests < @limit + verdict = @admission.detail + unless verdict.granted? + @headlines << "Detail fallback deferred — #{verdict.reason}" + return + end + + choice = schedule.next_claimable(->(lane) { + lanes.include?(lane) && @detail_claim.claimable?(EntityType.fetch(lane)) + }) + return if choice.nil? + + lane, borrowed = choice + result = @detail_runner.call(entity_class: lane, borrow: borrowed) + @tally = @tally.record_detail(result) + case result.status + when "idle" + idle_streak += 1 + return if idle_streak >= MAX_CONSECUTIVE_IDLE + when "deferred" + @headlines << "Detail fallback deferred — #{result.reason}" + return + else + idle_streak = 0 + @headlines << detail_headline(result) + end + end + end - DECIDED_STATUSES.include?(result.enrichment_status) ? SUCCESS : FAILURE + def lanes_for(entity_class) + return EntityType.keys if entity_class.nil? + + [ EntityType.resolve(entity_class).key ] + end + + def batch_headline(result) + if result.status == "completed" + "Search batch #{result.entity_type} ##{result.batch_id} — requested #{result.requested_count}, " \ + "valid #{result.valid_count}, fallback #{result.fallback_count}" + else + "Search batch #{result.entity_type} ##{result.batch_id} failed — #{result.deferral_reason}; retry scheduled" + end + end + + def detail_headline(result) + case result.status + when "completed" then "Detail #{result.entity_type} #{result.github_id} — complete" + when "terminal" then "Detail #{result.entity_type} #{result.github_id} — terminal: #{result.reason}" + when "lease_lost" then "Detail #{result.entity_type} #{result.github_id} — claimed by another worker" + else "Detail #{result.entity_type} #{result.github_id} — retry scheduled: #{result.reason}" + end end # Printed on every path that has a database, including the deferred and refused ones, # and composed as one string so a concurrent log line cannot split it. Both snapshots - # are captured after the runner returns, so they reflect this invocation's own writes. + # are captured after the runners return, so they reflect this invocation's own writes. def report(tally = nil) enrichment = Summary.capture state = Ingestion::StateSummary.capture @@ -128,26 +221,6 @@ def report(tally = nil) @error_output.puts("State summary unavailable: #{error.class.name}") end - def headline(result) - case result.status - when "enriched" then "Enriched #{entity_label(result)} — complete" - when "failed" then failure_line(result) - when "deferred" then "Enrichment deferred — #{result.deferral_reason}" - when "lease_lost" then "Enrichment discarded — #{entity_label(result)} was claimed by another worker" - else "Nothing to enrich — no eligible candidate" - end - end - - def failure_line(result) - scheduled = result.next_retry_at ? " — retry after #{Ingestion::Report.timestamp(result.next_retry_at)}" : "" - - "#{entity_label(result).capitalize} #{result.enrichment_status}#{scheduled}: #{result.last_error}" - end - - def entity_label(result) - "#{result.entity_type} #{result.github_id}" - end - def help @output.puts(USAGE) Result.new(outcome: :help, exit_code: SUCCESS, tally: nil) @@ -155,17 +228,21 @@ def help # @return [Hash, nil] nil when the invocation itself was wrong def parse_options - options = { limit: DEFAULT_LIMIT, entity_class: nil, help: false } + options = { limit: DEFAULT_LIMIT, entity_class: nil, stage: nil, help: false } parser = OptionParser.new do |parser| parser.on("--limit N", Integer) { |value| options[:limit] = value } parser.on("--class CLASS") { |value| options[:entity_class] = value } + parser.on("--stage STAGE") { |value| options[:stage] = value } parser.on("-h", "--help") { options[:help] = true } end remaining = parser.parse(@argv.dup) return usage_error("unexpected argument #{remaining.first.inspect}") unless remaining.empty? return usage_error("--limit must be greater than 0, got #{options[:limit]}") unless options[:limit].positive? + unless options[:stage].nil? || STAGES.include?(options[:stage]) + return usage_error("--stage must be one of #{STAGES.join(", ")}, got #{options[:stage].inspect}") + end return options if options[:entity_class].nil? || EntityType.resolve(options[:entity_class]) usage_error("--class must be one of #{EntityType.keys.join(", ")}, got #{options[:entity_class].inspect}") diff --git a/app/services/github/enrichment/parser.rb b/app/services/github/enrichment/parser.rb index 7a9e561..eddbbe8 100644 --- a/app/services/github/enrichment/parser.rb +++ b/app/services/github/enrichment/parser.rb @@ -29,6 +29,9 @@ def parse(body, github_id:) end return Document.identity_mismatch(expected: github_id, actual: identity) unless identity == github_id + contract_error = respond_to?(:contract_error, true) ? contract_error(document) : nil + return contract_error if contract_error + Document.ok(attributes: attributes_from(document).merge(raw_payload: document)) end @@ -65,6 +68,17 @@ def optional_string(value) def optional_integer(value) value if value.is_a?(Integer) end + + def optional_boolean(value) + value if value == true || value == false + end + + def required_string(value, field) + return value if value.is_a?(String) && value.present? + + Document.malformed(error_code: "invalid_contract_field", + error_message: "#{field} is not a non-empty string") + end end end end diff --git a/app/services/github/enrichment/repository_document.rb b/app/services/github/enrichment/repository_document.rb index 3612892..b527aaf 100644 --- a/app/services/github/enrichment/repository_document.rb +++ b/app/services/github/enrichment/repository_document.rb @@ -1,7 +1,8 @@ module Github module Enrichment - # §7's repository mapping: "(enrichment populates description, language, - # owner_github_id, raw_payload)". + # The repository half of Appendix G's useful-data completion contract: description, + # primary language, owner GitHub id, fork and archived status, default branch, and + # GitHub's creation time, alongside the complete raw item. # # `name` and `full_name` are deliberately absent, and this is the mapping §7 is most # explicit about: the envelope's repo.name is the qualified owner/repository form and @@ -9,8 +10,10 @@ module Enrichment # segment it yields are envelope-owned by GithubRepository::IDENTITY_MERGE. Writing # the API's short name here would put two writers on one column. # - # owner_github_id is nullable with no foreign key, so an absent or malformed owner - # object degrades to NULL rather than failing the document. + # Unlike the pre-contract mapping, an absent or malformed owner no longer degrades to + # NULL: #contract_error refuses any document without an integer owner id, because a + # repository whose owner cannot be identified has not met the contract. Nullable + # contract fields — description and language — stay nullable when GitHub returns null. module RepositoryDocument extend Parser @@ -23,9 +26,35 @@ def attributes_from(document) { description: optional_string(document["description"]), language: optional_string(document["language"]), - owner_github_id: owner.is_a?(Hash) ? optional_integer(owner["id"]) : nil + owner_github_id: owner.fetch("id"), + owner_login: optional_string(owner["login"]), + fork: document["fork"], + archived: document["archived"], + default_branch: document["default_branch"], + github_created_at: Time.iso8601(document["created_at"]).utc } end + + def contract_error(document) + owner = document["owner"] + return malformed("owner.id must be an integer") unless owner.is_a?(Hash) && owner["id"].is_a?(Integer) + return malformed("fork must be boolean") unless [ true, false ].include?(document["fork"]) + return malformed("archived must be boolean") unless [ true, false ].include?(document["archived"]) + return required_string(document["default_branch"], "default_branch") unless document["default_branch"].is_a?(String) && document["default_branch"].present? + return malformed("description must be a string or null") unless document["description"].nil? || document["description"].is_a?(String) + return malformed("language must be a string or null") unless document["language"].nil? || document["language"].is_a?(String) + + Time.iso8601(document["created_at"]) if document["created_at"].is_a?(String) + return malformed("created_at must be ISO-8601") unless document["created_at"].is_a?(String) + + nil + rescue ArgumentError + malformed("created_at must be ISO-8601") + end + + def malformed(message) + Document.malformed(error_code: "invalid_contract_field", error_message: message) + end end end end diff --git a/app/services/github/enrichment/search_query.rb b/app/services/github/enrichment/search_query.rb new file mode 100644 index 0000000..f58776b --- /dev/null +++ b/app/services/github/enrichment/search_query.rb @@ -0,0 +1,34 @@ +module Github + module Enrichment + # Builds the one Search URL a batch claim will fetch: repeated exact qualifiers + # (`user:` / `repo:`) joined by spaces — never `OR`, which the live probe showed + # answers HTTP 422 (issue #45). per_page equals the batch size so the requested + # and returned sets are comparable one-to-one. + # + # Mode-aware for the same reason Github::EventSources splits PublicEvents from + # FixtureEvents: these are application-origin URLs, and Github::UrlPolicy accepts + # only the fixture scheme in fixture mode — fail closed, never a live fallback. + class SearchQuery + ENDPOINT_PATHS = { + actor: "/search/users", + repository: "/search/repositories" + }.freeze + ORIGINS = { + live: "https://api.github.com", + fixture: "fixture://api.github.com" + }.freeze + QUALIFIERS = { actor: "user", repository: "repo" }.freeze + + def self.build(entity_type, identifiers, mode: Github.configuration.mode) + identifiers = identifiers.map(&:to_s) + raise ArgumentError, "Search batch cannot be empty" if identifiers.empty? + + origin = ORIGINS.fetch(mode.to_sym) do + raise ArgumentError, "unknown mode #{mode.inspect}" + end + query = identifiers.map { |identifier| "#{QUALIFIERS.fetch(entity_type.key)}:#{identifier}" }.join(" ") + "#{origin}#{ENDPOINT_PATHS.fetch(entity_type.key)}?#{URI.encode_www_form(q: query, per_page: identifiers.length)}" + end + end + end +end diff --git a/app/services/github/enrichment/search_response.rb b/app/services/github/enrichment/search_response.rb new file mode 100644 index 0000000..1f4bc28 --- /dev/null +++ b/app/services/github/enrichment/search_response.rb @@ -0,0 +1,28 @@ +module Github + module Enrichment + class SearchResponse < Data.define(:ok, :items, :total_count, :incomplete_results, + :error_message) + def self.parse(body) + document = body.is_a?(Hash) ? body : JSON.parse(body.to_s) + return failure("Search response is not an object") unless document.is_a?(Hash) + return failure("Search response items is not an array") unless document["items"].is_a?(Array) + unless document["total_count"].is_a?(Integer) && [ true, false ].include?(document["incomplete_results"]) + return failure("Search response metadata is malformed") + end + + new(ok: true, items: document["items"].freeze, + total_count: document["total_count"], + incomplete_results: document["incomplete_results"], error_message: nil) + rescue JSON::ParserError => error + failure(error.message) + end + + def self.failure(message) + new(ok: false, items: [].freeze, total_count: nil, + incomplete_results: nil, error_message: message) + end + + def ok? = ok + end + end +end diff --git a/app/services/github/enrichment/summary.rb b/app/services/github/enrichment/summary.rb index 30d1831..cbcbae2 100644 --- a/app/services/github/enrichment/summary.rb +++ b/app/services/github/enrichment/summary.rb @@ -1,87 +1,73 @@ module Github module Enrichment # What bin/enrich prints to prove enrichment state, alongside the block bin/ingest - # already prints (Github::Ingestion::StateSummary). + # already prints (Github::Ingestion::StateSummary), and the /status enrichment block. # - # A second object rather than more members on that one, because the two answer - # different questions and §13 splits them across two PRs: StateSummary is §9's - # proof-of-state for the *polling* command, and /status owns the coverage percentages. - # What lands here is the part an enrichment outcome would otherwise leave invisible: - # the durable per-class backlog, its oldest wait, and reserved allowance usage. - # The percentages themselves are Github::Enrichment::Coverage, which needs - # ENRICHMENT_COVERAGE_WINDOW_SECONDS and a join against push_events; both arrived with - # PR 10, and /status renders the two objects side by side. - # - # **It never initiates a GitHub request**, structurally and for StateSummary's reason: - # no executor, no transport, no ledger — read statements over Active Record - # models. Reading the ledger row with find_by rather than through - # Github::BudgetLedger matters for the same reason PollSchedule gives: a read path must - # not create the row. - class Summary < Data.define(:actor_counts, :repository_counts, - :actor_backlog_count, - :repository_backlog_count, - :actor_oldest_pending_at, :repository_oldest_pending_at, - :actor_oldest_pending_age_seconds, - :repository_oldest_pending_age_seconds, - :actor_share_used, - :repository_share_used, :actor_guarantee, - :repository_guarantee, :enrichment_used, - :enrichment_allowance, :window_status, :window_ready, - :work_waiting, :claimable_now, - :next_enrichment_at) + # **It never initiates a GitHub request**, structurally: no executor, no transport, + # no ledger writer — read statements over Active Record models. Ledger rows are read + # with find_by rather than through the ledgers, because a read path must not create + # them. + class Summary < Data.define(:actor, :repository, :detail_used, :detail_allowance, + :actor_share_used, :repository_share_used, + :actor_guarantee, :repository_guarantee, + :window_status, :window_ready, + :search_present, :search_used, :search_spendable, + :search_remaining, :search_blocked_until, :next_search_at, + :work_waiting, :claimable_now, :next_enrichment_at) NO_LEDGER = "not yet initialized".freeze DUE_NOW = "due now".freeze WAITING_FOR_WINDOW = "waiting for authoritative poll".freeze # next_enrichment_at is nil in two states that are not the same fact: something is # claimable *right now*, and nothing will ever become claimable without new ingest - # activity. Printing "due now" for both was wrong on an empty backlog — bin/enrich - # would say work was due in the same breath it reported nothing to enrich — and - # publishing the same nil as JSON would hand /status's consumers the identical - # ambiguity. claimable_now is the member that separates them; this is the label for + # activity. claimable_now is the member that separates them; this is the label for # the other side. NOTHING_WAITING = "nothing waiting".freeze class << self - # @param budget [GithubApiBudget, nil] the ledger row, when the caller already holds - # it. Github::Status::Snapshot passes one so /status reads the singleton exactly - # once: three independent find_by calls could straddle a committing reservation - # and produce one response whose poll block contradicts its ledger block. The - # default keeps every existing caller reading it here, and keeps reading it with - # find_by rather than through Github::BudgetLedger, because a read path must not - # create the row. + # @param budget [GithubApiBudget, nil] passed by Github::Status::Snapshot so + # /status reads each singleton exactly once — independent find_by calls could + # straddle a committing reservation and publish blocks that contradict each + # other. def capture(now: Time.current, configuration: Github.configuration, - selector: CandidateSelector.new(configuration: configuration), budget: GithubApiBudget.find_by(id: GithubApiBudget::SINGLETON_ID), - backlog: BacklogMetrics.capture(now: now)) + search_budget: GithubSearchBudget.find_by(id: GithubSearchBudget::SINGLETON_ID), + backlog: BacklogMetrics.capture(now: now, configuration: configuration), + admission: Admission.new(configuration: configuration), + batch_claim: BatchClaim.new(configuration: configuration), + detail_claim: DetailClaim.new(configuration: configuration)) guarantees = guarantees_for(budget, configuration) - backlog_waiting = backlog.actor.backlog_count.positive? || - backlog.repository.backlog_count.positive? - work_waiting = backlog_waiting || complete_rows?(backlog) - claimable = claimable_now?( - budget, selector, backlog_waiting: backlog_waiting, now: now - ) + + search_verdict = admission.search(now: now) + detail_verdict = admission.detail(now: now) + batch_work = EntityType.all.select { |type| batch_claim.claimable?(type, now: now) } + detail_work = EntityType.all.select { |type| detail_claim.claimable?(type, now: now) } + + claimable = (admissible_now?(search_verdict) && batch_work.any?) || + (detail_verdict.granted? && detail_work.any?) + work_waiting = work_waiting?(backlog) new( - actor_counts: backlog.actor.status_counts, - repository_counts: backlog.repository.status_counts, - actor_backlog_count: backlog.actor.backlog_count, - repository_backlog_count: backlog.repository.backlog_count, - actor_oldest_pending_at: backlog.actor.oldest_pending_at, - repository_oldest_pending_at: backlog.repository.oldest_pending_at, - actor_oldest_pending_age_seconds: backlog.actor.oldest_pending_age_seconds, - repository_oldest_pending_age_seconds: backlog.repository.oldest_pending_age_seconds, + actor: backlog.actor, repository: backlog.repository, + detail_used: budget&.enrichment_used, + detail_allowance: budget&.enrichment_allowance, actor_share_used: budget&.actor_share_used, repository_share_used: budget&.repository_share_used, actor_guarantee: guarantees[:actor], repository_guarantee: guarantees[:repository], - enrichment_used: budget&.enrichment_used, - enrichment_allowance: budget&.enrichment_allowance, window_status: budget&.window_status, - window_ready: !window_unavailable?(budget, now: now), + window_ready: detail_verdict.reason != :window_uninitialized && + detail_verdict.reason != :window_elapsed, + search_present: !search_budget.nil?, + search_used: search_budget&.used, + search_spendable: search_budget && (search_budget.request_ceiling - search_budget.reserve), + search_remaining: search_budget&.remaining, + search_blocked_until: search_budget&.blocked_until, + next_search_at: next_search_at(search_verdict, now: now), work_waiting: work_waiting, claimable_now: claimable, - next_enrichment_at: next_enrichment_at( - budget, selector, claimable, backlog_waiting: backlog_waiting, now: now + next_enrichment_at: claimable ? nil : next_enrichment_at( + search_verdict, detail_verdict, now: now, work_waiting: work_waiting, + configuration: configuration ) ) end @@ -94,106 +80,101 @@ def guarantees_for(budget, configuration) Allowances.split(budget.enrichment_allowance, configuration.actor_enrichment_share) end - # §9's effective_enrichment_time, answered for the *pool* rather than for one row: - # nil when something is claimable right now, otherwise the soonest instant at which - # something will be. - # - # Both pools, and the claimable question asked first. Reading "due now" off an - # absent *pending* retry instant was wrong three ways, and all three are states the - # deterministic fixture run reaches: a fully enriched backlog reported "due now" - # while `bin/enrich` in the same breath reported nothing to enrich, because the next - # legal action was a refresh at fetched_at + TTL and nothing looked there; a - # complete row deferred by a failed refresh was invisible for the same reason; and - # one candidate due now beside one deferred printed the deferred instant while work - # was in fact claimable. - def next_enrichment_at(budget, selector, claimable, backlog_waiting:, now:) - blocked = blocked_until(budget, now: now) - return blocked if blocked - return nil if claimable - return nil if window_unavailable?(budget, now: now) - - # Fairness reserves refresh capacity for the durable first-time backlog. If all - # of that backlog is in backoff, a stale refresh is not the next legal action; - # the earliest pending retry is. - if backlog_waiting - return EntityType.all.filter_map do |type| - selector.earliest_pending_at(type, now: now) - end.min - end + # Pacing is a wait, not a refusal — a cycle sleeps through it. + def admissible_now?(search_verdict) + search_verdict.granted? || search_verdict.reason == :search_pacing + end - EntityType.all.filter_map { |type| selector.earliest_claimable_at(type, now: now) }.min + def next_search_at(search_verdict, now:) + return nil if search_verdict.granted? + return nil if search_verdict.retry_in_seconds.nil? + + now + search_verdict.retry_in_seconds end - # "A request would be issued if the runner ran right now." The ledger is asked - # first, then the same first-time-before-refresh rule as Fairness: a deferred - # backlog row suppresses otherwise-due refresh work. - def claimable_now?(budget, selector, backlog_waiting:, now:) - return false if blocked_until(budget, now: now) - return false if window_unavailable?(budget, now: now) - - if backlog_waiting - EntityType.all.any? { |type| selector.pending_available?(type, now: now) } - else - EntityType.all.any? { |type| selector.refresh_available?(type, now: now) } + # The soonest instant at which any staged work could legally proceed: a ledger + # block or pacing clearing, a retry backoff expiring, a live lease expiring, or + # the earliest refresh becoming TTL-due. nil when no such instant exists — which + # with claimable_now false means nothing will happen without new ingest activity. + def next_enrichment_at(search_verdict, detail_verdict, now:, work_waiting:, + configuration:) + candidates = [] + + if work_waiting + [ search_verdict, detail_verdict ].each do |verdict| + candidates << now + verdict.retry_in_seconds if verdict.retry_in_seconds + end end - end - # nil unless the instant is genuinely still ahead: BudgetLedger derives blocking - # from the timestamp rather than from the label precisely so an expired one cannot - # strand the row, and this reader has to agree with it. - def blocked_until(budget, now:) - blocked = [ budget&.global_blocked_until, - budget&.enrichment_class_blocked_until(now: now) ].compact.max + EntityType.all.each do |type| + model = type.model + candidates << model.where.not(enrichment_stage: "terminal") + .where(next_retry_at: now...).minimum(:next_retry_at) + candidates << model.where(leased_until: now...).minimum(:leased_until) + candidates << earliest_refresh_at(type, now: now, configuration: configuration) + end - blocked if blocked&.>(now) + candidates.compact.min end - # A missing row is unavailable too: the first attempted reservation would create - # it, then be denied until a real poll supplies authoritative rate-limit headers. - # An elapsed window is equivalent: reserve! rolls it to uninitialized before it can - # authorize enrichment, so only a poll can make the new window usable. - def window_unavailable?(budget, now:) - budget.nil? || budget.window_initialized_at.nil? || - (budget.reset_at.present? && now >= budget.reset_at) + # The instant the oldest eligible completed row crosses its TTL. Only the + # fetched_at clock matters here; activity eligibility is applied in the scope. + def earliest_refresh_at(type, now:, configuration:) + oldest_fetched = type.model + .where(enrichment_status: "complete", + enrichment_stage: "contract_complete") + .where(last_seen_at: (now - configuration.refresh_active_within_seconds)..) + .minimum(:fetched_at) + return nil if oldest_fetched.nil? + + due_at = oldest_fetched + type.refresh_ttl_seconds(configuration) + [ due_at, now ].max end - def complete_rows?(backlog) + def work_waiting?(backlog) [ backlog.actor, backlog.repository ].any? do |entry| - entry.status_counts.fetch("complete", 0).positive? + entry.contract_backlog_count.positive? || + entry.stage_counts.values.sum > entry.stage_counts.fetch("terminal", 0) end end end def to_s [ - Ingestion::Report.line("Actor backlog", Ingestion::Report.count(actor_backlog_count)), - Ingestion::Report.line("Repository backlog", - Ingestion::Report.count(repository_backlog_count)), + Ingestion::Report.line("Actor contract backlog", + Ingestion::Report.count(actor.contract_backlog_count)), + Ingestion::Report.line("Repository contract backlog", + Ingestion::Report.count(repository.contract_backlog_count)), Ingestion::Report.line("Oldest actor pending", - oldest_pending(actor_oldest_pending_at, - actor_oldest_pending_age_seconds)), + oldest_pending(actor.oldest_pending_at, + actor.oldest_pending_age_seconds)), Ingestion::Report.line("Oldest repository pending", - oldest_pending(repository_oldest_pending_at, - repository_oldest_pending_age_seconds)), - Ingestion::Report.line("Actor requests used", share_line(actor_share_used, actor_guarantee)), - Ingestion::Report.line("Repository requests used", share_line(repository_share_used, repository_guarantee)), - Ingestion::Report.line("Enrichment backlog budget", - backlog_budget(enrichment_used, enrichment_allowance)), + oldest_pending(repository.oldest_pending_at, + repository.oldest_pending_age_seconds)), + Ingestion::Report.line("Search budget", search_line), + Ingestion::Report.line("Detail fallback budget", + backlog_budget(detail_used, detail_allowance)), + Ingestion::Report.line("Detail actor share", share_line(actor_share_used, actor_guarantee)), + Ingestion::Report.line("Detail repository share", + share_line(repository_share_used, repository_guarantee)), Ingestion::Report.line("Next enrichment attempt", next_enrichment) ].join("\n") end def to_log - { actor_counts: actor_counts, repository_counts: repository_counts, - actor_backlog_count: actor_backlog_count, - repository_backlog_count: repository_backlog_count, - actor_oldest_pending_at: Ingestion::Report.timestamp(actor_oldest_pending_at), - repository_oldest_pending_at: Ingestion::Report.timestamp(repository_oldest_pending_at), - actor_oldest_pending_age_seconds: actor_oldest_pending_age_seconds, - repository_oldest_pending_age_seconds: repository_oldest_pending_age_seconds, + { actor_counts: actor.status_counts, repository_counts: repository.status_counts, + actor_stage_counts: actor.stage_counts, + repository_stage_counts: repository.stage_counts, + actor_contract_backlog_count: actor.contract_backlog_count, + repository_contract_backlog_count: repository.contract_backlog_count, + actor_oldest_pending_at: Ingestion::Report.timestamp(actor.oldest_pending_at), + repository_oldest_pending_at: Ingestion::Report.timestamp(repository.oldest_pending_at), + detail_used: detail_used, detail_allowance: detail_allowance, actor_share_used: actor_share_used, repository_share_used: repository_share_used, - actor_guarantee: actor_guarantee, repository_guarantee: repository_guarantee, - enrichment_used: enrichment_used, enrichment_allowance: enrichment_allowance, + search_used: search_used, search_spendable: search_spendable, + search_remaining: search_remaining, + search_blocked_until: Ingestion::Report.timestamp(search_blocked_until), + next_search_at: Ingestion::Report.timestamp(next_search_at), window_status: window_status, claimable_now: claimable_now, next_enrichment_at: Ingestion::Report.timestamp(next_enrichment_at) }.compact end @@ -207,7 +188,9 @@ def oldest_pending(timestamp, age_seconds) end # NO_LEDGER for the same reason StateSummary spells its unknowns out: a fabricated - # zero on a fresh install would claim a quota window had been observed when it had not. + # zero on a fresh install would claim a quota window had been observed when it had + # not. The search row is different — it is configuration-born, so "not yet + # initialized" simply means no search request has ever been attempted. def share_line(used, allowance) return NO_LEDGER if used.nil? @@ -220,15 +203,22 @@ def backlog_budget(used, allowance) value == NO_LEDGER ? value : "#{value} used" end + def search_line + return NO_LEDGER unless search_present + + line = "#{Ingestion::Report.count(search_used)} of " \ + "#{Ingestion::Report.count(search_spendable)} spendable used" + line += ", next request #{Ingestion::Report.timestamp(next_search_at)}" if next_search_at + line + end + # Three answers, not two. A nil instant means "no deferral applies", which is true - # both when a candidate is claimable this second and when the backlog is empty — - # and an operator reads those two states completely differently. claimable_now is - # what tells them apart; without it this line said "due now" to a reviewer whose - # very next line of output was "nothing to enrich". + # both when work is claimable this second and when the backlog is empty — and an + # operator reads those two states completely differently. def next_enrichment return DUE_NOW if claimable_now return NOTHING_WAITING unless work_waiting - return WAITING_FOR_WINDOW unless window_ready + return WAITING_FOR_WINDOW unless window_ready || search_present return NOTHING_WAITING if next_enrichment_at.nil? Ingestion::Report.timestamp(next_enrichment_at) diff --git a/app/services/github/enrichment/tally.rb b/app/services/github/enrichment/tally.rb index ebefaea..4feeaf1 100644 --- a/app/services/github/enrichment/tally.rb +++ b/app/services/github/enrichment/tally.rb @@ -1,38 +1,64 @@ module Github module Enrichment - # What one `bin/enrich` invocation did, across however many cycles it ran. + # What one `bin/enrich` invocation did, across however many requests it ran. # # Immutable, like Github::Ingestion::Tally: #record returns a new value rather than # mutating, so a partially accumulated count can never be observed and a caller cannot # hold a stale reference that later changes underneath it. - class Tally < Data.define(:cycles, :enriched, :failed, :deferred, :idle, :lease_lost) - # Github::EnrichmentRunner::Result::STATUSES, as counters. Keyed by the same strings - # so a new status cannot be silently uncounted — #record fetches and raises. - COUNTERS = { - "enriched" => :enriched, "failed" => :failed, "deferred" => :deferred, - "idle" => :idle, "lease_lost" => :lease_lost - }.freeze - + class Tally < Data.define(:requests, :batches_completed, :batches_failed, + :items_requested, :items_valid, :fallbacks_admitted, + :details_completed, :details_terminal, :details_retrying, + :deferred, :idle, :lease_lost) def self.empty - new(cycles: 0, enriched: 0, failed: 0, deferred: 0, idle: 0, lease_lost: 0) + new(requests: 0, batches_completed: 0, batches_failed: 0, + items_requested: 0, items_valid: 0, fallbacks_admitted: 0, + details_completed: 0, details_terminal: 0, details_retrying: 0, + deferred: 0, idle: 0, lease_lost: 0) end - # @param result [Github::EnrichmentRunner::Result] - def record(result) - counter = COUNTERS.fetch(result.status) { raise ArgumentError, "unknown status #{result.status.inspect}" } + # @param result [BatchRunner::Result] + def record_batch(result) + case result.status + when "completed" + with(requests: requests + 1, batches_completed: batches_completed + 1, + items_requested: items_requested + result.requested_count, + items_valid: items_valid + result.valid_count, + fallbacks_admitted: fallbacks_admitted + result.fallback_count) + when "failed" + with(requests: requests + 1, batches_failed: batches_failed + 1, + items_requested: items_requested + result.requested_count) + when "deferred" then with(requests: requests + 1, deferred: deferred + 1) + when "idle" then with(idle: idle + 1) + else raise ArgumentError, "unknown batch status #{result.status.inspect}" + end + end - with(cycles: cycles + 1, counter => public_send(counter) + 1) + # @param result [DetailRunner::Result] + def record_detail(result) + case result.status + when "completed" then with(requests: requests + 1, details_completed: details_completed + 1) + when "terminal" then with(requests: requests + 1, details_terminal: details_terminal + 1) + when "retry_scheduled" then with(requests: requests + 1, details_retrying: details_retrying + 1) + when "deferred" then with(requests: requests + 1, deferred: deferred + 1) + when "lease_lost" then with(requests: requests + 1, lease_lost: lease_lost + 1) + when "idle" then with(idle: idle + 1) + else raise ArgumentError, "unknown detail status #{result.status.inspect}" + end end def to_log = to_h def to_s [ - Ingestion::Report.line("Enrichment cycles", Ingestion::Report.count(cycles)), - Ingestion::Report.line("Entities enriched", Ingestion::Report.count(enriched)), - Ingestion::Report.line("Entities failed", Ingestion::Report.count(failed)), - Ingestion::Report.line("Cycles deferred", Ingestion::Report.count(deferred)), - Ingestion::Report.line("Cycles with nothing eligible", Ingestion::Report.count(idle)) + Ingestion::Report.line("Requests attempted", Ingestion::Report.count(requests)), + Ingestion::Report.line("Search batches completed", Ingestion::Report.count(batches_completed)), + Ingestion::Report.line("Batch items requested", Ingestion::Report.count(items_requested)), + Ingestion::Report.line("Batch items applied", Ingestion::Report.count(items_valid)), + Ingestion::Report.line("Fallbacks admitted", Ingestion::Report.count(fallbacks_admitted)), + Ingestion::Report.line("Detail completions", Ingestion::Report.count(details_completed)), + Ingestion::Report.line("Detail terminal outcomes", Ingestion::Report.count(details_terminal)), + Ingestion::Report.line("Requests deferred", Ingestion::Report.count(deferred)), + Ingestion::Report.line("Claims with nothing eligible", Ingestion::Report.count(idle)) ].join("\n") end end diff --git a/app/services/github/enrichment/throughput.rb b/app/services/github/enrichment/throughput.rb new file mode 100644 index 0000000..bda1083 --- /dev/null +++ b/app/services/github/enrichment/throughput.rb @@ -0,0 +1,117 @@ +module Github + module Enrichment + # Issue #45's catch-up accounting: measured arrivals versus measured completions + # over the trailing metrics window, and the honest tri-state verdict. Built entirely + # from the BacklogMetrics aggregate — no queries of its own — so this block can + # never contradict the per-stage counts published beside it. + # + # There is deliberately no drained-by estimate anywhere: the measured numbers are + # the whole claim, and when the service is not keeping up the verdict says so. + class Throughput < Data.define(:window_seconds, :window_start, :sample_started_at, + :sample_seconds, :actor, :repository, :combined, + :catch_up_state, :min_sample_seconds) + Lane = Data.define(:arrivals, :completions, :terminals, :exits, + :arrival_rate_per_hour, :completion_rate_per_hour, + :backlog_delta) + + STATES = %w[ keeping_up not_keeping_up insufficient_sample ].freeze + + class << self + # @param backlog [BacklogMetrics] the shared aggregate Snapshot captured once. + def from(backlog, now: Time.current, configuration: Github.configuration) + window_seconds = backlog.window_seconds + window_start = now - window_seconds + earliest = [ backlog.actor.earliest_created_at, + backlog.repository.earliest_created_at ].compact.min + sample_started_at = earliest.nil? ? nil : [ window_start, earliest ].max + sample_seconds = sample_started_at && [ (now - sample_started_at).floor, 0 ].max + + actor = lane(backlog.actor, sample_seconds) + repository = lane(backlog.repository, sample_seconds) + combined = combine(actor, repository, sample_seconds) + contract_backlog = backlog.actor.contract_backlog_count + + backlog.repository.contract_backlog_count + + new(window_seconds: window_seconds, window_start: window_start, + sample_started_at: sample_started_at, sample_seconds: sample_seconds, + actor: actor, repository: repository, combined: combined, + catch_up_state: state(combined, contract_backlog, sample_seconds, configuration), + min_sample_seconds: configuration.catch_up_min_sample_seconds) + end + + private + + def lane(entry, sample_seconds) + exits = entry.completions + entry.terminals + Lane.new( + arrivals: entry.arrivals, completions: entry.completions, + terminals: entry.terminals, exits: exits, + arrival_rate_per_hour: rate(entry.arrivals, sample_seconds), + completion_rate_per_hour: rate(entry.completions, sample_seconds), + # Arrivals minus exits over the window: the backlog slope as a counted + # number, negative while draining. Not a fit, not a forecast. + backlog_delta: entry.arrivals - exits + ) + end + + def combine(actor, repository, sample_seconds) + arrivals = actor.arrivals + repository.arrivals + completions = actor.completions + repository.completions + terminals = actor.terminals + repository.terminals + Lane.new( + arrivals: arrivals, completions: completions, terminals: terminals, + exits: completions + terminals, + arrival_rate_per_hour: rate(arrivals, sample_seconds), + completion_rate_per_hour: rate(completions, sample_seconds), + backlog_delta: arrivals - (completions + terminals) + ) + end + + def rate(count, sample_seconds) + return nil if sample_seconds.nil? || sample_seconds.zero? + + (count * 3600.0 / sample_seconds).round(2) + end + + # insufficient_sample until the observed history spans the configured minimum; + # keeping_up when the backlog shrank over the window, or held level with zero + # contract debt; not_keeping_up otherwise — including a flat nonzero backlog. + def state(combined, contract_backlog, sample_seconds, configuration) + if sample_seconds.nil? || sample_seconds < configuration.catch_up_min_sample_seconds + return "insufficient_sample" + end + return "keeping_up" if combined.backlog_delta.negative? + return "keeping_up" if combined.backlog_delta.zero? && contract_backlog.zero? + + "not_keeping_up" + end + end + + def payload + { + window_seconds: window_seconds, + window_start: Ingestion::Report.timestamp(window_start), + sample_started_at: Ingestion::Report.timestamp(sample_started_at), + sample_seconds: sample_seconds, + actors: actor.to_h, + repositories: repository.to_h, + combined: combined.to_h, + catch_up: { state: catch_up_state, min_sample_seconds: min_sample_seconds } + } + end + + def to_s + case catch_up_state + when "keeping_up" + "Keeping up: yes (completions #{combined.completion_rate_per_hour}/hr vs " \ + "arrivals #{combined.arrival_rate_per_hour}/hr, backlog delta #{combined.backlog_delta})" + when "not_keeping_up" + "Keeping up: NO (completions #{combined.completion_rate_per_hour}/hr vs " \ + "arrivals #{combined.arrival_rate_per_hour}/hr, backlog delta #{combined.backlog_delta})" + else + "Keeping up: insufficient sample (#{sample_seconds || 0}s < #{min_sample_seconds}s)" + end + end + end + end +end diff --git a/app/services/github/ingestion/page_writer.rb b/app/services/github/ingestion/page_writer.rb index 0b491c7..19e7608 100644 --- a/app/services/github/ingestion/page_writer.rb +++ b/app/services/github/ingestion/page_writer.rb @@ -124,6 +124,7 @@ def persist(outcome, run_id:, received_at:) next nil if id.nil? touch_activity(outcome, received_at: received_at) + record_event_observations(outcome, push_event_id: id, received_at: received_at) id end @@ -142,6 +143,52 @@ def touch_activity(outcome, received_at:) ) end + # The accepted event and both event-native identity fragments commit together. The + # entity row coalesces future demand by stable GitHub id; these rows preserve the + # distinct raw evidence supplied by every accepted event. + def record_event_observations(outcome, push_event_id:, received_at:) + raw_event = outcome.raw_payload + + observations = [ + [ "actor", outcome.actor_attributes.fetch(:github_id), raw_event.fetch("actor") ], + [ "repository", outcome.repository_attributes.fetch(:github_id), raw_event.fetch("repo") ] + ].map do |kind, github_id, raw_item| + { + entity_kind: kind, + entity_github_id: github_id, + source: "event", + observed_at: received_at, + raw_payload: raw_item, + payload_fingerprint: Events::PayloadFingerprint.fingerprint(raw_item), + push_event_id: push_event_id, + validation_outcome: "event_native", + created_at: received_at, + updated_at: received_at + } + end + + EnrichmentObservation.insert_all!(observations) + + mark_derived(GithubActor, outcome.actor_attributes.fetch(:github_id), received_at) + mark_derived(GithubRepository, outcome.repository_attributes.fetch(:github_id), received_at) + end + + # COALESCE keeps the first-ever instant: a replayed identity refreshes nothing + # here, and a repeat event for a known entity does not restart its pipeline clock. + def mark_derived(model, github_id, observed_at) + model.where(github_id: github_id).update_all( + event_native_at: keep_first(model, :event_native_at, observed_at), + derived_at: keep_first(model, :derived_at, observed_at), + batch_pending_at: keep_first(model, :batch_pending_at, observed_at) + ) + end + + def keep_first(model, column, instant) + Arel::Nodes::NamedFunction.new( + "COALESCE", [ model.arel_table[column], Arel::Nodes.build_quoted(instant) ] + ) + end + def log_persisted(outcome, run_id:, created_id:) if created_id.nil? Rails.logger.debug(event: "ingestion.event_duplicate", run_id: run_id, **outcome.to_log) diff --git a/app/services/github/ingestion/poll_state.rb b/app/services/github/ingestion/poll_state.rb index 9c3842b..47c4bbc 100644 --- a/app/services/github/ingestion/poll_state.rb +++ b/app/services/github/ingestion/poll_state.rb @@ -137,8 +137,8 @@ def projected(event_source, attributes, now:) # is reported without anyone remembering to extend this. # # This class logs rather than Github::IngestionRunner doing it on its behalf, for - # three reasons. Github::Enrichment::EntityState is this class's entity-side mirror and - # already logs three of its own anomalies, so a state writer announcing its own + # three reasons. The enrichment runners are this class's entity-side mirror and + # already log their own anomalies, so a state writer announcing its own # transitions is the established shape here. The runner would have to re-derive the # predicate, and two readers of one rule is drift waiting to happen. And only this # class knows the delay — backoff_seconds is retry_not_before_at minus its own `now`. diff --git a/app/services/github/request.rb b/app/services/github/request.rb index a8dbb29..9448cfd 100644 --- a/app/services/github/request.rb +++ b/app/services/github/request.rb @@ -13,8 +13,10 @@ class Request < Data.define(:url, :request_class, :origin, :http_method, :custom :context, :borrow) # §7: every outbound attempt debits its class counter. :poll comes from an event # source, :actor and :repository from enrichment (PR 7). - CLASSES = %i[ poll actor repository ].freeze - ENRICHMENT_CLASSES = %i[ actor repository ].freeze + CLASSES = %i[ poll actor repository actor_search repository_search ].freeze + ENRICHMENT_CLASSES = %i[ actor repository actor_search repository_search ].freeze + DETAIL_CLASSES = %i[ actor repository ].freeze + SEARCH_CLASSES = %i[ actor_search repository_search ].freeze # Where the URL came from, which decides how strictly Github::UrlPolicy validates it. # @@ -52,9 +54,9 @@ def initialize(url:, request_class:, origin: :application, http_method: :get, # Validated here as well as in Github::BudgetLedger#reserve!, for the reason # Github::RequestExecutor validates a URL twice: the earlier of the two guards # fails before the gate is ever taken. - if borrow && !ENRICHMENT_CLASSES.include?(request_class) + if borrow && !DETAIL_CLASSES.include?(request_class) raise ArgumentError, - "borrow applies to #{ENRICHMENT_CLASSES.inspect}, got #{request_class.inspect}" + "borrow applies to #{DETAIL_CLASSES.inspect}, got #{request_class.inspect}" end super( @@ -84,6 +86,10 @@ def enrichment? ENRICHMENT_CLASSES.include?(request_class) end + def search? + SEARCH_CLASSES.include?(request_class) + end + def payload_supplied? origin == :payload end diff --git a/app/services/github/request_executor.rb b/app/services/github/request_executor.rb index 67daf35..f0d5c82 100644 --- a/app/services/github/request_executor.rb +++ b/app/services/github/request_executor.rb @@ -22,6 +22,7 @@ module Github class RequestExecutor def initialize(transport: Github.transport, ledger: BudgetLedger.new, + search_ledger: SearchBudgetLedger.new, retry_policy: RetryPolicy.new, mode: Github.configuration.mode, max_redirects: Github.configuration.max_redirects, @@ -30,6 +31,7 @@ def initialize(transport: Github.transport, clock: -> { Time.current }) @transport = transport @ledger = ledger + @search_ledger = search_ledger @retry_policy = retry_policy @mode = mode.to_sym @max_redirects = max_redirects @@ -98,7 +100,8 @@ def gated_attempt(request, attempt:) # which reserve again — stay authorized under the same fairness decision the # caller made once (§10). This class does not interpret it and could not # compute it: it is a fact about the entity tables. - @ledger.reserve!(request.request_class, now: @clock.call, borrow: request.borrow) + ledger = ledger_for(request) + ledger.reserve!(request.request_class, now: @clock.call, borrow: request.borrow) # Authoritative, in-chain validation: its return value is what the transport # receives, so an unvalidated URL cannot physically reach a socket. @@ -109,8 +112,8 @@ def gated_attempt(request, attempt:) # The class travels with the reconciliation so that a response proving the # rate-limit window has moved on can carry this request's debit into the window # GitHub actually counted it in. - @ledger.reconcile!(rate_limit_from(response), request_class: request.request_class, - now: @clock.call) + ledger.reconcile!(rate_limit_from(response), request_class: request.request_class, + now: @clock.call) log_result(FetchResult.from_response( request: request, status: response.status, headers: response.headers, @@ -151,6 +154,10 @@ def rate_limit_from(response) RateLimitSnapshot.from_headers(response.headers, observed_at: @clock.call) end + def ledger_for(request) + request.search? ? @search_ledger : @ledger + end + def failure(request, error, attempt:, classification: nil) FetchResult.from_error( request: request, error: error, attempt: attempt, diff --git a/app/services/github/search_budget_ledger.rb b/app/services/github/search_budget_ledger.rb new file mode 100644 index 0000000..f7f81ed --- /dev/null +++ b/app/services/github/search_budget_ledger.rb @@ -0,0 +1,186 @@ +module Github + # A separate persisted ledger for GitHub's minute-scoped Search resource. Core polling + # and detail fallback never read this row, so Search pressure cannot consume or block + # the polling allocation. + # + # Unlike the core ledger, this one self-initializes from configuration defaults: the + # search lane needs no bootstrap poll, because the ceiling/reserve pair is a + # configured budget and the observed x-ratelimit-search headers only tighten it. + class SearchBudgetLedger + SINGLETON_ID = GithubSearchBudget::SINGLETON_ID + SEARCH_CLASSES = %i[actor_search repository_search].freeze + + # GitHub's Search rate-limit window is one minute (a documented API fact, like + # Configuration::UNAUTHENTICATED_CORE_LIMIT). Used only as the fallback rollover + # horizon when no response ever supplied reset_at — without it, a run of + # header-less transport failures would pin `used` at the ceiling forever. + SEARCH_WINDOW_SECONDS = 60 + + DENIAL_REASONS = %i[search_blocked search_pacing search_reserve_reached + search_ceiling_exhausted].freeze + + def initialize(configuration: Github.configuration) + @configuration = configuration + end + + attr_reader :configuration + + def reserve!(request_class, now: Time.current, borrow: false) + raise ArgumentError, "Search reservations do not borrow" if borrow + unless SEARCH_CLASSES.include?(request_class) + raise ArgumentError, "unknown Search request class #{request_class.inspect}" + end + + bootstrap!(now: now) + reason = nil + + GithubSearchBudget.transaction do + budget = GithubSearchBudget.lock.find(SINGLETON_ID) + if window_elapsed?(budget, now: now) + roll_window!(budget, now: now) + budget.reload + end + reason = denial_reason(budget, now: now) + if reason + log_denial(budget, reason) + next + end + + budget.update!( + used: budget.used + 1, + actor_used: budget.actor_used + (request_class == :actor_search ? 1 : 0), + repository_used: budget.repository_used + (request_class == :repository_search ? 1 : 0), + remaining: budget.remaining.nil? ? nil : [ budget.remaining - 1, 0 ].max, + last_request_at: now, + updated_at: now + ) + end + + raise Errors::BudgetExhausted.new(request_class, reason) if reason + + GithubSearchBudget.find(SINGLETON_ID) + end + + def reconcile!(snapshot, request_class: nil, now: Time.current) + return :no_headers if snapshot.nil? + return :resource_mismatch if snapshot.resource.present? && snapshot.resource != "search" + + GithubSearchBudget.transaction do + budget = GithubSearchBudget.lock.find_by(id: SINGLETON_ID) + next :no_ledger if budget.nil? + next :partial_headers unless snapshot.quantitative? + + # The response proves GitHub counted this request in a window later than the one + # stored here. Carry only the in-flight request's own debit forward — a + # reconciliation with no request behind it (there is no such caller today, but + # the signature permits one) carries nothing, keeping + # used == actor_used + repository_used. The previous window's clamped remaining + # is dropped with its counters, or the monotonic minimum below would import it + # into a window GitHub says is fresh. + if budget.reset_at.present? && snapshot.reset_at > budget.reset_at + budget.assign_attributes( + used: request_class ? 1 : 0, + actor_used: request_class == :actor_search ? 1 : 0, + repository_used: request_class == :repository_search ? 1 : 0, + remaining: nil + ) + end + + budget.limit = snapshot.limit + budget.remaining = budget.remaining.nil? ? snapshot.remaining : + [ budget.remaining, snapshot.remaining ].min + budget.reset_at = snapshot.reset_at + budget.observed_at = snapshot.observed_at || now + budget.blocked_until = snapshot.reset_at if budget.remaining <= budget.reserve + budget.updated_at = now + budget.save! + :updated + end + end + + def block_from!(fetched, now: Time.current) + return unless %i[rate_limited secondary_limited].include?(fetched.classification) + + snapshot = fetched.rate_limit(observed_at: now) + retry_seconds = snapshot.retry_after_seconds + until_at = if retry_seconds.is_a?(Integer) && retry_seconds.positive? + now + retry_seconds + else + snapshot.reset_at || now + SEARCH_WINDOW_SECONDS + end + + # GREATEST ignores NULL, so a block only ever moves later — the core ledger's + # BLOCK_SQL rule, restated for the search row. + GithubSearchBudget.where(id: SINGLETON_ID).update_all( + blocked_until: Arel::Nodes::NamedFunction.new( + "GREATEST", + [ GithubSearchBudget.arel_table[:blocked_until], Arel::Nodes.build_quoted(until_at) ] + ), + updated_at: now + ) + Rails.logger.info(event: "search_budget.blocked", + classification: fetched.classification, + blocked_until: until_at.utc.iso8601) + end + + def bootstrap!(now: Time.current) + GithubSearchBudget.insert_all( + [ { + id: SINGLETON_ID, + request_ceiling: configuration.search_request_ceiling, + reserve: configuration.search_safety_reserve, + created_at: now, + updated_at: now + } ], + unique_by: :id + ) + end + + private + + # The window has moved on when GitHub's own reset instant passed, or — when no + # response ever told us one — when a full Search window elapsed since the last + # outbound attempt. + def window_elapsed?(budget, now:) + return now >= budget.reset_at if budget.reset_at.present? + + budget.last_request_at.present? && + budget.last_request_at <= now - SEARCH_WINDOW_SECONDS && + budget.used.positive? + end + + def denial_reason(budget, now:) + return :search_blocked if budget.blocked_until.present? && budget.blocked_until > now + if budget.last_request_at.present? && + budget.last_request_at + configuration.search_pacing_seconds > now + return :search_pacing + end + return :search_reserve_reached if budget.remaining.present? && budget.remaining <= budget.reserve + + :search_ceiling_exhausted if budget.used >= budget.request_ceiling - budget.reserve + end + + def roll_window!(budget, now:) + Rails.logger.info(event: "search_budget.window_rolled", + previous_used: budget.used, + previous_actor_used: budget.actor_used, + previous_repository_used: budget.repository_used, + reset_at: budget.reset_at&.utc&.iso8601) + budget.update!(used: 0, actor_used: 0, repository_used: 0, remaining: nil, + reset_at: nil, blocked_until: nil, updated_at: now) + end + + def log_denial(budget, reason) + case reason + when :search_pacing + Rails.logger.debug(event: "search_budget.pacing_deferred", + last_request_at: budget.last_request_at&.utc&.iso8601, + pacing_seconds: configuration.search_pacing_seconds) + when :search_reserve_reached, :search_ceiling_exhausted + Rails.logger.info(event: "search_budget.reserve_reached", reason: reason, + used: budget.used, request_ceiling: budget.request_ceiling, + reserve: budget.reserve, remaining: budget.remaining) + end + end + end +end diff --git a/app/services/github/status/ledger_state.rb b/app/services/github/status/ledger_state.rb index 88d49b0..bb4feee 100644 --- a/app/services/github/status/ledger_state.rb +++ b/app/services/github/status/ledger_state.rb @@ -83,7 +83,11 @@ def payload reserve: reserve, global_blocked_until: Ingestion::Report.timestamp(global_blocked_until), poll: { used: poll_used, allowance: poll_allowance }, - enrichment: { used: enrichment_used, allowance: enrichment_allowance }, + # Named for what the number now is (Appendix G): the explicit core + # detail-fallback allowance, not "everything after poll + reserve". The + # Search budget is a different rate-limit resource and lives in the + # search_ledger block. + detail_fallback: { used: enrichment_used, allowance: enrichment_allowance }, actor_requests: { used: actor_share_used, guarantee: actor_guarantee, available: actor_available }, repository_requests: { used: repository_share_used, diff --git a/app/services/github/status/scheduler_settings.rb b/app/services/github/status/scheduler_settings.rb new file mode 100644 index 0000000..36a5261 --- /dev/null +++ b/app/services/github/status/scheduler_settings.rb @@ -0,0 +1,49 @@ +module Github + module Status + # Issue #45: every staged-enrichment scheduler knob, published. A pure projection of + # the validated configuration — zero reads, always fully populated, so an operator + # can see the numbers the ledgers and workers are actually enforcing without a shell. + class SchedulerSettings < Data.define(:configuration) + def self.from(configuration = Github.configuration) + new(configuration: configuration) + end + + def payload + { + search: { + request_ceiling: configuration.search_request_ceiling, + safety_reserve: configuration.search_safety_reserve, + batch_size: configuration.search_batch_size, + pacing_seconds: configuration.search_pacing_seconds, + worker_concurrency: configuration.search_worker_concurrency + }, + fairness: { + actor_weight: configuration.actor_enrichment_weight, + repository_weight: configuration.repository_enrichment_weight, + actor_enrichment_share: configuration.actor_enrichment_share.to_f + }, + core: { + detail_fallback_allowance: configuration.core_detail_fallback_allowance, + rate_limit_reserve: configuration.rate_limit_reserve + }, + retry: { + base_seconds: configuration.enrichment_retry_base_seconds, + max_seconds: configuration.enrichment_retry_max_seconds, + detail_fallback_max_attempts: configuration.detail_fallback_max_attempts, + lease_seconds: configuration.enrichment_lease_seconds, + cycle_budget_seconds: configuration.enrichment_cycle_budget_seconds + }, + refresh: { + actor_ttl_seconds: configuration.actor_refresh_ttl_seconds, + repository_ttl_seconds: configuration.repository_refresh_ttl_seconds, + active_within_seconds: configuration.refresh_active_within_seconds + }, + metrics: { + window_seconds: configuration.enrichment_metrics_window_seconds, + catch_up_min_sample_seconds: configuration.catch_up_min_sample_seconds + } + } + end + end + end +end diff --git a/app/services/github/status/search_ledger_state.rb b/app/services/github/status/search_ledger_state.rb new file mode 100644 index 0000000..a5a7fc3 --- /dev/null +++ b/app/services/github/status/search_ledger_state.rb @@ -0,0 +1,64 @@ +module Github + module Status + # The Search-resource half of the budget story: a projection of the one + # github_search_budget row plus the pacing arithmetic. It issues no query of its + # own — Github::Status::Snapshot reads the singleton once and hands the row down, + # the same single-read discipline LedgerState holds for core. + class SearchLedgerState < Data.define(:present, :resource, :limit, :remaining, + :reset_at, :observed_at, :request_ceiling, + :reserve, :spendable, :used, :actor_used, + :repository_used, :available, :blocked_until, + :last_request_at, :next_request_earliest_at) + class << self + # @param budget [GithubSearchBudget, nil] nil until the first search reservation + # creates the row from configuration — unlike core, no poll is involved. + def from(budget, configuration: Github.configuration, now: Time.current) + return absent if budget.nil? + + new(present: true, resource: budget.resource, limit: budget.limit, + remaining: budget.remaining, reset_at: budget.reset_at, + observed_at: budget.observed_at, request_ceiling: budget.request_ceiling, + reserve: budget.reserve, + spendable: budget.request_ceiling - budget.reserve, + used: budget.used, actor_used: budget.actor_used, + repository_used: budget.repository_used, available: budget.available, + blocked_until: budget.blocked_until, last_request_at: budget.last_request_at, + next_request_earliest_at: next_request_earliest_at(budget, configuration, now)) + end + + def absent + new(present: false, resource: nil, limit: nil, remaining: nil, reset_at: nil, + observed_at: nil, request_ceiling: nil, reserve: nil, spendable: nil, + used: nil, actor_used: nil, repository_used: nil, available: nil, + blocked_until: nil, last_request_at: nil, next_request_earliest_at: nil) + end + + private + + # The instant pacing and any block next permit a Search request; null when a + # request is permitted right now. + def next_request_earliest_at(budget, configuration, now) + candidates = [ budget.blocked_until ] + if budget.last_request_at.present? + candidates << budget.last_request_at + configuration.search_pacing_seconds + end + earliest = candidates.compact.max + + earliest if earliest&.>(now) + end + end + + def payload + { present: present, resource: resource, limit: limit, remaining: remaining, + reset_at: Ingestion::Report.timestamp(reset_at), + observed_at: Ingestion::Report.timestamp(observed_at), + request_ceiling: request_ceiling, reserve: reserve, spendable: spendable, + used: used, actor_used: actor_used, repository_used: repository_used, + available: available, + blocked_until: Ingestion::Report.timestamp(blocked_until), + last_request_at: Ingestion::Report.timestamp(last_request_at), + next_request_earliest_at: Ingestion::Report.timestamp(next_request_earliest_at) } + end + end + end +end diff --git a/app/services/github/status/snapshot.rb b/app/services/github/status/snapshot.rb index 701f634..d0cd1de 100644 --- a/app/services/github/status/snapshot.rb +++ b/app/services/github/status/snapshot.rb @@ -1,26 +1,27 @@ module Github module Status - # Everything IMPLEMENTATION_PLAN.md §11 asks GET /status to report, taken as one - # snapshot of persisted state. + # Everything IMPLEMENTATION_PLAN.md §11 (as amended by Appendix G) asks GET /status + # to report, taken as one snapshot of persisted state. # # **It never initiates a GitHub request**, and it is structural rather than a promise, # exactly as Github::Ingestion::StateSummary states it: this class holds no executor, no # transport and no ledger, and its only collaborators are Active Record models and pure - # value objects. Github::BudgetLedger is absent by construction — all four of its public - # methods write, and #bootstrap! would create from a read path the very row a - # reservation owns. Every ledger read here is find_by. Four specs pin it: a recording - # transport that must see nothing, an unchanged github_api_budget count, an unchanged - # event_sources count, and a SQL subscriber that must see no write statement. + # value objects. Github::BudgetLedger and Github::SearchBudgetLedger are absent by + # construction — their public methods write, and bootstrap! would create from a read + # path the very row a reservation owns. Every ledger read here is find_by. The specs + # pin it: a recording transport that must see nothing, unchanged singleton counts, and + # a SQL subscriber that must see no write statement. # - # ## Why one aggregate rather than StateSummary + Summary side by side + # ## Read inventory, and why each read happens once # - # Three parts of this response need github_api_budget: the poll schedule, §11's ledger - # block, and the enrichment block. Composing the two existing summaries would read the - # singleton three times, so a reservation committing mid-request could produce one body - # whose poll block contradicts its ledger block. This reads the row **once** and passes - # it down. StateSummary additionally runs an unbounded PushEvent.count that §11 does not - # ask for here, collapses PollSchedule to a single instant behind a private method when - # §11 wants the components, and exposes neither poll_used, poll_allowance nor reserve. + # - github_api_budget and github_search_budget: one find_by each, handed down to every + # block that needs them — independent reads could straddle a committing reservation + # and publish blocks that contradict each other. + # - one aggregate statement per entity table (Enrichment::BacklogMetrics), captured + # once and shared by the enrichment and throughput blocks so they cannot disagree. + # - one grouped statement over enrichment_batches (Enrichment::BatchQuality). + # - one statement for coverage (unchanged §11 percentages). + # - zero reads: scheduler settings (pure configuration), both ledger projections. # # ## #payload, not #to_log # @@ -28,10 +29,12 @@ module Status # CLI's column-aligned block and #to_log is the INFO stream's projection — and both # #to_log implementations call .compact, dropping nil keys. A JSON client needs a fixed # key set: a field that appears and disappears makes every consumer handle two shapes. - # Naming the three apart is what stops one being quietly changed to suit another. - class Snapshot < Data.define(:captured_at, :sources, :ledger, :enrichment, :coverage) + class Snapshot < Data.define(:captured_at, :sources, :ledger, :search_ledger, + :scheduler, :enrichment, :batches, :throughput, :coverage) def self.capture(now: Time.current, configuration: Github.configuration) budget = GithubApiBudget.find_by(id: GithubApiBudget::SINGLETON_ID) + search_budget = GithubSearchBudget.find_by(id: GithubSearchBudget::SINGLETON_ID) + backlog = Enrichment::BacklogMetrics.capture(now: now, configuration: configuration) runs = latest_runs new( @@ -41,8 +44,16 @@ def self.capture(now: Time.current, configuration: Github.configuration) last_run: runs[event_source.id], now: now) end, ledger: LedgerState.from(budget, configuration: configuration), + search_ledger: SearchLedgerState.from(search_budget, configuration: configuration, + now: now), + scheduler: SchedulerSettings.from(configuration), enrichment: Enrichment::Summary.capture(now: now, configuration: configuration, - budget: budget), + budget: budget, + search_budget: search_budget, + backlog: backlog), + batches: Enrichment::BatchQuality.capture(now: now, configuration: configuration), + throughput: Enrichment::Throughput.from(backlog, now: now, + configuration: configuration), coverage: Enrichment::Coverage.capture(now: now, configuration: configuration) ) end @@ -75,56 +86,62 @@ def self.latest_runs end private_class_method :latest_runs - # §11's key set, in §11's order: poll state, ledger state, then the enrichment - # counters and coverage percentages. - # # `null` throughout, never a sentinel string and never a missing key. §16's rule is # that an unknown must not read as a zero, and the way to honour that in JSON is not # to swap the type — a `remaining` that is sometimes an Integer and sometimes # "not yet initialized" forces every consumer to type-check. It is: a counted zero # prints 0, a number that does not exist prints null, and wherever null would carry - # two meanings the disambiguating fact gets its own field — ledger.present, due_now, - # claimable_now. + # two meanings the disambiguating fact gets its own field — ledger.present, + # search_ledger.present, due_now, claimable_now, catch_up.state. def payload { captured_at: Ingestion::Report.timestamp(captured_at), sources: sources.map(&:payload), ledger: ledger.payload, + search_ledger: search_ledger.payload, + scheduler: scheduler.payload, enrichment: enrichment_payload, + batches: batches.payload, + throughput: throughput.payload, coverage: coverage.payload } end private # Each entity class exposes the durable backlog separately from its raw status - # counts. A backlog row may be temporarily deferred by retry backoff, so this number - # intentionally differs from claimable_now. Queue depth is not published: - # jobs are bounded wake-up hints and entity rows are the source of truth. + # counts, plus the staged-pipeline view: every stage with its count and oldest + # FIFO instant, and the contract backlog — rows not yet at the useful-data + # contract or a terminal outcome. A backlog row may be temporarily deferred by + # retry backoff, so these numbers intentionally differ from claimable_now. Queue + # depth is not published: jobs are bounded wake-up hints and entity rows are the + # source of truth. def enrichment_payload - { actors: entity_counts(enrichment.actor_counts, - backlog_count: enrichment.actor_backlog_count, - oldest_pending_at: enrichment.actor_oldest_pending_at, - oldest_pending_age_seconds: - enrichment.actor_oldest_pending_age_seconds), - repositories: entity_counts( - enrichment.repository_counts, - backlog_count: enrichment.repository_backlog_count, - oldest_pending_at: enrichment.repository_oldest_pending_at, - oldest_pending_age_seconds: enrichment.repository_oldest_pending_age_seconds - ), + { actors: entity_counts(enrichment.actor), + repositories: entity_counts(enrichment.repository), claimable_now: enrichment.claimable_now, next_enrichment_at: Ingestion::Report.timestamp(enrichment.next_enrichment_at) } end - # fetch(status, 0) because GROUP BY returns no key for a status with no rows. These - # zeros are counted, not fabricated: the table was read and held nothing. - def entity_counts(counts, backlog_count:, oldest_pending_at:, - oldest_pending_age_seconds:) + # fetch(status, 0) because the aggregate drops a status with no rows. These zeros + # are counted, not fabricated: the table was read and held nothing. Stage counts + # arrive with their zeros already present. + def entity_counts(entry) Enrichable::ENRICHMENT_STATUSES - .index_with { |status| counts.fetch(status, 0) } + .index_with { |status| entry.status_counts.fetch(status, 0) } .symbolize_keys - .merge(backlog_count: backlog_count, - oldest_pending_at: Ingestion::Report.timestamp(oldest_pending_at), - oldest_pending_age_seconds: oldest_pending_age_seconds) + .merge(backlog_count: entry.backlog_count, + contract_backlog_count: entry.contract_backlog_count, + oldest_pending_at: Ingestion::Report.timestamp(entry.oldest_pending_at), + oldest_pending_age_seconds: entry.oldest_pending_age_seconds, + stages: stage_payload(entry)) + end + + def stage_payload(entry) + Enrichable::ENRICHMENT_STAGES.index_with do |stage| + oldest = entry.stage_oldest.fetch(stage, nil) + { count: entry.stage_counts.fetch(stage, 0), + oldest_created_at: Ingestion::Report.timestamp(oldest), + oldest_age_seconds: Enrichment::BacklogMetrics.age_seconds(oldest, now: captured_at) } + end.symbolize_keys end end end diff --git a/bin/enrich b/bin/enrich index 530174e..f00c89b 100755 --- a/bin/enrich +++ b/bin/enrich @@ -9,10 +9,10 @@ # ends a process. # # Separate from bin/ingest rather than a flag on it, because §5 gives enrichment its own -# path: it belongs to no event source, never takes a source lock, and spends a different -# class of the budget. EnrichActorJob and EnrichRepositoryJob wrap the same -# Github::EnrichmentRunner one cycle at a time; this stays the operator's handle on it, -# for a burst of cycles on demand rather than the worker's budgeted trickle. +# path: it belongs to no event source, never takes a source lock, and spends the search +# ledger plus the bounded core detail allowance. EnrichmentCycleJob drives the same +# batch and detail runners on the worker's cadence; this stays the operator's handle on +# them, for a bounded burst of requests on demand rather than the worker's paced loop. # # config/environment rather than config/boot: this command uses models, so Rails has to be # initialized rather than merely loadable. diff --git a/db/migrate/20260802010000_add_staged_batch_enrichment.rb b/db/migrate/20260802010000_add_staged_batch_enrichment.rb new file mode 100644 index 0000000..45490ef --- /dev/null +++ b/db/migrate/20260802010000_add_staged_batch_enrichment.rb @@ -0,0 +1,210 @@ +class AddStagedBatchEnrichment < ActiveRecord::Migration[8.1] + ENTITY_TABLES = %i[github_actors github_repositories].freeze + + def up + create_table :enrichment_batches do |t| + t.uuid :correlation_id, null: false, default: -> { "gen_random_uuid()" } + t.text :request_kind, null: false + t.text :entity_kind, null: false + t.text :status, null: false, default: "in_flight" + t.jsonb :requested_github_ids, null: false, default: [] + t.jsonb :requested_identifiers, null: false, default: [] + t.text :request_url + t.integer :response_status + t.text :response_body + t.integer :total_count + t.boolean :incomplete_results + t.integer :requested_count, null: false, default: 0 + t.integer :returned_count, null: false, default: 0 + t.integer :valid_count, null: false, default: 0 + t.integer :missing_count, null: false, default: 0 + t.integer :invalid_count, null: false, default: 0 + t.datetime :started_at, null: false + t.datetime :completed_at + t.text :rate_limit_resource + t.integer :rate_limit_limit + t.integer :rate_limit_remaining + t.integer :rate_limit_used + t.datetime :rate_limit_reset_at + t.text :last_error + t.timestamps + end + add_index :enrichment_batches, :correlation_id, unique: true + add_index :enrichment_batches, %i[request_kind entity_kind started_at] + # The /status batch-quality window filters on started_at alone; the composite + # index above cannot range-scan it because started_at is its last column. + add_index :enrichment_batches, :started_at + add_check_constraint :enrichment_batches, + "request_kind IN ('search', 'detail')", + name: "enrichment_batches_request_kind_check" + add_check_constraint :enrichment_batches, + "entity_kind IN ('actor', 'repository')", + name: "enrichment_batches_entity_kind_check" + add_check_constraint :enrichment_batches, + "status IN ('in_flight', 'succeeded', 'failed', 'deferred', 'stale_lease')", + name: "enrichment_batches_status_check" + add_check_constraint :enrichment_batches, + "requested_count >= 0 AND returned_count >= 0 AND valid_count >= 0 " \ + "AND missing_count >= 0 AND invalid_count >= 0", + name: "enrichment_batches_counters_nonnegative" + + create_table :enrichment_observations do |t| + t.text :entity_kind, null: false + t.bigint :entity_github_id + t.text :source, null: false + t.datetime :observed_at, null: false + t.jsonb :raw_payload, null: false + t.text :payload_fingerprint, null: false + t.references :enrichment_batch, foreign_key: true + t.references :push_event, foreign_key: true + t.uuid :request_correlation_id + t.text :requested_identifier + t.text :validation_outcome, null: false + t.timestamps + end + add_index :enrichment_observations, + %i[entity_kind entity_github_id observed_at], + name: "index_enrichment_observations_on_entity_and_time" + add_index :enrichment_observations, :payload_fingerprint + add_check_constraint :enrichment_observations, + "entity_kind IN ('actor', 'repository')", + name: "enrichment_observations_entity_kind_check" + add_check_constraint :enrichment_observations, + "source IN ('event', 'search', 'detail')", + name: "enrichment_observations_source_check" + + create_table :github_search_budget, id: :integer, default: 1 do |t| + t.text :resource, null: false, default: "search" + t.integer :limit + t.integer :remaining + t.datetime :reset_at + t.datetime :observed_at + t.integer :request_ceiling, null: false, default: 10 + t.integer :reserve, null: false, default: 2 + t.integer :used, null: false, default: 0 + t.integer :actor_used, null: false, default: 0 + t.integer :repository_used, null: false, default: 0 + t.datetime :blocked_until + t.datetime :last_request_at + t.integer :lock_version, null: false, default: 0 + t.timestamps + end + add_check_constraint :github_search_budget, "id = 1", + name: "github_search_budget_singleton" + add_check_constraint :github_search_budget, + "request_ceiling > 0 AND reserve >= 0 AND used >= 0 " \ + "AND actor_used >= 0 AND repository_used >= 0 " \ + "AND (\"limit\" IS NULL OR \"limit\" >= 0) " \ + "AND (remaining IS NULL OR remaining >= 0)", + name: "github_search_budget_counters_valid" + + ENTITY_TABLES.each { |table| add_staged_columns(table) } + + add_column :github_actors, :account_type, :text + + add_column :github_repositories, :owner_login, :text + add_column :github_repositories, :fork, :boolean + add_column :github_repositories, :archived, :boolean + add_column :github_repositories, :default_branch, :text + add_column :github_repositories, :github_created_at, :datetime + + # Legacy rows joined the staged pipeline where their business outcome placed + # them. A pre-staged `complete` row maps to contract_complete even though the + # staged contract columns are NULL: its completion under the previous contract + # is a fact, and the TTL refresh path re-evaluates it against the staged + # contract within one refresh cycle. + ENTITY_TABLES.each do |table| + execute <<~SQL.squish + UPDATE #{table} + SET enrichment_stage = CASE enrichment_status + WHEN 'complete' THEN 'contract_complete' + WHEN 'permanent_failure' THEN 'terminal' + WHEN 'retryable_failure' THEN 'retry_scheduled' + ELSE 'batch_pending' + END, + event_native_at = COALESCE(first_seen_at, created_at), + derived_at = COALESCE(first_seen_at, created_at), + batch_pending_at = CASE WHEN enrichment_status IN ('pending', 'retryable_failure') + THEN COALESCE(first_seen_at, created_at) END, + retry_scheduled_at = CASE WHEN enrichment_status = 'retryable_failure' + THEN updated_at END, + contract_completed_at = CASE WHEN enrichment_status = 'complete' + THEN fetched_at END, + terminal_at = CASE WHEN enrichment_status = 'permanent_failure' + THEN updated_at END + SQL + + # Only resting stages are representable. Event-native persistence, local + # derivation, and batch application are instants recorded by event_native_at, + # derived_at, and batch_applied_at — no row ever rests in them, so they are + # deliberately not enum values a metric or spec would have to enumerate. + add_check_constraint table, + "enrichment_stage IN ('batch_pending', 'batch_in_flight', " \ + "'detail_pending', 'detail_in_flight', 'retry_scheduled', " \ + "'contract_complete', 'terminal')", + name: "#{table}_enrichment_stage_check" + end + end + + def down + remove_column :github_repositories, :github_created_at + remove_column :github_repositories, :default_branch + remove_column :github_repositories, :archived + remove_column :github_repositories, :fork + remove_column :github_repositories, :owner_login + remove_column :github_actors, :account_type + + ENTITY_TABLES.each { |table| remove_staged_columns(table) } + + drop_table :github_search_budget + drop_table :enrichment_observations + drop_table :enrichment_batches + end + + private + + def add_staged_columns(table) + add_column table, :enrichment_stage, :text, null: false, default: "batch_pending" + add_column table, :detail_attempts, :integer, null: false, default: 0 + add_column table, :event_native_at, :datetime + add_column table, :derived_at, :datetime + add_column table, :batch_pending_at, :datetime + add_column table, :batch_applied_at, :datetime + add_column table, :detail_pending_at, :datetime + add_column table, :retry_scheduled_at, :datetime + add_column table, :contract_completed_at, :datetime + add_column table, :terminal_at, :datetime + add_reference table, :latest_observation, foreign_key: { to_table: :enrichment_observations } + add_column table, :latest_observation_source, :text + add_column table, :latest_observed_at, :datetime + add_column table, :lease_token, :uuid + add_column table, :leased_until, :datetime + add_reference table, :current_enrichment_batch, foreign_key: { to_table: :enrichment_batches } + add_index table, %i[enrichment_stage created_at id], name: "index_#{table}_on_stage_fifo" + add_index table, :leased_until + add_check_constraint table, "detail_attempts >= 0", name: "#{table}_detail_attempts_nonnegative" + end + + def remove_staged_columns(table) + remove_check_constraint table, name: "#{table}_enrichment_stage_check" + remove_check_constraint table, name: "#{table}_detail_attempts_nonnegative" + remove_index table, :leased_until + remove_index table, name: "index_#{table}_on_stage_fifo" + remove_reference table, :current_enrichment_batch, foreign_key: { to_table: :enrichment_batches } + remove_column table, :leased_until + remove_column table, :lease_token + remove_column table, :latest_observed_at + remove_column table, :latest_observation_source + remove_reference table, :latest_observation, foreign_key: { to_table: :enrichment_observations } + remove_column table, :terminal_at + remove_column table, :contract_completed_at + remove_column table, :retry_scheduled_at + remove_column table, :detail_pending_at + remove_column table, :batch_applied_at + remove_column table, :batch_pending_at + remove_column table, :derived_at + remove_column table, :event_native_at + remove_column table, :enrichment_stage + remove_column table, :detail_attempts + end +end diff --git a/db/schema.rb b/db/schema.rb index 72c5db2..4dee343 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,10 +10,68 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.1].define(version: 2026_08_02_000000) do +ActiveRecord::Schema[8.1].define(version: 2026_08_02_010000) do # These are extensions that must be enabled in order to support this database enable_extension "pg_catalog.plpgsql" + create_table "enrichment_batches", force: :cascade do |t| + t.datetime "completed_at" + t.uuid "correlation_id", default: -> { "gen_random_uuid()" }, null: false + t.datetime "created_at", null: false + t.text "entity_kind", null: false + t.boolean "incomplete_results" + t.integer "invalid_count", default: 0, null: false + t.text "last_error" + t.integer "missing_count", default: 0, null: false + t.integer "rate_limit_limit" + t.integer "rate_limit_remaining" + t.datetime "rate_limit_reset_at" + t.text "rate_limit_resource" + t.integer "rate_limit_used" + t.text "request_kind", null: false + t.text "request_url" + t.integer "requested_count", default: 0, null: false + t.jsonb "requested_github_ids", default: [], null: false + t.jsonb "requested_identifiers", default: [], null: false + t.text "response_body" + t.integer "response_status" + t.integer "returned_count", default: 0, null: false + t.datetime "started_at", null: false + t.text "status", default: "in_flight", null: false + t.integer "total_count" + t.datetime "updated_at", null: false + t.integer "valid_count", default: 0, null: false + t.index ["correlation_id"], name: "index_enrichment_batches_on_correlation_id", unique: true + t.index ["request_kind", "entity_kind", "started_at"], name: "idx_on_request_kind_entity_kind_started_at_62b9f3c5e7" + t.index ["started_at"], name: "index_enrichment_batches_on_started_at" + t.check_constraint "entity_kind = ANY (ARRAY['actor'::text, 'repository'::text])", name: "enrichment_batches_entity_kind_check" + t.check_constraint "request_kind = ANY (ARRAY['search'::text, 'detail'::text])", name: "enrichment_batches_request_kind_check" + t.check_constraint "requested_count >= 0 AND returned_count >= 0 AND valid_count >= 0 AND missing_count >= 0 AND invalid_count >= 0", name: "enrichment_batches_counters_nonnegative" + t.check_constraint "status = ANY (ARRAY['in_flight'::text, 'succeeded'::text, 'failed'::text, 'deferred'::text, 'stale_lease'::text])", name: "enrichment_batches_status_check" + end + + create_table "enrichment_observations", force: :cascade do |t| + t.datetime "created_at", null: false + t.bigint "enrichment_batch_id" + t.bigint "entity_github_id" + t.text "entity_kind", null: false + t.datetime "observed_at", null: false + t.text "payload_fingerprint", null: false + t.bigint "push_event_id" + t.jsonb "raw_payload", null: false + t.uuid "request_correlation_id" + t.text "requested_identifier" + t.text "source", null: false + t.datetime "updated_at", null: false + t.text "validation_outcome", null: false + t.index ["enrichment_batch_id"], name: "index_enrichment_observations_on_enrichment_batch_id" + t.index ["entity_kind", "entity_github_id", "observed_at"], name: "index_enrichment_observations_on_entity_and_time" + t.index ["payload_fingerprint"], name: "index_enrichment_observations_on_payload_fingerprint" + t.index ["push_event_id"], name: "index_enrichment_observations_on_push_event_id" + t.check_constraint "entity_kind = ANY (ARRAY['actor'::text, 'repository'::text])", name: "enrichment_observations_entity_kind_check" + t.check_constraint "source = ANY (ARRAY['event'::text, 'search'::text, 'detail'::text])", name: "enrichment_observations_source_check" + end + create_table "event_sources", force: :cascade do |t| t.datetime "cadence_due_at" t.jsonb "configuration", default: {}, null: false @@ -36,27 +94,50 @@ end create_table "github_actors", force: :cascade do |t| + t.text "account_type" t.text "api_url" t.text "avatar_url" + t.datetime "batch_applied_at" + t.datetime "batch_pending_at" + t.datetime "contract_completed_at" t.datetime "created_at", null: false + t.bigint "current_enrichment_batch_id" + t.datetime "derived_at" + t.integer "detail_attempts", default: 0, null: false + t.datetime "detail_pending_at" t.text "display_login" t.integer "enrichment_attempts", default: 0, null: false + t.text "enrichment_stage", default: "batch_pending", null: false t.text "enrichment_status", default: "pending", null: false + t.datetime "event_native_at" t.datetime "fetched_at" t.datetime "first_seen_at" t.bigint "github_id", null: false t.text "last_error" t.datetime "last_seen_at" t.datetime "latest_event_at" + t.bigint "latest_observation_id" + t.text "latest_observation_source" + t.datetime "latest_observed_at" + t.uuid "lease_token" + t.datetime "leased_until" t.text "login", null: false t.text "name" t.datetime "next_retry_at" t.jsonb "raw_payload" + t.datetime "retry_scheduled_at" + t.datetime "terminal_at" t.datetime "updated_at", null: false t.index ["created_at", "id"], name: "index_github_actors_on_enrichment_candidates", where: "(enrichment_status = ANY (ARRAY['pending'::text, 'retryable_failure'::text]))" + t.index ["current_enrichment_batch_id"], name: "index_github_actors_on_current_enrichment_batch_id" + t.index ["enrichment_stage", "created_at", "id"], name: "index_github_actors_on_stage_fifo" t.index ["fetched_at", "next_retry_at"], name: "index_github_actors_on_enrichment_refresh", where: "(enrichment_status = 'complete'::text)" t.index ["github_id"], name: "index_github_actors_on_github_id", unique: true + t.index ["latest_observation_id"], name: "index_github_actors_on_latest_observation_id" + t.index ["leased_until"], name: "index_github_actors_on_leased_until" + t.check_constraint "detail_attempts >= 0", name: "github_actors_detail_attempts_nonnegative" t.check_constraint "enrichment_attempts >= 0", name: "github_actors_enrichment_attempts_nonnegative" + t.check_constraint "enrichment_stage = ANY (ARRAY['batch_pending'::text, 'batch_in_flight'::text, 'detail_pending'::text, 'detail_in_flight'::text, 'retry_scheduled'::text, 'contract_complete'::text, 'terminal'::text])", name: "github_actors_enrichment_stage_check" t.check_constraint "enrichment_status = ANY (ARRAY['pending'::text, 'complete'::text, 'retryable_failure'::text, 'permanent_failure'::text])", name: "github_actors_enrichment_status_check" end @@ -88,30 +169,77 @@ create_table "github_repositories", force: :cascade do |t| t.text "api_url" + t.boolean "archived" + t.datetime "batch_applied_at" + t.datetime "batch_pending_at" + t.datetime "contract_completed_at" t.datetime "created_at", null: false + t.bigint "current_enrichment_batch_id" + t.text "default_branch" + t.datetime "derived_at" t.text "description" + t.integer "detail_attempts", default: 0, null: false + t.datetime "detail_pending_at" t.integer "enrichment_attempts", default: 0, null: false + t.text "enrichment_stage", default: "batch_pending", null: false t.text "enrichment_status", default: "pending", null: false + t.datetime "event_native_at" t.datetime "fetched_at" t.datetime "first_seen_at" + t.boolean "fork" t.text "full_name", null: false + t.datetime "github_created_at" t.bigint "github_id", null: false t.text "language" t.text "last_error" t.datetime "last_seen_at" t.datetime "latest_event_at" + t.bigint "latest_observation_id" + t.text "latest_observation_source" + t.datetime "latest_observed_at" + t.uuid "lease_token" + t.datetime "leased_until" t.text "name" t.datetime "next_retry_at" t.bigint "owner_github_id" + t.text "owner_login" t.jsonb "raw_payload" + t.datetime "retry_scheduled_at" + t.datetime "terminal_at" t.datetime "updated_at", null: false t.index ["created_at", "id"], name: "index_github_repositories_on_enrichment_candidates", where: "(enrichment_status = ANY (ARRAY['pending'::text, 'retryable_failure'::text]))" + t.index ["current_enrichment_batch_id"], name: "index_github_repositories_on_current_enrichment_batch_id" + t.index ["enrichment_stage", "created_at", "id"], name: "index_github_repositories_on_stage_fifo" t.index ["fetched_at", "next_retry_at"], name: "index_github_repositories_on_enrichment_refresh", where: "(enrichment_status = 'complete'::text)" t.index ["github_id"], name: "index_github_repositories_on_github_id", unique: true + t.index ["latest_observation_id"], name: "index_github_repositories_on_latest_observation_id" + t.index ["leased_until"], name: "index_github_repositories_on_leased_until" + t.check_constraint "detail_attempts >= 0", name: "github_repositories_detail_attempts_nonnegative" t.check_constraint "enrichment_attempts >= 0", name: "github_repositories_enrichment_attempts_nonnegative" + t.check_constraint "enrichment_stage = ANY (ARRAY['batch_pending'::text, 'batch_in_flight'::text, 'detail_pending'::text, 'detail_in_flight'::text, 'retry_scheduled'::text, 'contract_complete'::text, 'terminal'::text])", name: "github_repositories_enrichment_stage_check" t.check_constraint "enrichment_status = ANY (ARRAY['pending'::text, 'complete'::text, 'retryable_failure'::text, 'permanent_failure'::text])", name: "github_repositories_enrichment_status_check" end + create_table "github_search_budget", id: :integer, default: 1, force: :cascade do |t| + t.integer "actor_used", default: 0, null: false + t.datetime "blocked_until" + t.datetime "created_at", null: false + t.datetime "last_request_at" + t.integer "limit" + t.integer "lock_version", default: 0, null: false + t.datetime "observed_at" + t.integer "remaining" + t.integer "repository_used", default: 0, null: false + t.integer "request_ceiling", default: 10, null: false + t.integer "reserve", default: 2, null: false + t.datetime "reset_at" + t.text "resource", default: "search", null: false + t.datetime "updated_at", null: false + t.integer "used", default: 0, null: false + t.check_constraint "id = 1", name: "github_search_budget_singleton" + t.check_constraint "request_ceiling > 0 AND reserve >= 0 AND used >= 0 AND actor_used >= 0 AND repository_used >= 0 AND (\"limit\" IS NULL OR \"limit\" >= 0) AND (remaining IS NULL OR remaining >= 0)", name: "github_search_budget_counters_valid" + end + create_table "ingestion_runs", force: :cascade do |t| t.datetime "completed_at" t.datetime "created_at", null: false @@ -169,6 +297,12 @@ t.check_constraint "occurrence_count >= 1", name: "quarantined_events_occurrence_count_positive" end + add_foreign_key "enrichment_observations", "enrichment_batches" + add_foreign_key "enrichment_observations", "push_events" + add_foreign_key "github_actors", "enrichment_batches", column: "current_enrichment_batch_id" + add_foreign_key "github_actors", "enrichment_observations", column: "latest_observation_id" + add_foreign_key "github_repositories", "enrichment_batches", column: "current_enrichment_batch_id" + add_foreign_key "github_repositories", "enrichment_observations", column: "latest_observation_id" add_foreign_key "ingestion_runs", "event_sources" add_foreign_key "push_events", "github_actors", primary_key: "github_id" add_foreign_key "push_events", "github_repositories", primary_key: "github_id" From a647d182935e4ee06d6f2bf86e6e5c236240df8f Mon Sep 17 00:00:00 2001 From: Umang Date: Sun, 2 Aug 2026 15:39:50 -0500 Subject: [PATCH 08/12] Add the offline Search corpus and its scenarios MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The staged path needs deterministic offline coverage for the cases live traffic produces: a partial batch, a rename, an unrequested extra item, incomplete_results, a malformed envelope, a rejected query, and both Search rate-limit shapes. Manifest keys are the canonical form the fixture transport actually computes, derived by running the claim and SearchQuery rather than by hand: per_page sorts before q, and encode_www_form's percent-encoding is reproduced exactly. A changed batch membership therefore fails closed as a FixtureMiss instead of silently matching. Search responses override the manifest's default headers so the corpus carries x-ratelimit-resource: search with its own limit and 60-second reset — the search ledger's header path is exercised offline. The two repository bodies gain "archived", the one contract field they lacked. The CI enrichment smoke step sets SEARCH_PACING_SECONDS=0 so a single one-shot can run both lanes back to back. Co-Authored-By: Claude Fable 5 --- .github/workflows/ci.yml | 4 + fixtures/github/README.md | 71 ++++- .../bodies/errors/validation-failed.json | 13 + .../bodies/repos/monalisa_spoon-knife.json | 1 + .../bodies/repos/octocat_hello-world.json | 1 + .../bodies/search/malformed-envelope.json | 5 + .../bodies/search/repositories-complete.json | 63 +++++ .../repositories-missing-hello-world.json | 25 ++ .../bodies/search/repositories-partial.json | 44 +++ .../bodies/search/repositories-renamed.json | 44 +++ .../search/repositories-unrequested.json | 63 +++++ .../github/bodies/search/users-complete.json | 39 +++ .../bodies/search/users-incomplete.json | 28 ++ .../github/bodies/search/users-partial.json | 28 ++ fixtures/github/manifest.json | 263 ++++++++++++++++++ 15 files changed, 687 insertions(+), 5 deletions(-) create mode 100644 fixtures/github/bodies/errors/validation-failed.json create mode 100644 fixtures/github/bodies/search/malformed-envelope.json create mode 100644 fixtures/github/bodies/search/repositories-complete.json create mode 100644 fixtures/github/bodies/search/repositories-missing-hello-world.json create mode 100644 fixtures/github/bodies/search/repositories-partial.json create mode 100644 fixtures/github/bodies/search/repositories-renamed.json create mode 100644 fixtures/github/bodies/search/repositories-unrequested.json create mode 100644 fixtures/github/bodies/search/users-complete.json create mode 100644 fixtures/github/bodies/search/users-incomplete.json create mode 100644 fixtures/github/bodies/search/users-partial.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 23f53bf..fb26f80 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,6 +118,10 @@ jobs: - name: Smoke test the one-shot enrichment command env: GITHUB_MODE: fixture + # The one-shot never sleeps pacing out; at the default 6-second search + # pacing the second batch would be reported as a pacing deferral instead + # of running, and the smoke would prove one lane instead of the chain. + SEARCH_PACING_SECONDS: "0" run: bin/enrich --limit 6 # Solid Queue's own validator over config/queue.yml and config/recurring.yml: a diff --git a/fixtures/github/README.md b/fixtures/github/README.md index 92e21e2..e3b05bb 100644 --- a/fixtures/github/README.md +++ b/fixtures/github/README.md @@ -25,7 +25,8 @@ fixtures/github/ ├── events/ event pages returned by /events ├── users/ actor documents returned by /users/:login ├── repos/ repository documents returned by /repos/:owner/:name - └── errors/ GitHub's error bodies for 403, 404, and 500 + ├── search/ Search envelopes returned by /search/users and /search/repositories + └── errors/ GitHub's error bodies for 403, 404, 422, and 500 ``` ## How a request finds a response @@ -35,6 +36,22 @@ parameters sorted by name. Scheme and host are omitted because `Github::UrlPolic already proved them — `https` and `api.github.com` in live mode, `fixture` and `api.github.com` offline. One entry therefore answers a request from either transport. +The two Search keys in `default` are the worked example: + +``` +/search/users?per_page=3&q=user%3Aoctocat+user%3Amonalisa+user%3Aghostuser +/search/repositories?per_page=3&q=repo%3Aoctocat%2FHello-World+repo%3Amonalisa%2FSpoon-Knife+repo%3Adeleted-org%2Fgone +``` + +`per_page` sorts before `q`, and the value is exactly what `URI.encode_www_form` +produces: the space between qualifiers becomes `+`, `:` becomes `%3A`, `/` becomes +`%2F`. A Search key therefore encodes the batch's exact membership and FIFO order +(`created_at, id` over the never-enriched backlog) — a claim that composes a different +batch, one entity more or fewer or in a different order, produces a different key and +fails closed as `Github::Errors::FixtureMiss` rather than quietly answering the wrong +batch. That strictness is deliberate: the corpus asserts the batch composition, not +just the endpoint. + Body paths are **authored in the manifest and never derived from a URL**. That is a security property rather than a convenience: `fixture://api.github.com/../../etc/passwd` parses with its path preserved, and a corpus that derived filenames from URLs would read @@ -55,6 +72,17 @@ ledger would then correctly roll the window on every single poll — so counters never accumulate and fixture mode would stop demonstrating the very accounting it exists to demonstrate. The transport's clock is injectable, so specs stay deterministic. +## Search responses override the core headers + +`default_headers` describes the core resource: `x-ratelimit-resource: core`, limit 60, +reset `+3600`. Search is a different rate-limit resource with a per-minute window, so +**every response scripted for a `/search/...` key overrides all five rate-limit headers +per response** — resource `search`, limit `10`, its own `remaining`/`used`, reset +`+60`. The override is per response rather than per scenario because header merging is +per response: the search budget ledger's `reconcile!` discards headers whose resource +is not `search`, and a search response that inherited the core defaults would either be +ignored by its own ledger or, worse, teach it core-window numbers. + ## Sequences A key's value is an ordered list. The *n*th request for that key gets the *n*th entry, @@ -87,19 +115,30 @@ readable in one place. | Scenario | What it exercises | |---|---| -| `default` | One page of events, then `304` with the same ETag forever. The reviewer scenario. | +| `default` | One page of events, then `304` with the same ETag forever; each Search batch answers two of its three entities, and the missing ones fall back to detail and meet `404`s. The reviewer scenario — the walkthrough below. | | `paginated` | `Link`-driven pagination: page 1 → page 2 → an empty page 3. Page 2 repeats page 1's first event on purpose, which is how the absence of a stop-on-known-event is proved. | | `paginated_final_page` | Page 1 → page 2, and page 2 carries no `Link` — the no-next-link stop, which `paginated` cannot show because its last page is empty first. | | `rate_limited` | `403` with `x-ratelimit-remaining: 0` — primary exhaustion, which blocks every live request until the window resets. | | `secondary_rate_limited` | `403` with `Retry-After` and quota remaining — a secondary limit, which blocks globally for the interval GitHub named. | | `transient_failure` | `500`, `500`, then `200`: two retries, each its own reservation. | | `transient_failure_exhausted` | `500` forever, so retries exhaust and the failure persists. | -| `redirecting_repository` | `301` to a renamed repository, re-validated before it is followed. | -| `hostile_redirect` | `301` off `api.github.com`, which `Github::UrlPolicy` must refuse. | +| `search_complete` | Both Search envelopes return every requested entity, so the whole backlog completes in two requests and no detail fallback is admitted. | +| `search_renamed_repository` | Hello-World comes back with its id intact but as `octocat/hello-world-renamed` — a rename, which the batch validator refuses (`renamed_repository`) and hands to the detail fallback. | +| `search_unrequested_result` | The repository envelope carries the two requested items plus `intruder/unasked`, which nobody asked for — preserved as an `unrequested_result` observation and never applied. | +| `search_incomplete_results` | The users envelope sets `incomplete_results: true` over the same two items as `default` — ID-valid items are applied regardless, and the missing `ghostuser` still falls back. The flag is an envelope fact, not an item fact. | +| `search_malformed` | Parseable JSON that fails the Search envelope contract (`total_count` not a number, `incomplete_results` not boolean, `items` not an array) — the batch fails and every member is rescheduled. | +| `search_rejected` | `422 Validation Failed` on both Search keys — GitHub refused the query itself. | +| `search_rate_limited` | `403` on the search resource with `x-ratelimit-remaining: 0` — the per-minute search window is exhausted. | +| `search_secondary_rate_limited` | `429` with `Retry-After` and search quota remaining — a secondary limit against Search. | +| `redirecting_repository` | Hello-World is absent from the Search envelope, so the detail fallback fetches it and meets a `301` to the renamed repository, re-validated before it is followed. | +| `hostile_redirect` | The same Search miss, but the `301` points off `api.github.com`, which `Github::UrlPolicy` must refuse. | Scenarios beyond `default` exist because §12 names them as corpus contents. The pagination and rate-limit ones are consumed by `Github::Ingestion::PageLoop` and -`Github::RateLimitPolicy`. The redirect ones are consumed by +`Github::RateLimitPolicy`. The two redirect scenarios also override the repository Search +key with `search/repositories-missing-hello-world.json`, because staged enrichment only +fetches a detail URL for an entity Search failed to answer — the Search miss is what keeps +the redirect boundary reachable. They are consumed by `spec/services/github/enrichment/redirect_boundary_spec.rb`, which drives them through the enrichment claim path rather than through the executor alone — so what is asserted is the consequence for an *entity*: a rename reaches `complete` and is debited for both hops, while a @@ -107,6 +146,28 @@ hostile `Location` reaches `permanent_failure` with the second hop never sent an source still in service. Both also appear in the README's fixture scenario matrix as reviewer commands. +## The default enrichment walkthrough + +`GITHUB_MODE=fixture bin/ingest` persists four push events and stubs three actors +(octocat, monalisa, ghostuser) and three repositories (octocat/Hello-World, +monalisa/Spoon-Knife, deleted-org/gone). `GITHUB_MODE=fixture SEARCH_PACING_SECONDS=0 +bin/enrich --limit 6` then walks the staged pipeline in four requests: + +1. The actor Search batch answers `search/users-partial.json`: octocat and monalisa + validate against their immutable ids and are applied; ghostuser is missing from the + envelope and is admitted to the detail fallback. +2. The repository batch answers `search/repositories-partial.json` the same way. + Spoon-Knife's item carries `"description": null` and `"language": null` — the proof + that the nullable contract fields pass validation as nulls rather than being refused. + `deleted-org/gone` is missing and falls back. +3. The two detail fallbacks fetch the stored payload URLs (`/users/ghostuser`, + `/repos/deleted-org/gone`), meet the corpus's `404`s, and go terminal immediately. + +End state, per class: two `complete`, one `permanent_failure` — from exactly two search +requests and two detail requests. `SEARCH_PACING_SECONDS=0` matters: the one-shot never +sleeps pacing out, so at the default 6 seconds the second batch would be reported as a +pacing deferral instead of running back to back. + ## What is in `bodies/events/page-1.json` Eight event envelopes, deliberately mixed so the tolerant parser and quarantine taxonomy diff --git a/fixtures/github/bodies/errors/validation-failed.json b/fixtures/github/bodies/errors/validation-failed.json new file mode 100644 index 0000000..5cbcf60 --- /dev/null +++ b/fixtures/github/bodies/errors/validation-failed.json @@ -0,0 +1,13 @@ +{ + "message": "Validation Failed", + "errors": [ + { + "message": "The listed users cannot be searched either because the users do not exist or you do not have permission to view the users.", + "resource": "Search", + "field": "q", + "code": "invalid" + } + ], + "documentation_url": "https://docs.github.com/v3/search/", + "status": "422" +} diff --git a/fixtures/github/bodies/repos/monalisa_spoon-knife.json b/fixtures/github/bodies/repos/monalisa_spoon-knife.json index 91828c8..a0cd4f7 100644 --- a/fixtures/github/bodies/repos/monalisa_spoon-knife.json +++ b/fixtures/github/bodies/repos/monalisa_spoon-knife.json @@ -22,6 +22,7 @@ "stargazers_count": 12894, "watchers_count": 12894, "open_issues_count": 17023, + "archived": false, "default_branch": "trunk", "visibility": "public" } diff --git a/fixtures/github/bodies/repos/octocat_hello-world.json b/fixtures/github/bodies/repos/octocat_hello-world.json index bed0907..047c3d9 100644 --- a/fixtures/github/bodies/repos/octocat_hello-world.json +++ b/fixtures/github/bodies/repos/octocat_hello-world.json @@ -22,6 +22,7 @@ "stargazers_count": 3011, "watchers_count": 3011, "open_issues_count": 1808, + "archived": false, "default_branch": "main", "visibility": "public" } diff --git a/fixtures/github/bodies/search/malformed-envelope.json b/fixtures/github/bodies/search/malformed-envelope.json new file mode 100644 index 0000000..a5160ef --- /dev/null +++ b/fixtures/github/bodies/search/malformed-envelope.json @@ -0,0 +1,5 @@ +{ + "total_count": "not-a-number", + "incomplete_results": "maybe", + "items": {} +} diff --git a/fixtures/github/bodies/search/repositories-complete.json b/fixtures/github/bodies/search/repositories-complete.json new file mode 100644 index 0000000..2981eb2 --- /dev/null +++ b/fixtures/github/bodies/search/repositories-complete.json @@ -0,0 +1,63 @@ +{ + "total_count": 3, + "incomplete_results": false, + "items": [ + { + "id": 1296269, + "node_id": "MDEwOlJlcG9zaXRvcnkxMjk2MjY5", + "name": "Hello-World", + "full_name": "octocat/Hello-World", + "owner": { + "login": "octocat", + "id": 583231, + "type": "User" + }, + "fork": false, + "archived": false, + "default_branch": "main", + "created_at": "2011-01-26T19:01:12Z", + "description": "My first repository on GitHub!", + "language": "Ruby", + "url": "https://api.github.com/repos/octocat/Hello-World", + "score": 1.0 + }, + { + "id": 1300192, + "node_id": "MDEwOlJlcG9zaXRvcnkxMzAwMTky", + "name": "Spoon-Knife", + "full_name": "monalisa/Spoon-Knife", + "owner": { + "login": "monalisa", + "id": 1024025, + "type": "User" + }, + "fork": false, + "archived": false, + "default_branch": "trunk", + "created_at": "2011-01-27T19:30:43Z", + "description": null, + "language": null, + "url": "https://api.github.com/repos/monalisa/Spoon-Knife", + "score": 1.0 + }, + { + "id": 1490033, + "node_id": "MDEwOlJlcG9zaXRvcnkxNDkwMDMz", + "name": "gone", + "full_name": "deleted-org/gone", + "owner": { + "login": "deleted-org", + "id": 9919137, + "type": "Organization" + }, + "fork": false, + "archived": true, + "default_branch": "master", + "created_at": "2013-07-19T04:12:29Z", + "description": null, + "language": null, + "url": "https://api.github.com/repos/deleted-org/gone", + "score": 1.0 + } + ] +} diff --git a/fixtures/github/bodies/search/repositories-missing-hello-world.json b/fixtures/github/bodies/search/repositories-missing-hello-world.json new file mode 100644 index 0000000..9915b07 --- /dev/null +++ b/fixtures/github/bodies/search/repositories-missing-hello-world.json @@ -0,0 +1,25 @@ +{ + "total_count": 1, + "incomplete_results": false, + "items": [ + { + "id": 1300192, + "node_id": "MDEwOlJlcG9zaXRvcnkxMzAwMTky", + "name": "Spoon-Knife", + "full_name": "monalisa/Spoon-Knife", + "owner": { + "login": "monalisa", + "id": 1024025, + "type": "User" + }, + "fork": false, + "archived": false, + "default_branch": "trunk", + "created_at": "2011-01-27T19:30:43Z", + "description": null, + "language": null, + "url": "https://api.github.com/repos/monalisa/Spoon-Knife", + "score": 1.0 + } + ] +} diff --git a/fixtures/github/bodies/search/repositories-partial.json b/fixtures/github/bodies/search/repositories-partial.json new file mode 100644 index 0000000..7d2cb01 --- /dev/null +++ b/fixtures/github/bodies/search/repositories-partial.json @@ -0,0 +1,44 @@ +{ + "total_count": 2, + "incomplete_results": false, + "items": [ + { + "id": 1296269, + "node_id": "MDEwOlJlcG9zaXRvcnkxMjk2MjY5", + "name": "Hello-World", + "full_name": "octocat/Hello-World", + "owner": { + "login": "octocat", + "id": 583231, + "type": "User" + }, + "fork": false, + "archived": false, + "default_branch": "main", + "created_at": "2011-01-26T19:01:12Z", + "description": "My first repository on GitHub!", + "language": "Ruby", + "url": "https://api.github.com/repos/octocat/Hello-World", + "score": 1.0 + }, + { + "id": 1300192, + "node_id": "MDEwOlJlcG9zaXRvcnkxMzAwMTky", + "name": "Spoon-Knife", + "full_name": "monalisa/Spoon-Knife", + "owner": { + "login": "monalisa", + "id": 1024025, + "type": "User" + }, + "fork": false, + "archived": false, + "default_branch": "trunk", + "created_at": "2011-01-27T19:30:43Z", + "description": null, + "language": null, + "url": "https://api.github.com/repos/monalisa/Spoon-Knife", + "score": 1.0 + } + ] +} diff --git a/fixtures/github/bodies/search/repositories-renamed.json b/fixtures/github/bodies/search/repositories-renamed.json new file mode 100644 index 0000000..9877c4f --- /dev/null +++ b/fixtures/github/bodies/search/repositories-renamed.json @@ -0,0 +1,44 @@ +{ + "total_count": 2, + "incomplete_results": false, + "items": [ + { + "id": 1296269, + "node_id": "MDEwOlJlcG9zaXRvcnkxMjk2MjY5", + "name": "hello-world-renamed", + "full_name": "octocat/hello-world-renamed", + "owner": { + "login": "octocat", + "id": 583231, + "type": "User" + }, + "fork": false, + "archived": false, + "default_branch": "main", + "created_at": "2011-01-26T19:01:12Z", + "description": "My first repository on GitHub!", + "language": "Ruby", + "url": "https://api.github.com/repos/octocat/hello-world-renamed", + "score": 1.0 + }, + { + "id": 1300192, + "node_id": "MDEwOlJlcG9zaXRvcnkxMzAwMTky", + "name": "Spoon-Knife", + "full_name": "monalisa/Spoon-Knife", + "owner": { + "login": "monalisa", + "id": 1024025, + "type": "User" + }, + "fork": false, + "archived": false, + "default_branch": "trunk", + "created_at": "2011-01-27T19:30:43Z", + "description": null, + "language": null, + "url": "https://api.github.com/repos/monalisa/Spoon-Knife", + "score": 1.0 + } + ] +} diff --git a/fixtures/github/bodies/search/repositories-unrequested.json b/fixtures/github/bodies/search/repositories-unrequested.json new file mode 100644 index 0000000..5798169 --- /dev/null +++ b/fixtures/github/bodies/search/repositories-unrequested.json @@ -0,0 +1,63 @@ +{ + "total_count": 3, + "incomplete_results": false, + "items": [ + { + "id": 1296269, + "node_id": "MDEwOlJlcG9zaXRvcnkxMjk2MjY5", + "name": "Hello-World", + "full_name": "octocat/Hello-World", + "owner": { + "login": "octocat", + "id": 583231, + "type": "User" + }, + "fork": false, + "archived": false, + "default_branch": "main", + "created_at": "2011-01-26T19:01:12Z", + "description": "My first repository on GitHub!", + "language": "Ruby", + "url": "https://api.github.com/repos/octocat/Hello-World", + "score": 1.0 + }, + { + "id": 1300192, + "node_id": "MDEwOlJlcG9zaXRvcnkxMzAwMTky", + "name": "Spoon-Knife", + "full_name": "monalisa/Spoon-Knife", + "owner": { + "login": "monalisa", + "id": 1024025, + "type": "User" + }, + "fork": false, + "archived": false, + "default_branch": "trunk", + "created_at": "2011-01-27T19:30:43Z", + "description": null, + "language": null, + "url": "https://api.github.com/repos/monalisa/Spoon-Knife", + "score": 1.0 + }, + { + "id": 9999999, + "node_id": "MDEwOlJlcG9zaXRvcnk5OTk5OTk5", + "name": "unasked", + "full_name": "intruder/unasked", + "owner": { + "login": "intruder", + "id": 8811560, + "type": "User" + }, + "fork": false, + "archived": false, + "default_branch": "main", + "created_at": "2019-03-05T11:47:02Z", + "description": "A result nobody asked for.", + "language": "Go", + "url": "https://api.github.com/repos/intruder/unasked", + "score": 0.4 + } + ] +} diff --git a/fixtures/github/bodies/search/users-complete.json b/fixtures/github/bodies/search/users-complete.json new file mode 100644 index 0000000..78f39bb --- /dev/null +++ b/fixtures/github/bodies/search/users-complete.json @@ -0,0 +1,39 @@ +{ + "total_count": 3, + "incomplete_results": false, + "items": [ + { + "login": "octocat", + "id": 583231, + "node_id": "MDQ6VXNlcjU4MzIzMQ==", + "avatar_url": "https://avatars.githubusercontent.com/u/583231?v=4", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "type": "User", + "site_admin": false, + "score": 1.0 + }, + { + "login": "monalisa", + "id": 1024025, + "node_id": "MDQ6VXNlcjEwMjQwMjU=", + "avatar_url": "https://avatars.githubusercontent.com/u/1024025?v=4", + "url": "https://api.github.com/users/monalisa", + "html_url": "https://github.com/monalisa", + "type": "User", + "site_admin": false, + "score": 1.0 + }, + { + "login": "ghostuser", + "id": 7700421, + "node_id": "MDQ6VXNlcjc3MDA0MjE=", + "avatar_url": "https://avatars.githubusercontent.com/u/7700421?v=4", + "url": "https://api.github.com/users/ghostuser", + "html_url": "https://github.com/ghostuser", + "type": "User", + "site_admin": false, + "score": 1.0 + } + ] +} diff --git a/fixtures/github/bodies/search/users-incomplete.json b/fixtures/github/bodies/search/users-incomplete.json new file mode 100644 index 0000000..efc9fbd --- /dev/null +++ b/fixtures/github/bodies/search/users-incomplete.json @@ -0,0 +1,28 @@ +{ + "total_count": 2, + "incomplete_results": true, + "items": [ + { + "login": "octocat", + "id": 583231, + "node_id": "MDQ6VXNlcjU4MzIzMQ==", + "avatar_url": "https://avatars.githubusercontent.com/u/583231?v=4", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "type": "User", + "site_admin": false, + "score": 1.0 + }, + { + "login": "monalisa", + "id": 1024025, + "node_id": "MDQ6VXNlcjEwMjQwMjU=", + "avatar_url": "https://avatars.githubusercontent.com/u/1024025?v=4", + "url": "https://api.github.com/users/monalisa", + "html_url": "https://github.com/monalisa", + "type": "User", + "site_admin": false, + "score": 1.0 + } + ] +} diff --git a/fixtures/github/bodies/search/users-partial.json b/fixtures/github/bodies/search/users-partial.json new file mode 100644 index 0000000..4838622 --- /dev/null +++ b/fixtures/github/bodies/search/users-partial.json @@ -0,0 +1,28 @@ +{ + "total_count": 2, + "incomplete_results": false, + "items": [ + { + "login": "octocat", + "id": 583231, + "node_id": "MDQ6VXNlcjU4MzIzMQ==", + "avatar_url": "https://avatars.githubusercontent.com/u/583231?v=4", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "type": "User", + "site_admin": false, + "score": 1.0 + }, + { + "login": "monalisa", + "id": 1024025, + "node_id": "MDQ6VXNlcjEwMjQwMjU=", + "avatar_url": "https://avatars.githubusercontent.com/u/1024025?v=4", + "url": "https://api.github.com/users/monalisa", + "html_url": "https://github.com/monalisa", + "type": "User", + "site_admin": false, + "score": 1.0 + } + ] +} diff --git a/fixtures/github/manifest.json b/fixtures/github/manifest.json index aa5b747..d358609 100644 --- a/fixtures/github/manifest.json +++ b/fixtures/github/manifest.json @@ -92,6 +92,32 @@ "x-ratelimit-used": "8" } } + ], + "/search/users?per_page=3&q=user%3Aoctocat+user%3Amonalisa+user%3Aghostuser": [ + { + "status": 200, + "body": "search/users-partial.json", + "headers": { + "x-ratelimit-resource": "search", + "x-ratelimit-limit": "10", + "x-ratelimit-remaining": "9", + "x-ratelimit-used": "1", + "x-ratelimit-reset": "+60" + } + } + ], + "/search/repositories?per_page=3&q=repo%3Aoctocat%2FHello-World+repo%3Amonalisa%2FSpoon-Knife+repo%3Adeleted-org%2Fgone": [ + { + "status": 200, + "body": "search/repositories-partial.json", + "headers": { + "x-ratelimit-resource": "search", + "x-ratelimit-limit": "10", + "x-ratelimit-remaining": "8", + "x-ratelimit-used": "2", + "x-ratelimit-reset": "+60" + } + } ] } }, @@ -238,9 +264,233 @@ ] } }, + "search_complete": { + "inherit": "default", + "responses": { + "/search/users?per_page=3&q=user%3Aoctocat+user%3Amonalisa+user%3Aghostuser": [ + { + "status": 200, + "body": "search/users-complete.json", + "headers": { + "x-ratelimit-resource": "search", + "x-ratelimit-limit": "10", + "x-ratelimit-remaining": "9", + "x-ratelimit-used": "1", + "x-ratelimit-reset": "+60" + } + } + ], + "/search/repositories?per_page=3&q=repo%3Aoctocat%2FHello-World+repo%3Amonalisa%2FSpoon-Knife+repo%3Adeleted-org%2Fgone": [ + { + "status": 200, + "body": "search/repositories-complete.json", + "headers": { + "x-ratelimit-resource": "search", + "x-ratelimit-limit": "10", + "x-ratelimit-remaining": "8", + "x-ratelimit-used": "2", + "x-ratelimit-reset": "+60" + } + } + ] + } + }, + "search_renamed_repository": { + "inherit": "default", + "responses": { + "/search/repositories?per_page=3&q=repo%3Aoctocat%2FHello-World+repo%3Amonalisa%2FSpoon-Knife+repo%3Adeleted-org%2Fgone": [ + { + "status": 200, + "body": "search/repositories-renamed.json", + "headers": { + "x-ratelimit-resource": "search", + "x-ratelimit-limit": "10", + "x-ratelimit-remaining": "9", + "x-ratelimit-used": "1", + "x-ratelimit-reset": "+60" + } + } + ] + } + }, + "search_unrequested_result": { + "inherit": "default", + "responses": { + "/search/repositories?per_page=3&q=repo%3Aoctocat%2FHello-World+repo%3Amonalisa%2FSpoon-Knife+repo%3Adeleted-org%2Fgone": [ + { + "status": 200, + "body": "search/repositories-unrequested.json", + "headers": { + "x-ratelimit-resource": "search", + "x-ratelimit-limit": "10", + "x-ratelimit-remaining": "9", + "x-ratelimit-used": "1", + "x-ratelimit-reset": "+60" + } + } + ] + } + }, + "search_incomplete_results": { + "inherit": "default", + "responses": { + "/search/users?per_page=3&q=user%3Aoctocat+user%3Amonalisa+user%3Aghostuser": [ + { + "status": 200, + "body": "search/users-incomplete.json", + "headers": { + "x-ratelimit-resource": "search", + "x-ratelimit-limit": "10", + "x-ratelimit-remaining": "9", + "x-ratelimit-used": "1", + "x-ratelimit-reset": "+60" + } + } + ] + } + }, + "search_malformed": { + "inherit": "default", + "responses": { + "/search/users?per_page=3&q=user%3Aoctocat+user%3Amonalisa+user%3Aghostuser": [ + { + "status": 200, + "body": "search/malformed-envelope.json", + "headers": { + "x-ratelimit-resource": "search", + "x-ratelimit-limit": "10", + "x-ratelimit-remaining": "9", + "x-ratelimit-used": "1", + "x-ratelimit-reset": "+60" + } + } + ], + "/search/repositories?per_page=3&q=repo%3Aoctocat%2FHello-World+repo%3Amonalisa%2FSpoon-Knife+repo%3Adeleted-org%2Fgone": [ + { + "status": 200, + "body": "search/malformed-envelope.json", + "headers": { + "x-ratelimit-resource": "search", + "x-ratelimit-limit": "10", + "x-ratelimit-remaining": "8", + "x-ratelimit-used": "2", + "x-ratelimit-reset": "+60" + } + } + ] + } + }, + "search_rejected": { + "inherit": "default", + "responses": { + "/search/users?per_page=3&q=user%3Aoctocat+user%3Amonalisa+user%3Aghostuser": [ + { + "status": 422, + "body": "errors/validation-failed.json", + "headers": { + "x-ratelimit-resource": "search", + "x-ratelimit-limit": "10", + "x-ratelimit-remaining": "9", + "x-ratelimit-used": "1", + "x-ratelimit-reset": "+60" + } + } + ], + "/search/repositories?per_page=3&q=repo%3Aoctocat%2FHello-World+repo%3Amonalisa%2FSpoon-Knife+repo%3Adeleted-org%2Fgone": [ + { + "status": 422, + "body": "errors/validation-failed.json", + "headers": { + "x-ratelimit-resource": "search", + "x-ratelimit-limit": "10", + "x-ratelimit-remaining": "8", + "x-ratelimit-used": "2", + "x-ratelimit-reset": "+60" + } + } + ] + } + }, + "search_rate_limited": { + "inherit": "default", + "responses": { + "/search/users?per_page=3&q=user%3Aoctocat+user%3Amonalisa+user%3Aghostuser": [ + { + "status": 403, + "body": "errors/rate-limit-exhausted.json", + "headers": { + "x-ratelimit-resource": "search", + "x-ratelimit-limit": "10", + "x-ratelimit-remaining": "0", + "x-ratelimit-used": "10", + "x-ratelimit-reset": "+60" + } + } + ], + "/search/repositories?per_page=3&q=repo%3Aoctocat%2FHello-World+repo%3Amonalisa%2FSpoon-Knife+repo%3Adeleted-org%2Fgone": [ + { + "status": 403, + "body": "errors/rate-limit-exhausted.json", + "headers": { + "x-ratelimit-resource": "search", + "x-ratelimit-limit": "10", + "x-ratelimit-remaining": "0", + "x-ratelimit-used": "10", + "x-ratelimit-reset": "+60" + } + } + ] + } + }, + "search_secondary_rate_limited": { + "inherit": "default", + "responses": { + "/search/users?per_page=3&q=user%3Aoctocat+user%3Amonalisa+user%3Aghostuser": [ + { + "status": 429, + "body": "errors/secondary-rate-limit.json", + "headers": { + "retry-after": "60", + "x-ratelimit-resource": "search", + "x-ratelimit-limit": "10", + "x-ratelimit-remaining": "7", + "x-ratelimit-used": "3", + "x-ratelimit-reset": "+60" + } + } + ], + "/search/repositories?per_page=3&q=repo%3Aoctocat%2FHello-World+repo%3Amonalisa%2FSpoon-Knife+repo%3Adeleted-org%2Fgone": [ + { + "status": 429, + "body": "errors/secondary-rate-limit.json", + "headers": { + "retry-after": "60", + "x-ratelimit-resource": "search", + "x-ratelimit-limit": "10", + "x-ratelimit-remaining": "7", + "x-ratelimit-used": "3", + "x-ratelimit-reset": "+60" + } + } + ] + } + }, "redirecting_repository": { "inherit": "default", "responses": { + "/search/repositories?per_page=3&q=repo%3Aoctocat%2FHello-World+repo%3Amonalisa%2FSpoon-Knife+repo%3Adeleted-org%2Fgone": [ + { + "status": 200, + "body": "search/repositories-missing-hello-world.json", + "headers": { + "x-ratelimit-resource": "search", + "x-ratelimit-limit": "10", + "x-ratelimit-remaining": "9", + "x-ratelimit-used": "1", + "x-ratelimit-reset": "+60" + } + } + ], "/repos/octocat/Hello-World": [ { "status": 301, @@ -266,6 +516,19 @@ "hostile_redirect": { "inherit": "default", "responses": { + "/search/repositories?per_page=3&q=repo%3Aoctocat%2FHello-World+repo%3Amonalisa%2FSpoon-Knife+repo%3Adeleted-org%2Fgone": [ + { + "status": 200, + "body": "search/repositories-missing-hello-world.json", + "headers": { + "x-ratelimit-resource": "search", + "x-ratelimit-limit": "10", + "x-ratelimit-remaining": "9", + "x-ratelimit-used": "1", + "x-ratelimit-reset": "+60" + } + } + ], "/repos/octocat/Hello-World": [ { "status": 301, From d1dcc0e7f924e1024c3c39ad00bb83c4daf3f770 Mon Sep 17 00:00:00 2001 From: Umang Date: Sun, 2 Aug 2026 15:39:50 -0500 Subject: [PATCH 09/12] Cover the staged pipeline, both ledgers, and the catch-up metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the coverage issue #45 requires and reworks what the retired per-entity path owned. Notable shapes: - Batch matrix: apply-by-stable-id with shuffled results, missing, renamed, identity-mismatched, unrequested, incomplete_results, malformed envelope, 422, and both rate-limit responses — each asserting where the members end up, not only what the call returned. - Dual-window deferral: one integration example drains 50 entities FIFO across rolled Search minute windows and a core hourly window, through the real executor and transport with WebMock echoing the requested identifiers back. - Durability: leases, SKIP LOCKED disjointness, expired-lease reclaim with its orphaned batch marked, and a late writer from a stolen lease matching zero rows. - A boundary spec asserts no quota-flavored terminal state exists in the vocabulary or the behavior: a denial on either ledger leaves every row bit-identical while the batch row remains as evidence. - /status: the nine-block contract, per-stage counts, batch quality, and all three catch-up states — including insufficient_sample on a fresh database and not_keeping_up on a flat nonzero backlog. Co-Authored-By: Claude Fable 5 --- spec/config/github_initializer_spec.rb | 26 +- spec/db/add_staged_batch_enrichment_spec.rb | 215 ++++++++ ...ove_skipped_budget_from_enrichment_spec.rb | 82 +-- spec/db/schema_spec.rb | 166 +++++- spec/jobs/application_job_spec.rb | 6 +- spec/jobs/enrich_actor_job_spec.rb | 35 -- spec/jobs/enrich_repository_job_spec.rb | 35 -- spec/jobs/enrichment_cycle_job_spec.rb | 96 ++++ spec/jobs/poll_event_source_job_spec.rb | 5 +- .../reconcile_pending_enrichments_job_spec.rb | 4 +- spec/models/enrichment_batch_spec.rb | 131 +++++ spec/models/enrichment_observation_spec.rb | 95 ++++ spec/models/github_actor_spec.rb | 26 +- spec/models/github_repository_spec.rb | 50 ++ spec/models/github_search_budget_spec.rb | 162 ++++++ spec/quota_outcome_boundary_spec.rb | 152 ++++++ spec/recovery/concurrent_write_spec.rb | 20 +- spec/recovery/crash_window_spec.rb | 43 +- spec/recovery/duplicate_job_execution_spec.rb | 111 ++-- spec/recovery/multi_poller_spec.rb | 46 +- .../pending_enrichment_recovery_spec.rb | 86 +-- spec/recovery/worker_crash_lease_spec.rb | 162 ++++-- spec/requests/status_spec.rb | 90 +++- spec/services/github/allowances_spec.rb | 107 +++- .../github/budget_ledger_bootstrap_spec.rb | 10 +- .../github/budget_ledger_shared_ip_spec.rb | 19 +- spec/services/github/budget_ledger_spec.rb | 65 +-- spec/services/github/configuration_spec.rb | 136 ++++- .../github/enrichment/actor_document_spec.rb | 48 +- .../github/enrichment/admission_spec.rb | 150 ++++++ .../github/enrichment/backlog_metrics_spec.rb | 241 +++++++-- .../github/enrichment/backoff_spec.rb | 73 ++- .../github/enrichment/batch_claim_spec.rb | 302 +++++++++++ .../github/enrichment/batch_quality_spec.rb | 138 +++++ .../github/enrichment/batch_runner_spec.rb | 500 +++++++++++++++++ .../enrichment/candidate_selector_spec.rb | 329 ------------ spec/services/github/enrichment/claim_spec.rb | 175 ------ .../github/enrichment/cycle_runner_spec.rb | 288 ++++++++++ .../github/enrichment/detail_claim_spec.rb | 117 ++++ .../github/enrichment/detail_runner_spec.rb | 329 ++++++++++++ .../github/enrichment/dispatch_spec.rb | 172 +++--- .../github/enrichment/end_to_end_spec.rb | 246 +++++---- .../github/enrichment/entity_state_spec.rb | 319 ----------- .../github/enrichment/entity_type_spec.rb | 25 +- .../event_native_derivation_spec.rb | 125 +++++ .../github/enrichment/fairness_spec.rb | 243 --------- .../github/enrichment/fairness_stress_spec.rb | 232 -------- .../github/enrichment/lane_fairness_spec.rb | 229 ++++++++ .../enrichment/multi_window_backlog_spec.rb | 159 ------ .../multi_window_batch_backlog_spec.rb | 305 +++++++++++ .../github/enrichment/one_shot_spec.rb | 295 +++++++--- .../enrichment/redirect_boundary_spec.rb | 170 +++--- .../enrichment/repository_document_spec.rb | 113 +++- .../github/enrichment/search_query_spec.rb | 103 ++++ .../github/enrichment/search_response_spec.rb | 111 ++++ .../github/enrichment/summary_spec.rb | 360 +++++++------ spec/services/github/enrichment/tally_spec.rb | 126 ++++- .../github/enrichment/throughput_spec.rb | 158 ++++++ .../services/github/enrichment_runner_spec.rb | 307 ----------- .../github/ingestion/page_writer_spec.rb | 88 +++ spec/services/github/ingestion_runner_spec.rb | 9 +- spec/services/github/request_executor_spec.rb | 92 +++- spec/services/github/request_spec.rb | 55 +- .../github/search_budget_ledger_spec.rb | 507 ++++++++++++++++++ .../github/status/ledger_state_spec.rb | 91 ++++ .../github/status/scheduler_settings_spec.rb | 70 +++ .../github/status/search_ledger_state_spec.rb | 118 ++++ spec/services/github/status/snapshot_spec.rb | 158 ++++-- spec/support/budget_helpers.rb | 29 +- spec/support/ingestion_helpers.rb | 75 ++- .../shared_examples/enrichable_entity.rb | 45 +- .../support/shared_examples/enrichment_job.rb | 70 --- 72 files changed, 7172 insertions(+), 2904 deletions(-) create mode 100644 spec/db/add_staged_batch_enrichment_spec.rb delete mode 100644 spec/jobs/enrich_actor_job_spec.rb delete mode 100644 spec/jobs/enrich_repository_job_spec.rb create mode 100644 spec/jobs/enrichment_cycle_job_spec.rb create mode 100644 spec/models/enrichment_batch_spec.rb create mode 100644 spec/models/enrichment_observation_spec.rb create mode 100644 spec/models/github_search_budget_spec.rb create mode 100644 spec/quota_outcome_boundary_spec.rb create mode 100644 spec/services/github/enrichment/admission_spec.rb create mode 100644 spec/services/github/enrichment/batch_claim_spec.rb create mode 100644 spec/services/github/enrichment/batch_quality_spec.rb create mode 100644 spec/services/github/enrichment/batch_runner_spec.rb delete mode 100644 spec/services/github/enrichment/candidate_selector_spec.rb delete mode 100644 spec/services/github/enrichment/claim_spec.rb create mode 100644 spec/services/github/enrichment/cycle_runner_spec.rb create mode 100644 spec/services/github/enrichment/detail_claim_spec.rb create mode 100644 spec/services/github/enrichment/detail_runner_spec.rb delete mode 100644 spec/services/github/enrichment/entity_state_spec.rb create mode 100644 spec/services/github/enrichment/event_native_derivation_spec.rb delete mode 100644 spec/services/github/enrichment/fairness_spec.rb delete mode 100644 spec/services/github/enrichment/fairness_stress_spec.rb create mode 100644 spec/services/github/enrichment/lane_fairness_spec.rb delete mode 100644 spec/services/github/enrichment/multi_window_backlog_spec.rb create mode 100644 spec/services/github/enrichment/multi_window_batch_backlog_spec.rb create mode 100644 spec/services/github/enrichment/search_query_spec.rb create mode 100644 spec/services/github/enrichment/search_response_spec.rb create mode 100644 spec/services/github/enrichment/throughput_spec.rb delete mode 100644 spec/services/github/enrichment_runner_spec.rb create mode 100644 spec/services/github/search_budget_ledger_spec.rb create mode 100644 spec/services/github/status/ledger_state_spec.rb create mode 100644 spec/services/github/status/scheduler_settings_spec.rb create mode 100644 spec/services/github/status/search_ledger_state_spec.rb delete mode 100644 spec/support/shared_examples/enrichment_job.rb diff --git a/spec/config/github_initializer_spec.rb b/spec/config/github_initializer_spec.rb index 4a17af5..51b6022 100644 --- a/spec/config/github_initializer_spec.rb +++ b/spec/config/github_initializer_spec.rb @@ -18,7 +18,8 @@ def boot = Rails.application.reloader.prepare! it "logs the resolved allowances at boot" do expect(Rails.logger).to receive(:info).with(hash_including( event: "config.budget_resolved", mode: "live", - poll_allowance: 12, enrichment_allowance: 40, reserve: 8 + poll_allowance: 12, enrichment_allowance: 4, reserve: 8, + actor_guarantee: 2, repository_guarantee: 2 )) boot @@ -27,7 +28,7 @@ def boot = Rails.application.reloader.prepare! it "leaves Github.configuration validated and memoized for the process" do boot - expect(Github.configuration.allowances).to have_attributes(poll_allowance: 12, enrichment_allowance: 40) + expect(Github.configuration.allowances).to have_attributes(poll_allowance: 12, enrichment_allowance: 4) end # The over-commitment the allowance formula cannot see: it counts one attempt per page, @@ -62,13 +63,24 @@ def stub_configuration(**environment) end end - # The rejection §10 states outright: "Startup validation rejects any configuration where - # poll_attempt_allowance + reserve >= effective_limit." Raising here stops the container - # rather than letting it poll into an over-commitment. - it "refuses to finish booting on a configuration that leaves no enrichment capacity" do + # The rejection Appendix F restates for the staged budget: polling, the configured + # detail-fallback allowance, and the reserve must fit inside the core limit together. + # Raising here stops the container rather than letting it poll into an over-commitment, + # and the message names CORE_DETAIL_FALLBACK_ALLOWANCE among the levers an operator + # can move. + it "refuses to finish booting on a configuration whose commitments exceed the core limit" do allow(Github).to receive(:configuration) .and_return(Github::Configuration.new("POLL_INTERVAL_SECONDS" => "60", "MAX_PAGES_PER_POLL" => "2")) - expect { boot }.to raise_error(Github::Errors::ConfigurationError) + expect { boot }.to raise_error(Github::Errors::ConfigurationError, /CORE_DETAIL_FALLBACK_ALLOWANCE/) + end + + # Appendix F's structural rules run in the same boot-time validation, so a container + # with a lease its own worst-case fetch could outlive never starts. + it "refuses to finish booting on a staged-enrichment rule violation" do + allow(Github).to receive(:configuration) + .and_return(Github::Configuration.new("ENRICHMENT_LEASE_SECONDS" => "585")) + + expect { boot }.to raise_error(Github::Errors::ConfigurationError, /ENRICHMENT_LEASE_SECONDS/) end end diff --git a/spec/db/add_staged_batch_enrichment_spec.rb b/spec/db/add_staged_batch_enrichment_spec.rb new file mode 100644 index 0000000..697b532 --- /dev/null +++ b/spec/db/add_staged_batch_enrichment_spec.rb @@ -0,0 +1,215 @@ +require "rails_helper" +require Rails.root.join("db/migrate/20260802010000_add_staged_batch_enrichment").to_s + +# The staged-batch migration is the one that decides what happens to every entity row +# that predates Appendix G: each legacy business outcome must land on the one staged +# resting stage that means the same thing, or the durable FIFO would silently forget +# work (a pending row without batch_pending_at is invisible to BatchClaim's order). +# +# Choreography note, shared with the removal migration's spec (20260802000000): every +# down/up here goes through ActiveRecord::MigrationContext rather than a bare +# Migration#migrate. The context replays the migrations *in version order* and keeps +# schema_migrations truthful, and the around hook's ensure runs `migrate` back to the +# top whatever the example did — so a failure at any point still leaves the suite's +# schema fully migrated, and no later randomly-ordered spec inherits a half-reverted +# database. While the schema sits below this migration the entity models are unusable +# (their enrichment_stage enum has no backing column), so every read and write against +# the reverted schema goes through raw SQL. +RSpec.describe AddStagedBatchEnrichment, type: :migration do + self.use_transactional_tests = false + + STAGED_ENTITY_COLUMNS = %w[ + enrichment_stage detail_attempts event_native_at derived_at batch_pending_at + batch_applied_at detail_pending_at retry_scheduled_at contract_completed_at + terminal_at latest_observation_id latest_observation_source latest_observed_at + lease_token leased_until current_enrichment_batch_id + ].freeze + + let(:connection) { ActiveRecord::Base.connection } + + # The version below this migration: `down(parent_version)` reverts exactly this + # migration and nothing older. + let(:parent_version) { 20260802000000 } + + let(:staged_actor_ids) { [ 99_300_001, 99_300_002, 99_300_003, 99_300_004 ] } + let(:staged_repository_ids) { [ 99_400_001, 99_400_002, 99_400_003, 99_400_004 ] } + + # One instant per role, so every backfill assertion names its source column. + let(:legacy_created_at) { Time.utc(2026, 7, 30, 9, 0, 0) } + let(:legacy_seen_at) { Time.utc(2026, 7, 30, 10, 0, 0) } + let(:legacy_updated_at) { Time.utc(2026, 7, 31, 11, 0, 0) } + let(:legacy_fetched_at) { Time.utc(2026, 7, 31, 12, 0, 0) } + + around do |example| + previous_verbose = ActiveRecord::Migration.verbose + ActiveRecord::Migration.verbose = false + + begin + example.run + ensure + # Whatever the example (or its failure) left behind, the suite continues on a + # fully migrated schema. + migration_context.migrate + reset_schema_caches! + GithubActor.where(github_id: staged_actor_ids).delete_all + GithubRepository.where(github_id: staged_repository_ids).delete_all + ActiveRecord::Migration.verbose = previous_verbose + end + end + + describe "the legacy-row backfill" do + it "maps every pre-staged business outcome onto its one staged resting stage" do + migration_context.down(parent_version) + reset_schema_caches! + insert_legacy_rows! + + migration_context.migrate + reset_schema_caches! + + [ GithubActor, GithubRepository ].zip([ staged_actor_ids, staged_repository_ids ]).each do |model, ids| + pending_row, complete, retryable, permanent = ids.map { model.find_by!(github_id: _1) } + + # pending → batch_pending: never-enriched work re-enters the FIFO, and the + # first_seen_at fallback proves COALESCE reaches created_at when the row + # predates activity tracking. + expect(pending_row).to have_attributes( + enrichment_status: "pending", enrichment_stage: "batch_pending", + event_native_at: legacy_created_at, derived_at: legacy_created_at, + batch_pending_at: legacy_created_at, retry_scheduled_at: nil, + contract_completed_at: nil, terminal_at: nil + ) + + # complete → contract_complete: completion under the previous contract is a + # fact, stamped with the fetch that established it. No batch_pending_at — a + # complete row is not backlog. + expect(complete).to have_attributes( + enrichment_status: "complete", enrichment_stage: "contract_complete", + event_native_at: legacy_seen_at, derived_at: legacy_seen_at, + batch_pending_at: nil, contract_completed_at: legacy_fetched_at, terminal_at: nil + ) + + # retryable_failure → retry_scheduled, still carrying batch_pending_at so the + # row remains visible backlog once its preserved next_retry_at clears. + expect(retryable).to have_attributes( + enrichment_status: "retryable_failure", enrichment_stage: "retry_scheduled", + event_native_at: legacy_seen_at, derived_at: legacy_seen_at, + batch_pending_at: legacy_seen_at, retry_scheduled_at: legacy_updated_at, + contract_completed_at: nil, terminal_at: nil + ) + + # permanent_failure → terminal, dated by the write that decided it. + expect(permanent).to have_attributes( + enrichment_status: "permanent_failure", enrichment_stage: "terminal", + event_native_at: legacy_seen_at, derived_at: legacy_seen_at, + batch_pending_at: nil, retry_scheduled_at: nil, + contract_completed_at: nil, terminal_at: legacy_updated_at + ) + end + end + end + + describe "the stage vocabulary" do + # event_native / derived / batch_applied are instants (*_at columns), not resting + # stages — a row that could rest in them would be invisible to both claims. + it "rejects a stage outside the seven resting stages" do + create_actor(github_id: staged_actor_ids.first) + + expect_violation(ActiveRecord::CheckViolation) do + GithubActor.where(github_id: staged_actor_ids.first) + .update_all(enrichment_stage: "event_native") + end + end + end + + describe "the down migration" do + it "removes every staged table, column, index and constraint, then restores them on re-up" do + migration_context.down(parent_version) + reset_schema_caches! + + %w[enrichment_batches enrichment_observations github_search_budget].each do |table| + expect(connection.table_exists?(table)).to be(false), "expected #{table} to be dropped" + end + + %w[github_actors github_repositories].each do |table| + columns = connection.columns(table).map(&:name) + expect(columns).not_to include(*STAGED_ENTITY_COLUMNS) + + index_names = connection.indexes(table).map(&:name) + expect(index_names).not_to include("index_#{table}_on_stage_fifo") + expect(index_names).not_to include("index_#{table}_on_leased_until") + + constraint_names = connection.check_constraints(table).map(&:name) + expect(constraint_names).not_to include("#{table}_enrichment_stage_check") + expect(constraint_names).not_to include("#{table}_detail_attempts_nonnegative") + end + + expect(connection.columns("github_actors").map(&:name)).not_to include("account_type") + expect(connection.columns("github_repositories").map(&:name)) + .not_to include("owner_login", "fork", "archived", "default_branch", "github_created_at") + + # The round trip: re-applying leaves the staged structures present, which is what + # lets the around hook's ensure restore the suite's schema unconditionally. + migration_context.migrate + reset_schema_caches! + + %w[enrichment_batches enrichment_observations github_search_budget].each do |table| + expect(connection.table_exists?(table)).to be(true), "expected #{table} to be recreated" + end + %w[github_actors github_repositories].each do |table| + expect(connection.columns(table).map(&:name)).to include(*STAGED_ENTITY_COLUMNS) + end + end + end + + private + + def migration_context + ActiveRecord::Base.connection_pool.migration_context + end + + # DDL invalidates cached column metadata; every choreography step re-reads it so a + # model built after the step sees the schema that step produced. + def reset_schema_caches! + connection.schema_cache.clear! + [ GithubActor, GithubRepository, EnrichmentBatch, + EnrichmentObservation, GithubSearchBudget ].each(&:reset_column_information) + end + + # Raw SQL, because the pre-staged schema has no staged columns and the entity models' + # enums cannot even be type-cast against it. + def insert_legacy_rows! + rows = [ + # [github_id offset, status, attempts, first_seen_at, fetched_at, last_error] + [ 0, "pending", 0, nil, nil, nil ], + [ 1, "complete", 0, legacy_seen_at, legacy_fetched_at, nil ], + [ 2, "retryable_failure", 2, legacy_seen_at, nil, "GitHub unavailable" ], + [ 3, "permanent_failure", 3, legacy_seen_at, nil, "HTTP 404" ] + ] + + rows.each do |offset, status, attempts, seen_at, fetched_at, last_error| + connection.execute(<<~SQL.squish) + INSERT INTO github_actors + (github_id, login, enrichment_status, enrichment_attempts, + first_seen_at, fetched_at, last_error, created_at, updated_at) + VALUES + (#{staged_actor_ids.fetch(offset)}, 'legacy-#{status}', + #{connection.quote(status)}, #{attempts}, + #{connection.quote(seen_at)}, #{connection.quote(fetched_at)}, + #{connection.quote(last_error)}, + #{connection.quote(legacy_created_at)}, #{connection.quote(legacy_updated_at)}) + SQL + + connection.execute(<<~SQL.squish) + INSERT INTO github_repositories + (github_id, full_name, enrichment_status, enrichment_attempts, + first_seen_at, fetched_at, last_error, created_at, updated_at) + VALUES + (#{staged_repository_ids.fetch(offset)}, 'legacy/#{status}', + #{connection.quote(status)}, #{attempts}, + #{connection.quote(seen_at)}, #{connection.quote(fetched_at)}, + #{connection.quote(last_error)}, + #{connection.quote(legacy_created_at)}, #{connection.quote(legacy_updated_at)}) + SQL + end + end +end diff --git a/spec/db/remove_skipped_budget_from_enrichment_spec.rb b/spec/db/remove_skipped_budget_from_enrichment_spec.rb index 05308bf..992e87d 100644 --- a/spec/db/remove_skipped_budget_from_enrichment_spec.rb +++ b/spec/db/remove_skipped_budget_from_enrichment_spec.rb @@ -1,14 +1,31 @@ require "rails_helper" require Rails.root.join("db/migrate/20260802000000_remove_skipped_budget_from_enrichment").to_s +# Choreography note, shared with add_staged_batch_enrichment_spec.rb: this migration no +# longer sits at the top of the ladder — 20260802010000 is stacked above it — so its +# down/up cycle must go through ActiveRecord::MigrationContext, which reverts the staged +# migration *first* and replays both in version order afterwards. A bare +# Migration#migrate(:down) on this class alone would exercise it against a schema shape +# it never ran on, and would leave schema_migrations lying about what is applied. The +# around hook's ensure migrates back to the top whatever the example did, so no later +# randomly-ordered spec inherits a half-reverted database. +# +# While the schema sits below the staged migration the entity models are unusable +# (their enrichment_stage enum has no backing column), so the legacy rows are seeded +# with raw SQL and every assertion runs after the ladder is fully re-applied. The +# restored statuses survive that composition untouched: 20260802010000 maps them onto +# stages, it never rewrites them. RSpec.describe RemoveSkippedBudgetFromEnrichment, type: :migration do self.use_transactional_tests = false - ACTOR_IDS = [ 99_100_001, 99_100_002 ].freeze - REPOSITORY_IDS = [ 99_200_001, 99_200_002 ].freeze - let(:connection) { ActiveRecord::Base.connection } - let(:migration) { described_class.new } + + # The version below this migration: `down(parent_version)` reverts the staged + # migration and then this one, restoring the pre-removal schema this spec seeds. + let(:parent_version) { 20260731120000 } + + let(:skipped_actor_ids) { [ 99_100_001, 99_100_002 ] } + let(:skipped_repository_ids) { [ 99_200_001, 99_200_002 ] } around do |example| previous_verbose = ActiveRecord::Migration.verbose @@ -17,30 +34,38 @@ begin example.run ensure - restore_current_schema! - GithubActor.where(github_id: ACTOR_IDS).delete_all - GithubRepository.where(github_id: REPOSITORY_IDS).delete_all + # Whatever the example (or its failure) left behind, the suite continues on a + # fully migrated schema. + migration_context.migrate + reset_schema_caches! + GithubActor.where(github_id: skipped_actor_ids).delete_all + GithubRepository.where(github_id: skipped_repository_ids).delete_all ActiveRecord::Migration.verbose = previous_verbose end end it "restores every discarded row to the durable FIFO and removes the old state" do - migration.migrate(:down) - reset_entity_schema_cache! + migration_context.down(parent_version) + reset_schema_caches! insert_legacy_rows! - migration.migrate(:up) - reset_entity_schema_cache! + migration_context.migrate + reset_schema_caches! - expect(GithubActor.where(github_id: ACTOR_IDS).order(:github_id).pluck(:enrichment_status)) + expect(GithubActor.where(github_id: skipped_actor_ids).order(:github_id).pluck(:enrichment_status)) .to eq(%w[pending retryable_failure]) - expect(GithubRepository.where(github_id: REPOSITORY_IDS).order(:github_id).pluck(:enrichment_status)) + expect(GithubRepository.where(github_id: skipped_repository_ids).order(:github_id).pluck(:enrichment_status)) .to eq(%w[pending retryable_failure]) - attempted = GithubActor.find_by!(github_id: ACTOR_IDS.last) + attempted = GithubActor.find_by!(github_id: skipped_actor_ids.last) expect(attempted).to have_attributes(enrichment_attempts: 2, last_error: "GitHub unavailable") expect(attempted.next_retry_at).to be <= Time.current + # The staged migration above carries the restored outcomes into the staged FIFO: + # quota delay ends as actionable backlog, never as a terminal state. + expect(GithubActor.where(github_id: skipped_actor_ids).order(:github_id).pluck(:enrichment_stage)) + .to eq(%w[batch_pending retry_scheduled]) + %w[github_actors github_repositories].each do |table| expect(connection.column_exists?(table, :skipped_at)).to be(false) @@ -51,13 +76,17 @@ end expect_violation(ActiveRecord::CheckViolation) do - GithubActor.where(github_id: ACTOR_IDS.first) + GithubActor.where(github_id: skipped_actor_ids.first) .update_all(enrichment_status: "skipped_budget") end end private + def migration_context + ActiveRecord::Base.connection_pool.migration_context + end + def insert_legacy_rows! now = connection.quote(Time.current - 2.days) future = connection.quote(Time.current + 1.day) @@ -67,9 +96,9 @@ def insert_legacy_rows! (github_id, login, enrichment_status, enrichment_attempts, next_retry_at, last_error, created_at, updated_at, skipped_at) VALUES - (#{ACTOR_IDS.first}, 'legacy-pending', 'skipped_budget', 0, #{future}, + (#{skipped_actor_ids.first}, 'legacy-pending', 'skipped_budget', 0, #{future}, NULL, #{now}, #{now}, #{now}), - (#{ACTOR_IDS.last}, 'legacy-retry', 'skipped_budget', 2, #{future}, + (#{skipped_actor_ids.last}, 'legacy-retry', 'skipped_budget', 2, #{future}, 'GitHub unavailable', #{now}, #{now}, #{now}) SQL @@ -78,25 +107,16 @@ def insert_legacy_rows! (github_id, full_name, enrichment_status, enrichment_attempts, next_retry_at, last_error, created_at, updated_at, skipped_at) VALUES - (#{REPOSITORY_IDS.first}, 'legacy/pending', 'skipped_budget', 0, #{future}, + (#{skipped_repository_ids.first}, 'legacy/pending', 'skipped_budget', 0, #{future}, NULL, #{now}, #{now}, #{now}), - (#{REPOSITORY_IDS.last}, 'legacy/retry', 'skipped_budget', 2, #{future}, + (#{skipped_repository_ids.last}, 'legacy/retry', 'skipped_budget', 2, #{future}, 'GitHub unavailable', #{now}, #{now}, #{now}) SQL end - def restore_current_schema! - if connection.column_exists?(:github_actors, :skipped_at) || - connection.column_exists?(:github_repositories, :skipped_at) - migration.migrate(:up) - end - - reset_entity_schema_cache! - end - - def reset_entity_schema_cache! + def reset_schema_caches! connection.schema_cache.clear! - GithubActor.reset_column_information - GithubRepository.reset_column_information + [ GithubActor, GithubRepository, EnrichmentBatch, + EnrichmentObservation, GithubSearchBudget ].each(&:reset_column_information) end end diff --git a/spec/db/schema_spec.rb b/spec/db/schema_spec.rb index 5897c45..6fb8536 100644 --- a/spec/db/schema_spec.rb +++ b/spec/db/schema_spec.rb @@ -9,7 +9,8 @@ it "defines every table plan §7 specifies" do expect(connection.tables).to include( "event_sources", "github_api_budget", "ingestion_runs", "push_events", - "quarantined_events", "github_actors", "github_repositories" + "quarantined_events", "github_actors", "github_repositories", + "github_search_budget", "enrichment_batches", "enrichment_observations" ) end @@ -34,6 +35,105 @@ end end + describe "github_search_budget" do + # Mirrors the core ledger's posture above: the Search ledger's columns are its + # contract, and Github::SearchBudgetLedger reserves against exactly these counters. + it "carries exactly the columns the search ledger reserves against" do + expect(connection.columns("github_search_budget").map(&:name)).to match_array(%w[ + id resource limit remaining reset_at observed_at + request_ceiling reserve used actor_used repository_used + blocked_until last_request_at + lock_version created_at updated_at + ]) + end + + it "constrains itself to a single row" do + names = connection.check_constraints("github_search_budget").map(&:name) + + expect(names).to include("github_search_budget_singleton") + end + + it "refuses negative counters at the schema level" do + names = connection.check_constraints("github_search_budget").map(&:name) + + expect(names).to include("github_search_budget_counters_valid") + end + end + + describe "enrichment_batches" do + it "carries the request envelope, counters, and rate-limit evidence columns" do + expect(connection.columns("enrichment_batches").map(&:name)).to include( + "correlation_id", "request_kind", "entity_kind", "status", + "requested_github_ids", "requested_identifiers", "request_url", + "response_status", "response_body", "total_count", "incomplete_results", + "requested_count", "returned_count", "valid_count", "missing_count", "invalid_count", + "started_at", "completed_at", + "rate_limit_resource", "rate_limit_limit", "rate_limit_remaining", + "rate_limit_used", "rate_limit_reset_at", "last_error" + ) + end + + it "makes the correlation id unique" do + index = connection.indexes("enrichment_batches") + .find { |i| i.columns == [ "correlation_id" ] } + + expect(index.unique).to be(true) + end + + # The /status batch-quality window filters on started_at alone; the composite index + # cannot range-scan it because started_at is its last column. + it "indexes both the per-kind history and the bare started_at window" do + columns = connection.indexes("enrichment_batches").map(&:columns) + + expect(columns).to include(%w[request_kind entity_kind started_at]) + expect(columns).to include([ "started_at" ]) + end + + it "constrains kind, entity, status, and counter signs" do + names = connection.check_constraints("enrichment_batches").map(&:name) + + expect(names).to include( + "enrichment_batches_request_kind_check", + "enrichment_batches_entity_kind_check", + "enrichment_batches_status_check", + "enrichment_batches_counters_nonnegative" + ) + end + end + + describe "enrichment_observations" do + it "carries the append-only evidence columns" do + expect(connection.columns("enrichment_observations").map(&:name)).to include( + "entity_kind", "entity_github_id", "source", "observed_at", + "raw_payload", "payload_fingerprint", "enrichment_batch_id", "push_event_id", + "request_correlation_id", "requested_identifier", "validation_outcome" + ) + end + + it "indexes the per-entity timeline and the fingerprint" do + indexes = connection.indexes("enrichment_observations") + + timeline = indexes.find { |i| i.name == "index_enrichment_observations_on_entity_and_time" } + expect(timeline.columns).to eq(%w[entity_kind entity_github_id observed_at]) + expect(indexes.map(&:columns)).to include([ "payload_fingerprint" ]) + end + + it "constrains entity kind and source to their vocabularies" do + names = connection.check_constraints("enrichment_observations").map(&:name) + + expect(names).to include( + "enrichment_observations_entity_kind_check", + "enrichment_observations_source_check" + ) + end + + it "references its batch and its push event with real foreign keys" do + tables = connection.foreign_keys("enrichment_observations").map(&:to_table) + + expect(tables).to contain_exactly("enrichment_batches", "push_events") + end + end + describe "push_events" do it "requires every structured column" do nullable = connection.columns("push_events").reject(&:null).map(&:name) @@ -73,6 +173,18 @@ end end + # The same posture for every raw_payload in the schema: retention is a durability + # decision, indexing is a query decision, and no query demands one yet. + describe "raw payload retention" do + it "carries no GIN index on any raw_payload column" do + %w[push_events quarantined_events enrichment_observations + github_actors github_repositories].each do |table| + expect(connection.indexes(table).map(&:columns)).not_to include([ "raw_payload" ]), + "expected no raw_payload index on #{table}" + end + end + end + describe "quarantined_events" do it "makes the payload fingerprint the only unique identity" do unique = connection.indexes("quarantined_events").select(&:unique).map(&:columns) @@ -96,6 +208,21 @@ end end + # Appendix G's staged pipeline: the stage machine's resting position, the instant + # columns for the three non-resting facts, and the lease that makes a claim durable. + it "each carry the same staged pipeline columns" do + staged_columns = %w[ + enrichment_stage detail_attempts event_native_at derived_at batch_pending_at + batch_applied_at detail_pending_at retry_scheduled_at contract_completed_at + terminal_at latest_observation_id latest_observation_source latest_observed_at + lease_token leased_until current_enrichment_batch_id + ] + + %w[github_actors github_repositories].each do |table| + expect(connection.columns(table).map(&:name)).to include(*staged_columns) + end + end + it "each carry a partial index over the reconciler's ordering columns" do %w[github_actors github_repositories].each do |table| index = connection.indexes(table) @@ -106,10 +233,41 @@ end end - it "stores enrichment status as text so the index predicate needs no cast" do + # BatchClaim's FIFO is `ORDER BY created_at, id` under a stage predicate; the index + # leads with the stage so the order it yields is the order the claim reads. + it "each index the staged FIFO in claim order" do + %w[github_actors github_repositories].each do |table| + index = connection.indexes(table).find { |i| i.name == "index_#{table}_on_stage_fifo" } + + expect(index).not_to be_nil, "expected index_#{table}_on_stage_fifo" + expect(index.columns).to eq(%w[enrichment_stage created_at id]) + end + end + + # The lease-expiry predicate (`leased_until IS NULL OR leased_until <= now`) is on + # every claim scope, so expiry re-admission never scans the table. + it "each index leased_until for lease-expiry reclaims" do + %w[github_actors github_repositories].each do |table| + expect(connection.indexes(table).map(&:columns)).to include([ "leased_until" ]), + "expected a leased_until index on #{table}" + end + end + + it "each constrain the stage to the seven resting stages" do + %w[github_actors github_repositories].each do |table| + names = connection.check_constraints(table).map(&:name) + + expect(names).to include("#{table}_enrichment_stage_check") + expect(names).to include("#{table}_detail_attempts_nonnegative") + end + end + + it "stores enrichment status and stage as text so the index predicates need no cast" do %w[github_actors github_repositories].each do |table| - column = connection.columns(table).find { |c| c.name == "enrichment_status" } - expect(column.type).to eq(:text) + %w[enrichment_status enrichment_stage].each do |name| + column = connection.columns(table).find { |c| c.name == name } + expect(column.type).to eq(:text) + end end end end diff --git a/spec/jobs/application_job_spec.rb b/spec/jobs/application_job_spec.rb index 6a9b930..069a4b1 100644 --- a/spec/jobs/application_job_spec.rb +++ b/spec/jobs/application_job_spec.rb @@ -73,12 +73,12 @@ def perform describe "retries" do # Every job here can spend GitHub budget, and both retry ladders are already durable and - # coordinated with the ledger (Github::Ingestion::PollState and - # Github::Enrichment::EntityState). A second, uncoordinated Active Job ladder would + # coordinated with the ledger (Github::Ingestion::PollState and the enrichment + # runners). A second, uncoordinated Active Job ladder would # re-poll a source whose backoff was just written. The 60-second recurring tick is the # retry. it "declares none, in any job in this application" do - jobs = [ ApplicationJob, PollEventSourceJob, EnrichActorJob, EnrichRepositoryJob, + jobs = [ ApplicationJob, PollEventSourceJob, EnrichmentCycleJob, ReconcilePendingEnrichmentsJob ] expect(jobs.map { |job| job.rescue_handlers.map(&:first) }.flatten).to be_empty diff --git a/spec/jobs/enrich_actor_job_spec.rb b/spec/jobs/enrich_actor_job_spec.rb deleted file mode 100644 index f05464c..0000000 --- a/spec/jobs/enrich_actor_job_spec.rb +++ /dev/null @@ -1,35 +0,0 @@ -require "rails_helper" - -RSpec.describe EnrichActorJob do - it_behaves_like "an enrichment job", - entity_class: GithubActor, entity_type: :actor, log_key: :github_actor_id - - # The whole job over the real runner and the offline corpus, so "one cycle" is known to mean - # one entity and one request rather than only to be stubbed that way. - describe "with the real runner in fixture mode", type: :integration do - before do - allow(Github).to receive(:configuration).and_return(configuration_with("GITHUB_MODE" => "fixture")) - allow(Github::EnrichmentRunner).to receive(:new) - .and_return(fixture_enrichment_runner(transport: fixture_transport, now: frozen_time)) - - active_budget_window(now: frozen_time) - create_actor(github_id: 583_231, last_seen_at: frozen_time, - api_url: "https://api.github.com/users/octocat") - end - - it "enriches one actor and spends one request" do - described_class.new.perform_now - - expect(GithubActor.sole).to have_attributes(enrichment_status: "complete", name: "The Octocat") - expect(current_budget).to have_attributes(enrichment_used: 1, actor_share_used: 1) - end - - it "leaves repositories alone, whatever the fairness policy would have preferred" do - create_repository(github_id: 1_296_269, last_seen_at: frozen_time) - - described_class.new.perform_now - - expect(GithubRepository.sole.enrichment_status).to eq("pending") - end - end -end diff --git a/spec/jobs/enrich_repository_job_spec.rb b/spec/jobs/enrich_repository_job_spec.rb deleted file mode 100644 index 9e42371..0000000 --- a/spec/jobs/enrich_repository_job_spec.rb +++ /dev/null @@ -1,35 +0,0 @@ -require "rails_helper" - -RSpec.describe EnrichRepositoryJob do - it_behaves_like "an enrichment job", - entity_class: GithubRepository, entity_type: :repository, log_key: :github_repository_id - - describe "with the real runner in fixture mode", type: :integration do - before do - allow(Github).to receive(:configuration).and_return(configuration_with("GITHUB_MODE" => "fixture")) - allow(Github::EnrichmentRunner).to receive(:new) - .and_return(fixture_enrichment_runner(transport: fixture_transport, now: frozen_time)) - - active_budget_window(now: frozen_time) - create_repository(github_id: 1_296_269, last_seen_at: frozen_time, - api_url: "https://api.github.com/repos/octocat/Hello-World") - end - - it "enriches one repository and spends one request" do - described_class.new.perform_now - - expect(GithubRepository.sole).to have_attributes( - enrichment_status: "complete", description: "My first repository on GitHub!", language: "Ruby" - ) - expect(current_budget).to have_attributes(enrichment_used: 1, repository_share_used: 1) - end - - it "leaves actors alone, whatever the fairness policy would have preferred" do - create_actor(github_id: 583_231, last_seen_at: frozen_time) - - described_class.new.perform_now - - expect(GithubActor.sole.enrichment_status).to eq("pending") - end - end -end diff --git a/spec/jobs/enrichment_cycle_job_spec.rb b/spec/jobs/enrichment_cycle_job_spec.rb new file mode 100644 index 0000000..55c8d11 --- /dev/null +++ b/spec/jobs/enrichment_cycle_job_spec.rb @@ -0,0 +1,96 @@ +require "rails_helper" + +# One staged-enrichment cycle (§5, Appendix G). The cycle's own behaviour — lanes, pacing, +# the wall-clock budget — is Github::Enrichment::CycleRunner's, specified there; what this +# file pins is the job boundary: which queue, that one delivery is one cycle, that the +# cycle's counters reach the job's completion line, that a duplicate delivery is harmless, +# and that no source lock is ever taken. +RSpec.describe EnrichmentCycleJob do + def cycle(overrides = {}) + defaults = { + batches_attempted: 2, batches_completed: 1, batches_deferred: 0, batches_failed: 1, + items_requested: 10, items_valid: 8, fallbacks_admitted: 2, details_attempted: 1, + details_completed: 1, details_terminal: 0, details_deferred: 0, + batch_stop_reason: "search_reserve_reached", detail_stop_reason: "no_detail_work", + duration_ms: 1234 + } + + Github::Enrichment::CycleRunner::Cycle.new(**defaults.merge(overrides)) + end + + def stub_runner(runner) + allow(Github::Enrichment::CycleRunner).to receive(:new).and_return(runner) + runner + end + + # Enrichment is deliberately isolated: a deep durable backlog may keep this queue busy + # for many quota windows, and it must never delay polling or the control tick. + it "runs on the dedicated enrichment queue" do + expect(described_class.new.queue_name).to eq("enrichment") + end + + it "runs exactly one cycle per delivery" do + runner = stub_runner(instance_double(Github::Enrichment::CycleRunner)) + expect(runner).to receive(:call).once.and_return(cycle) + + described_class.new.perform_now + end + + # §11: the cycle's counters ride the job's own completion line, so a reviewer's trace + # from job_id to what the cycle did is zero hops. + it "joins the cycle's counters to the job on one line" do + stub_runner(instance_double(Github::Enrichment::CycleRunner, call: cycle)) + allow(Rails.logger).to receive(:info) + + job = described_class.new + job.perform_now + + expect(Rails.logger).to have_received(:info).with( + hash_including(event: "job.completed", job_id: job.job_id, job_class: "EnrichmentCycleJob", + batches_attempted: 2, batches_completed: 1, items_requested: 10, + items_valid: 8, fallbacks_admitted: 2, details_completed: 1, + batch_stop_reason: "search_reserve_reached", + detail_stop_reason: "no_detail_work") + ) + end + + # Duplicate deliveries are harmless by construction: the surplus cycle's admission + # checks and claims read the committed entity rows and the ledgers, find nothing to do, + # and exit having created no batch row and spent no budget. Asserted over the real + # runner and the real database rather than a stub, because the guarantee lives in the + # committed state, not in the queue. + it "makes a duplicate delivery a fast no-op against committed state" do + active_budget_window(now: Time.current) + create_actor(github_id: 583_231, enrichment_status: "complete", + enrichment_stage: "contract_complete", fetched_at: Time.current, + last_seen_at: Time.current) + + 2.times { described_class.new.perform_now } + + expect(EnrichmentBatch.count).to eq(0) + expect(current_budget.enrichment_used).to eq(0) + expect(WebMock).not_to have_requested(:any, //) + end + + # §8 step 1: "Enrichment jobs skip this step — they take only the request gate." + # Asserted at the job boundary as well as inside the runners, because this is where a + # future "just lock the source while we enrich it" would be written. + it "never takes a source lock" do + stub_runner(instance_double(Github::Enrichment::CycleRunner, call: cycle)) + expect(Github::SourceLock).not_to receive(:acquire) + + described_class.new.perform_now + + expect(Github::LockOrder.held_keys).to be_empty + end + + # §6 requires a corpus gap to be raised rather than laundered into a failed fetch; the + # batch runner has already finalized the batch row and released the lease by the time + # it arrives here. + it "lets a fixture corpus gap fail the job" do + runner = stub_runner(instance_double(Github::Enrichment::CycleRunner)) + allow(runner).to receive(:call).and_raise(Github::Errors::FixtureMiss, "no such body") + + expect { described_class.new.perform_now }.to raise_error(Github::Errors::FixtureMiss) + end +end diff --git a/spec/jobs/poll_event_source_job_spec.rb b/spec/jobs/poll_event_source_job_spec.rb index 70c4dfe..38c7047 100644 --- a/spec/jobs/poll_event_source_job_spec.rb +++ b/spec/jobs/poll_event_source_job_spec.rb @@ -195,9 +195,10 @@ def poll!(job = described_class.new) expect(PushEvent.count).to eq(4) end + # One cycle job, however much the page created: Github::Enrichment::CycleRunner + # loops until a ledger denies, so the enqueue is a hint rather than a unit of work. it "hands the run's enrichment work to the queue" do - expect { poll! }.to have_enqueued_job(EnrichActorJob).exactly(:once) - .and have_enqueued_job(EnrichRepositoryJob).exactly(:once) + expect { poll! }.to have_enqueued_job(EnrichmentCycleJob).exactly(:once) end end end diff --git a/spec/jobs/reconcile_pending_enrichments_job_spec.rb b/spec/jobs/reconcile_pending_enrichments_job_spec.rb index 62b8eab..cf0c67f 100644 --- a/spec/jobs/reconcile_pending_enrichments_job_spec.rb +++ b/spec/jobs/reconcile_pending_enrichments_job_spec.rb @@ -18,10 +18,10 @@ allow(Rails.logger).to receive(:info) job = described_class.new - expect { job.perform_now }.to have_enqueued_job(EnrichActorJob).exactly(:once) + expect { job.perform_now }.to have_enqueued_job(EnrichmentCycleJob).exactly(:once) expect(Rails.logger).to have_received(:info).with( - hash_including(event: "job.completed", job_id: job.job_id, reason: "reconcile", actor_enqueued: 1) + hash_including(event: "job.completed", job_id: job.job_id, reason: "reconcile", cycle_enqueued: 1) ) end diff --git a/spec/models/enrichment_batch_spec.rb b/spec/models/enrichment_batch_spec.rb new file mode 100644 index 0000000..79224ac --- /dev/null +++ b/spec/models/enrichment_batch_spec.rb @@ -0,0 +1,131 @@ +require "rails_helper" + +RSpec.describe EnrichmentBatch do + # One row per REQUEST attempt (Appendix F): the durable envelope a search or detail + # fetch is audited against. correlation_id is passed explicitly because the model + # validates presence before the database default could fill it in. + def create_batch(**overrides) + described_class.create!({ + request_kind: "search", entity_kind: "actor", started_at: frozen_time, + correlation_id: SecureRandom.uuid + }.merge(overrides)) + end + + def observation_for(batch) + EnrichmentObservation.create!( + enrichment_batch: batch, entity_kind: "actor", source: "search", + observed_at: frozen_time, raw_payload: { "id" => 583_231, "login" => "octocat" }, + payload_fingerprint: "0" * 64, validation_outcome: "valid" + ) + end + + describe "validations" do + it "accepts both documented request kinds and nothing else" do + described_class::REQUEST_KINDS.each do |kind| + expect(create_batch(request_kind: kind)).to be_persisted + end + expect { create_batch(request_kind: "bulk") } + .to raise_error(ActiveRecord::RecordInvalid, /Request kind/) + end + + it "accepts both entity kinds and nothing else" do + expect { create_batch(entity_kind: "organization") } + .to raise_error(ActiveRecord::RecordInvalid, /Entity kind/) + end + + it "accepts every documented status and rejects an invented one" do + batch = create_batch + + described_class::STATUSES.each do |status| + expect { batch.update!(status: status) }.not_to raise_error + end + expect { batch.update!(status: "abandoned") } + .to raise_error(ActiveRecord::RecordInvalid, /Status/) + end + + it "opens in flight, because the row is written before the request is issued" do + expect(create_batch.status).to eq("in_flight") + end + + it "requires the instant the attempt started" do + expect { create_batch(started_at: nil) } + .to raise_error(ActiveRecord::RecordInvalid, /Started at/) + end + + it "rejects a negative counter" do + %i[ requested_count returned_count valid_count missing_count invalid_count ].each do |counter| + expect { create_batch(counter => -1) } + .to raise_error(ActiveRecord::RecordInvalid) + end + end + end + + describe "the correlation id" do + # The column default is the server-side gen_random_uuid(), which Active Record + # cannot evaluate before validation — so the model assigns one rather than letting + # a caller that omitted it fail the presence rule below. Every claim site relies on + # this: the observations written beside a batch carry its correlation id. + it "is assigned when the caller supplies none" do + batch = create_batch(correlation_id: nil) + + expect(batch.correlation_id).to be_present + end + + # The uuid column casts a blank string to nil before validation, so the assignment + # above catches that case too. The presence validation stays as the backstop that + # would fail loudly if the assignment were ever removed. + it "assigns one over a blank value rather than storing it" do + batch = create_batch(correlation_id: "") + + expect(batch.correlation_id).to match(/\A[0-9a-f-]{36}\z/) + end + + it "refuses a duplicate through the validation" do + existing = create_batch + + expect { create_batch(correlation_id: existing.correlation_id) } + .to raise_error(ActiveRecord::RecordInvalid, /Correlation/) + end + + # The unique index is the arbiter under concurrency, where two validations can + # both pass before either insert lands. + it "refuses a duplicate at the database level even past the validation" do + existing = create_batch + rival = described_class.new(request_kind: "search", entity_kind: "actor", + started_at: frozen_time, + correlation_id: existing.correlation_id) + + expect_violation(ActiveRecord::RecordNotUnique) { rival.save!(validate: false) } + end + end + + # Audit evidence outlives its envelope's usefulness, never the other way around: + # a batch with observations recorded against it cannot be deleted out from under + # them. + describe "deleting a batch that has observations" do + it "is refused with an error rather than cascading or orphaning" do + batch = create_batch + observation_for(batch) + + expect(batch.destroy).to be(false) + expect(batch.errors[:base]).to be_present + expect(described_class.exists?(batch.id)).to be(true) + expect(EnrichmentObservation.count).to eq(1) + end + + it "permits deleting a batch nothing observed against" do + batch = create_batch + + expect { batch.destroy! }.to change(described_class, :count).from(1).to(0) + end + end + + describe ".during" do + it "selects by the instant the attempt started" do + inside = create_batch(started_at: frozen_time) + create_batch(started_at: frozen_time - 7200) + + expect(described_class.during((frozen_time - 3600)..frozen_time)).to eq([ inside ]) + end + end +end diff --git a/spec/models/enrichment_observation_spec.rb b/spec/models/enrichment_observation_spec.rb new file mode 100644 index 0000000..996c891 --- /dev/null +++ b/spec/models/enrichment_observation_spec.rb @@ -0,0 +1,95 @@ +require "rails_helper" + +RSpec.describe EnrichmentObservation do + def create_observation(**overrides) + described_class.create!({ + entity_kind: "actor", source: "event", observed_at: frozen_time, + raw_payload: { "id" => 583_231, "login" => "octocat" }, + payload_fingerprint: "0" * 64, validation_outcome: "event_native", + entity_github_id: 583_231 + }.merge(overrides)) + end + + describe "validations" do + it "accepts each documented source and rejects an invented one" do + described_class::SOURCES.each do |source| + expect(create_observation(source: source)).to be_persisted + end + expect { create_observation(source: "webhook") } + .to raise_error(ActiveRecord::RecordInvalid, /Source/) + end + + it "accepts both entity kinds and rejects an invented one" do + expect(create_observation(entity_kind: "repository")).to be_persisted + expect { create_observation(entity_kind: "organization") } + .to raise_error(ActiveRecord::RecordInvalid, /Entity kind/) + end + + it "requires the audit fields evidence is useless without" do + { + observed_at: nil, raw_payload: nil, payload_fingerprint: nil, validation_outcome: nil + }.each do |field, blank| + expect { create_observation(field => blank) } + .to raise_error(ActiveRecord::RecordInvalid), "expected #{field} to be required" + end + end + + # Both parents are optional on purpose: an event-sourced observation belongs to a + # push event and no batch, a search-sourced one to a batch and no push event. + it "persists without either parent" do + expect(create_observation.enrichment_batch).to be_nil + expect(create_observation(payload_fingerprint: "1" * 64).push_event).to be_nil + end + + it "associates with the batch whose request produced it" do + batch = EnrichmentBatch.create!(request_kind: "search", entity_kind: "actor", + started_at: frozen_time, + correlation_id: SecureRandom.uuid) + + expect(create_observation(enrichment_batch: batch).enrichment_batch_id).to eq(batch.id) + end + end + + # Audit evidence is append-only. Batch envelopes are updated as their request + # finishes; individual observations never are — enforced by the model rather than + # by convention, so no code path can rewrite history quietly. + describe "the append-only contract" do + it "refuses an update once persisted" do + observation = create_observation + + expect { observation.update!(validation_outcome: "rewritten") } + .to raise_error(ActiveRecord::ReadOnlyRecord) + expect(observation.reload.validation_outcome).to eq("event_native") + end + + it "refuses a destroy once persisted" do + observation = create_observation + + expect { observation.destroy }.to raise_error(ActiveRecord::ReadOnlyRecord) + expect(described_class.count).to eq(1) + end + + it "is writable exactly once: the create itself" do + expect(create_observation).to be_persisted + end + end + + describe "the retained payload" do + # Content-equivalent, not byte-equal: jsonb normalizes key order and whitespace, + # and hash equality is the contract the fingerprint is computed over. + it "round-trips the raw payload as structured data" do + observation = create_observation(raw_payload: { "login" => "octocat", "id" => 583_231 }) + + expect(observation.reload.raw_payload).to eq("id" => 583_231, "login" => "octocat") + end + end + + describe ".during" do + it "selects by the instant the payload was observed" do + inside = create_observation(observed_at: frozen_time) + create_observation(observed_at: frozen_time - 7200, payload_fingerprint: "2" * 64) + + expect(described_class.during((frozen_time - 3600)..frozen_time)).to eq([ inside ]) + end + end +end diff --git a/spec/models/github_actor_spec.rb b/spec/models/github_actor_spec.rb index 126f855..4fd6d14 100644 --- a/spec/models/github_actor_spec.rb +++ b/spec/models/github_actor_spec.rb @@ -50,18 +50,36 @@ it "never clears an enrichment payload already stored" do described_class.upsert_stub!(github_id: 4242, login: "octocat", now: frozen_time) described_class.find_by(github_id: 4242).update_columns( - name: "The Octocat", - raw_payload: { "login" => "octocat", "name" => "The Octocat" }, + account_type: "User", + raw_payload: { "login" => "octocat", "type" => "User" }, enrichment_status: "complete", + enrichment_stage: "contract_complete", fetched_at: frozen_time ) described_class.upsert_stub!(github_id: 4242, login: "octocat", now: frozen_time + 60) actor = described_class.find_by(github_id: 4242) - expect(actor.name).to eq("The Octocat") - expect(actor.raw_payload).to eq("login" => "octocat", "name" => "The Octocat") + expect(actor.account_type).to eq("User") + expect(actor.raw_payload).to eq("login" => "octocat", "type" => "User") expect(actor.enrichment_status).to eq("complete") + expect(actor.enrichment_stage).to eq("contract_complete") + end + + # account_type is enrichment-owned (§7): the Search document contract writes it, and + # the envelope merge must neither clear nor overwrite it — even while the same + # observation legitimately refreshes the login. + it "keeps account_type through an identity merge that renames the actor" do + described_class.upsert_stub!(github_id: 4242, login: "octocat", now: frozen_time) + described_class.where(github_id: 4242).update_all(account_type: "Organization", + updated_at: frozen_time) + + described_class.upsert_stub!(github_id: 4242, login: "renamed-octocat", + now: frozen_time + 60) + + actor = described_class.find_by(github_id: 4242) + expect(actor.login).to eq("renamed-octocat") + expect(actor.account_type).to eq("Organization") end it "refreshes identity without clearing a retryable entity's failure state" do diff --git a/spec/models/github_repository_spec.rb b/spec/models/github_repository_spec.rb index 81a0c84..b3108a5 100644 --- a/spec/models/github_repository_spec.rb +++ b/spec/models/github_repository_spec.rb @@ -30,6 +30,43 @@ expect(described_class.find(id).name).to be_nil end + # Appendix F's derivation-first rule: the owner segment of the qualified name is + # locally derivable, so it is stamped at ingest with no fetch — it is what the + # Search batch query is later built from. + it "derives owner_login locally from the qualified name" do + described_class.upsert_stub!(github_id: 8484, full_name: "octocat/hello-world", + now: frozen_time) + + expect(described_class.find_by(github_id: 8484).owner_login).to eq("octocat") + end + + it "leaves owner_login null for an unqualified name rather than guessing" do + described_class.upsert_stub!(github_id: 8484, full_name: "hello-world", + now: frozen_time) + + expect(described_class.find_by(github_id: 8484).owner_login).to be_nil + end + + it "refreshes owner_login when a later observation shows a transferred repository" do + described_class.upsert_stub!(github_id: 8484, full_name: "octocat/hello-world", + now: frozen_time) + described_class.upsert_stub!(github_id: 8484, full_name: "new-owner/hello-world", + now: frozen_time + 60) + + repository = described_class.find_by(github_id: 8484) + expect(repository.full_name).to eq("new-owner/hello-world") + expect(repository.owner_login).to eq("new-owner") + end + + it "does not let an older envelope regress owner_login" do + described_class.upsert_stub!(github_id: 8484, full_name: "new-owner/hello-world", + now: frozen_time + 300) + described_class.upsert_stub!(github_id: 8484, full_name: "octocat/hello-world", + now: frozen_time) + + expect(described_class.find_by(github_id: 8484).owner_login).to eq("new-owner") + end + it "refreshes full_name on a later observation" do described_class.upsert_stub!(github_id: 8484, full_name: "octocat/hello-world", now: frozen_time) @@ -93,6 +130,9 @@ expect(described_class.count).to eq(0) end + # The contract columns are enrichment-owned: only a validated Search item or detail + # document writes them, so an envelope refresh — which knows nothing about forks or + # branches — must leave every one of them exactly as the last enrichment left it. it "never clears enrichment-owned fields" do described_class.upsert_stub!(github_id: 8484, full_name: "octocat/hello-world", now: frozen_time) @@ -100,8 +140,13 @@ description: "My first repository", language: "Ruby", owner_github_id: 1, + fork: true, + archived: false, + default_branch: "main", + github_created_at: Time.utc(2011, 1, 26, 19, 1, 12), raw_payload: { "full_name" => "octocat/hello-world" }, enrichment_status: "complete", + enrichment_stage: "contract_complete", fetched_at: frozen_time ) @@ -112,8 +157,13 @@ expect(repository.description).to eq("My first repository") expect(repository.language).to eq("Ruby") expect(repository.owner_github_id).to eq(1) + expect(repository.fork).to be(true) + expect(repository.archived).to be(false) + expect(repository.default_branch).to eq("main") + expect(repository.github_created_at).to eq(Time.utc(2011, 1, 26, 19, 1, 12)) expect(repository.raw_payload).to eq("full_name" => "octocat/hello-world") expect(repository.enrichment_status).to eq("complete") + expect(repository.enrichment_stage).to eq("contract_complete") end it "refreshes identity without clearing a retryable entity's failure state" do diff --git a/spec/models/github_search_budget_spec.rb b/spec/models/github_search_budget_spec.rb new file mode 100644 index 0000000..dd9d2c2 --- /dev/null +++ b/spec/models/github_search_budget_spec.rb @@ -0,0 +1,162 @@ +require "rails_helper" + +RSpec.describe GithubSearchBudget do + # No shared builder on purpose: the search row is created by exactly one class in + # production (Github::SearchBudgetLedger#bootstrap!), and these examples are about + # the schema surface, so the column defaults are the baseline. + def create_search_budget(**overrides) + described_class.create!(overrides) + end + + it "is stored in a singular table because it holds exactly one row" do + expect(described_class.table_name).to eq("github_search_budget") + end + + describe "the singleton constraint" do + it "defaults the primary key to the singleton id" do + expect(create_search_budget.id).to eq(described_class::SINGLETON_ID) + end + + # Enforced in the schema rather than the application, so no process — worker or + # one-shot — can create a second search ledger to reserve against. + it "rejects a second row at the database level" do + create_search_budget + + expect_violation(ActiveRecord::CheckViolation) do + described_class.connection.execute(<<~SQL.squish) + INSERT INTO github_search_budget (id, created_at, updated_at) + VALUES (2, NOW(), NOW()) + SQL + end + + expect(described_class.count).to eq(1) + end + end + + describe "column defaults" do + it "opens on the configured ceiling and reserve with nothing spent or observed" do + budget = create_search_budget + + expect(budget).to have_attributes( + resource: "search", request_ceiling: 10, reserve: 2, + used: 0, actor_used: 0, repository_used: 0, + limit: nil, remaining: nil, reset_at: nil, observed_at: nil, + blocked_until: nil, last_request_at: nil + ) + end + end + + describe "counter constraints" do + it "rejects a negative counter at the database level" do + budget = create_search_budget + + %i[ reserve used actor_used repository_used ].each do |counter| + expect_violation(ActiveRecord::CheckViolation) do + described_class.where(id: budget.id).update_all(counter => -1) + end + end + end + + # A zero ceiling is not a conservative setting: spendable is ceiling - reserve, so + # zero silently derives a search budget of nothing. + it "rejects a non-positive ceiling at the database level" do + budget = create_search_budget + + expect_violation(ActiveRecord::CheckViolation) do + described_class.where(id: budget.id).update_all(request_ceiling: 0) + end + end + + it "rejects a negative header value while permitting an unobserved null" do + budget = create_search_budget + + %i[ limit remaining ].each do |column| + expect_violation(ActiveRecord::CheckViolation) do + described_class.where(id: budget.id).update_all(column => -1) + end + end + + expect { budget.update!(limit: nil, remaining: nil) }.not_to raise_error + end + + it "mirrors the same boundaries as model validations" do + expect(described_class.new(request_ceiling: 0)).not_to be_valid + expect(described_class.new(reserve: -1)).not_to be_valid + expect(described_class.new(used: -1)).not_to be_valid + expect(described_class.new(actor_used: -1)).not_to be_valid + expect(described_class.new(repository_used: -1)).not_to be_valid + expect(described_class.new).to be_valid + end + end + + # What the ledger may still spend, from one row: the local ceiling-minus-reserve + # budget, tightened by GitHub's own remaining when one has been observed. + describe "#available" do + it "derives from the local counters alone while remaining is unobserved" do + expect(create_search_budget(used: 3).available).to eq(5) + end + + it "reaches zero once the local spendable budget is gone" do + expect(create_search_budget(used: 8).available).to eq(0) + end + + it "never goes negative when used has passed the spendable boundary" do + expect(create_search_budget(used: 9).available).to eq(0) + end + + it "is bounded by the observed remaining less the reserve" do + expect(create_search_budget(used: 0, remaining: 4).available).to eq(2) + end + + it "clamps to zero when the observed remaining is inside the reserve" do + expect(create_search_budget(used: 0, remaining: 1).available).to eq(0) + end + + it "never exceeds the local budget however generous the observed remaining" do + expect(create_search_budget(used: 0, remaining: 100).available).to eq(8) + end + + it "takes the tighter of the two bounds" do + expect(create_search_budget(used: 7, remaining: 9).available).to eq(1) + end + end + + # One rendering of the row for the structured stream and /status, so the search + # budget lines cannot describe the same row with different field names. + describe "#to_log" do + it "reports the whole search-budget state an operator reads a quiet system against" do + budget = create_search_budget( + limit: 10, remaining: 4, reset_at: frozen_time + 60, observed_at: frozen_time, + used: 3, actor_used: 2, repository_used: 1, + blocked_until: frozen_time + 120, last_request_at: frozen_time + ) + + expect(budget.to_log).to eq( + resource: "search", limit: 10, remaining: 4, + reset_at: "2026-07-29T12:01:00Z", observed_at: "2026-07-29T12:00:00Z", + request_ceiling: 10, reserve: 2, used: 3, actor_used: 2, repository_used: 1, + available: 2, blocked_until: "2026-07-29T12:02:00Z", + last_request_at: "2026-07-29T12:00:00Z" + ) + end + + # Kept rather than compacted: "remaining is unknown" and "remaining was not + # reported" are different facts to an operator reading one line. + it "keeps an unknown value visible instead of dropping the key" do + expect(create_search_budget.to_log) + .to include(limit: nil, remaining: nil, reset_at: nil, blocked_until: nil) + end + end + + describe "optimistic locking" do + it "refuses a stale write so concurrent reservations cannot be lost" do + create_search_budget + first = described_class.sole + second = described_class.sole + + first.update!(used: 1) + + expect { second.update!(used: 2) }.to raise_error(ActiveRecord::StaleObjectError) + end + end +end diff --git a/spec/quota_outcome_boundary_spec.rb b/spec/quota_outcome_boundary_spec.rb new file mode 100644 index 0000000..b4f54e9 --- /dev/null +++ b/spec/quota_outcome_boundary_spec.rb @@ -0,0 +1,152 @@ +require "rails_helper" + +# Appendix F's one-sentence law, asserted rather than assumed: quota delay is never an +# entity outcome. The skipped_budget status and its skipped_at column were removed by +# migration 20260802000000, and nothing may reintroduce them — or any quota-flavored +# terminal value — without failing here first. +# +# Written in the idiom of spec/network_boundary_spec.rb's grep guard: the static half +# scans the code the suite can reach, and the behavioral half proves the property the +# vocabulary implies — a ledger denial, from either ledger, leaves every entity row +# bit-identical. +RSpec.describe "the quota-outcome boundary" do + describe "the retired skipped_budget state" do + # Everything executable. Documentation (IMPLEMENTATION_PLAN.md, docs/evidence/) may + # tell the removal's story; code may not re-live it. + CODE_GLOBS = %w[ + app/**/*.rb bin/* config/**/*.rb config/**/*.yml db/**/*.rb lib/**/*.rb spec/**/*.rb + ].freeze + + # Each entry names why the mention is legitimate: + # * the two create migrations are frozen history — they built the column the + # removal migration later dropped, and rewriting history breaks replays; + # * the removal migration and its spec are the removal itself; + # * schema_spec and the enrichable_entity shared examples assert the *absence* — + # the column gone, the status refused at both the model and the database. + ALLOWED_MENTIONS = %w[ + db/migrate/20260729210001_create_github_actors.rb + db/migrate/20260729210002_create_github_repositories.rb + db/migrate/20260802000000_remove_skipped_budget_from_enrichment.rb + spec/db/remove_skipped_budget_from_enrichment_spec.rb + spec/db/schema_spec.rb + spec/support/shared_examples/enrichable_entity.rb + spec/quota_outcome_boundary_spec.rb + ].freeze + + it "appears nowhere in reachable code outside the removal's own history" do + mentioning = Dir[*CODE_GLOBS.map { |glob| Rails.root.join(glob).to_s }].select do |path| + File.file?(path) && File.read(path).match?(/skipped_budget|skipped_at/) + end + + relative = mentioning.map { |path| Pathname.new(path).relative_path_from(Rails.root).to_s } + + expect(relative - ALLOWED_MENTIONS).to eq([]) + end + end + + describe "the entity vocabularies" do + # A quota denial defers; it never names an entity state. If either list ever grows + # a member spelled like a budget, the durable-backlog guarantee is being reversed. + it "contain no quota-flavored value among the statuses" do + expect(Enrichable::ENRICHMENT_STATUSES.grep(/skip|budget|quota/i)).to eq([]) + end + + it "contain no quota-flavored value among the stages" do + expect(Enrichable::ENRICHMENT_STAGES.grep(/skip|budget|quota/i)).to eq([]) + end + end + + # The behavioral half. Both ledgers are spent to their denial thresholds, the real + # runners attempt both lanes for both classes, and every entity row must come back + # bit-identical — same status, same stage, same timestamps, same everything. The + # attempt itself is still evidenced (a deferred enrichment_batches row), because a + # deferral is an operational fact about the *window*, never about the entity. + describe "a ledger denial, behaviorally", type: :integration do + let(:now) { frozen_time } + let(:configuration) { configuration_with("GITHUB_MODE" => "fixture") } + + # Frozen creation instants, so the release path's updated_at write is provably a + # restore rather than an unnoticed same-second overwrite. + let!(:actor) do + create_actor(github_id: 583_231, last_seen_at: now, + created_at: now, updated_at: now) + end + let!(:repository) do + create_repository(github_id: 1_296_269, full_name: "octocat/Hello-World", + last_seen_at: now, created_at: now, updated_at: now) + end + + def entity_fingerprints + [ actor.reload.attributes, repository.reload.attributes ] + end + + describe "from the search ledger, with zero headroom" do + before do + # used 8 of ceiling 10 with reserve 2: the next reservation is exactly the + # first one the ceiling refuses. + active_search_window(now: now, used: 8) + end + + it "is the admission verdict the cycle would defer on" do + verdict = Github::Enrichment::Admission.new(configuration: configuration).search(now: now) + + expect(verdict.reason).to eq(:search_ceiling_exhausted) + end + + it "defers both classes and leaves every entity row bit-identical" do + before_rows = entity_fingerprints + runner = fixture_batch_runner(configuration: configuration) + + [ GithubActor, GithubRepository ].each do |entity_class| + result = runner.call(entity_class: entity_class) + + expect(result.status).to eq("deferred") + expect(result.deferral_reason).to eq("budget_denied") + end + + expect(entity_fingerprints).to eq(before_rows) + expect(current_search_budget.used).to eq(8) + expect(EnrichmentBatch.where(request_kind: "search").pluck(:status).uniq) + .to eq([ "deferred" ]) + end + end + + describe "from the core ledger, with the detail allowance spent" do + before do + active_budget_window(now: now, enrichment_used: 4) + + # Rows already admitted to the bounded fallback lane — the shape a search miss + # leaves behind — with every write frozen at the same instant. + GithubActor.where(id: actor.id).update_all( + enrichment_stage: "detail_pending", detail_pending_at: now, updated_at: now + ) + GithubRepository.where(id: repository.id).update_all( + enrichment_stage: "detail_pending", detail_pending_at: now, updated_at: now + ) + end + + it "is the admission verdict the cycle would defer on" do + verdict = Github::Enrichment::Admission.new(configuration: configuration).detail(now: now) + + expect(verdict.reason).to eq(:class_exhausted) + end + + it "defers both classes and leaves every entity row bit-identical" do + before_rows = entity_fingerprints + runner = fixture_detail_runner(configuration: configuration) + + [ GithubActor, GithubRepository ].each do |entity_class| + result = runner.call(entity_class: entity_class) + + expect(result.status).to eq("deferred") + expect(result.reason).to eq("budget_denied") + end + + expect(entity_fingerprints).to eq(before_rows) + expect(current_budget.enrichment_used).to eq(4) + expect(EnrichmentBatch.where(request_kind: "detail").pluck(:status).uniq) + .to eq([ "deferred" ]) + end + end + end +end diff --git a/spec/recovery/concurrent_write_spec.rb b/spec/recovery/concurrent_write_spec.rb index 5a48a7e..5d2cacc 100644 --- a/spec/recovery/concurrent_write_spec.rb +++ b/spec/recovery/concurrent_write_spec.rb @@ -53,6 +53,12 @@ # to PageWriter. With no fixture transaction to revert it, this RESET is the only thing # that stops a 250ms lock timeout riding the pooled connection into the rest of the run. connection.execute("RESET lock_timeout") + # The event-native observations first: they carry a real foreign key to the + # push_events rows deleted next. + connection.execute( + "DELETE FROM enrichment_observations WHERE push_event_id IN " \ + "(SELECT id FROM push_events WHERE github_event_id IN ('#{EVENT_ID}', '#{OTHER_EVENT_ID}'))" + ) connection.execute( "DELETE FROM push_events WHERE github_event_id IN ('#{EVENT_ID}', '#{OTHER_EVENT_ID}')" ) @@ -119,6 +125,14 @@ def commit_push_event! expect(GithubActor.find_by(github_id: ACTOR_ID).last_seen_at).to eq(frozen_time) expect(GithubActor.find_by(github_id: ACTOR_ID).latest_event_at).to be_nil end + + # The observation ledger rides the same RETURNING gate: a duplicate that registered + # no activity appended no event-native evidence either. + it "appends no observation for the loser" do + writer.write([ well_formed_envelope ], run_id: SecureRandom.uuid) + + expect(EnrichmentObservation.where(source: "event")).to be_empty + end end describe "a retryable entity whose event another poller committed first" do @@ -205,13 +219,15 @@ def capture_sql end # ADR 0005's per-envelope transaction, seen from the failure side: the envelope leaves no - # half-written entity pair behind, because the repository upsert was inside the same - # transaction the timeout rolled back. + # half-written entity pair behind — and no orphan observation — because the repository + # upsert and the observation appends were inside the same transaction the timeout + # rolled back. it "leaves no partial pair behind" do writer.write([ well_formed_envelope ], run_id: SecureRandom.uuid) expect(GithubRepository.where(github_id: REPOSITORY_ID)).to be_empty expect(PushEvent.where(github_event_id: EVENT_ID)).to be_empty + expect(EnrichmentObservation.where(source: "event")).to be_empty end # ADR 0005's per-envelope transaction, against a real contended row rather than the diff --git a/spec/recovery/crash_window_spec.rb b/spec/recovery/crash_window_spec.rb index a04958a..afc7d9b 100644 --- a/spec/recovery/crash_window_spec.rb +++ b/spec/recovery/crash_window_spec.rb @@ -241,7 +241,10 @@ def replay_whole_page # This is as close as RSpec gets to §15 step 8. The transcript in docs/evidence/ is the rest. describe "the whole container-kill cycle, without the container" do let(:source_namespace) { Github::AdvisoryLock::SOURCE_LOCK_NAMESPACE } - let(:claim) { Github::Enrichment::Claim.new(configuration: Github.configuration) } + let(:recovery_configuration) do + configuration_with("GITHUB_MODE" => "fixture", "SEARCH_PACING_SECONDS" => "0") + end + let(:claim) { Github::Enrichment::BatchClaim.new(configuration: recovery_configuration) } let(:actor_type) { Github::Enrichment::EntityType.fetch(:actor) } let!(:event_source) { fixture_event_source } @@ -251,29 +254,32 @@ def replay_whole_page before { active_budget_window(now: crashed_at) } - # Everything a worker container was holding at the instant it was killed. + # Everything a worker container was holding at the instant it was killed. The batch + # claim leases every actor in the FIFO and opens an in_flight enrichment_batches + # row — the exact durable residue a kill mid-batch leaves. def crash! fixture_runner(transport: transport, now: crashed_at).call(event_source: event_source) - claim.acquire(actor_type, pool: :pending, now: crashed_at) # a lease with no worker - clear_enqueued_jobs # the lost enqueue - acquire_in_other_session(source_namespace, # the dead poller's lock + @abandoned_lease = claim.acquire(actor_type, now: crashed_at) # a lease with no worker + clear_enqueued_jobs # the lost enqueue + acquire_in_other_session(source_namespace, # the dead poller's lock Github::AdvisoryLock.key_for(event_source.id)) - terminate_second_session! # the kill + terminate_second_session! # the kill wait_for_advisory_lock_release(source_namespace, Github::AdvisoryLock.key_for(event_source.id)) end # The restart, running only the two recurring tasks a real worker runs, at a clock past - # the abandoned lease's expiry — so the entity the dead worker was holding is reachable - # again by arithmetic alone, with no cleanup step. + # the abandoned lease's expiry — so the entities the dead worker was holding are + # reachable again by arithmetic alone, with no cleanup step. def restart! - allow(Github).to receive(:configuration).and_return(configuration_with("GITHUB_MODE" => "fixture")) - allow(Github::EnrichmentRunner).to receive(:new).and_call_original - allow(Github::EnrichmentRunner).to receive(:new) - .and_return(fixture_enrichment_runner(transport: transport, - now: crashed_at + claim.lease_seconds + 1)) + revived_at = crashed_at + recovery_configuration.enrichment_lease_seconds + 1 - 6.times do + allow(Github).to receive(:configuration).and_return(recovery_configuration) + allow(Github::Enrichment::CycleRunner).to receive(:new) + .and_return(fixture_cycle_runner(transport: transport, now: revived_at, + configuration: recovery_configuration)) + + 3.times do ReconcilePendingEnrichmentsJob.perform_now perform_enqueued_jobs end @@ -285,11 +291,18 @@ def restart! expect(PushEvent.count).to eq(4) expect(GithubActor.find_by(github_id: IngestionHelpers::ACTOR_GITHUB_ID)) - .to have_attributes(enrichment_status: "complete", name: "The Octocat") + .to have_attributes(enrichment_status: "complete", enrichment_stage: "contract_complete") expect(GithubRepository.find_by(github_id: IngestionHelpers::REPOSITORY_GITHUB_ID).enrichment_status) .to eq("complete") end + it "finalizes the dead worker's batch as stale_lease evidence, never deleting it" do + crash! + restart! + + expect(@abandoned_lease.batch.reload.status).to eq("stale_lease") + end + it "needs no operator step, no cleanup job and no sweeper to get there" do crash! restart! diff --git a/spec/recovery/duplicate_job_execution_spec.rb b/spec/recovery/duplicate_job_execution_spec.rb index fc39622..6f45576 100644 --- a/spec/recovery/duplicate_job_execution_spec.rb +++ b/spec/recovery/duplicate_job_execution_spec.rb @@ -2,87 +2,122 @@ # §12's "Enrichment job executed twice" exercises one redelivery case. The ingestion-wide # guarantees are narrower: a duplicate event ID cannot create another push_events row or -# register new entity activity. Executions, run summaries, quarantine counters, budget use, -# and logs can repeat. Here the second execution really happens, and this scenario asserts -# only that an already-complete actor is left unchanged by the freshness check. +# register new entity activity. Executions, run summaries, quarantine counters, budget +# use, and logs can repeat. Here the second execution really happens, and the claims are +# the staged pipeline's: a surplus cycle finds no claimable stage and no admissible +# budget, so it cannot double-apply a projection or double-spend past either ledger's cap. # # The redelivery is modelled as the *same job instance* performed twice: one job id, two # executions, which is what Solid Queue produces when a worker dies after the job ran and # before its claim was released. -RSpec.describe "an enrichment job delivered twice", type: :integration do +RSpec.describe "an enrichment cycle delivered twice", type: :integration do + let(:now) { frozen_time } let(:transport) { fixture_transport } - let(:job) { EnrichActorJob.new } + let(:configuration) { configuration_with("GITHUB_MODE" => "fixture", "SEARCH_PACING_SECONDS" => "0") } + let(:job) { EnrichmentCycleJob.new } before do - active_budget_window(now: frozen_time) - create_actor(github_id: 583_231, last_seen_at: frozen_time, - api_url: "https://api.github.com/users/octocat") + fixture_runner(transport: transport, now: now).call(event_source: fixture_event_source) - allow(Github).to receive(:configuration).and_return(configuration_with("GITHUB_MODE" => "fixture")) - allow(Github::EnrichmentRunner).to receive(:new) - .and_return(fixture_enrichment_runner(transport: transport, now: frozen_time)) + allow(Github).to receive(:configuration).and_return(configuration) + allow(Github::Enrichment::CycleRunner).to receive(:new) + .and_return(fixture_cycle_runner(transport: transport, now: now, + configuration: configuration)) job.perform_now end - it "enriched the actor on the first delivery" do - expect(GithubActor.sole) - .to have_attributes(enrichment_status: "complete", name: "The Octocat", fetched_at: frozen_time) - expect(current_budget.enrichment_used).to eq(1) + it "did the whole staged pipeline's work on the first delivery" do + expect(GithubActor.find_by(github_id: 583_231)) + .to have_attributes(enrichment_status: "complete", enrichment_stage: "contract_complete") + expect(GithubActor.find_by(github_id: 7_700_421).enrichment_stage).to eq("terminal") + expect(current_search_budget.used).to eq(2) + expect(current_budget.enrichment_used).to eq(2) end it "leaves the durable state byte-identical after the second" do - before_attributes = GithubActor.sole.attributes + before_rows = [ GithubActor.order(:id).map(&:attributes), + GithubRepository.order(:id).map(&:attributes) ] job.perform_now - expect(GithubActor.sole.attributes).to eq(before_attributes) + expect([ GithubActor.order(:id).map(&:attributes), + GithubRepository.order(:id).map(&:attributes) ]).to eq(before_rows) end - # The freshness cache is what makes this true — not a dedup table, and not the queue. A - # fresh record is not a candidate, so the second delivery has nothing to claim. - it "spends no second request and no second reservation" do - expect { job.perform_now }.not_to change { current_budget.enrichment_used }.from(1) - expect(transport.requests.size).to eq(1) + # The staged FIFO is what makes this true — not a dedup table, and not the queue. A + # contract_complete row is not claimable and a terminal row never is, so the second + # delivery has nothing to claim, nothing to observe, and nothing to spend. + it "spends no second request on either ledger and appends no observation" do + before_counts = [ current_search_budget.used, current_budget.enrichment_used, + EnrichmentObservation.count, EnrichmentBatch.count ] + + job.perform_now + + expect([ current_search_budget.used, current_budget.enrichment_used, + EnrichmentObservation.count, EnrichmentBatch.count ]).to eq(before_counts) + expect(transport.requests.size).to eq(5) end - it "reports idle rather than pretending it did the work again" do + it "reports an empty cycle rather than pretending it did the work again" do job.perform_now - expect(job.outcome).to include(enrichment_outcome: "idle") + expect(job.outcome).to include(batches_attempted: 0, details_attempted: 0, + batch_stop_reason: "no_batch_work") end it "creates no duplicate entity row" do - expect { job.perform_now }.not_to change(GithubActor, :count).from(1) + expect { job.perform_now }.not_to change(GithubActor, :count).from(3) end # A redelivery that lands while the first execution is still in flight: the lease on - # next_retry_at excludes the row from all four selector queries, so the second finds nothing - # rather than fetching the same entity twice. + # lease_token/leased_until makes the rows invisible to every claim scope, so the + # second cycle finds nothing rather than fetching the same batch twice. describe "arriving while the first execution still holds the lease" do - it "finds nothing to do and spends nothing" do - GithubActor.update_all(enrichment_status: "pending", fetched_at: nil, - next_retry_at: frozen_time + 600) + before do + GithubActor.update_all(enrichment_status: "pending", enrichment_stage: "batch_in_flight", + lease_token: SecureRandom.uuid, leased_until: now + 600, + fetched_at: nil) + end + + it "finds nothing to claim and spends nothing" do + expect { job.perform_now }.not_to change { current_search_budget.used }.from(2) + expect(job.outcome).to include(batches_attempted: 0) + end + end + + # The ledgers are the last line even when a surplus delivery does find claimable work: + # a re-pended backlog against spent windows stops at admission, before any claim. + describe "arriving after the caps are already spent" do + before do + GithubActor.update_all(enrichment_status: "pending", enrichment_stage: "batch_pending", + batch_pending_at: now, fetched_at: nil) + GithubSearchBudget.where(id: GithubSearchBudget::SINGLETON_ID).update_all(used: 8) + GithubApiBudget.where(id: GithubApiBudget::SINGLETON_ID).update_all(enrichment_used: 4) + end - expect { job.perform_now }.not_to change { current_budget.enrichment_used }.from(1) - expect(job.outcome).to include(enrichment_outcome: "idle") + it "cannot spend past either cap" do + job.perform_now + + expect(current_search_budget.used).to eq(8) + expect(current_budget.enrichment_used).to eq(4) + expect(job.outcome).to include(batches_attempted: 0, + batch_stop_reason: "search_ceiling_exhausted") end end - # §10: "actor or repo URL returns 404/410 → entity permanent_failure". A redelivery must not - # re-attempt a decided entity, and must not reset the attempt counter that decided it. + # §10: "actor or repo URL returns 404/410 → entity permanent_failure". A redelivery + # must not re-attempt a decided entity, and must not reset the ladder that decided it. describe "after a permanent failure" do let(:ghost) { GithubActor.find_by(github_id: 7_700_421) } it "does not re-attempt the entity" do - create_actor(github_id: 7_700_421, login: "ghostuser", last_seen_at: frozen_time, - api_url: "https://api.github.com/users/ghostuser") - job.perform_now before_attributes = ghost.attributes job.perform_now - expect(ghost.reload).to have_attributes(enrichment_status: "permanent_failure") + expect(ghost.reload).to have_attributes(enrichment_status: "permanent_failure", + enrichment_stage: "terminal") expect(ghost.attributes).to eq(before_attributes) end end diff --git a/spec/recovery/multi_poller_spec.rb b/spec/recovery/multi_poller_spec.rb index b46852b..da9acc0 100644 --- a/spec/recovery/multi_poller_spec.rb +++ b/spec/recovery/multi_poller_spec.rb @@ -131,17 +131,22 @@ def poll_b(force: false, wait_seconds: 5) # to put back. describe "the global request gate, across both request paths" do let(:transport) { fixture_transport } + let(:configuration) { configuration_with("GITHUB_MODE" => "fixture") } let!(:actor) do create_actor(github_id: IngestionHelpers::ACTOR_GITHUB_ID, last_seen_at: frozen_time, - enrichment_status: "pending") + enrichment_status: "pending", + created_at: frozen_time, updated_at: frozen_time) end # 0.1s, never 0: PostgreSQL reads lock_timeout = 0 as "no timeout", so a zero wait would # block forever rather than defer. - def enrichment_cycle - fixture_enrichment_runner( - executor: fixture_executor(transport: transport, request_gate_wait: 0.1) - ).call + def batch_attempt + fixture_batch_runner( + configuration: configuration, + executor: fixture_executor(transport: transport, request_gate_wait: 0.1, + ledger: ledger_for(configuration), + search_ledger: search_ledger_for(configuration)) + ).call(entity_class: GithubActor) end def poll_cycle @@ -149,46 +154,49 @@ def poll_cycle .call(event_source: fixture_event_source) end - it "defers an enrichment cycle rather than failing it" do - result = other_session_holding(gate_namespace, gate_key) { enrichment_cycle } + it "defers a Search batch rather than failing it" do + result = other_session_holding(gate_namespace, gate_key) { batch_attempt } - expect(result).to be_deferred + expect(result.status).to eq("deferred") expect(result.deferral_reason).to eq("gate_unavailable") end it "spends nothing and issues no request while the gate is held" do - other_session_holding(gate_namespace, gate_key) { enrichment_cycle } + other_session_holding(gate_namespace, gate_key) { batch_attempt } expect(current_budget.enrichment_used).to eq(0) - expect(current_budget.actor_share_used).to eq(0) + expect(GithubSearchBudget.find_by(id: GithubSearchBudget::SINGLETON_ID)&.used.to_i).to eq(0) expect(transport.requests).to be_empty end - # The assertion this file exists for. A deferred cycle still *claimed* the row — it - # wrote next_retry_at as a lease before discovering the gate was busy — so proving the - # entity is untouched proves Github::Enrichment::Claim#release! restored the exact prior - # instant rather than clearing it or leaving the lease stranded for its full 594s. + # The assertion this file exists for. A deferred batch still *claimed* the row — it + # wrote the lease columns and opened a batch row before discovering the gate was + # busy — so proving the entity is untouched proves BatchClaim#release! restored the + # row exactly rather than leaving the lease stranded for its full 600s. The batch + # row itself survives as `deferred` evidence, which is about the window, never the + # entity. it "gives the lease back exactly, leaving the entity byte-identical" do before = actor.reload.attributes - other_session_holding(gate_namespace, gate_key) { enrichment_cycle } + other_session_holding(gate_namespace, gate_key) { batch_attempt } expect(actor.reload.attributes).to eq(before) + expect(EnrichmentBatch.sole.status).to eq("deferred") end # One gate, application-wide — asserted as a single fact rather than as two separate # claims about two subsystems. it "stops the poller and the enrichment worker alike" do poll_result = nil - enrichment_result = nil + batch_result = nil other_session_holding(gate_namespace, gate_key) do poll_result = poll_cycle - enrichment_result = enrichment_cycle + batch_result = batch_attempt end expect(poll_result).to be_deferred - expect(enrichment_result).to be_deferred + expect(batch_result.status).to eq("deferred") expect(current_budget.poll_used).to eq(0) expect(current_budget.enrichment_used).to eq(0) expect(transport.requests).to be_empty @@ -197,7 +205,7 @@ def poll_cycle it "leaves the gate free and the lock order clean after both deferrals" do other_session_holding(gate_namespace, gate_key) do poll_cycle - enrichment_cycle + batch_attempt end expect(advisory_lock_holders(gate_namespace, gate_key)).to be_empty diff --git a/spec/recovery/pending_enrichment_recovery_spec.rb b/spec/recovery/pending_enrichment_recovery_spec.rb index 97941bd..10fbb8d 100644 --- a/spec/recovery/pending_enrichment_recovery_spec.rb +++ b/spec/recovery/pending_enrichment_recovery_spec.rb @@ -4,12 +4,15 @@ # "Pending enrichment rediscovered (entity-scoped)". # # The crash is expressed the way a crash actually presents itself: rows committed, queue -# empty. Nothing is stubbed to raise, because a SIGKILL runs no ensure block and no rescue — -# what it leaves behind is exactly this state, and §2A's claim is that this state is -# recoverable *because* the entity rows are the durable record of pending work. +# empty. Nothing is stubbed to raise, because a SIGKILL runs no ensure block and no +# rescue — what it leaves behind is exactly this state, and §2A's claim is that this +# state is recoverable *because* the entity rows are the durable record of pending work: +# the staged FIFO (enrichment_stage plus batch_pending_at) is readable by any later +# reconciler tick. # -# The page is ingested at Time.current because the reconciler reads the worker's clock and -# Solid Queue constructs the job, so there is no test clock to inject into that boundary. +# The page is ingested at Time.current because the reconciler reads the worker's clock +# and Solid Queue constructs the job, so there is no test clock to inject into that +# boundary. RSpec.describe "recovering enrichment work that was never enqueued", type: :integration do let(:transport) { fixture_transport } let(:ingested_at) { Time.current } @@ -25,15 +28,14 @@ it "leaves the work durable in the business tables, where nothing could lose it" do expect(PushEvent.count).to eq(4) - expect(GithubActor.enrichment_candidates.count).to eq(3) - expect(GithubRepository.enrichment_candidates.count).to eq(3) + expect(GithubActor.where(enrichment_stage: "batch_pending").count).to eq(3) + expect(GithubRepository.where(enrichment_stage: "batch_pending").count).to eq(3) expect(enqueued_jobs).to be_empty end it "rediscovers it on the next reconciler tick" do expect { ReconcilePendingEnrichmentsJob.perform_now } - .to have_enqueued_job(EnrichActorJob).exactly(:once) - .and have_enqueued_job(EnrichRepositoryJob).exactly(:once) + .to have_enqueued_job(EnrichmentCycleJob).exactly(:once) end it "rediscovers pending rows even when their durable insertion time is very old" do @@ -43,56 +45,66 @@ last_seen_at: ingested_at - 30.days) expect { ReconcilePendingEnrichmentsJob.perform_now } - .to have_enqueued_job(EnrichActorJob).exactly(:once) - .and have_enqueued_job(EnrichRepositoryJob).exactly(:once) + .to have_enqueued_job(EnrichmentCycleJob).exactly(:once) end - # §8: "a small, entity-scoped set, not N event rows per entity." Three actors behind four - # events are one cycle, not three and not four — the queue is not where the backlog lives. - it "schedules one cycle per class, not one per pending entity or per event" do + # §8: "a small, entity-scoped set, not N event rows per entity." Six entities behind + # four events are one cycle — not one job per class, per entity, or per event. The + # cycle works both lanes itself; queue depth is set by the budgets, not the backlog. + it "schedules one cycle, not one per class, per pending entity, or per event" do ReconcilePendingEnrichmentsJob.perform_now - expect(enqueued_jobs.map { _1[:job] }).to contain_exactly(EnrichActorJob, EnrichRepositoryJob) + expect(enqueued_jobs.map { _1[:job] }).to eq([ EnrichmentCycleJob ]) end - # The sweep reads state and schedules; it never writes entity rows. If it did, a worker that - # came back after an hour would silently reset the backoff of everything it found. + # The sweep reads state and schedules; it never writes entity rows. If it did, a worker + # that came back after an hour would silently reset the backoff of everything it found. it "changes no entity row while rediscovering the work" do - before_rows = GithubActor.order(:id).pluck(:id, :enrichment_status, :enrichment_attempts, :updated_at) + before_rows = GithubActor.order(:id) + .pluck(:id, :enrichment_status, :enrichment_stage, + :enrichment_attempts, :updated_at) ReconcilePendingEnrichmentsJob.perform_now - expect(GithubActor.order(:id).pluck(:id, :enrichment_status, :enrichment_attempts, :updated_at)) + expect(GithubActor.order(:id) + .pluck(:id, :enrichment_status, :enrichment_stage, + :enrichment_attempts, :updated_at)) .to eq(before_rows) end it "keeps scheduling on every tick until the work is actually done" do 2.times { ReconcilePendingEnrichmentsJob.perform_now } - expect(enqueued_jobs.count { _1[:job] == EnrichActorJob }).to eq(2) + expect(enqueued_jobs.count { _1[:job] == EnrichmentCycleJob }).to eq(2) end - # The end of the story rather than the middle: the rediscovered work runs, and the entities - # reach the same durable state the un-crashed run would have produced. - it "completes the recovered work when the scheduled cycles run" do - allow(Github).to receive(:configuration).and_return(configuration_with("GITHUB_MODE" => "fixture")) - allow(Github::EnrichmentRunner).to receive(:new) - .and_return(fixture_enrichment_runner(transport: transport, now: frozen_time)) - - 6.times do + # The end of the story rather than the middle: the rediscovered work runs, and the + # entities reach the same durable state the un-crashed run would have produced. + it "completes the recovered work when the scheduled cycle runs" do + recovery_configuration = configuration_with("GITHUB_MODE" => "fixture", + "SEARCH_PACING_SECONDS" => "0") + allow(Github).to receive(:configuration).and_return(recovery_configuration) + allow(Github::Enrichment::CycleRunner).to receive(:new) + .and_return(fixture_cycle_runner(transport: transport, now: Time.current, + configuration: recovery_configuration)) + + 3.times do ReconcilePendingEnrichmentsJob.perform_now perform_enqueued_jobs end expect(GithubActor.find_by(github_id: 583_231)) - .to have_attributes(enrichment_status: "complete", name: "The Octocat") + .to have_attributes(enrichment_status: "complete", enrichment_stage: "contract_complete") expect(GithubRepository.find_by(github_id: 1_296_269).enrichment_status).to eq("complete") + expect(GithubActor.find_by(github_id: 7_700_421).enrichment_stage).to eq("terminal") end describe "when there is nothing left to recover" do before do - GithubActor.update_all(enrichment_status: "complete", fetched_at: ingested_at) - GithubRepository.update_all(enrichment_status: "complete", fetched_at: ingested_at) + GithubActor.update_all(enrichment_status: "complete", enrichment_stage: "contract_complete", + fetched_at: ingested_at) + GithubRepository.update_all(enrichment_status: "complete", enrichment_stage: "contract_complete", + fetched_at: ingested_at) end it "schedules nothing" do @@ -100,13 +112,15 @@ end end - # A live worker's in-flight rows are not pending work: the claim lease is written onto - # next_retry_at, and every one of the selector's queries excludes it. Without this the - # reconciler would pile cycles onto entities another thread already holds. + # A live worker's in-flight rows are not pending work: the batch lease is written onto + # lease_token/leased_until, and every claim scope excludes a live lease. Without this + # the reconciler would pile cycles onto entities another worker already holds. describe "while a live worker holds every candidate" do before do - GithubActor.update_all(next_retry_at: ingested_at + 600) - GithubRepository.update_all(next_retry_at: ingested_at + 600) + lease = { enrichment_stage: "batch_in_flight", lease_token: SecureRandom.uuid, + leased_until: ingested_at + 600 } + GithubActor.update_all(lease) + GithubRepository.update_all(lease) end it "schedules nothing" do diff --git a/spec/recovery/worker_crash_lease_spec.rb b/spec/recovery/worker_crash_lease_spec.rb index 858c726..91405d0 100644 --- a/spec/recovery/worker_crash_lease_spec.rb +++ b/spec/recovery/worker_crash_lease_spec.rb @@ -1,93 +1,143 @@ require "rails_helper" # §12's "Worker failure before completion". A real crash runs no ensure block, so it is -# modelled as what it leaves behind — a claim lease with no worker behind it — rather than by -# stubbing an exception, which would exercise the release path the crash skipped. +# modelled as what it leaves behind — a batch lease with no worker behind it — rather +# than by stubbing an exception, which would exercise the abandon path the crash skipped. # -# The recovery has no cleanup code anywhere, and that is the design: the lease is written onto -# next_retry_at, so it expires by arithmetic. Nothing has to notice the worker died. +# The staged lease is three columns and one row of evidence: lease_token + leased_until +# + current_enrichment_batch_id on every claimed entity, and an in_flight +# enrichment_batches row. Recovery still has no cleanup code anywhere: the lease expires +# by arithmetic, the next claim reclaims the rows, and the orphaned batch row is +# finalized as stale_lease evidence rather than deleted. RSpec.describe "a lease left behind by a crashed worker", type: :integration do + let(:now) { frozen_time } + let(:configuration) { configuration_with("GITHUB_MODE" => "fixture", "SEARCH_PACING_SECONDS" => "0") } let(:transport) { fixture_transport } - let(:configuration) { Github.configuration } - let(:claim) { Github::Enrichment::Claim.new(configuration: configuration) } + let(:claim) { Github::Enrichment::BatchClaim.new(configuration: configuration) } let(:actor_type) { Github::Enrichment::EntityType.fetch(:actor) } - let!(:actor) do - create_actor(github_id: 583_231, last_seen_at: frozen_time, - api_url: "https://api.github.com/users/octocat") + + # The corpus Search key carries exactly this trio in this order. + let!(:actors) do + [ + create_actor(github_id: 583_231, login: "octocat", display_login: "octocat", + api_url: "https://api.github.com/users/octocat", + last_seen_at: now, created_at: now - 3), + create_actor(github_id: 1_024_025, login: "monalisa", display_login: "monalisa", + api_url: "https://api.github.com/users/monalisa", + last_seen_at: now, created_at: now - 2), + create_actor(github_id: 7_700_421, login: "ghostuser", display_login: "ghostuser", + api_url: "https://api.github.com/users/ghostuser", + last_seen_at: now, created_at: now - 1) + ] end - # The crash: a lease taken, and then nothing. - let!(:lease) { claim.acquire(actor_type, pool: :pending, now: frozen_time) } + # The crash: a batch claimed, and then nothing at all. + let!(:lease) { claim.acquire(actor_type, now: now) } - before do - active_budget_window(now: frozen_time) - allow(Github).to receive(:configuration).and_return(configuration_with("GITHUB_MODE" => "fixture")) + def batch_runner(at:) + fixture_batch_runner(transport: transport, now: at, configuration: configuration) end - # The stub has to come off before the next runner is built: #fixture_enrichment_runner goes - # through Github::EnrichmentRunner.new itself, so a second call with the previous stub still - # in place would hand back the previous runner — and its clock, which is the one thing every - # example here is varying. - def enrich_at(instant) - allow(Github::EnrichmentRunner).to receive(:new).and_call_original - runner = fixture_enrichment_runner(transport: transport, now: instant) - allow(Github::EnrichmentRunner).to receive(:new).and_return(runner) - - job = EnrichActorJob.new - job.perform_now - job + it "holds every claimed entity for exactly the configured lease window" do + expect(lease.leased_until - now).to eq(configuration.enrichment_lease_seconds) + expect(GithubActor.where(lease_token: lease.token).count).to eq(3) + expect(GithubActor.pluck(:enrichment_stage).uniq).to eq([ "batch_in_flight" ]) end - # Derived from §2A's pinned defaults — attempts × redirect hops × (gate wait + open + read) - # plus the backoff — so a configuration change moves this expectation with the code instead - # of leaving a stale literal behind. - it "holds the entity for exactly the derived lease window" do - expect(lease.leased_until - frozen_time).to eq(claim.lease_seconds) + it "leaves the in-flight batch row behind as evidence" do + expect(lease.batch.reload).to have_attributes( + status: "in_flight", request_kind: "search", entity_kind: "actor", + requested_count: 3, completed_at: nil + ) end describe "while the lease is still live" do - it "finds nothing to do, and says so" do - expect(enrich_at(frozen_time).outcome).to include(enrichment_outcome: "idle") + it "is invisible to another claim" do + expect(claim.acquire(actor_type, now: now)).to be_nil end - it "leaves the entity exactly as the dead worker left it" do - before_attributes = actor.reload.attributes - - enrich_at(frozen_time) + it "makes a full runner cycle report idle, and spend nothing" do + result = batch_runner(at: now).call(entity_class: GithubActor) - expect(actor.reload.attributes).to eq(before_attributes) - expect(actor.enrichment_attempts).to eq(0) - expect(actor.last_error).to be_nil + expect(result.status).to eq("idle") + expect(transport.requests).to be_empty + expect(GithubSearchBudget.find_by(id: GithubSearchBudget::SINGLETON_ID)&.used.to_i).to eq(0) end - it "spends no budget on an entity it cannot claim" do - expect { enrich_at(frozen_time) }.not_to change { current_budget.enrichment_used }.from(0) - expect(transport.requests).to be_empty + it "leaves the entities exactly as the dead worker left them" do + before_rows = GithubActor.order(:id).map(&:attributes) + + batch_runner(at: now).call(entity_class: GithubActor) + + expect(GithubActor.order(:id).map(&:attributes)).to eq(before_rows) end end describe "once the lease expires" do - it "enriches the entity, with no sweeper and no cleanup step in between" do - enrich_at(lease.leased_until) + let(:expiry) { lease.leased_until } + + it "reclaims the rows, finalizes the orphan batch as stale_lease, and completes the work" do + result = batch_runner(at: expiry).call(entity_class: GithubActor) - expect(actor.reload) - .to have_attributes(enrichment_status: "complete", name: "The Octocat") + expect(result).to have_attributes(status: "completed", requested_count: 3, + valid_count: 2, fallback_count: 1) + expect(lease.batch.reload).to have_attributes(status: "stale_lease", completed_at: expiry) + expect(GithubActor.find_by(github_id: 583_231)) + .to have_attributes(enrichment_status: "complete", enrichment_stage: "contract_complete") + expect(GithubActor.find_by(github_id: 7_700_421).enrichment_stage).to eq("detail_pending") end - it "spends exactly one request for the whole crash-and-recovery sequence" do - enrich_at(frozen_time) - enrich_at(lease.leased_until) + it "spends exactly one search request for the whole crash-and-recovery sequence" do + batch_runner(at: now).call(entity_class: GithubActor) # idle, while leased + batch_runner(at: expiry).call(entity_class: GithubActor) - expect(current_budget.enrichment_used).to eq(1) + expect(current_search_budget.used).to eq(1) expect(transport.requests.size).to eq(1) end - # The crash cost the entity nothing: attempts count attempts *since the last success*, and - # a claim that was never used is not one. - it "charges the entity no attempt for the crash" do - enrich_at(lease.leased_until) + # The crash cost the entities nothing: attempts count attempts *since the last + # success*, and a claim that was never used is not one. + it "charges the entities no attempt for the crash" do + batch_runner(at: expiry).call(entity_class: GithubActor) + + expect(GithubActor.where(github_id: [ 583_231, 1_024_025 ]).pluck(:enrichment_attempts).uniq) + .to eq([ 0 ]) + end + + # The token guard: if the dead worker were merely slow and finished after the + # reclaim, its guarded writes name a lease_token and batch id the rows no longer + # carry, so they match zero rows — the same UPDATE shape BatchRunner uses. + it "makes the old writer's late guarded write match nothing" do + batch_runner(at: expiry).call(entity_class: GithubActor) + + late_write = GithubActor.where(id: lease.items.first.id, lease_token: lease.token, + current_enrichment_batch_id: lease.batch.id) + .update_all(last_error: "late write from a dead worker") + + expect(late_write).to eq(0) + expect(GithubActor.find_by(github_id: 583_231).last_error).to be_nil + end + end + + # The same arithmetic on the one-row detail lane: a live detail lease is invisible, + # an expired one is reclaimed and its orphan batch finalized as evidence. + describe "a crashed detail-lane worker" do + let(:detail_claim) { Github::Enrichment::DetailClaim.new(configuration: configuration) } + + before do + claim.release!(lease, now: now) + GithubActor.where(github_id: 7_700_421) + .update_all(enrichment_stage: "detail_pending", detail_pending_at: now) + end + + it "expires by arithmetic and finalizes the orphan batch as stale_lease" do + dead = detail_claim.acquire(actor_type, now: now) + + expect(detail_claim.acquire(actor_type, now: now)).to be_nil - expect(actor.reload.enrichment_attempts).to eq(0) + reclaimed = detail_claim.acquire(actor_type, now: dead.leased_until) + expect(reclaimed.item.github_id).to eq(7_700_421) + expect(dead.batch.reload.status).to eq("stale_lease") end end end diff --git a/spec/requests/status_spec.rb b/spec/requests/status_spec.rb index 21712e7..1c79226 100644 --- a/spec/requests/status_spec.rb +++ b/spec/requests/status_spec.rb @@ -3,9 +3,10 @@ RSpec.describe "GET /status", type: :request do # §11: "reports persisted state only; never initiates a GitHub request." # - # Four examples rather than one, because the guarantee has four distinct ways to break - # and each fails silently. They are the pair Github::Ingestion::StateSummary's spec - # already carries, plus the two that only a controller can get wrong. + # Five examples rather than one, because the guarantee has distinct ways to break and + # each fails silently. They are the pair Github::Ingestion::StateSummary's spec + # already carries, plus the ones only a controller can get wrong — now including the + # Search ledger, whose bootstrap! would create its configuration-born row from a GET. describe "the guarantee that reading state costs nothing (plan §11)" do it "initiates no GitHub request" do transport = fixture_transport @@ -21,36 +22,87 @@ # The subtle one: Github::BudgetLedger#bootstrap! is public and issues an INSERT even # when it inserts nothing, so reaching for the ledger instead of find_by would create # from a read path the very row a reservation owns. - it "does not create the ledger row it reports on" do + it "does not create the core ledger row it reports on" do expect { get "/status" }.not_to change(GithubApiBudget, :count).from(0) end + # The same hazard one table over: Github::SearchBudgetLedger#bootstrap! creates the + # search row from configuration, so a read path reaching it would make /status the + # first "search request" the installation ever recorded. + it "does not create the search ledger row it reports on" do + expect { get "/status" }.not_to change(GithubSearchBudget, :count).from(0) + end + # The mirror hazard: reaching for Github::Ingestion::SourceProvisioner to find "the" # event source would provision one from a GET. it "does not provision the event source it reports on" do expect { get "/status" }.not_to change(EventSource, :count).from(0) end - # Belt and braces over the three above: whatever the implementation reaches for, and - # whatever a future collaborator adds to it, no statement it issues may write. + # Belt and braces over the four above, now proven across the staged-enrichment + # tables too: whatever the implementation reaches for, and whatever a future + # collaborator adds to it, no statement it issues may write. it "issues no write statement at all" do create_event_source create_actor(github_id: 1) + create_actor(github_id: 2, login: "two", enrichment_stage: "detail_pending", + detail_pending_at: Time.current - 60) active_budget_window + active_search_window + EnrichmentBatch.create!(request_kind: "search", entity_kind: "actor", + status: "succeeded", correlation_id: SecureRandom.uuid, + started_at: Time.current - 60, requested_count: 1, + returned_count: 1, valid_count: 1) expect(write_statements { get "/status" }).to be_empty end end describe "the response" do - it "answers 200 with §11's blocks on a clean checkout" do + it "answers 200 with the amended §11 blocks on a clean checkout" do get "/status" expect(response).to have_http_status(:ok) expect(response.parsed_body.keys) - .to eq(%w[captured_at sources ledger enrichment coverage]) + .to eq(%w[captured_at sources ledger search_ledger scheduler enrichment + batches throughput coverage]) expect(response.parsed_body["sources"]).to eq([]) expect(response.parsed_body.dig("ledger", "present")).to be(false) + expect(response.parsed_body.dig("search_ledger", "present")).to be(false) + expect(response.parsed_body.dig("throughput", "catch_up", "state")) + .to eq("insufficient_sample") + end + + # The scheduler block is pure configuration — the one block a clean checkout can + # and must answer in full, so an operator can always read the enforced knobs. + it "publishes the full scheduler block on a clean checkout" do + get "/status" + + scheduler = response.parsed_body["scheduler"] + + expect(scheduler.keys).to eq(%w[search fairness core retry refresh metrics]) + expect(scheduler["search"]) + .to eq("request_ceiling" => 10, "safety_reserve" => 2, "batch_size" => 10, + "pacing_seconds" => 6, "worker_concurrency" => 1) + expect(scheduler["core"]) + .to eq("detail_fallback_allowance" => 4, "rate_limit_reserve" => 8) + expect(scheduler["metrics"]) + .to eq("window_seconds" => 3600, "catch_up_min_sample_seconds" => 900) + end + + it "publishes the staged pipeline for each entity class" do + create_actor(github_id: 1, created_at: Time.current - 600) + + get "/status" + + actors = response.parsed_body.dig("enrichment", "actors") + + expect(actors).to include("pending" => 1, "backlog_count" => 1, + "contract_backlog_count" => 1) + expect(actors["stages"].keys).to eq(Enrichable::ENRICHMENT_STAGES) + expect(actors.dig("stages", "batch_pending")).to include("count" => 1) + expect(actors.dig("stages", "terminal")) + .to eq("count" => 0, "oldest_created_at" => nil, "oldest_age_seconds" => nil) end # A snapshot is true for the instant it was taken. An intermediary serving a stale @@ -76,6 +128,28 @@ expect(response.parsed_body["sources"].first["status"]).to eq("failed") end + # The populated catch-up story: rows that arrived before the metrics window and + # completed inside it are pure drain, so the counted backlog slope is negative and + # the verdict is keeping_up — measured numbers, no ETA anywhere. + it "reports keeping_up with a negative backlog delta after completions drain the window" do + now = Time.current + create_actor(github_id: 1001, created_at: now - 7200, + enrichment_status: "complete", enrichment_stage: "contract_complete", + contract_completed_at: now - 120, fetched_at: now - 120) + create_repository(github_id: 2001, created_at: now - 7200, + enrichment_status: "complete", enrichment_stage: "contract_complete", + contract_completed_at: now - 60, fetched_at: now - 60) + + get "/status" + + throughput = response.parsed_body["throughput"] + + expect(throughput["combined"]).to include("arrivals" => 0, "completions" => 2, + "exits" => 2, "backlog_delta" => -2) + expect(throughput.dig("catch_up", "state")).to eq("keeping_up") + expect(response.parsed_body.dig("enrichment", "actors", "complete")).to eq(1) + end + it "reports coverage over persisted events, joined to their entities" do actor = create_actor(github_id: 1001) repository = create_repository(github_id: 2001) diff --git a/spec/services/github/allowances_spec.rb b/spec/services/github/allowances_spec.rb index 814199a..3bfbd89 100644 --- a/spec/services/github/allowances_spec.rb +++ b/spec/services/github/allowances_spec.rb @@ -5,13 +5,28 @@ def configuration(**overrides) Github::Configuration.new(overrides.transform_keys(&:to_s)) end - describe "the one authoritative formula (plan §10)" do + describe "the one authoritative formula (plan §10, Appendix F)" do it "derives twelve poll attempts an hour from the pinned defaults" do expect(described_class.derive(configuration: configuration, limit: 60).poll_allowance).to eq(12) end - it "derives forty enrichment attempts as limit minus reserve minus polling" do - expect(described_class.derive(configuration: configuration, limit: 60).enrichment_allowance).to eq(40) + # Appendix F: the detail-fallback allowance is a configured cap, not the remainder + # of the limit. Search batches carry the enrichment volume on their own per-minute + # ledger, so the core budget only funds the few per-entity fallback fetches. + it "takes the configured detail-fallback allowance rather than deriving it from the limit" do + expect(described_class.derive(configuration: configuration, limit: 60).enrichment_allowance).to eq(4) + end + + it "keeps that allowance fixed whatever limit GitHub reports" do + expect(described_class.derive(configuration: configuration, limit: 5000).enrichment_allowance).to eq(4) + end + + it "reads CORE_DETAIL_FALLBACK_ALLOWANCE, so the cap is an operator's number" do + derived = described_class.derive( + configuration: configuration(CORE_DETAIL_FALLBACK_ALLOWANCE: "6"), limit: 60 + ) + + expect(derived.enrichment_allowance).to eq(6) end # A 450-second cadence fits seven whole intervals in an hour plus a remainder, and @@ -35,25 +50,44 @@ def configuration(**overrides) expect(derived.poll_allowance).to eq(24) end - it "reports a negative enrichment allowance rather than hiding an over-commitment" do + # The allowance is configured, so an over-committed cadence no longer shows up as a + # negative allowance — the derivation stays honest and #feasible? carries the verdict. + it "keeps the configured allowance under an over-committed cadence, leaving #feasible? the verdict" do derived = described_class.derive(configuration: configuration(POLL_INTERVAL_SECONDS: "60"), limit: 60) - expect(derived.enrichment_allowance).to eq(-8) + expect(derived.enrichment_allowance).to eq(4) + expect(derived).not_to be_feasible end end describe "#feasible?" do - it "accepts the pinned defaults, which leave forty enrichment attempts" do + it "accepts the pinned defaults, which commit twenty-four of sixty attempts" do expect(described_class.derive(configuration: configuration, limit: 60)).to be_feasible end - # Plan §10 rejects on >=, not >: a configuration that leaves exactly zero - # enrichment capacity cannot satisfy Story 3 either. - it "rejects a split that leaves exactly zero enrichment capacity" do - derived = described_class.derive(configuration: configuration(RATE_LIMIT_RESERVE: "48"), limit: 60) + # Appendix F's predicate is poll + reserve + allowance <= limit: every configured + # number is a real commitment now, so a sum that lands exactly on the limit is a + # fully-funded plan rather than a starved one. + it "accepts the exact boundary, where the three commitments fill the limit precisely" do + derived = described_class.derive(configuration: configuration(RATE_LIMIT_RESERVE: "44"), limit: 60) - expect(derived.enrichment_allowance).to eq(0) - expect(derived).not_to be_feasible + expect(derived.poll_allowance + derived.reserve + derived.enrichment_allowance).to eq(60) + expect(derived).to be_feasible + end + + it "rejects the first sum past the limit" do + expect(described_class.derive(configuration: configuration(RATE_LIMIT_RESERVE: "45"), limit: 60)) + .not_to be_feasible + end + + # Detail fallback is the exception path behind search batches, so an operator may + # turn it off outright; the batch lane still enriches on its own ledger. + it "accepts a zero detail-fallback allowance as a legal operating point" do + derived = described_class.derive( + configuration: configuration(CORE_DETAIL_FALLBACK_ALLOWANCE: "0"), limit: 60 + ) + + expect(derived).to be_feasible end it "rejects a polling requirement that exceeds the limit outright" do @@ -71,15 +105,31 @@ def configuration(**overrides) # A live x-ratelimit-limit lower than the configured default is GitHub's business, # not an operator error, so runtime degrades instead of crash-looping the worker. - # Polling wins the clamp because enrichment reaching zero is a documented outcome + # Polling wins the clamp because detail fallback reaching zero is a documented outcome # while polling stopping is a Story 1 failure. - it "keeps polling whole and gives enrichment what is left when the limit is lower" do + it "keeps polling first and gives detail fallback what is left when the limit is lower" do clamped = described_class.derive(configuration: configuration, limit: 15).clamped expect(clamped.poll_allowance).to eq(7) expect(clamped.enrichment_allowance).to eq(0) end + it "funds part of the allowance when the limit covers polling and some fallback" do + clamped = described_class.derive(configuration: configuration, limit: 22).clamped + + expect(clamped.poll_allowance).to eq(12) + expect(clamped.enrichment_allowance).to eq(2) + end + + # The allowance is a cap, not a remainder: spendable headroom above it belongs to + # nobody. Raising it to the leftover would silently turn a generous observed limit + # into detail-fallback spending the operator never asked for. + it "never raises the allowance above its configured cap, however much headroom the limit leaves" do + clamped = described_class.derive(configuration: configuration, limit: 5000).clamped + + expect(clamped.enrichment_allowance).to eq(4) + end + it "never derives a negative allowance, which the schema's CHECK would reject" do clamped = described_class.derive(configuration: configuration, limit: 4).clamped @@ -105,12 +155,12 @@ def configuration(**overrides) end # The single strongest argument for deriving the guarantees rather than storing them: - # members computed at .derive would have frozen the pre-clamp numbers into the clamped - # copy, and -3 actor requests is not a thing the ledger could enforce. + # members computed at .derive would have frozen the pre-clamp 2/2 into the clamped + # copy, and the ledger would enforce guarantees the clamped allowance cannot fund. it "recomputes the guarantees from the clamped allowance rather than from the derived one" do derived = described_class.derive(configuration: configuration, limit: 15) - expect(derived.actor_guarantee).to eq(-3) + expect(derived).to have_attributes(actor_guarantee: 2, repository_guarantee: 2) expect(derived.clamped).to have_attributes(actor_guarantee: 0, repository_guarantee: 0) end end @@ -118,10 +168,10 @@ def configuration(**overrides) describe "the fairness split (plan §10)" do def split(allowance, share) = described_class.split(allowance, Rational(share)) - it "splits the pinned defaults into twenty actor and twenty repository attempts" do + it "splits the pinned defaults into two actor and two repository fallback attempts" do derived = described_class.derive(configuration: configuration, limit: 60) - expect(derived).to have_attributes(actor_guarantee: 20, repository_guarantee: 20) + expect(derived).to have_attributes(actor_guarantee: 2, repository_guarantee: 2) end # §10 writes the formula as a floor and a subtraction, so the odd attempt goes to @@ -148,7 +198,7 @@ def split(allowance, share) = described_class.split(allowance, Rational(share)) expect(split(100, "0.29")).to eq(actor: 29, repository: 71) end - it "gives the single attempt of the smallest feasible allowance to repository" do + it "gives the single attempt of the smallest allowance to repository" do expect(split(1, "0.5")).to eq(actor: 0, repository: 1) end @@ -160,14 +210,19 @@ def split(allowance, share) = described_class.split(allowance, Rational(share)) expect(split(40, "1.0")).to eq(actor: 40, repository: 0) end - it "keys the split by request class, so the ledger fetches rather than branching" do - expect(split(40, "0.5").keys).to match_array(Github::Request::ENRICHMENT_CLASSES) + # Keyed by the two detail classes: only :actor and :repository spend core detail + # fallback, while the search pair debits its own per-minute ledger with no share. + it "keys the split by detail request class, so the ledger fetches rather than branching" do + expect(split(40, "0.5").keys).to match_array(Github::Request::DETAIL_CLASSES) end - # §10's rejection rule is about capacity for Story 3, and one attempt of capacity is - # one attempt whichever class ends up holding it. + # Feasibility is about the total, and one attempt of capacity is one attempt of + # capacity whichever class ends up holding it — a zero guarantee is relieved by + # borrowing, not by rejecting the configuration. it "leaves feasibility to the total, so a zero guarantee is not an infeasible configuration" do - derived = described_class.derive(configuration: configuration(RATE_LIMIT_RESERVE: "47"), limit: 60) + derived = described_class.derive( + configuration: configuration(CORE_DETAIL_FALLBACK_ALLOWANCE: "1"), limit: 60 + ) expect(derived).to be_feasible expect(derived).to have_attributes(enrichment_allowance: 1, actor_guarantee: 0, repository_guarantee: 1) @@ -176,7 +231,7 @@ def split(allowance, share) = described_class.split(allowance, Rational(share)) it "reports both guarantees in the line the ledger logs at window initialization" do derived = described_class.derive(configuration: configuration, limit: 60) - expect(derived.to_log).to include(actor_guarantee: 20, repository_guarantee: 20) + expect(derived.to_log).to include(actor_guarantee: 2, repository_guarantee: 2) end end end diff --git a/spec/services/github/budget_ledger_bootstrap_spec.rb b/spec/services/github/budget_ledger_bootstrap_spec.rb index e95a52b..60e2c4d 100644 --- a/spec/services/github/budget_ledger_bootstrap_spec.rb +++ b/spec/services/github/budget_ledger_bootstrap_spec.rb @@ -163,22 +163,24 @@ def bootstrap_poll! # configuration created it. ADR 0004 accepts that: allowances change at window # boundaries, "the price of keeping the change atomic with the counter reset". it "does not restate the allowances of a row another configuration created" do - described_class.new(configuration: configuration_with(POLL_INTERVAL_SECONDS: "600")) + described_class.new(configuration: configuration_with(POLL_INTERVAL_SECONDS: "600", + CORE_DETAIL_FALLBACK_ALLOWANCE: "6")) .bootstrap!(now: frozen_time) ledger.bootstrap!(now: frozen_time) - expect(budget).to have_attributes(poll_allowance: 6, enrichment_allowance: 46) + expect(budget).to have_attributes(poll_allowance: 6, enrichment_allowance: 6) end it "adopts the current configuration at the first window it opens" do - described_class.new(configuration: configuration_with(POLL_INTERVAL_SECONDS: "600")) + described_class.new(configuration: configuration_with(POLL_INTERVAL_SECONDS: "600", + CORE_DETAIL_FALLBACK_ALLOWANCE: "6")) .bootstrap!(now: frozen_time) ledger.reserve!(:poll, now: frozen_time) ledger.reconcile!(snapshot, request_class: :poll, now: frozen_time) - expect(budget).to have_attributes(poll_allowance: 12, enrichment_allowance: 40) + expect(budget).to have_attributes(poll_allowance: 12, enrichment_allowance: 4) end # A block set before the window was ever initialized. denial_reason derives blocking diff --git a/spec/services/github/budget_ledger_shared_ip_spec.rb b/spec/services/github/budget_ledger_shared_ip_spec.rb index 33c6803..5c396b9 100644 --- a/spec/services/github/budget_ledger_shared_ip_spec.rb +++ b/spec/services/github/budget_ledger_shared_ip_spec.rb @@ -76,27 +76,30 @@ def snapshot(reset_at: window_reset, observed_at: frozen_time, **overrides) describe "an observed limit that changes mid-window" do # ADR 0004: "allowances are re-derived at window rollover and initialization, not # mid-window... the price of keeping the change atomic with the counter reset." + # A limit of 22 cannot fund the full 12 + 4 + 8 commitment, so a re-derivation here + # would have clamped the detail allowance to 2 — which is how staying at 4 proves + # nothing was re-derived. it "stores the new limit without re-deriving the allowances under it" do active_window - ledger.reconcile!(snapshot("x-ratelimit-limit" => "30", "x-ratelimit-remaining" => "20"), + ledger.reconcile!(snapshot("x-ratelimit-limit" => "22", "x-ratelimit-remaining" => "20"), now: frozen_time) - expect(budget).to have_attributes(limit: 30, poll_allowance: 12, enrichment_allowance: 40) + expect(budget).to have_attributes(limit: 22, poll_allowance: 12, enrichment_allowance: 4) end - it "derives from the new limit at the next rollover" do + it "derives from the new limit at the next rollover, clamping the detail allowance to what it funds" do active_window - ledger.reconcile!(snapshot("x-ratelimit-limit" => "30", "x-ratelimit-remaining" => "20"), + ledger.reconcile!(snapshot("x-ratelimit-limit" => "22", "x-ratelimit-remaining" => "20"), now: frozen_time) ledger.reserve!(:poll, now: window_reset + 1) - expect(budget).to have_attributes(poll_allowance: 12, enrichment_allowance: 10) + expect(budget).to have_attributes(poll_allowance: 12, enrichment_allowance: 2) end - # The over-commitment in between is real and is what the reserve guard covers: the row - # still authorises 52 attempts against a limit of 30, and `remaining <= reserve` is what - # actually stops it. + # The gap in between is real and is what the reserve guard covers: the row's own + # counters would still authorise sixteen attempts, and `remaining <= reserve` is what + # actually stops them against the drained window GitHub reports. it "leaves the reserve guard, not the allowances, holding the line until then" do active_window ledger.reconcile!(snapshot("x-ratelimit-limit" => "30", "x-ratelimit-remaining" => "8"), diff --git a/spec/services/github/budget_ledger_spec.rb b/spec/services/github/budget_ledger_spec.rb index 6e3fa8a..4ac2a89 100644 --- a/spec/services/github/budget_ledger_spec.rb +++ b/spec/services/github/budget_ledger_spec.rb @@ -29,7 +29,7 @@ def snapshot(**overrides) ledger.bootstrap!(now: frozen_time) expect(budget).to have_attributes( - window_status: "uninitialized", poll_allowance: 12, enrichment_allowance: 40, + window_status: "uninitialized", poll_allowance: 12, enrichment_allowance: 4, reserve: 8, poll_used: 0, enrichment_used: 0, limit: nil, remaining: nil, reset_at: nil ) end @@ -63,9 +63,9 @@ def snapshot(**overrides) it "records each enrichment request against its own class share" do active_window - 3.times { ledger.reserve!(:actor, now: frozen_time) } + 2.times { ledger.reserve!(:actor, now: frozen_time) } - expect(budget).to have_attributes(actor_share_used: 3, repository_share_used: 0) + expect(budget).to have_attributes(actor_share_used: 2, repository_share_used: 0) end it "decrements the local remaining estimate, so a failure is spent against it too" do @@ -84,8 +84,8 @@ def snapshot(**overrides) end describe "class isolation (plan §10)" do - # §10: enrichment exhausting its forty attempts never stops polling, and polling - # exhausting its twelve never stops enrichment. + # §10: detail fallback exhausting its four attempts never stops polling, and polling + # exhausting its twelve never stops the fallback. it "denies polling once its allowance is spent, without touching enrichment" do active_window(poll_used: 12) @@ -95,7 +95,7 @@ def snapshot(**overrides) end it "denies enrichment once its allowance is spent, without touching polling" do - active_window(enrichment_used: 40) + active_window(enrichment_used: 4) expect { ledger.reserve!(:actor, now: frozen_time) } .to raise_error(Github::Errors::BudgetExhausted, /class_allowance_exhausted/) @@ -199,7 +199,7 @@ def snapshot(**overrides) describe "window rollover" do it "resets the counters when the stored window boundary has passed" do - active_window(poll_used: 12, enrichment_used: 40, actor_share_used: 20, repository_share_used: 20) + active_window(poll_used: 12, enrichment_used: 4, actor_share_used: 2, repository_share_used: 2) ledger.reserve!(:poll, now: window_reset + 1) @@ -420,7 +420,7 @@ def superseding_snapshot # The clock-driven rollover inside reserve! has no in-flight request to carry, so it # still starts the window clean. it "still starts a clean window when the rollover happens before a reservation" do - active_window(poll_used: 12, enrichment_used: 40) + active_window(poll_used: 12, enrichment_used: 4) ledger.reserve!(:poll, now: window_reset + 1) @@ -682,7 +682,7 @@ def block(reason, until_at: frozen_time + 60) ledger.reserve!(:poll, now: frozen_time) ledger.reconcile!(snapshot, now: frozen_time) - expect(budget).to have_attributes(poll_allowance: 12, enrichment_allowance: 40, reserve: 8) + expect(budget).to have_attributes(poll_allowance: 12, enrichment_allowance: 4, reserve: 8) end # A limit lower than the configured default is GitHub's business, not an operator @@ -709,7 +709,7 @@ def block(reason, until_at: frozen_time + 60) expect(Rails.logger).to have_received(:warn).with( hash_including(event: "budget.allowances_clamped", - requested_poll_allowance: 12, requested_enrichment_allowance: -5, + requested_poll_allowance: 12, requested_enrichment_allowance: 4, poll_allowance: 7, enrichment_allowance: 0) ) end @@ -742,7 +742,7 @@ def block(reason, until_at: frozen_time + 60) observed_at: later ), now: later) - expect(budget).to have_attributes(poll_allowance: 12, enrichment_allowance: 40, limit: 60) + expect(budget).to have_attributes(poll_allowance: 12, enrichment_allowance: 4, limit: 60) end end @@ -761,7 +761,7 @@ def live_sources(count) ledger.reconcile!(snapshot, now: frozen_time) - expect(budget).to have_attributes(poll_allowance: 24, enrichment_allowance: 28) + expect(budget).to have_attributes(poll_allowance: 24, enrichment_allowance: 4) end it "re-derives it at rollover, so an added source takes effect within the hour" do @@ -770,7 +770,7 @@ def live_sources(count) ledger.reserve!(:poll, now: window_reset + 1) - expect(budget).to have_attributes(poll_allowance: 24, enrichment_allowance: 28) + expect(budget).to have_attributes(poll_allowance: 24, enrichment_allowance: 4) end # #bootstrap! runs ahead of every reservation, so asking event_sources there would put a @@ -780,7 +780,7 @@ def live_sources(count) ledger.bootstrap!(now: frozen_time) - expect(budget).to have_attributes(poll_allowance: 12, enrichment_allowance: 40) + expect(budget).to have_attributes(poll_allowance: 12, enrichment_allowance: 4) end it "ignores a disabled or failed source, which will never spend a poll attempt" do @@ -792,18 +792,18 @@ def live_sources(count) ledger.reserve!(:poll, now: frozen_time) ledger.reconcile!(snapshot, now: frozen_time) - expect(budget).to have_attributes(poll_allowance: 12, enrichment_allowance: 40) + expect(budget).to have_attributes(poll_allowance: 12, enrichment_allowance: 4) end end describe "fairness shares (plan §10)" do # §10's reason for the split, in one example: one observed live page held ~92 - # repositories against ~89 actors, so a repository-first policy would spend the whole - # hourly allowance before a single actor was enriched. + # repositories against ~89 actors, so a repository-first fallback policy would spend + # the whole detail allowance before a single actor was fetched. it "stops a repository flood at its guarantee, leaving the actor share untouched" do active_window - 20.times { ledger.reserve!(:repository, now: frozen_time) } + 2.times { ledger.reserve!(:repository, now: frozen_time) } expect { ledger.reserve!(:repository, now: frozen_time) } .to raise_error(Github::Errors::BudgetExhausted, /share_exhausted/) @@ -811,39 +811,42 @@ def live_sources(count) end it "grants an actor reservation right up to its guarantee" do - active_window(actor_share_used: 19, enrichment_used: 19) + active_window(actor_share_used: 1, enrichment_used: 1) expect { ledger.reserve!(:actor, now: frozen_time) } - .to change { budget.actor_share_used }.from(19).to(20) + .to change { budget.actor_share_used }.from(1).to(2) end it "refuses the next one while the caller has not reported the other class quiet" do - active_window(actor_share_used: 20, enrichment_used: 20) + active_window(actor_share_used: 2, enrichment_used: 2) expect { ledger.reserve!(:actor, now: frozen_time) } .to raise_error(Github::Errors::BudgetExhausted, /share_exhausted/) end it "spends nothing when it refuses a share, so a denied reservation costs no quota" do - active_window(actor_share_used: 20, enrichment_used: 20) + active_window(actor_share_used: 2, enrichment_used: 2) expect { suppress(Github::Errors::BudgetExhausted) { ledger.reserve!(:actor, now: frozen_time) } } - .not_to change { budget.enrichment_used }.from(20) + .not_to change { budget.enrichment_used }.from(2) end it "grants the same reservation once the caller reports no eligible repository candidate" do - active_window(actor_share_used: 20, enrichment_used: 20) + active_window(actor_share_used: 2, enrichment_used: 2) expect { ledger.reserve!(:actor, now: frozen_time, borrow: true) } - .to change { budget.actor_share_used }.from(20).to(21) + .to change { budget.actor_share_used }.from(2).to(3) end # The plan's phrasing is "borrow the other's unused capacity", and capping at the whole # enrichment allowance authorizes exactly that set: actor_share_used + # repository_share_used == enrichment_used is an invariant of the debit statements, so # the class guard already limits a borrower to allowance - other_share_used. + # An explicit forty-attempt window: this example is about the arithmetic of the cap + # itself, so it keeps the roomier volume rather than the four-attempt default. it "lets a borrowing class spend the whole enrichment allowance and not one request more" do - active_window(actor_share_used: 20, repository_share_used: 5, enrichment_used: 25) + active_window(enrichment_allowance: 40, actor_share_used: 20, repository_share_used: 5, + enrichment_used: 25) 15.times { ledger.reserve!(:actor, now: frozen_time, borrow: true) } @@ -855,21 +858,23 @@ def live_sources(count) # The ordering is not arbitrary: both conditions are true here, and naming the share # would send an operator to ACTOR_ENRICHMENT_SHARE when the answer is the window. it "names the class allowance rather than the share once the whole budget is gone" do - active_window(actor_share_used: 40, enrichment_used: 40) + active_window(actor_share_used: 4, enrichment_used: 4) expect { ledger.reserve!(:actor, now: frozen_time) } .to raise_error(Github::Errors::BudgetExhausted, /class_allowance_exhausted/) end it "never applies a share to a poll, which has none to spend" do - active_window(actor_share_used: 40, repository_share_used: 40) + active_window(actor_share_used: 2, repository_share_used: 2, enrichment_used: 4) expect { ledger.reserve!(:poll, now: frozen_time) }.to change { budget.poll_used }.from(0).to(1) end + # At a quarter share of the four-attempt allowance the actor guarantee floors to one + # while repository keeps three, so one spent attempt each splits the two verdicts. it "derives the guarantees from the configured share rather than from a fixed half" do quarter = described_class.new(configuration: configuration_with(ACTOR_ENRICHMENT_SHARE: "0.25")) - active_window(actor_share_used: 10, repository_share_used: 10, enrichment_used: 20) + active_window(actor_share_used: 1, repository_share_used: 1, enrichment_used: 2) expect { quarter.reserve!(:actor, now: frozen_time) } .to raise_error(Github::Errors::BudgetExhausted, /share_exhausted/) @@ -902,7 +907,7 @@ def live_sources(count) it "keeps the two shares summing to the class counter, whichever class spends" do active_window - 3.times { ledger.reserve!(:actor, now: frozen_time) } + 2.times { ledger.reserve!(:actor, now: frozen_time) } 2.times { ledger.reserve!(:repository, now: frozen_time) } expect(budget.actor_share_used + budget.repository_share_used).to eq(budget.enrichment_used) diff --git a/spec/services/github/configuration_spec.rb b/spec/services/github/configuration_spec.rb index e0fcf1a..f7a4a64 100644 --- a/spec/services/github/configuration_spec.rb +++ b/spec/services/github/configuration_spec.rb @@ -30,7 +30,25 @@ def configuration(**overrides) # §11's coverage window, pinned at 86400 by §10. It arrives with the rich /status # rather than earlier because until Github::Enrichment::Coverage existed nothing # read it, and §16 forbids a knob with no consumer. - enrichment_coverage_window_seconds: 86_400 + enrichment_coverage_window_seconds: 86_400, + # Appendix F's staged-enrichment block: the search lane's per-minute budget, the + # core detail-fallback cap, and the cycle/retry/lease timings around them. + core_detail_fallback_allowance: 4, + search_request_ceiling: 10, + search_safety_reserve: 2, + search_batch_size: 10, + search_pacing_seconds: 6, + search_worker_concurrency: 1, + enrichment_cycle_budget_seconds: 55, + actor_enrichment_weight: 1, + repository_enrichment_weight: 1, + detail_fallback_max_attempts: 3, + enrichment_lease_seconds: 600, + enrichment_retry_base_seconds: 60, + enrichment_retry_max_seconds: 3_600, + enrichment_metrics_window_seconds: 3_600, + catch_up_min_sample_seconds: 900, + refresh_active_within_seconds: 604_800 ) end @@ -97,15 +115,91 @@ def configuration(**overrides) end end - it "accepts zero where zero is meaningful: no reserve, no retries, no redirects" do - config = configuration(RATE_LIMIT_RESERVE: "0", MAX_HTTP_RETRIES: "0", MAX_REDIRECTS: "0") + it "accepts zero where zero is meaningful: no reserve, no retries, no redirects, no fallback" do + config = configuration(RATE_LIMIT_RESERVE: "0", MAX_HTTP_RETRIES: "0", MAX_REDIRECTS: "0", + CORE_DETAIL_FALLBACK_ALLOWANCE: "0", SEARCH_SAFETY_RESERVE: "0") expect { config.validate! }.not_to raise_error end - it "rejects a negative retry or redirect count" do - expect { configuration(MAX_REDIRECTS: "-1").validate! } - .to raise_error(Github::Errors::ConfigurationError, /MAX_REDIRECTS/) + # Zero pacing is the offline fixture walkthrough's operating point: a one-shot runs + # both lanes back to back with no wait between search requests. + it "accepts zero pacing, which disables the wait rather than breaking it" do + expect { configuration(SEARCH_PACING_SECONDS: "0").validate! }.not_to raise_error + end + + it "rejects a negative value in every member of the non-negative group" do + described_class::NON_NEGATIVE_INTEGERS.each_value do |variable| + expect { configuration(variable.to_sym => "-1").validate! } + .to raise_error(Github::Errors::ConfigurationError, /#{variable}/), + "expected #{variable}=-1 to be rejected" + end + end + end + + # Appendix F's structural rules between the staged-enrichment knobs: each one relates + # two numbers, so the group validates after the sign checks and names the variable an + # operator must move. + describe "validation of the staged-enrichment knobs (Appendix F)" do + it "rejects a search reserve that swallows the whole ceiling" do + expect { configuration(SEARCH_SAFETY_RESERVE: "10").validate! } + .to raise_error(Github::Errors::ConfigurationError, /SEARCH_SAFETY_RESERVE/) + end + + it "accepts a reserve one below the ceiling, which leaves one spendable request" do + expect { configuration(SEARCH_SAFETY_RESERVE: "9").validate! }.not_to raise_error + end + + # GitHub caps repeated qualifiers well below the per_page maximum; ten is the batch + # size the live probes behind Appendix F verified. + it "accepts a batch size of ten and rejects eleven" do + expect { configuration(SEARCH_BATCH_SIZE: "10").validate! }.not_to raise_error + expect { configuration(SEARCH_BATCH_SIZE: "11").validate! } + .to raise_error(Github::Errors::ConfigurationError, /SEARCH_BATCH_SIZE/) + end + + # The global request gate serialises outbound calls, so a second search worker could + # only queue behind the first while holding claims whose leases are burning down. + it "requires exactly one search worker while the request gate serialises outbound calls" do + expect { configuration(SEARCH_WORKER_CONCURRENCY: "2").validate! } + .to raise_error(Github::Errors::ConfigurationError, /SEARCH_WORKER_CONCURRENCY/) + end + + it "rejects a retry base above the retry ceiling, and accepts them equal" do + expect { configuration(ENRICHMENT_RETRY_BASE_SECONDS: "3601").validate! } + .to raise_error(Github::Errors::ConfigurationError, /ENRICHMENT_RETRY_BASE_SECONDS/) + expect { configuration(ENRICHMENT_RETRY_BASE_SECONDS: "3600").validate! }.not_to raise_error + end + + # The cycle runs inside a 60-second dispatch tick, so a budget at or past the tick + # would let one cycle overlap the next dispatch decision. + it "rejects a cycle budget at or above the sixty-second dispatch tick" do + expect { configuration(ENRICHMENT_CYCLE_BUDGET_SECONDS: "60").validate! } + .to raise_error(Github::Errors::ConfigurationError, /ENRICHMENT_CYCLE_BUDGET_SECONDS/) + expect { configuration(ENRICHMENT_CYCLE_BUDGET_SECONDS: "59").validate! }.not_to raise_error + end + + # A pacing wait the cycle budget cannot contain would defer every batch after the + # first, forever: no cycle could ever wait out its own pacing. + it "rejects a pacing interval the cycle budget cannot wait out" do + expect { configuration(SEARCH_PACING_SECONDS: "55").validate! } + .to raise_error(Github::Errors::ConfigurationError, /SEARCH_PACING_SECONDS/) + expect { configuration(SEARCH_PACING_SECONDS: "54").validate! }.not_to raise_error + end + + # The lease must outlive the worst-case single fetch — (MAX_HTTP_RETRIES + 1) x + # (MAX_REDIRECTS + 1) attempts, each waiting out the gate and both HTTP timeouts — + # or a slow-but-alive worker loses its claimed rows to a stale-lease reclaim + # mid-request. 585 at the pinned defaults, and the boundary is strict. + it "rejects a lease that the worst-case fetch could outlive, at the exact boundary" do + expect { configuration(ENRICHMENT_LEASE_SECONDS: "585").validate! } + .to raise_error(Github::Errors::ConfigurationError, /ENRICHMENT_LEASE_SECONDS/) + expect { configuration(ENRICHMENT_LEASE_SECONDS: "586").validate! }.not_to raise_error + end + + it "derives that worst case from the gate wait, the timeouts, the retries and the redirects" do + expect(configuration.worst_case_fetch_seconds).to eq(585) + expect(configuration(MAX_HTTP_RETRIES: "0", MAX_REDIRECTS: "0").worst_case_fetch_seconds).to eq(65) end end @@ -163,27 +257,32 @@ def configuration(**overrides) end end - describe "startup validation of the allowance split (plan §10)" do - it "accepts the pinned defaults, which leave forty enrichment attempts an hour" do + describe "startup validation of the allowance split (plan §10, Appendix F)" do + it "accepts the pinned defaults, which commit twelve poll and four fallback attempts" do expect(configuration.validate!.allowances) - .to have_attributes(poll_allowance: 12, enrichment_allowance: 40) + .to have_attributes(poll_allowance: 12, enrichment_allowance: 4) end # Polling every 60 seconds is 60 attempts an hour — the entire unauthenticated # limit — which is exactly the V1 defect Appendix A item 2 records. it "rejects a cadence that would spend the whole hourly limit on polling" do expect { configuration(POLL_INTERVAL_SECONDS: "60").validate! } - .to raise_error(Github::Errors::ConfigurationError, /no capacity for enrichment/) + .to raise_error(Github::Errors::ConfigurationError, /exceed the core limit/) end + # The message names CORE_DETAIL_FALLBACK_ALLOWANCE among the levers: the allowance + # is a configured commitment now, so lowering it is a legitimate way out. it "names the offending numbers so an operator can fix it without reading the code" do expect { configuration(POLL_INTERVAL_SECONDS: "60").validate! } - .to raise_error(/poll_allowance \(60\).*RATE_LIMIT_RESERVE \(8\)/m) + .to raise_error(/poll_allowance \(60\).*CORE_DETAIL_FALLBACK_ALLOWANCE \(4\).*RATE_LIMIT_RESERVE \(8\)/m) end - it "rejects the exact boundary, because a zero enrichment allowance fails Story 3" do - expect { configuration(RATE_LIMIT_RESERVE: "48").validate! } - .to raise_error(Github::Errors::ConfigurationError) + # Appendix F's predicate is <= : the three commitments may fill the limit exactly, + # because each is a real, funded plan — and the first request past it is rejected. + it "accepts the sum landing exactly on the limit, and rejects one attempt more" do + expect { configuration(RATE_LIMIT_RESERVE: "44").validate! }.not_to raise_error + expect { configuration(RATE_LIMIT_RESERVE: "45").validate! } + .to raise_error(Github::Errors::ConfigurationError, /CORE_DETAIL_FALLBACK_ALLOWANCE/) end it "returns itself when valid, so the initializer can validate and assign in one line" do @@ -250,9 +349,14 @@ def configuration(**overrides) end # It is a report, not a rule: §7 already accepts that these are request-*attempt* - # allowances, so an amplifying configuration must still boot. + # allowances, so an amplifying configuration must still boot. The lease is raised + # alongside, because the same retries and redirects stretch the worst-case fetch to + # 2340 seconds and the lease rule is a genuine rejection. it "is not a rejection, so an amplifying configuration still validates" do - expect { configuration(MAX_HTTP_RETRIES: "5", MAX_REDIRECTS: "5").validate! }.not_to raise_error + amplified = configuration(MAX_HTTP_RETRIES: "5", MAX_REDIRECTS: "5", + ENRICHMENT_LEASE_SECONDS: "2341") + + expect { amplified.validate! }.not_to raise_error end end diff --git a/spec/services/github/enrichment/actor_document_spec.rb b/spec/services/github/enrichment/actor_document_spec.rb index 95ca2f0..a182e81 100644 --- a/spec/services/github/enrichment/actor_document_spec.rb +++ b/spec/services/github/enrichment/actor_document_spec.rb @@ -11,12 +11,12 @@ def parse(document, id: github_id) described_class.parse(document.is_a?(String) ? document : JSON.generate(document), github_id: id) end - describe "the §7 mapping" do - it "populates name and the whole document, which is all §7 assigns to enrichment" do + describe "the useful-data contract" do + it "populates account_type and the whole document, which is all the contract assigns" do document = described_class.parse(body, github_id: github_id) expect(document).to be_ok - expect(document.attributes[:name]).to eq("The Octocat") + expect(document.attributes[:account_type]).to eq("User") expect(document.attributes[:raw_payload]).to include("login" => "octocat", "id" => github_id) end @@ -26,35 +26,39 @@ def parse(document, id: github_id) it "never writes login or avatar_url, which the envelope owns" do document = described_class.parse(body, github_id: github_id) - expect(document.attributes.keys).to contain_exactly(:name, :raw_payload) + expect(document.attributes.keys).to contain_exactly(:account_type, :raw_payload) end + # raw_payload is the full item, always: whatever a projection ignores today, the + # enriched truth is preserved verbatim for a later reading. it "keeps unknown fields in the payload without mapping them to columns" do - document = parse({ "id" => github_id, "name" => "Octo", "invented_field" => true }) + document = parse({ "id" => github_id, "type" => "User", "invented_field" => true }) expect(document.attributes[:raw_payload]).to include("invented_field" => true) - expect(document.attributes.keys).to contain_exactly(:name, :raw_payload) + expect(document.attributes.keys).to contain_exactly(:account_type, :raw_payload) end end - # §7's tolerant-parser doctrine: identity fields are strict, everything else degrades to - # NULL. Refusing a whole document over an optional field would throw away what did - # arrive, and a malformed verdict is destructive — it writes permanent_failure. - describe "tolerance for optional fields" do - it "stores a null name rather than refusing the document" do - expect(parse({ "id" => github_id, "name" => nil }).attributes[:name]).to be_nil - end + # `type` is a contract field, not an optional one: a Search item that cannot say whether + # the account is a User or an Organization has not satisfied the useful-data contract, + # and the row goes to the bounded detail fallback rather than being marked complete. + describe "the type contract" do + it "refuses a document with no type at all" do + document = parse({ "id" => github_id }) - it "stores a null name when the field is absent entirely" do - expect(parse({ "id" => github_id }).attributes[:name]).to be_nil + expect(document).not_to be_ok + expect(document.error_code).to eq("invalid_contract_field") + expect(document.error_message).to include("type") end - it "degrades a non-String name to null rather than storing a number in a text column" do - expect(parse({ "id" => github_id, "name" => 42 }).attributes[:name]).to be_nil + it "refuses a blank type rather than storing an empty string" do + expect(parse({ "id" => github_id, "type" => " " }).error_code).to eq("invalid_contract_field") + expect(parse({ "id" => github_id, "type" => "" }).error_code).to eq("invalid_contract_field") end - it "treats a blank name as absent" do - expect(parse({ "id" => github_id, "name" => " " }).attributes[:name]).to be_nil + it "refuses a non-String type rather than coercing it" do + expect(parse({ "id" => github_id, "type" => 42 }).error_code).to eq("invalid_contract_field") + expect(parse({ "id" => github_id, "type" => nil }).error_code).to eq("invalid_contract_field") end end @@ -75,8 +79,8 @@ def parse(document, id: github_id) end it "refuses a document with no integer id, which cannot prove which row it describes" do - expect(parse({ "login" => "octocat" }).error_code).to eq("missing_identity") - expect(parse({ "id" => "583231" }).error_code).to eq("missing_identity") + expect(parse({ "login" => "octocat", "type" => "User" }).error_code).to eq("missing_identity") + expect(parse({ "id" => "583231", "type" => "User" }).error_code).to eq("missing_identity") end # GitHub logins are recyclable, so a stale actor.url can legitimately resolve to a @@ -84,7 +88,7 @@ def parse(document, id: github_id) # push_events.github_actor_id still relies on, which is why it gets its own kind rather # than being folded into "malformed". it "refuses another entity's document under its own error code" do - document = parse({ "id" => 999, "name" => "Someone Else" }) + document = parse({ "id" => 999, "type" => "User" }) expect(document.kind).to eq(:identity_mismatch) expect(document.error_message).to include("999", "583231") diff --git a/spec/services/github/enrichment/admission_spec.rb b/spec/services/github/enrichment/admission_spec.rb new file mode 100644 index 0000000..6ac24f0 --- /dev/null +++ b/spec/services/github/enrichment/admission_spec.rb @@ -0,0 +1,150 @@ +require "rails_helper" + +# Read-side admission for the two enrichment lanes. Advisory only: the ledgers re-check +# everything under their row locks, so what these examples pin down is the churn +# contract — a denied tick enqueues no cycle, names its reason, and says when to ask +# again — plus the read-only discipline that a pre-check must never create ledger rows. +RSpec.describe Github::Enrichment::Admission do + subject(:admission) { described_class.new } + + describe "#search" do + # The search ledger self-bootstraps from configuration, unlike core — so a missing + # row means "nothing has ever been denied", not "nothing is known". + it "grants when no search ledger row exists, and creates none" do + verdict = admission.search(now: frozen_time) + + expect(verdict).to be_granted + expect(verdict).to have_attributes(reason: nil, retry_in_seconds: nil) + expect(GithubSearchBudget.count).to eq(0) + end + + it "grants against a fresh mid-window row" do + active_search_window + + expect(admission.search(now: frozen_time)).to be_granted + end + + it "denies :search_blocked with the seconds until the block lifts" do + active_search_window(blocked_until: frozen_time + 30) + + expect(admission.search(now: frozen_time)) + .to have_attributes(reason: :search_blocked, retry_in_seconds: 30.0) + end + + it "denies :search_pacing with the seconds until the pacing interval elapses" do + active_search_window(last_request_at: frozen_time - 2) + + expect(admission.search(now: frozen_time)) + .to have_attributes(reason: :search_pacing, retry_in_seconds: 4.0) + end + + it "denies :search_reserve_reached until the reset GitHub named" do + active_search_window(remaining: 2) + + expect(admission.search(now: frozen_time)) + .to have_attributes(reason: :search_reserve_reached, retry_in_seconds: 60.0) + end + + # No reset was ever observed, so there is no instant to name — nil is "ask the + # ledger again later", not "ask in zero seconds". + it "denies :search_reserve_reached with no instant when no reset is known" do + active_search_window(remaining: 2, reset_at: nil) + + expect(admission.search(now: frozen_time)) + .to have_attributes(reason: :search_reserve_reached, retry_in_seconds: nil) + end + + it "denies :search_ceiling_exhausted until the reset" do + active_search_window(remaining: nil, used: 8) + + expect(admission.search(now: frozen_time)) + .to have_attributes(reason: :search_ceiling_exhausted, retry_in_seconds: 60.0) + end + + it "denies :search_ceiling_exhausted with no instant when no reset is known" do + active_search_window(remaining: nil, used: 8, reset_at: nil) + + expect(admission.search(now: frozen_time)) + .to have_attributes(reason: :search_ceiling_exhausted, retry_in_seconds: nil) + end + + # Counters from an elapsed window are stale: the ledger will roll them on its next + # reservation, so a pre-check denying on them would withhold a grantable cycle. + it "grants over spent counters once GitHub's reset instant has passed" do + active_search_window(remaining: 2, used: 8, reset_at: frozen_time - 1) + + expect(admission.search(now: frozen_time)).to be_granted + end + + it "grants over a header-less window once a full search window has passed in silence" do + active_search_window(remaining: nil, used: 8, reset_at: nil, + last_request_at: frozen_time - 61) + + expect(admission.search(now: frozen_time)).to be_granted + end + + # Pacing outranks window staleness deliberately, mirroring the ledger: the roll + # resets counters, never the wire-spacing contract. + it "still paces even when the window behind the counters has elapsed" do + active_search_window(used: 8, reset_at: frozen_time - 1, + last_request_at: frozen_time - 2) + + expect(admission.search(now: frozen_time)).to have_attributes(reason: :search_pacing) + end + end + + describe "#detail" do + # The detail lane keeps the core ledger's bootstrap discipline: no initialized + # window, no enrichment (§7) — and a read path must never create the row. + it "denies :window_uninitialized when no core ledger row exists, and creates none" do + verdict = admission.detail(now: frozen_time) + + expect(verdict).to have_attributes(reason: :window_uninitialized, retry_in_seconds: nil) + expect(GithubApiBudget.count).to eq(0) + end + + it "denies :window_uninitialized while the row exists but no response opened it" do + active_budget_window(window_status: "uninitialized", window_initialized_at: nil) + + expect(admission.detail(now: frozen_time)) + .to have_attributes(reason: :window_uninitialized) + end + + # A dead window's counters prove nothing about the current hour; the ledger rolls + # them on its next reservation, and until then the honest answer is "elapsed". + it "denies :window_elapsed once the stored reset has passed" do + active_budget_window(reset_at: frozen_time - 1) + + expect(admission.detail(now: frozen_time)) + .to have_attributes(reason: :window_elapsed, retry_in_seconds: nil) + end + + it "denies :globally_blocked with the seconds until the block lifts" do + active_budget_window(global_blocked_until: frozen_time + 120) + + expect(admission.detail(now: frozen_time)) + .to have_attributes(reason: :globally_blocked, retry_in_seconds: 120.0) + end + + it "grants again once the global block has passed" do + active_budget_window(global_blocked_until: frozen_time - 1) + + expect(admission.detail(now: frozen_time)).to be_granted + end + + # enrichment_allowance now means CORE_DETAIL_FALLBACK_ALLOWANCE (=4), and the class + # block is derived from the counters — spent allowance defers to the window reset. + it "denies :class_exhausted once the detail-fallback allowance is spent" do + active_budget_window(enrichment_used: 4) + + expect(admission.detail(now: frozen_time)) + .to have_attributes(reason: :class_exhausted, retry_in_seconds: 3600.0) + end + + it "grants while the detail-fallback allowance has attempts left" do + active_budget_window(enrichment_used: 3) + + expect(admission.detail(now: frozen_time)).to be_granted + end + end +end diff --git a/spec/services/github/enrichment/backlog_metrics_spec.rb b/spec/services/github/enrichment/backlog_metrics_spec.rb index 9620811..3e55f50 100644 --- a/spec/services/github/enrichment/backlog_metrics_spec.rb +++ b/spec/services/github/enrichment/backlog_metrics_spec.rb @@ -3,48 +3,183 @@ RSpec.describe Github::Enrichment::BacklogMetrics do let(:now) { frozen_time } - def capture = described_class.capture(now: now) - - it "counts pending and retryable rows even when their next attempt is deferred" do - create_actor(github_id: 1, enrichment_status: "pending") - create_actor(github_id: 2, enrichment_status: "retryable_failure", - next_retry_at: now + 3600) - create_actor(github_id: 3, enrichment_status: "complete", fetched_at: now) - create_actor(github_id: 4, enrichment_status: "permanent_failure") - - expect(capture.actor).to have_attributes( - status_counts: { - "pending" => 1, "complete" => 1, - "retryable_failure" => 1, "permanent_failure" => 1 - }, - backlog_count: 2 - ) + def capture(configuration: Github.configuration) + described_class.capture(now: now, configuration: configuration) end - it "uses created_at for the oldest wait, matching FIFO selection" do - create_actor(github_id: 1, created_at: now - 300, last_seen_at: now) - create_actor(github_id: 2, created_at: now - 900, last_seen_at: now - 10) + describe "status counts" do + it "drops statuses with no rows, publishing only counted facts" do + create_actor(github_id: 1) + + expect(capture.actor.status_counts).to eq("pending" => 1) + end + + it "counts pending and retryable rows even when their next attempt is deferred" do + create_actor(github_id: 1, enrichment_status: "pending") + create_actor(github_id: 2, enrichment_status: "retryable_failure", + enrichment_stage: "retry_scheduled", next_retry_at: now + 3600) + create_actor(github_id: 3, enrichment_status: "complete", + enrichment_stage: "contract_complete", fetched_at: now) + create_actor(github_id: 4, enrichment_status: "permanent_failure", + enrichment_stage: "terminal") + + expect(capture.actor).to have_attributes( + status_counts: { + "pending" => 1, "complete" => 1, + "retryable_failure" => 1, "permanent_failure" => 1 + }, + backlog_count: 2 + ) + end + end - expect(capture.actor).to have_attributes( - backlog_count: 2, - oldest_pending_at: now - 900, - oldest_pending_age_seconds: 900 - ) + describe "stage counts and oldest instants" do + # The opposite convention from status_counts, deliberately: the payload publishes + # every stage, so a consumer never has to distinguish "absent key" from "counted + # zero" when reading the pipeline. + it "keeps all seven stages with their zeros" do + create_actor(github_id: 1, enrichment_stage: "detail_pending", + detail_pending_at: now - 60) + + expect(capture.actor.stage_counts).to eq( + "batch_pending" => 0, "batch_in_flight" => 0, "detail_pending" => 1, + "detail_in_flight" => 0, "retry_scheduled" => 0, "contract_complete" => 0, + "terminal" => 0 + ) + end + + # created_at is the immutable FIFO clock BatchClaim orders by, so "oldest" here is + # the row a worker would actually choose next from that stage. + it "reports each stage's oldest created_at, nil where the stage is empty" do + create_actor(github_id: 1, created_at: now - 300) + create_actor(github_id: 2, created_at: now - 900) + create_actor(github_id: 3, enrichment_stage: "detail_pending", + detail_pending_at: now - 60, created_at: now - 600) + + stage_oldest = capture.actor.stage_oldest + + expect(stage_oldest.fetch("batch_pending")).to eq(now - 900) + expect(stage_oldest.fetch("detail_pending")).to eq(now - 600) + expect(stage_oldest.fetch("terminal")).to be_nil + expect(stage_oldest.keys).to eq(Enrichable::ENRICHMENT_STAGES) + end end - it "excludes older terminal rows from the oldest backlog wait" do - create_actor(github_id: 1, enrichment_status: "complete", - fetched_at: now, created_at: now - 1800) - create_actor(github_id: 2, enrichment_status: "permanent_failure", - created_at: now - 1200) - create_actor(github_id: 3, enrichment_status: "pending", - created_at: now - 300) - - expect(capture.actor).to have_attributes( - backlog_count: 1, - oldest_pending_at: now - 300, - oldest_pending_age_seconds: 300 - ) + describe "the contract backlog" do + # Appendix G's rule: not yet at the useful-data contract or a terminal outcome, and + # not a completed row transiting a refresh. A complete row mid-refresh sits in + # batch_in_flight or detail_pending without owing the contract anything. + it "excludes complete-status refresh transits and terminal rows" do + create_actor(github_id: 1, enrichment_status: "pending", + enrichment_stage: "batch_pending") + create_actor(github_id: 2, enrichment_status: "retryable_failure", + enrichment_stage: "retry_scheduled", next_retry_at: now + 60) + create_actor(github_id: 3, enrichment_status: "pending", + enrichment_stage: "detail_in_flight", detail_pending_at: now - 60) + create_actor(github_id: 4, enrichment_status: "complete", + enrichment_stage: "contract_complete", fetched_at: now) + create_actor(github_id: 5, enrichment_status: "complete", + enrichment_stage: "batch_in_flight", fetched_at: now - 90_000) + create_actor(github_id: 6, enrichment_status: "complete", + enrichment_stage: "detail_pending", detail_pending_at: now - 60, + fetched_at: now - 90_000) + create_actor(github_id: 7, enrichment_status: "permanent_failure", + enrichment_stage: "terminal", terminal_at: now - 60) + + expect(capture.actor).to have_attributes(contract_backlog_count: 3, + backlog_count: 3) + end + end + + describe "the oldest pending wait" do + it "uses created_at for the oldest wait, matching FIFO selection" do + create_actor(github_id: 1, created_at: now - 300, last_seen_at: now) + create_actor(github_id: 2, created_at: now - 900, last_seen_at: now - 10) + + expect(capture.actor).to have_attributes( + backlog_count: 2, + oldest_pending_at: now - 900, + oldest_pending_age_seconds: 900 + ) + end + + it "excludes older terminal rows from the oldest backlog wait" do + create_actor(github_id: 1, enrichment_status: "complete", + enrichment_stage: "contract_complete", + fetched_at: now, created_at: now - 1800) + create_actor(github_id: 2, enrichment_status: "permanent_failure", + enrichment_stage: "terminal", created_at: now - 1200) + create_actor(github_id: 3, enrichment_status: "pending", + created_at: now - 300) + + expect(capture.actor).to have_attributes( + backlog_count: 1, + oldest_pending_at: now - 300, + oldest_pending_age_seconds: 300 + ) + end + + it "reports nil oldest metrics for an empty backlog" do + expect(capture.actor).to have_attributes(backlog_count: 0, oldest_pending_at: nil, + oldest_pending_age_seconds: nil) + end + + it "clamps harmless database-clock skew instead of reporting a negative age" do + create_actor(github_id: 1, created_at: now + 1) + + expect(capture.actor.oldest_pending_age_seconds).to eq(0) + end + end + + describe "the windowed flow counters" do + # The three clocks are deliberately different columns: arrival is the row's + # immutable created_at, completion is contract_completed_at, and a terminal outcome + # is terminal_at. Each counter answers only its own clock. + it "counts arrivals strictly inside the trailing window" do + create_actor(github_id: 1, created_at: now - 3599) + create_actor(github_id: 2, created_at: now - 3600) + create_actor(github_id: 3, created_at: now - 3601) + + metrics = capture + + expect(metrics.window_seconds).to eq(3600) + expect(metrics.actor.arrivals).to eq(1) + end + + it "counts completions and terminals on their own clocks, not on arrival" do + create_actor(github_id: 1, enrichment_status: "complete", + enrichment_stage: "contract_complete", + created_at: now - 7200, contract_completed_at: now - 60) + create_actor(github_id: 2, enrichment_status: "permanent_failure", + enrichment_stage: "terminal", + created_at: now - 7200, terminal_at: now - 10) + create_actor(github_id: 3, enrichment_status: "complete", + enrichment_stage: "contract_complete", + created_at: now - 9000, contract_completed_at: now - 7200) + + expect(capture.actor).to have_attributes(arrivals: 0, completions: 1, terminals: 1) + end + + it "reads the window from the configuration it was given" do + create_actor(github_id: 1, created_at: now - 700) + configuration = configuration_with(ENRICHMENT_METRICS_WINDOW_SECONDS: "600") + + expect(capture(configuration: configuration)) + .to have_attributes(window_seconds: 600) + expect(capture(configuration: configuration).actor.arrivals).to eq(0) + expect(capture.actor.arrivals).to eq(1) + end + + # The throughput sample truncates to this instant, so it spans every row the table + # has ever held — terminal ones included. + it "reports the earliest created_at across every row, whatever its outcome" do + create_actor(github_id: 1, enrichment_status: "permanent_failure", + enrichment_stage: "terminal", created_at: now - 7200) + create_actor(github_id: 2, created_at: now - 300) + + expect(capture.actor.earliest_created_at).to eq(now - 7200) + expect(capture.repository.earliest_created_at).to be_nil + end end it "reports each entity class independently" do @@ -57,19 +192,6 @@ def capture = described_class.capture(now: now) oldest_pending_at: now - 600) end - it "reports nil oldest metrics for an empty backlog" do - entry = capture.actor - - expect(entry).to have_attributes(backlog_count: 0, oldest_pending_at: nil, - oldest_pending_age_seconds: nil) - end - - it "clamps harmless database-clock skew instead of reporting a negative age" do - create_actor(github_id: 1, created_at: now + 1) - - expect(capture.actor.oldest_pending_age_seconds).to eq(0) - end - it "reads persisted state without writing or initiating a GitHub request" do create_actor(github_id: 1) transport = fixture_transport @@ -79,17 +201,18 @@ def capture = described_class.capture(now: now) expect(transport.requests).to be_empty end - it "captures counts and the oldest row in one aggregate statement per entity class" do - create_actor(github_id: 1) + # A worker may commit between statements, so numbers taken from separate reads could + # publish a combination that never existed. One aggregate per table is the guarantee, + # and counting the SELECTs is how it stays true under refactoring. + it "captures everything in exactly two SELECT statements, one per entity table" do + create_actor(github_id: 1, enrichment_status: "complete", + enrichment_stage: "contract_complete", contract_completed_at: now - 60) create_repository(github_id: 2) - statements = capture_sql { capture } - actor_reads = statements.grep(/FROM "github_actors"/) - repository_reads = statements.grep(/FROM "github_repositories"/) + selects = capture_sql { capture }.grep(/\A\s*SELECT/i) - expect(actor_reads.one?).to be(true) - expect(repository_reads.one?).to be(true) - expect(actor_reads.first).to include("COUNT(CASE WHEN", "MIN(CASE WHEN") - expect(repository_reads.first).to include("COUNT(CASE WHEN", "MIN(CASE WHEN") + expect(selects.length).to eq(2) + expect(selects.count { |statement| statement.include?('FROM "github_actors"') }).to eq(1) + expect(selects.count { |statement| statement.include?('FROM "github_repositories"') }).to eq(1) end end diff --git a/spec/services/github/enrichment/backoff_spec.rb b/spec/services/github/enrichment/backoff_spec.rb index a09ad61..84667d5 100644 --- a/spec/services/github/enrichment/backoff_spec.rb +++ b/spec/services/github/enrichment/backoff_spec.rb @@ -5,15 +5,65 @@ # reaching into Kernel — the technique Github::PollBackoff's own spec uses. subject(:backoff) { described_class.new(random: Random.new(1234)) } - let(:no_jitter) { described_class.new(random: instance_double(Random, rand: 0.0)) } - let(:full_jitter) { described_class.new(random: instance_double(Random, rand: 1.0)) } + def no_jitter(**options) + described_class.new(random: instance_double(Random, rand: 0.0), **options) + end + + def full_jitter(**options) + described_class.new(random: instance_double(Random, rand: 1.0), **options) + end + + describe "the configured ladder" do + # Issue #45 makes the retry ladder configurable; the constants remain as the documented + # defaults ENRICHMENT_RETRY_BASE_SECONDS / ENRICHMENT_RETRY_MAX_SECONDS start from, and + # the two sources must not drift apart. + it "defaults to the one-minute floor and the one-window cap" do + configured = no_jitter(configuration: configuration_with) + + expect(configured.base_seconds).to eq(60) + expect(configured.max_seconds).to eq(3600) + expect(configured.base_seconds).to eq(described_class::BASE_SECONDS) + expect(configured.max_seconds).to eq(described_class::MAX_SECONDS) + end + + it "reads a tuned ladder from the configuration" do + configured = no_jitter(configuration: configuration_with( + "ENRICHMENT_RETRY_BASE_SECONDS" => "30", "ENRICHMENT_RETRY_MAX_SECONDS" => "120" + )) + + expect(configured.delay_for(1)).to eq(30.0) + expect(configured.delay_for(2)).to eq(60.0) + expect(configured.delay_for(3)).to eq(120.0) + expect(configured.delay_for(4)).to eq(120.0) + end + + it "lets explicit keyword arguments override the configuration" do + configured = no_jitter(base_seconds: 10, max_seconds: 40, + configuration: configuration_with( + "ENRICHMENT_RETRY_BASE_SECONDS" => "900" + )) + + expect(configured.delay_for(1)).to eq(10.0) + expect(configured.delay_for(2)).to eq(20.0) + expect(configured.delay_for(3)).to eq(40.0) + end + + # With both bounds supplied there is nothing left to read, so the process-wide + # configuration must not be touched — a spec-built Backoff with explicit bounds must + # not depend on the environment it happens to run in. + it "never reads the process configuration when both bounds are explicit" do + expect(Github).not_to receive(:configuration) + + expect(no_jitter(base_seconds: 10, max_seconds: 40).delay_for(1)).to eq(10.0) + end + end describe "#delay_for" do # §10 states the floor numerically ("≥ 1 minute"), and it is the one property that must # hold for every attempt count. - it "never schedules sooner than the one-minute floor, whatever the jitter" do + it "never schedules sooner than the configured floor, whatever the jitter" do (1..10).each do |attempts| - expect(backoff.delay_for(attempts)).to be >= described_class::BASE_SECONDS + expect(backoff.delay_for(attempts)).to be >= backoff.base_seconds end end @@ -28,14 +78,19 @@ expect(no_jitter.delay_for(0)).to eq(60.0) end - # Capped *after* jitter, so MAX_SECONDS is an honest bound rather than a bound plus up + # Capped *after* jitter, so max_seconds is an honest bound rather than a bound plus up # to 25 per cent. - it "caps at one rate-limit window even at full jitter" do - expect(full_jitter.delay_for(20)).to eq(described_class::MAX_SECONDS.to_f) + it "caps at the configured maximum even at full jitter" do + expect(full_jitter.delay_for(20)).to eq(3600.0) end - it "caps at one rate-limit window so a repeatedly failing row periodically rejoins the FIFO" do - expect(described_class::MAX_SECONDS).to eq(3600) + it "honours the cap even when jitter alone would exceed it" do + tight = full_jitter(configuration: configuration_with( + "ENRICHMENT_RETRY_BASE_SECONDS" => "60", "ENRICHMENT_RETRY_MAX_SECONDS" => "70" + )) + + # 60 + 25% jitter is 75; the configured 70-second cap still binds. + expect(tight.delay_for(1)).to eq(70.0) end # Additive only: subtracting could schedule a retry sooner than the floor, and the diff --git a/spec/services/github/enrichment/batch_claim_spec.rb b/spec/services/github/enrichment/batch_claim_spec.rb new file mode 100644 index 0000000..f3281ce --- /dev/null +++ b/spec/services/github/enrichment/batch_claim_spec.rb @@ -0,0 +1,302 @@ +require "rails_helper" + +# The two-step batch claim (plan Appendix F/G): never-enriched backlog fills a batch +# FIFO by created_at, id; TTL-stale refresh candidates are admitted only when neither +# class has claimable backlog left. The entity table itself is the work record — the +# lease columns are the claim, so every property here is asserted from committed rows. +RSpec.describe Github::Enrichment::BatchClaim do + let(:now) { frozen_time } + let(:configuration) { Github.configuration } + let(:claim) { described_class.new(configuration: configuration) } + let(:actor_type) { Github::Enrichment::EntityType.fetch(:actor) } + let(:repository_type) { Github::Enrichment::EntityType.fetch(:repository) } + + # A batch row another worker started and never finished. correlation_id is passed + # explicitly because the model validates its presence while the column's + # gen_random_uuid() default is database-side. + def orphan_batch(entity_kind: "actor") + EnrichmentBatch.create!( + request_kind: "search", entity_kind: entity_kind, started_at: now - 700, + correlation_id: SecureRandom.uuid, + request_url: "https://api.github.com/search/users?q=user%3Aoctocat&per_page=1" + ) + end + + def pending_actor(github_id:, created_at: now - 60, **overrides) + create_actor(github_id: github_id, login: "user-#{github_id}", + created_at: created_at, **overrides) + end + + # A row the refresh scope should admit at `now`: complete, contract-complete, a day + # past the default actor TTL, and active within the last week. + def refreshable_actor(github_id:, fetched_at: now - 172_800, last_seen_at: now - 60, **overrides) + create_actor(github_id: github_id, login: "user-#{github_id}", + enrichment_status: "complete", enrichment_stage: "contract_complete", + fetched_at: fetched_at, last_seen_at: last_seen_at, **overrides) + end + + describe "#acquire on the never-enriched backlog" do + # Claim order is the plan's one FIFO: immutable created_at, id as the tiebreak. + it "claims the oldest rows first by created_at then id, at most SEARCH_BATCH_SIZE" do + small = described_class.new(configuration: configuration_with(SEARCH_BATCH_SIZE: "3")) + pending_actor(github_id: 1010, created_at: now - 10) + pending_actor(github_id: 1030, created_at: now - 30) + pending_actor(github_id: 1020, created_at: now - 20) + pending_actor(github_id: 1001, created_at: now - 1) + + lease = small.acquire(actor_type, now: now) + + expect(lease.items.map(&:github_id)).to eq([ 1030, 1020, 1010 ]) + expect(GithubActor.find_by(github_id: 1001).enrichment_stage).to eq("batch_pending") + end + + it "breaks a created_at tie by id, so equal arrivals still claim deterministically" do + second = pending_actor(github_id: 2002, created_at: now - 30) + first = pending_actor(github_id: 2001, created_at: now - 30) + + lease = claim.acquire(actor_type, now: now) + + expect(lease.items.map(&:id)).to eq([ second.id, first.id ].sort) + end + + # One row per GitHub id is what makes the entity table a coalescing work queue: + # however many events demanded the entity, one Search slot answers all of them. + it "yields one item per github id however many event observations demanded it" do + 3.times { |n| GithubActor.upsert_stub!(github_id: 583_231, login: "octocat", now: now + n) } + + lease = claim.acquire(actor_type, now: now) + + expect(lease.items.map(&:github_id)).to eq([ 583_231 ]) + expect(GithubActor.where(github_id: 583_231).count).to eq(1) + end + + it "leases every claimed row: stage, token, expiry, and the batch row it belongs to" do + pending_actor(github_id: 583_231) + + lease = claim.acquire(actor_type, now: now) + row = GithubActor.find_by(github_id: 583_231) + + expect(lease.leased_until).to eq(now + configuration.enrichment_lease_seconds) + expect(row).to have_attributes( + enrichment_stage: "batch_in_flight", lease_token: lease.token, + leased_until: now + configuration.enrichment_lease_seconds, + current_enrichment_batch_id: lease.batch.id + ) + expect(lease.batch).to have_attributes( + request_kind: "search", entity_kind: "actor", status: "in_flight", + requested_count: 1, requested_github_ids: [ 583_231 ], + requested_identifiers: [ "user-583231" ], started_at: now, + request_url: "https://api.github.com/search/users?q=user%3Auser-583231&per_page=1" + ) + end + + it "returns nil on an empty table without creating a batch row" do + expect(claim.acquire(actor_type, now: now)).to be_nil + expect(EnrichmentBatch.count).to eq(0) + end + + it "keeps rows under a live lease invisible to a second claim" do + pending_actor(github_id: 3001) + claim.acquire(actor_type, now: now) + + expect(claim.acquire(actor_type, now: now)).to be_nil + expect(claim.claimable?(actor_type, now: now)).to be(false) + end + + # A worker that died mid-batch left the row batch_in_flight and its batch in_flight. + # Lease expiry re-admits the row, and the orphaned batch is finalized as evidence. + it "reclaims an expired lease and marks its orphaned in-flight batch stale_lease" do + stale = orphan_batch + pending_actor(github_id: 4001, enrichment_stage: "batch_in_flight", + lease_token: SecureRandom.uuid, leased_until: now - 1, + current_enrichment_batch_id: stale.id) + allow(Rails.logger).to receive(:warn) + + lease = claim.acquire(actor_type, now: now) + + expect(lease.items.map(&:github_id)).to eq([ 4001 ]) + expect(stale.reload).to have_attributes(status: "stale_lease", completed_at: now) + expect(Rails.logger).to have_received(:warn).with( + hash_including(event: "enrichment.stale_lease_reclaimed", + enrichment_batch_ids: [ stale.id ], count: 1) + ) + end + + it "re-claims a retry_scheduled row once its next_retry_at is due, but not before" do + pending_actor(github_id: 5001, enrichment_status: "retryable_failure", + enrichment_stage: "retry_scheduled", next_retry_at: now - 1) + pending_actor(github_id: 5002, created_at: now - 900, + enrichment_status: "retryable_failure", + enrichment_stage: "retry_scheduled", next_retry_at: now + 300) + + lease = claim.acquire(actor_type, now: now) + + expect(lease.items.map(&:github_id)).to eq([ 5001 ]) + end + + it "raises rather than building a Search query around a blank identifier" do + pending_actor(github_id: 6001) + GithubActor.where(github_id: 6001).update_all(login: "") + + expect { claim.acquire(actor_type, now: now) } + .to raise_error(ArgumentError, /has no Search identifier/) + end + end + + describe "refresh admission (the two-step top-up policy)" do + it "claims a refresh-only batch when neither class has never-enriched backlog" do + refreshable_actor(github_id: 7001) + + lease = claim.acquire(actor_type, now: now) + + expect(lease.items.map(&:github_id)).to eq([ 7001 ]) + expect(lease.items.first.enrichment_status).to eq("complete") + expect(GithubActor.find_by(github_id: 7001).enrichment_stage).to eq("batch_in_flight") + end + + it "orders refresh claims by fetched_at then id — stalest data first" do + refreshable_actor(github_id: 7011, fetched_at: now - 200_000) + refreshable_actor(github_id: 7012, fetched_at: now - 400_000) + + lease = claim.acquire(actor_type, now: now) + + expect(lease.items.map(&:github_id)).to eq([ 7012, 7011 ]) + end + + # Spare slots belong to backlog, not to refresh: while this class still has + # never-enriched work, a claim takes only that work. + it "does not top a backlog batch up with refresh candidates" do + pending_actor(github_id: 7021) + refreshable_actor(github_id: 7022) + + lease = claim.acquire(actor_type, now: now) + + expect(lease.items.map(&:github_id)).to eq([ 7021 ]) + expect(GithubActor.find_by(github_id: 7022).enrichment_stage).to eq("contract_complete") + end + + # The deny case the policy exists for: the OTHER class still has never-enriched + # rows, so the spare Search request belongs to that backlog rather than to refresh. + it "denies refresh while the other class has claimable backlog" do + refreshable_actor(github_id: 7031) + create_repository(github_id: 7032) + + expect(claim.acquire(actor_type, now: now)).to be_nil + expect(claim.claimable?(actor_type, now: now)).to be(false) + end + + it "denies refresh while fetched_at is still inside the class TTL" do + refreshable_actor(github_id: 7041, fetched_at: now - 600) + + expect(claim.acquire(actor_type, now: now)).to be_nil + end + + it "denies refresh for an entity not seen within REFRESH_ACTIVE_WITHIN_SECONDS" do + refreshable_actor(github_id: 7051, last_seen_at: now - 700_000) + + expect(claim.acquire(actor_type, now: now)).to be_nil + end + + it "denies refresh while a failed refresh batch's backoff is still holding the row" do + refreshable_actor(github_id: 7061, next_retry_at: now + 120) + + expect(claim.acquire(actor_type, now: now)).to be_nil + end + end + + describe "#claimable? and #claimable_backlog?" do + it "reports backlog for a due pending row, and overall claimability with it" do + pending_actor(github_id: 8001) + + expect(claim.claimable_backlog?(actor_type, now: now)).to be(true) + expect(claim.claimable?(actor_type, now: now)).to be(true) + expect(claim.claimable_backlog?(repository_type, now: now)).to be(false) + end + + it "reports refresh-only work as claimable but not as backlog" do + refreshable_actor(github_id: 8011) + + expect(claim.claimable_backlog?(actor_type, now: now)).to be(false) + expect(claim.claimable?(actor_type, now: now)).to be(true) + end + end + + describe "#release!" do + # A hand-built lease over rows staged exactly as #acquire leaves them, so release + # semantics are provable without a round trip through a claim. + def leased_lease(rows) + batch = orphan_batch + token = SecureRandom.uuid + items = rows.map do |row| + GithubActor.where(id: row.id).update_all( + enrichment_stage: "batch_in_flight", lease_token: token, + leased_until: now + 600, current_enrichment_batch_id: batch.id + ) + described_class::Item.new(id: row.id, github_id: row.github_id, identifier: row.login, + api_url: row.api_url, previous_stage: "batch_in_flight", + enrichment_status: row.enrichment_status, + enrichment_attempts: row.enrichment_attempts) + end + described_class::Lease.new(entity_type: actor_type, batch: batch, token: token, + leased_until: now + 600, items: items.freeze) + end + + it "restores a never-enriched member to batch_pending and a complete one to contract_complete" do + pending = pending_actor(github_id: 9001) + complete = refreshable_actor(github_id: 9002) + + claim.release!(leased_lease([ pending, complete ]), now: now) + + expect(pending.reload).to have_attributes( + enrichment_stage: "batch_pending", lease_token: nil, leased_until: nil, + current_enrichment_batch_id: nil + ) + expect(complete.reload).to have_attributes( + enrichment_stage: "contract_complete", lease_token: nil, leased_until: nil, + current_enrichment_batch_id: nil + ) + end + + it "leaves a row alone when its lease has since been taken by someone else" do + row = pending_actor(github_id: 9011) + lease = leased_lease([ row ]) + foreign_token = SecureRandom.uuid + GithubActor.where(id: row.id).update_all(lease_token: foreign_token) + + claim.release!(lease, now: now) + + expect(row.reload).to have_attributes(enrichment_stage: "batch_in_flight", + lease_token: foreign_token) + end + end + + # FOR UPDATE SKIP LOCKED is what lets two workers claim concurrently without either + # blocking or double-claiming. Real threads on real sessions require transactional + # fixtures off (spec/support/concurrency_helpers.rb), so this group owns its cleanup. + describe "two concurrent claims" do + self.use_transactional_tests = false + + after do + GithubActor.where(github_id: 700_001..700_015).delete_all + EnrichmentBatch.delete_all + restore_connection_pool! + end + + it "hands each claim a disjoint row set that together covers the backlog" do + ids = (1..15).map do |n| + create_actor(github_id: 700_000 + n, login: "worker-#{n}", + created_at: now - 60 + n).github_id + end + + # One stateless claim built on the main thread; each worker thread runs + # #acquire on its own pooled connection. + shared_claim = described_class.new(configuration: configuration) + results = in_parallel(2, threads: 2) { shared_claim.acquire(actor_type, now: now) } + + expect(results).to all(be_a(described_class::Lease)) + claimed = results.map { |lease| lease.items.map(&:github_id) } + expect(claimed[0] & claimed[1]).to eq([]) + expect(claimed.flatten).to match_array(ids) + end + end +end diff --git a/spec/services/github/enrichment/batch_quality_spec.rb b/spec/services/github/enrichment/batch_quality_spec.rb new file mode 100644 index 0000000..41d2b91 --- /dev/null +++ b/spec/services/github/enrichment/batch_quality_spec.rb @@ -0,0 +1,138 @@ +require "rails_helper" + +RSpec.describe Github::Enrichment::BatchQuality do + let(:now) { frozen_time } + + def capture(configuration: Github.configuration) + described_class.capture(now: now, configuration: configuration) + end + + # correlation_id is passed explicitly because the model validates its presence while + # the column's gen_random_uuid() default is database-side, so a bare create! never + # sees a value to validate. + def create_batch(request_kind: "search", entity_kind: "actor", status: "succeeded", + started_at: now - 60, **overrides) + EnrichmentBatch.create!( + request_kind: request_kind, entity_kind: entity_kind, status: status, + started_at: started_at, correlation_id: SecureRandom.uuid, **overrides + ) + end + + describe "the four groups" do + # An empty table was read and held nothing — that is a counted zero, not an absent + # key, so all four request-kind x entity-kind groups are always published. + it "publishes every group with counted zeros on an empty table" do + payload = capture.payload + + expect(payload.keys).to eq(%i[window_seconds window_start search detail]) + %i[search detail].each do |kind| + %i[actors repositories].each do |entity| + expect(payload.dig(kind, entity)).to eq(described_class::EMPTY_GROUP.to_h) + end + end + expect(payload.dig(:search, :actors, :fill_ratio)).to be_nil + end + + it "routes each batch to its own request-kind x entity-kind group" do + create_batch(request_kind: "search", entity_kind: "actor") + create_batch(request_kind: "search", entity_kind: "repository") + create_batch(request_kind: "detail", entity_kind: "actor") + create_batch(request_kind: "detail", entity_kind: "actor") + + payload = capture.payload + + expect(payload.dig(:search, :actors, :attempts)).to eq(1) + expect(payload.dig(:search, :repositories, :attempts)).to eq(1) + expect(payload.dig(:detail, :actors, :attempts)).to eq(2) + expect(payload.dig(:detail, :repositories, :attempts)).to eq(0) + end + end + + describe "the status tally" do + it "counts each of the five batch outcomes separately" do + EnrichmentBatch::STATUSES.each { |status| create_batch(status: status) } + + expect(capture.payload.dig(:search, :actors)).to include( + attempts: 5, in_flight: 1, succeeded: 1, failed: 1, deferred: 1, stale_lease: 1 + ) + end + end + + describe "the item sums and fill ratio" do + it "sums the item counters across the group's batches" do + create_batch(requested_count: 10, returned_count: 9, valid_count: 8, + missing_count: 1, invalid_count: 1) + create_batch(requested_count: 10, returned_count: 7, valid_count: 6, + missing_count: 3, invalid_count: 1) + + expect(capture.payload.dig(:search, :actors)).to include( + requested_items: 20, returned_items: 16, valid_items: 14, + missing_items: 4, invalid_items: 2, fill_ratio: 0.8 + ) + end + + it "rounds the fill ratio to three decimals" do + create_batch(requested_count: 3, returned_count: 1) + + expect(capture.payload.dig(:search, :actors, :fill_ratio)).to eq(0.333) + end + + # A ratio with a zero denominator is undefined, never 0.0 — §16's fabricated-zero + # rule. The denominator is published beside it, so null is self-explanatory. + it "reports a null fill ratio when nothing was requested" do + create_batch(status: "deferred", requested_count: 0, returned_count: 0) + + expect(capture.payload.dig(:search, :actors)) + .to include(attempts: 1, requested_items: 0, fill_ratio: nil) + end + end + + describe "the incomplete_results count" do + # GitHub's Search envelope flag: recorded per batch, counted only when it was + # actually true — an absent flag (detail batches never carry one) is not a false. + it "counts only batches whose envelope said incomplete_results true" do + create_batch(incomplete_results: true) + create_batch(incomplete_results: false) + create_batch(incomplete_results: nil) + + expect(capture.payload.dig(:search, :actors, :incomplete_results_count)).to eq(1) + end + end + + describe "the trailing window" do + it "filters on started_at with an inclusive floor" do + create_batch(started_at: now - 3600) + create_batch(started_at: now - 3601) + + quality = capture + + expect(quality.payload.dig(:search, :actors, :attempts)).to eq(1) + expect(quality.window_start).to eq(now - 3600) + end + + it "reads the window from the configuration it was given" do + create_batch(started_at: now - 700) + configuration = configuration_with(ENRICHMENT_METRICS_WINDOW_SECONDS: "600") + + expect(capture(configuration: configuration).payload) + .to include(window_seconds: 600, window_start: (now - 600).utc.iso8601) + expect(capture(configuration: configuration).payload.dig(:search, :actors, :attempts)).to eq(0) + end + end + + describe "consistency" do + # Batches commit while /status reads. One grouped statement means the counts of one + # group can never describe a different instant than another group's. + it "captures every group in one grouped statement, writing nothing" do + create_batch + create_batch(request_kind: "detail", entity_kind: "repository") + + statements = capture_sql { capture } + selects = statements.grep(/\A\s*SELECT/i) + + expect(selects.length).to eq(1) + expect(selects.first).to include('FROM "enrichment_batches"', "GROUP BY") + expect(statements.grep(SqlHelpers::WRITE)).to be_empty + end + end +end diff --git a/spec/services/github/enrichment/batch_runner_spec.rb b/spec/services/github/enrichment/batch_runner_spec.rb new file mode 100644 index 0000000..2116fda --- /dev/null +++ b/spec/services/github/enrichment/batch_runner_spec.rb @@ -0,0 +1,500 @@ +require "rails_helper" + +# One Search request against a claimed batch, applied item-by-item on the stable +# GitHub id. The executor is a hand-rolled recorder (the request_executor_spec +# anonymous-class idiom) returning FetchResults this file constructs, so every +# branch of the response matrix is exercised without a transport or a corpus. +RSpec.describe Github::Enrichment::BatchRunner do + let(:now) { frozen_time } + let(:configuration) { Github.configuration } + + # Records every Request it is handed and answers from the example's script — + # the state of the world at call time is the recording, never an assumption. + let(:executor_class) do + Class.new do + attr_reader :requests + + def initialize(&responder) + @responder = responder + @requests = [] + end + + def call(request) + @requests << request + @responder.call(request, @requests.length) + end + end + end + + def recording_executor(&responder) + executor_class.new(&responder) + end + + def runner(executor, now: frozen_time) + described_class.new( + executor: executor, configuration: configuration, + claim: Github::Enrichment::BatchClaim.new(configuration: configuration), + backoff: jitterless_backoff(configuration: configuration), + clock: -> { now } + ) + end + + def search_headers(remaining: 9, used: 1) + { "x-ratelimit-resource" => "search", "x-ratelimit-limit" => "10", + "x-ratelimit-remaining" => remaining.to_s, "x-ratelimit-used" => used.to_s, + "x-ratelimit-reset" => (frozen_time + 60).to_i.to_s } + end + + def search_success(request, items:, total_count: items.length, incomplete_results: false) + Github::FetchResult.from_response( + request: request, status: 200, headers: search_headers, + body: JSON.generate("total_count" => total_count, + "incomplete_results" => incomplete_results, "items" => items), + duration_ms: 1.0 + ) + end + + def failure_response(request, status:, headers: search_headers, body: "") + Github::FetchResult.from_response(request: request, status: status, headers: headers, + body: body, duration_ms: 1.0) + end + + def actor_item(github_id:, login:, type: "User") + { "id" => github_id, "login" => login, "type" => type } + end + + def repository_item(github_id:, full_name:, description: "a fixture repository") + { "id" => github_id, "full_name" => full_name, "description" => description, + "language" => "Ruby", "owner" => { "id" => 42, "login" => full_name.split("/").first }, + "fork" => false, "archived" => false, "default_branch" => "main", + "created_at" => "2020-01-01T00:00:00Z" } + end + + def create_pending_actor(github_id:, login:, created_at: now - 60, **overrides) + create_actor(github_id: github_id, login: login, created_at: created_at, **overrides) + end + + describe "an idle claim" do + it "returns idle without spending a request or creating a batch row" do + executor = recording_executor { raise "must not be called" } + + result = runner(executor).call(entity_class: GithubActor) + + expect(result).to have_attributes(status: "idle", entity_type: :actor, + batch_id: nil, requested_count: 0) + expect(executor.requests).to be_empty + expect(EnrichmentBatch.count).to eq(0) + end + end + + describe "a successful batch" do + it "applies every item by its stable id even when the response order is shuffled" do + create_pending_actor(github_id: 101, login: "alpha", created_at: now - 30) + create_pending_actor(github_id: 102, login: "beta", created_at: now - 20) + create_pending_actor(github_id: 103, login: "gamma", created_at: now - 10) + executor = recording_executor do |request, _call| + search_success(request, items: [ actor_item(github_id: 103, login: "gamma"), + actor_item(github_id: 101, login: "alpha"), + actor_item(github_id: 102, login: "beta") ]) + end + + result = runner(executor).call(entity_class: GithubActor) + + expect(result).to have_attributes(status: "completed", requested_count: 3, + returned_count: 3, valid_count: 3, fallback_count: 0) + [ [ 101, "alpha" ], [ 102, "beta" ], [ 103, "gamma" ] ].each do |github_id, login| + row = GithubActor.find_by(github_id: github_id) + expect(row).to have_attributes( + enrichment_status: "complete", enrichment_stage: "contract_complete", + account_type: "User", fetched_at: now, batch_applied_at: now, + contract_completed_at: now, latest_observation_source: "search", + latest_observed_at: now, enrichment_attempts: 0, + lease_token: nil, leased_until: nil, current_enrichment_batch_id: nil + ) + expect(row.raw_payload).to eq(actor_item(github_id: github_id, login: login)) + end + end + + it "sends one application-origin Search request through the executor" do + create_pending_actor(github_id: 111, login: "alpha") + executor = recording_executor do |request, _call| + search_success(request, items: [ actor_item(github_id: 111, login: "alpha") ]) + end + + runner(executor).call(entity_class: GithubActor) + + expect(executor.requests.length).to eq(1) + expect(executor.requests.first).to have_attributes( + request_class: :actor_search, origin: :application, + url: "https://api.github.com/search/users?q=user%3Aalpha&per_page=1" + ) + end + + it "appends one applied observation per item, linked to the batch" do + create_pending_actor(github_id: 121, login: "alpha") + executor = recording_executor do |request, _call| + search_success(request, items: [ actor_item(github_id: 121, login: "alpha") ]) + end + + result = runner(executor).call(entity_class: GithubActor) + observation = EnrichmentObservation.find_by(entity_github_id: 121) + + expect(observation).to have_attributes( + entity_kind: "actor", source: "search", validation_outcome: "applied", + enrichment_batch_id: result.batch_id, requested_identifier: "alpha", + observed_at: now + ) + expect(observation.raw_payload).to eq(actor_item(github_id: 121, login: "alpha")) + expect(GithubActor.find_by(github_id: 121).latest_observation_id).to eq(observation.id) + end + + it "finalizes the batch envelope with counts and the response's rate-limit headers" do + create_pending_actor(github_id: 131, login: "alpha") + executor = recording_executor do |request, _call| + search_success(request, items: [ actor_item(github_id: 131, login: "alpha") ]) + end + + result = runner(executor).call(entity_class: GithubActor) + batch = EnrichmentBatch.find(result.batch_id) + + expect(batch).to have_attributes( + status: "succeeded", completed_at: now, requested_count: 1, returned_count: 1, + valid_count: 1, missing_count: 0, invalid_count: 0, total_count: 1, + incomplete_results: false, response_status: 200, response_body: nil, + rate_limit_resource: "search", rate_limit_limit: 10, rate_limit_remaining: 9, + rate_limit_used: 1, rate_limit_reset_at: Time.zone.at((frozen_time + 60).to_i) + ) + end + + # §45 throughput counts first completions; a refresh must re-stamp fetched_at + # without re-counting the entity, which is what the COALESCE keep-first proves. + it "preserves contract_completed_at across a refresh re-application" do + create_pending_actor(github_id: 141, login: "alpha") + respond = lambda do |request, _call| + search_success(request, items: [ actor_item(github_id: 141, login: "alpha") ]) + end + runner(recording_executor(&respond)).call(entity_class: GithubActor) + GithubActor.where(github_id: 141) + .update_all(fetched_at: now - 172_800, last_seen_at: now) + + later = now + 60 + runner(recording_executor(&respond), now: later).call(entity_class: GithubActor) + row = GithubActor.find_by(github_id: 141) + + expect(row.contract_completed_at).to eq(now) + expect(row).to have_attributes(fetched_at: later, batch_applied_at: later, + enrichment_status: "complete") + end + end + + describe "items the response could not validate" do + it "admits a missing item to the detail fallback and counts it missing" do + create_pending_actor(github_id: 201, login: "alpha", created_at: now - 30) + create_pending_actor(github_id: 202, login: "vanished", created_at: now - 20) + allow(Rails.logger).to receive(:info) + executor = recording_executor do |request, _call| + search_success(request, items: [ actor_item(github_id: 201, login: "alpha") ]) + end + + result = runner(executor).call(entity_class: GithubActor) + row = GithubActor.find_by(github_id: 202) + + expect(result).to have_attributes(status: "completed", valid_count: 1, fallback_count: 1) + expect(row).to have_attributes( + enrichment_status: "pending", enrichment_stage: "detail_pending", + detail_pending_at: now, last_error: "missing_search_result", + lease_token: nil, current_enrichment_batch_id: nil + ) + expect(EnrichmentBatch.find(result.batch_id).missing_count).to eq(1) + expect(Rails.logger).to have_received(:info).with( + hash_including(event: "enrichment.fallback_admitted", entity_type: :actor, + github_actor_id: 202, reason: "missing_search_result") + ) + end + + # The id matched but the name no longer does, even case-insensitively: GitHub + # renamed the repository, and the projection must not adopt the new identity. + it "routes a renamed repository to the fallback instead of projecting it" do + create_repository(github_id: 301, full_name: "octocat/hello-world") + executor = recording_executor do |request, _call| + search_success(request, items: [ repository_item(github_id: 301, + full_name: "octocat/renamed") ]) + end + + result = runner(executor).call(entity_class: GithubRepository) + row = GithubRepository.find_by(github_id: 301) + + expect(row).to have_attributes(enrichment_status: "pending", + enrichment_stage: "detail_pending", + description: nil, last_error: "renamed_repository") + expect(EnrichmentObservation.find_by(entity_github_id: 301).validation_outcome) + .to eq("renamed_repository") + expect(EnrichmentBatch.find(result.batch_id)).to have_attributes(valid_count: 0, + invalid_count: 1) + end + + # casecmp, not ==: GitHub canonicalizes case freely, and a case-only difference + # is the same repository. + it "still applies a repository whose full_name differs only in case" do + create_repository(github_id: 311, full_name: "octocat/hello-world") + executor = recording_executor do |request, _call| + search_success(request, items: [ repository_item(github_id: 311, + full_name: "Octocat/Hello-World") ]) + end + + runner(executor).call(entity_class: GithubRepository) + + expect(GithubRepository.find_by(github_id: 311)).to have_attributes( + enrichment_status: "complete", description: "a fixture repository" + ) + end + + # Logins are recyclable: an item whose login matches but whose id does not is a + # different account, and projecting it would corrupt the identity join. + it "records an identity mismatch when the identifier matches but the id differs" do + create_pending_actor(github_id: 401, login: "octocat") + executor = recording_executor do |request, _call| + search_success(request, items: [ actor_item(github_id: 999, login: "octocat") ]) + end + + result = runner(executor).call(entity_class: GithubActor) + row = GithubActor.find_by(github_id: 401) + + expect(row).to have_attributes(enrichment_stage: "detail_pending", + last_error: "identity_mismatch", account_type: nil) + expect(EnrichmentObservation.find_by(entity_github_id: 401).validation_outcome) + .to eq("identity_mismatch") + expect(result.fallback_count).to eq(1) + end + + it "retains an unrequested extra item as evidence without projecting it" do + create_pending_actor(github_id: 501, login: "alpha") + executor = recording_executor do |request, _call| + search_success(request, items: [ actor_item(github_id: 501, login: "alpha"), + actor_item(github_id: 777, login: "stranger") ]) + end + + result = runner(executor).call(entity_class: GithubActor) + extra = EnrichmentObservation.find_by(entity_github_id: 777) + + expect(extra).to have_attributes(validation_outcome: "unrequested_result", + enrichment_batch_id: result.batch_id) + expect(GithubActor.find_by(github_id: 777)).to be_nil + expect(EnrichmentBatch.find(result.batch_id)).to have_attributes(valid_count: 1, + invalid_count: 1) + end + + # incomplete_results is a fact about the query timing out, not about any item + # that did come back: id-validated items apply, absent ones fall back. + it "applies id-valid items under incomplete_results while missing ones fall back" do + create_pending_actor(github_id: 601, login: "alpha", created_at: now - 30) + create_pending_actor(github_id: 602, login: "slow", created_at: now - 20) + executor = recording_executor do |request, _call| + search_success(request, items: [ actor_item(github_id: 601, login: "alpha") ], + total_count: 2, incomplete_results: true) + end + + result = runner(executor).call(entity_class: GithubActor) + + expect(GithubActor.find_by(github_id: 601).enrichment_status).to eq("complete") + expect(GithubActor.find_by(github_id: 602).enrichment_stage).to eq("detail_pending") + expect(EnrichmentBatch.find(result.batch_id)).to have_attributes( + incomplete_results: true, valid_count: 1, missing_count: 1 + ) + end + end + + describe "a failed request" do + it "fails the batch on a malformed envelope and schedules every member's retry" do + create_pending_actor(github_id: 701, login: "alpha", created_at: now - 40) + create_pending_actor(github_id: 702, login: "beta", created_at: now - 30, + enrichment_attempts: 2) + create_pending_actor(github_id: 703, login: "gamma", created_at: now - 20, + enrichment_attempts: 10) + executor = recording_executor do |request, _call| + failure_response(request, status: 200, body: "not json at all") + end + + result = runner(executor).call(entity_class: GithubActor) + + expect(result).to have_attributes(status: "failed", + deferral_reason: "malformed_search_response") + expect(EnrichmentBatch.find(result.batch_id).status).to eq("failed") + # Jitterless backoff makes the ladder exact: 60s doubling per prior attempt, + # capped at the configured hour. + expect(GithubActor.find_by(github_id: 701)).to have_attributes( + enrichment_status: "retryable_failure", enrichment_stage: "retry_scheduled", + enrichment_attempts: 1, next_retry_at: now + 60, retry_scheduled_at: now + ) + expect(GithubActor.find_by(github_id: 702)).to have_attributes( + enrichment_attempts: 3, next_retry_at: now + 240 + ) + expect(GithubActor.find_by(github_id: 703)).to have_attributes( + enrichment_attempts: 11, next_retry_at: now + 3600 + ) + end + + # A refresh member already holds a completed contract; a failed refresh batch + # must not demote it — the backoff rides next_retry_at while the row rests + # back in contract_complete. + it "restores a complete refresh member to contract_complete with its backoff set" do + create_actor(github_id: 711, login: "alpha", enrichment_status: "complete", + enrichment_stage: "contract_complete", + fetched_at: now - 172_800, last_seen_at: now - 60) + executor = recording_executor do |request, _call| + failure_response(request, status: 200, body: "{}") + end + + runner(executor).call(entity_class: GithubActor) + + expect(GithubActor.find_by(github_id: 711)).to have_attributes( + enrichment_status: "complete", enrichment_stage: "contract_complete", + enrichment_attempts: 1, next_retry_at: now + 60, + lease_token: nil, current_enrichment_batch_id: nil + ) + end + + it "fails the batch on a 422 and leaves the debited request spent" do + create_pending_actor(github_id: 721, login: "alpha") + allow(Rails.logger).to receive(:warn) + executor = recording_executor do |request, _call| + failure_response(request, status: 422, body: "Validation Failed") + end + + result = runner(executor).call(entity_class: GithubActor) + batch = EnrichmentBatch.find(result.batch_id) + + expect(result.status).to eq("failed") + # One attempt was made and there is no refund path: the executor saw exactly + # one request, and the batch retains the failure evidence. + expect(executor.requests.length).to eq(1) + expect(batch).to have_attributes(status: "failed", response_status: 422, + response_body: "Validation Failed") + expect(GithubActor.find_by(github_id: 721).enrichment_stage).to eq("retry_scheduled") + expect(Rails.logger).to have_received(:warn).with( + hash_including(event: "enrichment.batch_failed", response_status: 422) + ) + end + + # Observed against the live API: Search answers 422 rather than an empty result set + # when every requested identifier is unsearchable — the state a renamed repository + # is in. Retrying the search would reproduce it forever, so the members take the + # same route an omitted item takes, and the stored payload URL resolves the rename. + it "admits every member to the detail lane when GitHub says none are searchable" do + create_pending_actor(github_id: 741, login: "alpha") + create_pending_actor(github_id: 742, login: "beta") + allow(Rails.logger).to receive(:warn) + allow(Rails.logger).to receive(:info) + executor = recording_executor do |request, _call| + failure_response( + request, status: 422, + body: '{"message":"Validation Failed","errors":[{"message":"The listed users ' \ + 'and repositories cannot be searched either because the resources do ' \ + 'not exist or you do not have permission to view them.","resource":' \ + '"Search","field":"q","code":"invalid"}]}' + ) + end + + result = runner(executor).call(entity_class: GithubActor) + batch = EnrichmentBatch.find(result.batch_id) + + expect(result).to have_attributes(status: "completed", returned_count: 0, + valid_count: 0, fallback_count: 2) + expect(GithubActor.where(github_id: [ 741, 742 ]).pluck(:enrichment_stage)) + .to all(eq("detail_pending")) + expect(GithubActor.where(github_id: [ 741, 742 ]).pluck(:last_error)) + .to all(eq("unsearchable_identifier")) + # No member is left on the search lane, and the batch records why. + expect(batch).to have_attributes(status: "failed", missing_count: 2, + returned_count: 0, + last_error: "unsearchable_identifiers") + expect(Rails.logger).to have_received(:warn).with( + hash_including(event: "enrichment.batch_unsearchable", response_status: 422) + ) + end + + it "stores a failure body truncated to the retention bound" do + create_pending_actor(github_id: 731, login: "alpha") + executor = recording_executor do |request, _call| + failure_response(request, status: 422, body: "x" * 70_000) + end + + result = runner(executor).call(entity_class: GithubActor) + + expect(EnrichmentBatch.find(result.batch_id).response_body.length) + .to eq(described_class::RESPONSE_BODY_LIMIT) + end + end + + describe "a rate-limited request" do + { 403 => { remaining: 0, reason: "rate_limited" }, + 429 => { remaining: 5, reason: "secondary_limited" } }.each do |status, expected| + it "defers the batch on #{status} and releases the rows unchanged" do + active_search_window(now: now) + create_pending_actor(github_id: 801, login: "alpha") + original = GithubActor.find_by(github_id: 801).attributes + allow(Rails.logger).to receive(:info) + executor = recording_executor do |request, _call| + failure_response(request, status: status, + headers: search_headers(remaining: expected[:remaining]), + body: "limited") + end + + result = runner(executor).call(entity_class: GithubActor) + restored = GithubActor.find_by(github_id: 801).attributes + + expect(result).to have_attributes(status: "deferred", + deferral_reason: expected[:reason]) + # Deferral is not an attempt: everything but updated_at is byte-identical. + expect(restored.except("updated_at")).to eq(original.except("updated_at")) + expect(EnrichmentBatch.find(result.batch_id)).to have_attributes( + status: "deferred", last_error: expected[:reason] + ) + # block_from! propagated the response's reset instant to the search ledger. + expect(current_search_budget.blocked_until).to eq(Time.zone.at((frozen_time + 60).to_i)) + expect(Rails.logger).to have_received(:info).with( + hash_including(event: "enrichment.batch_deferred", deferral_reason: expected[:reason]) + ) + end + end + end + + describe "a lease lost mid-flight" do + it "counts the item invalid rather than applying over a foreign lease" do + create_pending_actor(github_id: 901, login: "alpha") + executor = recording_executor do |request, _call| + # Another claimant stole the row between our request and our write. + GithubActor.where(github_id: 901).update_all(lease_token: SecureRandom.uuid) + search_success(request, items: [ actor_item(github_id: 901, login: "alpha") ]) + end + + result = runner(executor).call(entity_class: GithubActor) + row = GithubActor.find_by(github_id: 901) + + expect(result).to have_attributes(status: "completed", valid_count: 0) + expect(row).to have_attributes(enrichment_status: "pending", account_type: nil) + expect(EnrichmentBatch.find(result.batch_id)).to have_attributes(valid_count: 0, + invalid_count: 1) + end + end + + describe "a crash between claim and outcome" do + [ RuntimeError, Github::Errors::FixtureMiss ].each do |error_class| + it "finalizes the batch, releases the rows, and re-raises #{error_class}" do + create_pending_actor(github_id: 951, login: "alpha") + executor = recording_executor { raise error_class, "boom" } + + expect { runner(executor).call(entity_class: GithubActor) } + .to raise_error(error_class, "boom") + + expect(EnrichmentBatch.sole).to have_attributes(status: "failed", + last_error: error_class.name, + completed_at: now) + expect(GithubActor.find_by(github_id: 951)).to have_attributes( + enrichment_stage: "batch_pending", enrichment_attempts: 0, + next_retry_at: nil, lease_token: nil, current_enrichment_batch_id: nil + ) + end + end + end +end diff --git a/spec/services/github/enrichment/candidate_selector_spec.rb b/spec/services/github/enrichment/candidate_selector_spec.rb deleted file mode 100644 index 31c181b..0000000 --- a/spec/services/github/enrichment/candidate_selector_spec.rb +++ /dev/null @@ -1,329 +0,0 @@ -require "rails_helper" - -RSpec.describe Github::Enrichment::CandidateSelector do - subject(:selector) { described_class.new(configuration: configuration) } - - let(:configuration) { configuration_with } - let(:now) { frozen_time } - let(:actor_type) { Github::Enrichment::EntityType.fetch(:actor) } - let(:repository_type) { Github::Enrichment::EntityType.fetch(:repository) } - - def pending_actor(github_id:, last_seen_at: now - 60, **overrides) - create_actor(github_id: github_id, last_seen_at: last_seen_at, **overrides) - end - - def next_pending(entity_type = actor_type) - selector.scope(entity_type, pool: :pending, now: now).first - end - - def next_refresh(entity_type = actor_type) - selector.scope(entity_type, pool: :refresh, now: now).first - end - - describe "the pending pool" do - it "enriches oldest-created first, so every durable backlog row advances toward service" do - oldest = pending_actor(github_id: 1, created_at: now - 600) - pending_actor(github_id: 2, created_at: now - 10) - - expect(next_pending).to eq(oldest) - end - - it "applies the same oldest-created FIFO to repositories" do - oldest = create_repository(github_id: 1, created_at: now - 600) - create_repository(github_id: 2, created_at: now - 10) - - expect(next_pending(repository_type)).to eq(oldest) - end - - # PageWriter creates many stubs in one page with one timestamp. The ascending id tie - # break preserves insertion order rather than letting PostgreSQL choose a plan-dependent - # winner on every reconciliation tick. - it "breaks a created-at tie by ascending id" do - first = pending_actor(github_id: 1, created_at: now - 60) - pending_actor(github_id: 2, created_at: now - 60) - - expect(next_pending).to eq(first) - end - - it "offers a retryable failure alongside a pending row, which is what the index predicate says" do - retryable = pending_actor(github_id: 1, enrichment_status: "retryable_failure") - - expect(next_pending).to eq(retryable) - end - - it "excludes a candidate whose retry has not come due" do - pending_actor(github_id: 1, next_retry_at: now + 60) - - expect(next_pending).to be_nil - end - - it "offers a candidate again the instant its retry is due" do - due = pending_actor(github_id: 1, next_retry_at: now) - - expect(next_pending).to eq(due) - end - - it "keeps an old candidate claimable until it is eventually enriched" do - old = pending_actor(github_id: 1, last_seen_at: now - 100_000, - created_at: now - 100_000) - - expect(next_pending).to eq(old) - end - - it "excludes a terminal status, which no amount of budget would help" do - pending_actor(github_id: 1, enrichment_status: "permanent_failure") - - expect(next_pending).to be_nil - end - - # A stub can be created with a NULL last_seen_at: PageWriter upserts the stub, the - # push_events insert returns nil on a duplicate, and the transaction still commits. - # created_at is total and immutable, so the durable FIFO can still order this work. - it "keeps a stub with no last_seen_at claimable regardless of age" do - stub = create_actor(github_id: 1, last_seen_at: nil, created_at: now - 100_000) - - expect(next_pending).to eq(stub) - end - - it "orders solely by durable insertion time, not later activity" do - oldest = pending_actor(github_id: 1, created_at: now - 600, last_seen_at: now - 10) - pending_actor(github_id: 2, created_at: now - 300, last_seen_at: now - 200) - - expect(next_pending).to eq(oldest) - end - - it "keeps the two classes apart, so a repository backlog never appears as actor work" do - create_repository(github_id: 1, last_seen_at: now - 60) - - expect(next_pending(actor_type)).to be_nil - expect(next_pending(repository_type)).not_to be_nil - end - end - - describe "the refresh pool (plan §10's freshness cache)" do - def complete_actor(github_id:, fetched_at:, **overrides) - create_actor(github_id: github_id, enrichment_status: "complete", fetched_at: fetched_at, - last_seen_at: now - 60, **overrides) - end - - it "treats a record inside its TTL as fresh and offers nothing" do - complete_actor(github_id: 1, fetched_at: now - 86_399) - - expect(next_refresh).to be_nil - end - - it "offers a refresh once the TTL has passed" do - stale = complete_actor(github_id: 1, fetched_at: now - 86_400) - - expect(next_refresh).to eq(stale) - end - - # Oldest-fetched first is the rule that terminates: a monotone refresh queue cannot - # starve a complete row behind a hotter neighbour. - it "refreshes the most stale record first, so no complete row can be starved" do - complete_actor(github_id: 1, fetched_at: now - 90_000) - oldest = complete_actor(github_id: 2, fetched_at: now - 200_000) - - expect(next_refresh).to eq(oldest) - end - - it "defers a refresh whose last attempt backed off" do - complete_actor(github_id: 1, fetched_at: now - 90_000, next_retry_at: now + 60) - - expect(next_refresh).to be_nil - end - - it "reads each class's own TTL" do - short = described_class.new(configuration: configuration_with(REPOSITORY_REFRESH_TTL_SECONDS: "60")) - create_repository(github_id: 1, enrichment_status: "complete", fetched_at: now - 120, - last_seen_at: now - 60) - - expect(short.scope(repository_type, pool: :refresh, now: now).first).not_to be_nil - expect(next_refresh(repository_type)).to be_nil - end - - it "offers a refresh regardless of how long ago the entity was referenced" do - stale = complete_actor(github_id: 1, fetched_at: now - 90_000, last_seen_at: now - 100_000) - - expect(next_refresh).to eq(stale) - end - end - - describe "#pending_available?" do - it "answers true for old durable work rather than treating age as ineligibility" do - pending_actor(github_id: 1, last_seen_at: now - 100_000, created_at: now - 100_000) - - expect(GithubActor.count).to eq(1) - expect(selector.pending_available?(actor_type, now: now)).to be(true) - end - - it "answers true while one eligible candidate remains" do - pending_actor(github_id: 1) - - expect(selector.pending_available?(actor_type, now: now)).to be(true) - end - - # §10's prioritization ladder ranks refreshing stale enrichment below enriching - # never-seen entities *globally*. Counting refreshes here would let one class decline - # to lend its idle capacity because the other had a refresh waiting, inverting it. - it "reports pending availability without counting refreshes, which rank below it" do - create_actor(github_id: 1, enrichment_status: "complete", fetched_at: now - 90_000) - - expect(selector.pending_available?(actor_type, now: now)).to be(false) - end - end - - describe "#pending_backlog?" do - it "sees never-enriched work even while its retry is deferred" do - pending_actor(github_id: 1, enrichment_status: "retryable_failure", - next_retry_at: now + 3600) - - expect(selector.pending_available?(actor_type, now: now)).to be(false) - expect(selector.pending_backlog?(actor_type)).to be(true) - end - - it "does not count complete or permanently failed rows as never-enriched backlog" do - create_actor(github_id: 1, enrichment_status: "complete", fetched_at: now) - create_actor(github_id: 2, enrichment_status: "permanent_failure") - - expect(selector.pending_backlog?(actor_type)).to be(false) - end - - it "keeps entity classes independent" do - create_repository(github_id: 1) - - expect(selector.pending_backlog?(actor_type)).to be(false) - expect(selector.pending_backlog?(repository_type)).to be(true) - end - end - - describe "#claimable?" do - it "is true while a pending candidate is eligible" do - pending_actor(github_id: 1) - - expect(selector.claimable?(actor_type, now: now)).to be(true) - end - - # The half that was missing, and the reason a fully enriched backlog reported "due - # now": a stale refresh is work, so a class holding one is not idle. - it "is true while only a TTL-stale refresh is waiting" do - create_actor(github_id: 1, enrichment_status: "complete", fetched_at: now - 90_000) - - expect(selector.claimable?(actor_type, now: now)).to be(true) - end - - it "is false when every record is enriched and still fresh" do - create_actor(github_id: 1, enrichment_status: "complete", fetched_at: now) - - expect(selector.claimable?(actor_type, now: now)).to be(false) - end - - it "is false when the only candidate is deferred" do - pending_actor(github_id: 1, next_retry_at: now + 60) - - expect(selector.claimable?(actor_type, now: now)).to be(false) - end - end - - describe "#earliest_pending_at" do - it "names the soonest instant at which a deferred candidate becomes claimable" do - pending_actor(github_id: 1, next_retry_at: now + 300) - pending_actor(github_id: 2, next_retry_at: now + 60) - - expect(selector.earliest_pending_at(actor_type, now: now)).to eq(now + 60) - end - - it "is nil when nothing is deferred, because the answer is not a pending instant" do - pending_actor(github_id: 1) - - expect(selector.earliest_pending_at(actor_type, now: now)).to be_nil - end - - it "names the retry instant even for a very old candidate" do - pending_actor(github_id: 1, next_retry_at: now + 60, - last_seen_at: now - 100_000, created_at: now - 100_000) - - expect(selector.earliest_pending_at(actor_type, now: now)).to eq(now + 60) - end - end - - describe "#earliest_refresh_at" do - def complete_actor(github_id:, fetched_at:, **overrides) - create_actor(github_id: github_id, enrichment_status: "complete", fetched_at: fetched_at, **overrides) - end - - it "names the instant the freshness cache lets go" do - complete_actor(github_id: 1, fetched_at: now) - - expect(selector.earliest_refresh_at(actor_type, now: now)).to eq(now + 86_400) - end - - it "names the earliest across every enriched record" do - complete_actor(github_id: 1, fetched_at: now) - complete_actor(github_id: 2, fetched_at: now - 600) - - expect(selector.earliest_refresh_at(actor_type, now: now)).to eq(now + 85_800) - end - - # A complete row can carry a retry instant: a retryable failure on a refresh keeps the - # status (a network blip must not drop coverage) and backs the row off. The next legal - # fetch is the later of the two. - it "defers to a failed refresh's backoff when it outlasts the TTL" do - complete_actor(github_id: 1, fetched_at: now - 90_000, next_retry_at: now + 300) - - expect(selector.earliest_refresh_at(actor_type, now: now)).to eq(now + 300) - end - - it "ignores a retry instant that has already passed, which no longer defers anything" do - complete_actor(github_id: 1, fetched_at: now, next_retry_at: now - 300) - - expect(selector.earliest_refresh_at(actor_type, now: now)).to eq(now + 86_400) - end - - # The minimum is taken over the whole expression rather than over fetched_at alone, - # because the oldest document may be the one carrying the longest backoff. - it "takes the minimum over both columns together, not over the oldest fetch" do - complete_actor(github_id: 1, fetched_at: now - 86_000, next_retry_at: now + 9_000) - complete_actor(github_id: 2, fetched_at: now - 85_000) - - expect(selector.earliest_refresh_at(actor_type, now: now)).to eq(now + 1_400) - end - - it "reads each class's own TTL" do - short = described_class.new(configuration: configuration_with(REPOSITORY_REFRESH_TTL_SECONDS: "60")) - create_repository(github_id: 1, enrichment_status: "complete", fetched_at: now) - - expect(short.earliest_refresh_at(repository_type, now: now)).to eq(now + 60) - end - - it "is nil when nothing has been enriched, because there is no refresh to name" do - pending_actor(github_id: 1) - - expect(selector.earliest_refresh_at(actor_type, now: now)).to be_nil - end - end - - describe "#earliest_claimable_at" do - it "keeps refresh timing subordinate to the deferred first-time backlog" do - pending_actor(github_id: 1, next_retry_at: now + 300) - create_actor(github_id: 2, enrichment_status: "complete", fetched_at: now - 86_340) - - expect(selector.earliest_claimable_at(actor_type, now: now)).to eq(now + 300) - end - - it "falls back to the refresh pool when nothing is pending at all" do - create_actor(github_id: 1, enrichment_status: "complete", fetched_at: now) - - expect(selector.earliest_claimable_at(actor_type, now: now)).to eq(now + 86_400) - end - - it "is nil for a class with nothing at all in either pool" do - expect(selector.earliest_claimable_at(actor_type, now: now)).to be_nil - end - end - - it "refuses an unknown pool rather than silently selecting nothing" do - expect { selector.scope(actor_type, pool: :everything, now: now) } - .to raise_error(ArgumentError, /everything/) - end -end diff --git a/spec/services/github/enrichment/claim_spec.rb b/spec/services/github/enrichment/claim_spec.rb deleted file mode 100644 index ac04b58..0000000 --- a/spec/services/github/enrichment/claim_spec.rb +++ /dev/null @@ -1,175 +0,0 @@ -require "rails_helper" - -RSpec.describe Github::Enrichment::Claim do - subject(:claim) { described_class.new(configuration: configuration) } - - let(:configuration) { configuration_with } - let(:now) { frozen_time } - let(:actor_type) { Github::Enrichment::EntityType.fetch(:actor) } - - def acquire(pool: :pending, at: now) - claim.acquire(actor_type, pool: pool, now: at) - end - - def pending_actor(github_id: 1, **overrides) - create_actor(github_id: github_id, last_seen_at: now - 60, **overrides) - end - - describe "#acquire" do - it "hands back the entity's identity and the URL enrichment will fetch" do - actor = pending_actor(api_url: "https://api.github.com/users/octocat") - - expect(acquire).to have_attributes(id: actor.id, github_id: actor.github_id, - api_url: "https://api.github.com/users/octocat", - pool: :pending, enrichment_status: "pending") - end - - it "pushes next_retry_at into the future, which is what marks the row in flight" do - pending_actor - - lease = acquire - - expect(lease.leased_until).to eq(now + claim.lease_seconds) - expect(GithubActor.sole.next_retry_at).to eq(lease.leased_until) - end - - it "returns nothing when no candidate is eligible" do - expect(acquire).to be_nil - end - - # S3.6: "Prevent duplicate concurrent enrichment (keyed by the entity row)." The lease - # is what makes the second claim find nothing rather than a second worker fetching the - # same entity and spending a second request on it. - it "prevents a second worker from claiming the same entity" do - pending_actor - - expect(acquire).not_to be_nil - expect(acquire).to be_nil - end - - it "hands the next-best candidate to a second claim when one exists" do - pending_actor(github_id: 1, last_seen_at: now - 10) - pending_actor(github_id: 2, last_seen_at: now - 600) - - expect(acquire.github_id).to eq(1) - expect(acquire.github_id).to eq(2) - end - - # A SIGKILL mid-fetch leaves nothing but a column value. There is no lock to release - # and no transaction to roll back, because none is held across the HTTP call. - it "makes a crashed worker's entity claimable again once the lease expires" do - pending_actor - lease = acquire - - expect(acquire(at: lease.leased_until - 1)).to be_nil - expect(acquire(at: lease.leased_until)).not_to be_nil - end - - # GithubActor::IDENTITY_MERGE gates every identity refresh on - # EXCLUDED.updated_at >= the stored value, so a lease that bumped updated_at would make - # a concurrently-processed page whose received_at predates it silently lose its - # refresh — to a write that may be released a second later. - it "does not move updated_at, because a lease is not observable state" do - actor = pending_actor - - expect { acquire }.not_to change { actor.reload.updated_at } - end - - it "leaves the enrichment status and the attempt count alone, because no fetch has happened" do - actor = pending_actor(enrichment_status: "retryable_failure", enrichment_attempts: 2) - - acquire - - expect(actor.reload).to have_attributes(enrichment_status: "retryable_failure", enrichment_attempts: 2) - end - - it "carries the attempt count forward so the backoff knows which attempt this is" do - pending_actor(enrichment_status: "retryable_failure", enrichment_attempts: 2) - - expect(acquire.enrichment_attempts).to eq(2) - end - - # The pending statement's guard names CANDIDATE_STATUSES, which excludes `complete`, so - # one statement provably cannot serve both pools. - it "claims a refresh candidate, which the pending statement's status guard excludes" do - create_actor(github_id: 1, enrichment_status: "complete", fetched_at: now - 90_000, - last_seen_at: now - 60) - - expect(acquire(pool: :pending)).to be_nil - expect(acquire(pool: :refresh)).to have_attributes(pool: :refresh, enrichment_status: "complete") - end - - it "refuses an unknown pool rather than claiming from the wrong one" do - expect { claim.acquire(actor_type, pool: :everything, now: now) } - .to raise_error(ArgumentError, /everything/) - end - end - - describe "#release!" do - # §7's "failures stay spent" has a mirror here: a deferral leaves no trace. That is - # what makes the whole-row assertion below possible, and it is why the release - # restores the prior instant rather than nulling it. - it "leaves the row byte-for-byte as it found it" do - actor = pending_actor - before = actor.reload.attributes - - claim.release!(acquire) - - expect(actor.reload.attributes).to eq(before) - end - - it "restores the exact prior retry instant rather than clearing it" do - actor = pending_actor(next_retry_at: now - 60) - - claim.release!(acquire) - - expect(actor.reload.next_retry_at).to eq(now - 60) - end - - it "makes the entity immediately claimable again" do - pending_actor - claim.release!(acquire) - - expect(acquire).not_to be_nil - end - - # lease_seconds is the worst-case runtime by construction, so a lease expiring - # mid-flight is reachable. Without the guard a late release would clear a *different* - # worker's fresh lease. - it "refuses to release a lease it no longer holds" do - pending_actor - lease = acquire - GithubActor.sole.update!(next_retry_at: now + 9_999) - - expect(claim.release!(lease)).to be(false) - expect(GithubActor.sole.next_retry_at).to eq(now + 9_999) - end - end - - describe "#lease_seconds" do - # Derived rather than chosen, following RequestGate::WAIT_SECONDS' precedent: every - # term is a real component's worst case, so retuning a timeout retunes the lease. - it "derives from the gate wait, both HTTP timeouts, the retries and the redirects" do - derived = described_class.new(configuration: configuration_with(MAX_HTTP_RETRIES: "0", - MAX_REDIRECTS: "0"), - gate_wait_seconds: 10) - - # One attempt x one hop x (10 + 5 + 15), plus one attempt's worth of backoff. - expect(derived.lease_seconds).to eq(32) - end - - it "grows with the retries a single fetch may make" do - fewer = described_class.new(configuration: configuration_with(MAX_HTTP_RETRIES: "0")) - - expect(claim.lease_seconds).to be > fewer.lease_seconds - end - - it "outlasts the worst case of one gated attempt, so a lease cannot expire mid-request" do - per_request = Github::RequestGate::WAIT_SECONDS + - configuration.http_open_timeout_seconds + - configuration.http_read_timeout_seconds - - expect(claim.lease_seconds).to be > per_request - end - end -end diff --git a/spec/services/github/enrichment/cycle_runner_spec.rb b/spec/services/github/enrichment/cycle_runner_spec.rb new file mode 100644 index 0000000..2c31a67 --- /dev/null +++ b/spec/services/github/enrichment/cycle_runner_spec.rb @@ -0,0 +1,288 @@ +require "rails_helper" + +# One worker cycle over the staged pipeline. The collaborators are verifying doubles: +# this file is about the loop — admission before every request, weighted lane rotation +# with borrowing, pacing waits inside the deadline, and the stop-reason bookkeeping — +# not about what the runners do once called (their own specs prove that). +RSpec.describe Github::Enrichment::CycleRunner do + let(:batch_runner) { instance_double(Github::Enrichment::BatchRunner) } + let(:detail_runner) { instance_double(Github::Enrichment::DetailRunner) } + let(:admission) { instance_double(Github::Enrichment::Admission) } + let(:batch_claim) { instance_double(Github::Enrichment::BatchClaim) } + let(:detail_claim) { instance_double(Github::Enrichment::DetailClaim) } + + let(:granted) { Github::Enrichment::Admission::GRANTED } + + def verdict(reason, retry_in: nil) + Github::Enrichment::Admission::Verdict.new(reason: reason, retry_in_seconds: retry_in) + end + + def batch_result(status:, entity_type: :actor, requested: 0, valid: 0, fallback: 0, reason: nil) + Github::Enrichment::BatchRunner::Result.new( + status: status, entity_type: entity_type, batch_id: 1, requested_count: requested, + returned_count: valid, valid_count: valid, fallback_count: fallback, + deferral_reason: reason + ) + end + + def detail_result(status:, entity_type: :actor, reason: nil) + Github::Enrichment::DetailRunner::Result.new(status: status, entity_type: entity_type, + github_id: 9, batch_id: 2, reason: reason) + end + + def cycle_runner(configuration: Github.configuration, monotonic: -> { 0.0 }, + sleeper: ->(_seconds) { }) + described_class.new(configuration: configuration, batch_runner: batch_runner, + detail_runner: detail_runner, admission: admission, + batch_claim: batch_claim, detail_claim: detail_claim, + clock: -> { frozen_time }, monotonic: monotonic, sleeper: sleeper) + end + + # Most examples focus on one phase; the other is stopped at its first admission. + def quiet_detail_phase + allow(admission).to receive(:detail).and_return(verdict(:class_exhausted, retry_in: 120.0)) + end + + def quiet_batch_phase + allow(admission).to receive(:search).and_return(verdict(:search_ceiling_exhausted, + retry_in: 30.0)) + end + + describe "the batch phase" do + it "runs batches until admission denies, and stops with the denial reason" do + quiet_detail_phase + allow(admission).to receive(:search) + .and_return(granted, granted, verdict(:search_ceiling_exhausted, retry_in: 30.0)) + allow(batch_claim).to receive(:claimable?).and_return(true) + lanes = [] + allow(batch_runner).to receive(:call) do |entity_class:| + lanes << entity_class + batch_result(status: "completed", requested: 10, valid: 9, fallback: 1) + end + + cycle = cycle_runner.call + + expect(cycle).to have_attributes( + batches_attempted: 2, batches_completed: 2, batches_failed: 0, + items_requested: 20, items_valid: 18, fallbacks_admitted: 2, + batch_stop_reason: "search_ceiling_exhausted" + ) + # Default weights 1/1 alternate the lanes. + expect(lanes).to eq([ :actor, :repository ]) + end + + it "sleeps out a pacing wait that fits the deadline, then resumes the loop" do + quiet_detail_phase + allow(admission).to receive(:search) + .and_return(verdict(:search_pacing, retry_in: 6.0), granted, + verdict(:search_reserve_reached, retry_in: 30.0)) + allow(batch_claim).to receive(:claimable?).and_return(true) + allow(batch_runner).to receive(:call) + .and_return(batch_result(status: "completed", requested: 10, valid: 10)) + sleeps = [] + elapsed = { seconds: 0.0 } + runner = cycle_runner(monotonic: -> { elapsed[:seconds] }, + sleeper: ->(seconds) { sleeps << seconds; elapsed[:seconds] += seconds }) + + cycle = runner.call + + expect(sleeps).to eq([ 6.0 ]) + expect(cycle).to have_attributes(batches_attempted: 1, + batch_stop_reason: "search_reserve_reached") + end + + it "does not sleep a pacing wait that would cross the deadline — it stops instead" do + quiet_detail_phase + allow(admission).to receive(:search).and_return(verdict(:search_pacing, retry_in: 60.0)) + sleeps = [] + runner = cycle_runner(sleeper: ->(seconds) { sleeps << seconds }) + + cycle = runner.call + + # batch_runner carries no stub here: a call would raise on the verifying double, + # so reaching the expectations is itself proof no batch was attempted. + expect(sleeps).to be_empty + expect(cycle.batch_stop_reason).to eq("search_pacing") + expect(cycle.batches_attempted).to eq(0) + end + + # The ledger re-checks under its row lock; its denial outranks the advisory + # pre-check that had already granted this iteration. + it "stops the phase with the ledger's reason when the batch itself was deferred" do + quiet_detail_phase + allow(admission).to receive(:search).and_return(granted) + allow(batch_claim).to receive(:claimable?).and_return(true) + allow(batch_runner).to receive(:call) + .and_return(batch_result(status: "deferred", requested: 10, reason: "rate_limited")) + + cycle = cycle_runner.call + + expect(cycle).to have_attributes(batches_attempted: 1, batches_deferred: 1, + batch_stop_reason: "rate_limited") + end + + it "stops after two consecutive idle claims rather than spinning" do + quiet_detail_phase + allow(admission).to receive(:search).and_return(granted) + allow(batch_claim).to receive(:claimable?).and_return(true) + allow(batch_runner).to receive(:call).and_return(batch_result(status: "idle")) + + cycle = cycle_runner.call + + expect(batch_runner).to have_received(:call).twice + expect(cycle).to have_attributes(batches_attempted: 0, batch_stop_reason: "no_batch_work") + end + + it "honors the configured lane weights: actor 2 / repository 1 schedules A A R" do + quiet_detail_phase + allow(admission).to receive(:search) + .and_return(granted, granted, granted, verdict(:search_ceiling_exhausted, retry_in: 30.0)) + allow(batch_claim).to receive(:claimable?).and_return(true) + lanes = [] + allow(batch_runner).to receive(:call) do |entity_class:| + lanes << entity_class + batch_result(status: "completed", requested: 10, valid: 10) + end + runner = cycle_runner(configuration: configuration_with(ACTOR_ENRICHMENT_WEIGHT: "2")) + + runner.call + + expect(lanes).to eq([ :actor, :actor, :repository ]) + end + + it "borrows the slot for the other lane when the scheduled lane has nothing claimable" do + quiet_detail_phase + allow(admission).to receive(:search) + .and_return(granted, verdict(:search_ceiling_exhausted, retry_in: 30.0)) + allow(batch_claim).to receive(:claimable?) do |entity_type, now:| + entity_type.key == :repository + end + lanes = [] + allow(batch_runner).to receive(:call) do |entity_class:| + lanes << entity_class + batch_result(status: "completed", requested: 10, valid: 10) + end + + cycle_runner.call + + expect(lanes).to eq([ :repository ]) + end + + it "stops with no_batch_work when neither lane has anything claimable" do + quiet_detail_phase + allow(admission).to receive(:search).and_return(granted) + allow(batch_claim).to receive(:claimable?).and_return(false) + + cycle = cycle_runner.call + + expect(cycle.batch_stop_reason).to eq("no_batch_work") + end + end + + describe "the detail phase" do + it "mirrors the loop: runs until admission denies, stopping with that reason" do + quiet_batch_phase + allow(admission).to receive(:detail) + .and_return(granted, granted, verdict(:class_exhausted, retry_in: 120.0)) + allow(detail_claim).to receive(:claimable?).and_return(true) + allow(detail_runner).to receive(:call) + .and_return(detail_result(status: "completed"), detail_result(status: "terminal", + reason: "entity_gone_404")) + + cycle = cycle_runner.call + + expect(cycle).to have_attributes( + details_attempted: 2, details_completed: 1, details_terminal: 1, + detail_stop_reason: "class_exhausted" + ) + end + + # A borrowed detail slot is a real authorization: it reaches the core ledger's + # share check through the runner's borrow flag. + it "passes borrow: true to the runner when the slot was borrowed from the other lane" do + quiet_batch_phase + allow(admission).to receive(:detail) + .and_return(granted, verdict(:class_exhausted, retry_in: 120.0)) + allow(detail_claim).to receive(:claimable?) do |entity_type, now:| + entity_type.key == :repository + end + allow(detail_runner).to receive(:call) + .and_return(detail_result(status: "completed", entity_type: :repository)) + + cycle_runner.call + + expect(detail_runner).to have_received(:call).with(entity_class: :repository, borrow: true) + end + + it "passes borrow: false when the scheduled lane claimed its own slot" do + quiet_batch_phase + allow(admission).to receive(:detail) + .and_return(granted, verdict(:class_exhausted, retry_in: 120.0)) + allow(detail_claim).to receive(:claimable?).and_return(true) + allow(detail_runner).to receive(:call).and_return(detail_result(status: "completed")) + + cycle_runner.call + + expect(detail_runner).to have_received(:call).with(entity_class: :actor, borrow: false) + end + + it "stops the phase when a detail request comes back deferred" do + quiet_batch_phase + allow(admission).to receive(:detail).and_return(granted) + allow(detail_claim).to receive(:claimable?).and_return(true) + allow(detail_runner).to receive(:call) + .and_return(detail_result(status: "deferred", reason: "rate_limited")) + + cycle = cycle_runner.call + + expect(cycle).to have_attributes(details_attempted: 1, details_deferred: 1, + detail_stop_reason: "rate_limited") + end + end + + describe "the cycle budget" do + # The deadline is monotonic: a clock already past it stops both phases before + # either consults admission. Neither admission method carries a stub here, so a + # single admission call would raise on the verifying double. + it "stops both phases with cycle_budget once the monotonic deadline has passed" do + ticks = [ 0.0, 100.0, 100.0, 100.0 ] + runner = cycle_runner(monotonic: -> { ticks.shift || 100.0 }) + + cycle = runner.call + + expect(cycle).to have_attributes(batch_stop_reason: "cycle_budget", + detail_stop_reason: "cycle_budget", + duration_ms: 100_000) + end + end + + describe "the completed cycle" do + it "aggregates both phases' counters and emits the INFO summary" do + allow(Rails.logger).to receive(:info) + allow(admission).to receive(:search) + .and_return(granted, granted, verdict(:search_ceiling_exhausted, retry_in: 30.0)) + allow(admission).to receive(:detail) + .and_return(granted, verdict(:class_exhausted, retry_in: 120.0)) + allow(batch_claim).to receive(:claimable?).and_return(true) + allow(detail_claim).to receive(:claimable?).and_return(true) + allow(batch_runner).to receive(:call) + .and_return(batch_result(status: "completed", requested: 10, valid: 8, fallback: 2), + batch_result(status: "failed", requested: 5)) + allow(detail_runner).to receive(:call).and_return(detail_result(status: "completed")) + + cycle = cycle_runner.call + + expect(cycle).to have_attributes( + batches_attempted: 2, batches_completed: 1, batches_failed: 1, batches_deferred: 0, + items_requested: 15, items_valid: 8, fallbacks_admitted: 2, + details_attempted: 1, details_completed: 1, details_terminal: 0, + batch_stop_reason: "search_ceiling_exhausted", detail_stop_reason: "class_exhausted" + ) + expect(cycle.duration_ms).to be >= 0 + expect(Rails.logger).to have_received(:info).with( + hash_including(event: "enrichment.cycle_completed", batches_attempted: 2, + details_attempted: 1, batch_stop_reason: "search_ceiling_exhausted") + ) + end + end +end diff --git a/spec/services/github/enrichment/detail_claim_spec.rb b/spec/services/github/enrichment/detail_claim_spec.rb new file mode 100644 index 0000000..3252e8f --- /dev/null +++ b/spec/services/github/enrichment/detail_claim_spec.rb @@ -0,0 +1,117 @@ +require "rails_helper" + +# The bounded detail-fallback lane's claim: one row per call, FIFO by the instant the +# batch admitted it. Its stage vocabulary (detail_pending/detail_in_flight) is disjoint +# from the batch path's by construction, so nothing here can double-claim a batch row. +RSpec.describe Github::Enrichment::DetailClaim do + let(:now) { frozen_time } + let(:configuration) { Github.configuration } + let(:claim) { described_class.new(configuration: configuration) } + let(:actor_type) { Github::Enrichment::EntityType.fetch(:actor) } + + def fallback_actor(github_id:, detail_pending_at: now - 60, **overrides) + create_actor(github_id: github_id, login: "user-#{github_id}", + api_url: "https://api.github.com/users/user-#{github_id}", + enrichment_stage: "detail_pending", + detail_pending_at: detail_pending_at, **overrides) + end + + # See batch_claim_spec: correlation_id is supplied because the model validates its + # presence while the column default is database-side. + def orphan_batch + EnrichmentBatch.create!(request_kind: "detail", entity_kind: "actor", + started_at: now - 700, correlation_id: SecureRandom.uuid, + request_url: "https://api.github.com/users/orphan") + end + + describe "#scope" do + it "admits only detail-stage rows that carry an admission instant and are due" do + fallback_actor(github_id: 1) + fallback_actor(github_id: 2, enrichment_stage: "detail_in_flight", + leased_until: now - 1) + # Backed off, still leased, never admitted, or resting on the batch path — all out. + fallback_actor(github_id: 3, next_retry_at: now + 300) + fallback_actor(github_id: 4, enrichment_stage: "detail_in_flight", + leased_until: now + 300) + create_actor(github_id: 5, login: "user-5") + fallback_actor(github_id: 6, detail_pending_at: nil) + + expect(claim.scope(actor_type, now: now).pluck(:github_id)).to match_array([ 1, 2 ]) + expect(claim.claimable?(actor_type, now: now)).to be(true) + end + end + + describe "#acquire" do + it "claims the oldest admission first, by detail_pending_at then id" do + fallback_actor(github_id: 11, detail_pending_at: now - 100) + fallback_actor(github_id: 12, detail_pending_at: now - 300) + + first = claim.acquire(actor_type, now: now) + second = claim.acquire(actor_type, now: now) + + expect(first.item.github_id).to eq(12) + expect(second.item.github_id).to eq(11) + end + + it "leases the row and records the attempt as a detail-kind batch row" do + fallback_actor(github_id: 21) + + lease = claim.acquire(actor_type, now: now) + row = GithubActor.find_by(github_id: 21) + + expect(row).to have_attributes( + enrichment_stage: "detail_in_flight", lease_token: lease.token, + leased_until: now + configuration.enrichment_lease_seconds, + current_enrichment_batch_id: lease.batch.id + ) + expect(lease.batch).to have_attributes( + request_kind: "detail", entity_kind: "actor", status: "in_flight", + requested_github_ids: [ 21 ], requested_identifiers: [ "user-21" ], + requested_count: 1, request_url: "https://api.github.com/users/user-21", + started_at: now + ) + expect(lease.item).to have_attributes(github_id: 21, identifier: "user-21", + api_url: "https://api.github.com/users/user-21") + end + + it "returns nil while the only candidate is under a live lease" do + fallback_actor(github_id: 31) + claim.acquire(actor_type, now: now) + + expect(claim.acquire(actor_type, now: now)).to be_nil + expect(claim.claimable?(actor_type, now: now)).to be(false) + end + + it "reclaims an expired detail lease and finalizes its orphaned batch as stale_lease" do + stale = orphan_batch + fallback_actor(github_id: 41, enrichment_stage: "detail_in_flight", + lease_token: SecureRandom.uuid, leased_until: now - 1, + current_enrichment_batch_id: stale.id) + allow(Rails.logger).to receive(:warn) + + lease = claim.acquire(actor_type, now: now) + + expect(lease.item.github_id).to eq(41) + expect(lease.batch.id).not_to eq(stale.id) + expect(stale.reload).to have_attributes(status: "stale_lease", completed_at: now) + expect(Rails.logger).to have_received(:warn).with( + hash_including(event: "enrichment.stale_lease_reclaimed", + enrichment_batch_ids: [ stale.id ], count: 1) + ) + end + end + + describe "#release!" do + it "restores the row to detail_pending with its lease cleared" do + fallback_actor(github_id: 51) + lease = claim.acquire(actor_type, now: now) + + claim.release!(lease, now: now) + + expect(GithubActor.find_by(github_id: 51)).to have_attributes( + enrichment_stage: "detail_pending", detail_pending_at: now - 60, + lease_token: nil, leased_until: nil, current_enrichment_batch_id: nil + ) + end + end +end diff --git a/spec/services/github/enrichment/detail_runner_spec.rb b/spec/services/github/enrichment/detail_runner_spec.rb new file mode 100644 index 0000000..a798101 --- /dev/null +++ b/spec/services/github/enrichment/detail_runner_spec.rb @@ -0,0 +1,329 @@ +require "rails_helper" + +# The bounded exception path: one core request against the api_url the event payload +# supplied, never a URL built from a login or name. Executor calls are recorded, so +# the URL discipline and the borrow flag are asserted from what was actually sent. +RSpec.describe Github::Enrichment::DetailRunner do + let(:now) { frozen_time } + let(:configuration) { Github.configuration } + + let(:executor_class) do + Class.new do + attr_reader :requests + + def initialize(&responder) + @responder = responder + @requests = [] + end + + def call(request) + @requests << request + @responder.call(request, @requests.length) + end + end + end + + def recording_executor(&responder) + executor_class.new(&responder) + end + + def runner(executor, now: frozen_time) + described_class.new( + executor: executor, configuration: configuration, + claim: Github::Enrichment::DetailClaim.new(configuration: configuration), + backoff: jitterless_backoff(configuration: configuration), + clock: -> { now } + ) + end + + def core_headers(remaining: 55, used: 5) + { "x-ratelimit-resource" => "core", "x-ratelimit-limit" => "60", + "x-ratelimit-remaining" => remaining.to_s, "x-ratelimit-used" => used.to_s, + "x-ratelimit-reset" => (frozen_time + 3600).to_i.to_s } + end + + def detail_success(request, body:) + Github::FetchResult.from_response(request: request, status: 200, + headers: core_headers, body: body, duration_ms: 1.0) + end + + def transport_failure(request) + Github::FetchResult.from_error(request: request, + error: Github::Errors::ConnectionFailed.new("connection refused"), + classification: :transport_error) + end + + def actor_body(github_id:, login: "octocat") + JSON.generate("id" => github_id, "login" => login, "type" => "User") + end + + def fallback_actor(github_id:, detail_pending_at: now - 60, **overrides) + create_actor(github_id: github_id, login: "user-#{github_id}", + api_url: "https://api.github.com/users/user-#{github_id}", + enrichment_stage: "detail_pending", + detail_pending_at: detail_pending_at, **overrides) + end + + describe "the request it sends" do + it "fetches only the stored api_url, payload-origin, on the core detail class" do + fallback_actor(github_id: 101) + executor = recording_executor do |request, _call| + detail_success(request, body: actor_body(github_id: 101)) + end + + runner(executor).call(entity_class: GithubActor) + + expect(executor.requests.length).to eq(1) + expect(executor.requests.first).to have_attributes( + url: "https://api.github.com/users/user-101", + request_class: :actor, origin: :payload, borrow: false + ) + end + + it "carries the caller's borrow authorization through to the request" do + fallback_actor(github_id: 102) + executor = recording_executor do |request, _call| + detail_success(request, body: actor_body(github_id: 102)) + end + + runner(executor).call(entity_class: GithubActor, borrow: true) + + expect(executor.requests.first.borrow).to be(true) + end + + it "returns idle without a request when nothing is claimable" do + executor = recording_executor { raise "must not be called" } + + result = runner(executor).call(entity_class: GithubActor) + + expect(result).to have_attributes(status: "idle", github_id: nil, batch_id: nil) + expect(executor.requests).to be_empty + end + end + + describe "a successful fetch" do + it "projects the document, appends a detail observation, and finalizes the batch" do + fallback_actor(github_id: 201) + allow(Rails.logger).to receive(:info) + executor = recording_executor do |request, _call| + detail_success(request, body: actor_body(github_id: 201)) + end + + result = runner(executor).call(entity_class: GithubActor) + row = GithubActor.find_by(github_id: 201) + observation = EnrichmentObservation.find_by(entity_github_id: 201) + + expect(result).to have_attributes(status: "completed", github_id: 201, reason: nil) + expect(row).to have_attributes( + enrichment_status: "complete", enrichment_stage: "contract_complete", + account_type: "User", detail_attempts: 1, enrichment_attempts: 0, + fetched_at: now, batch_applied_at: now, contract_completed_at: now, + latest_observation_id: observation.id, latest_observation_source: "detail", + lease_token: nil, current_enrichment_batch_id: nil + ) + expect(observation).to have_attributes(source: "detail", validation_outcome: "applied", + enrichment_batch_id: result.batch_id) + expect(EnrichmentBatch.find(result.batch_id)).to have_attributes( + status: "succeeded", returned_count: 1, valid_count: 1, response_status: 200, + response_body: nil, rate_limit_resource: "core", rate_limit_remaining: 55 + ) + expect(Rails.logger).to have_received(:info).with( + hash_including(event: "enrichment.detail_completed", github_actor_id: 201, + detail_attempts: 1) + ) + end + end + + describe "a confirmed-gone entity" do + it "is terminal immediately, even on the first attempt, with the evidence retained" do + row = fallback_actor(github_id: 301) + prior = Github::Enrichment::ObservationRecorder.record!( + entity_type: Github::Enrichment::EntityType.fetch(:actor), entity_github_id: 301, + source: :event, raw_payload: { "id" => 301 }, observed_at: now - 600, + validation_outcome: "applied" + ) + allow(Rails.logger).to receive(:warn) + executor = recording_executor do |request, _call| + Github::FetchResult.from_response(request: request, status: 404, + headers: core_headers, + body: '{"message":"Not Found"}', duration_ms: 1.0) + end + + result = runner(executor).call(entity_class: GithubActor) + + expect(result).to have_attributes(status: "terminal", reason: "entity_gone_404") + expect(row.reload).to have_attributes( + enrichment_status: "permanent_failure", enrichment_stage: "terminal", + terminal_at: now, last_error: "entity_gone_404", detail_attempts: 1, + next_retry_at: nil, lease_token: nil, + # The event-native identity survives the terminal outcome. + login: "user-301", api_url: "https://api.github.com/users/user-301" + ) + expect(EnrichmentObservation.exists?(prior.id)).to be(true) + expect(EnrichmentBatch.find(result.batch_id).status).to eq("failed") + expect(Rails.logger).to have_received(:warn).with( + hash_including(event: "enrichment.detail_terminal", reason: "entity_gone_404") + ) + end + + # Found in live traffic, not in design: the actor `github-actions[bot]` carries a + # login with brackets, so the URL its own event supplied is unparsable and + # Github::UrlPolicy refuses it before the gate. §10 classifies a policy violation + # as permanent, and the ladder would otherwise spend the scarce core detail + # allowance three times refusing the same stored string. + it "is terminal immediately when the stored URL cannot pass the SSRF policy" do + row = fallback_actor(github_id: 302, + api_url: "https://api.github.com/users/github-actions[bot]") + allow(Rails.logger).to receive(:warn) + + # The real executor, so the refusal comes from Github::UrlPolicy itself rather + # than from a double asserting what it would have done. Nothing reaches a socket: + # validation happens before the gate, and WebMock would refuse it regardless. + result = runner( + Github::RequestExecutor.new(transport: Github::Transports::Faraday.new, + mode: :live, sleeper: ->(_seconds) { }, + clock: -> { now }) + ).call(entity_class: GithubActor) + + expect(result.status).to eq("terminal") + expect(row.reload).to have_attributes( + enrichment_status: "permanent_failure", enrichment_stage: "terminal", + terminal_at: now, detail_attempts: 1, next_retry_at: nil + ) + expect(row.last_error).to match(/unparsable/) + end + end + + describe "the retry ladder" do + it "reschedules a transport failure in detail_pending with the backoff instant" do + fallback_actor(github_id: 401) + allow(Rails.logger).to receive(:warn) + executor = recording_executor { |request, _call| transport_failure(request) } + + result = runner(executor).call(entity_class: GithubActor) + row = GithubActor.find_by(github_id: 401) + + expect(result.status).to eq("retry_scheduled") + # detail_pending, never retry_scheduled: that stage belongs to the batch path, + # and re-batching would spend Search budget reproducing the same miss. + expect(row).to have_attributes( + enrichment_status: "retryable_failure", enrichment_stage: "detail_pending", + detail_attempts: 1, enrichment_attempts: 1, + next_retry_at: now + 60, retry_scheduled_at: now, + last_error: "connection refused", lease_token: nil + ) + expect(EnrichmentBatch.sole).to have_attributes(status: "failed", invalid_count: 1) + expect(Rails.logger).to have_received(:warn).with( + hash_including(event: "enrichment.detail_retry_scheduled", detail_attempts: 1) + ) + end + + it "walks a malformed document up the same ladder, retaining it as an observation" do + fallback_actor(github_id: 411) + executor = recording_executor do |request, _call| + detail_success(request, body: "not json") + end + + result = runner(executor).call(entity_class: GithubActor) + observation = EnrichmentObservation.find_by(entity_github_id: 411) + + expect(result.status).to eq("retry_scheduled") + expect(observation.validation_outcome).to eq("unparsable_document") + expect(observation.raw_payload).to eq("unparsed_body" => "not json") + expect(GithubActor.find_by(github_id: 411)).to have_attributes( + enrichment_stage: "detail_pending", detail_attempts: 1, next_retry_at: now + 60 + ) + end + + it "goes terminal at DETAIL_FALLBACK_MAX_ATTEMPTS instead of retrying forever" do + fallback_actor(github_id: 421, detail_attempts: configuration.detail_fallback_max_attempts - 1) + executor = recording_executor { |request, _call| transport_failure(request) } + + result = runner(executor).call(entity_class: GithubActor) + + expect(result.status).to eq("terminal") + expect(GithubActor.find_by(github_id: 421)).to have_attributes( + enrichment_status: "permanent_failure", enrichment_stage: "terminal", + terminal_at: now, detail_attempts: configuration.detail_fallback_max_attempts, + next_retry_at: nil + ) + end + + # A refresh row that fell back keeps its completed contract: a retryable failure + # delays the refresh without demoting the business outcome. + it "keeps a complete refresh row complete while its detail retry backs off" do + fallback_actor(github_id: 431, enrichment_status: "complete", + fetched_at: now - 172_800) + executor = recording_executor { |request, _call| transport_failure(request) } + + result = runner(executor).call(entity_class: GithubActor) + + expect(result.status).to eq("retry_scheduled") + expect(GithubActor.find_by(github_id: 431)).to have_attributes( + enrichment_status: "complete", enrichment_stage: "detail_pending", + next_retry_at: now + 60 + ) + end + end + + describe "a rate-limited fetch" do + it "defers and releases without counting an attempt" do + active_budget_window(now: now) + fallback_actor(github_id: 501) + allow(Rails.logger).to receive(:info) + executor = recording_executor do |request, _call| + Github::FetchResult.from_response(request: request, status: 403, + headers: core_headers(remaining: 0), + body: '{"message":"rate limited"}', + duration_ms: 1.0) + end + + result = runner(executor).call(entity_class: GithubActor) + row = GithubActor.find_by(github_id: 501) + + expect(result).to have_attributes(status: "deferred", reason: "rate_limited") + expect(row).to have_attributes(enrichment_stage: "detail_pending", + detail_attempts: 0, enrichment_attempts: 0, + next_retry_at: nil, lease_token: nil) + expect(EnrichmentBatch.find(result.batch_id).status).to eq("deferred") + expect(Rails.logger).to have_received(:info).with( + hash_including(event: "enrichment.detail_deferred", deferral_reason: "rate_limited") + ) + end + end + + describe "a lease lost mid-flight" do + it "reports lease_lost rather than double-applying the projection" do + fallback_actor(github_id: 601) + executor = recording_executor do |request, _call| + GithubActor.where(github_id: 601).update_all(lease_token: SecureRandom.uuid) + detail_success(request, body: actor_body(github_id: 601)) + end + + result = runner(executor).call(entity_class: GithubActor) + + expect(result).to have_attributes(status: "lease_lost", reason: "lease_lost") + expect(GithubActor.find_by(github_id: 601).account_type).to be_nil + expect(EnrichmentBatch.find(result.batch_id)).to have_attributes( + status: "succeeded", valid_count: 0, invalid_count: 1 + ) + end + end + + describe "a crash between claim and outcome" do + it "finalizes the batch as failed, releases the row, and re-raises" do + fallback_actor(github_id: 701) + executor = recording_executor { raise RuntimeError, "boom" } + + expect { runner(executor).call(entity_class: GithubActor) } + .to raise_error(RuntimeError, "boom") + + expect(EnrichmentBatch.sole).to have_attributes(status: "failed", + last_error: "RuntimeError") + expect(GithubActor.find_by(github_id: 701)).to have_attributes( + enrichment_stage: "detail_pending", detail_attempts: 0, lease_token: nil, + current_enrichment_batch_id: nil + ) + end + end +end diff --git a/spec/services/github/enrichment/dispatch_spec.rb b/spec/services/github/enrichment/dispatch_spec.rb index 88c3e36..23fb0b6 100644 --- a/spec/services/github/enrichment/dispatch_spec.rb +++ b/spec/services/github/enrichment/dispatch_spec.rb @@ -1,8 +1,8 @@ require "rails_helper" -# The one rule both enqueue paths share (§8 steps 10 and 11): "is there durable enrichment -# work this class could do right now?", answered from the committed entity rows and the -# ledger, never from the queue. +# The one rule both enqueue paths share (§8 steps 10 and 11): "is there staged enrichment +# work a cycle could do right now?", answered from the committed entity rows and the two +# ledgers, never from the queue. RSpec.describe Github::Enrichment::Dispatch do subject(:dispatch) { described_class.new(clock: -> { frozen_time }) } @@ -14,62 +14,108 @@ def repository(**overrides) create_repository(github_id: 1_296_269, last_seen_at: frozen_time, **overrides) end + def detail_pending!(model) + model.update_all(enrichment_stage: "detail_pending", detail_pending_at: frozen_time) + end + before { active_budget_window(now: frozen_time) } - describe "when a class has work" do - it "enqueues one cycle for each class that does, and none for the class that does not" do + describe "when there is claimable batch work" do + # A missing search-ledger row is a grant: the search ledger self-bootstraps from + # configuration, so a clean checkout's first committed entity is immediately claimable. + it "enqueues exactly one cycle, whichever classes have work" do actor + repository expect { dispatch.call(reason: "reconcile") } - .to have_enqueued_job(EnrichActorJob).exactly(:once) - expect(ActiveJob::Base.queue_adapter.enqueued_jobs.map { _1[:job] }).to eq([ EnrichActorJob ]) + .to have_enqueued_job(EnrichmentCycleJob).exactly(:once) + expect(ActiveJob::Base.queue_adapter.enqueued_jobs.map { _1[:job] }).to eq([ EnrichmentCycleJob ]) end - # However deep the backlog. Github::EnrichmentRunner enriches at most one entity per call, - # so queue depth is set by §10's hourly allowance and not by how many rows are waiting — - # 90 pending actors would otherwise become 90 cycles the ledger refuses 40 requests in. + # However deep the backlog. EnrichmentCycleJob loops until a ledger denies, so queue + # depth is set by the budgets, not by how many rows are waiting — 90 pending actors + # would otherwise become 90 cycles the ledgers refuse most of. it "enqueues one cycle whether one entity is pending or fifty" do 50.times { |index| create_actor(github_id: 1_000 + index, last_seen_at: frozen_time) } - expect { dispatch.call(reason: "reconcile") }.to have_enqueued_job(EnrichActorJob).exactly(:once) + expect { dispatch.call(reason: "reconcile") } + .to have_enqueued_job(EnrichmentCycleJob).exactly(:once) end - it "reports what it scheduled" do + # Pacing is a wait, not a refusal: the cycle sleeps it out on its own worker thread, + # so a paced ledger is still admissible work. + it "still enqueues while the search ledger is only pacing" do actor - repository + active_search_window(now: frozen_time, last_request_at: frozen_time - 2) - expect(dispatch.call(reason: "ingestion")) - .to eq(actor_enqueued: 1, repository_enqueued: 1, reason: "ingestion") + expect { dispatch.call(reason: "reconcile") } + .to have_enqueued_job(EnrichmentCycleJob).exactly(:once) end - # §11 lists "reconciliation summaries" among the INFO events, and PR 7's Summary is what - # fills it — per-status counts, per-class share usage, the window state. - it "logs the summary at INFO, because a tick that scheduled work is worth reading" do + it "reports what it scheduled, and nothing that blocked" do + actor + + expect(dispatch.call(reason: "ingestion")).to eq(cycle_enqueued: 1, reason: "ingestion") + end + + # INFO only when it scheduled something. The rich per-stage summary lives on /status + # and the one-shot now — this line carries the decision alone, so the exact-argument + # match below is also the proof that no summary is merged onto it anymore. + it "logs the bare decision at INFO, with no summary merged onto the line" do actor allow(Rails.logger).to receive(:info) dispatch.call(reason: "reconcile") expect(Rails.logger).to have_received(:info).with( - hash_including(event: "enrichment.dispatched", reason: "reconcile", actor_enqueued: 1, - enrichment_used: 0, enrichment_allowance: 40, window_status: "active") + event: "enrichment.dispatched", cycle_enqueued: 1, reason: "reconcile" ) end end - describe "when there is nothing to do" do - it "enqueues nothing when every entity is decided" do - actor(enrichment_status: "complete", fetched_at: frozen_time) - repository(enrichment_status: "permanent_failure") + describe "when only detail-fallback work is claimable" do + before do + # Search denied outright (remaining at the reserve), so batch work alone could not + # justify a cycle — the detail lane has to carry the decision. + active_search_window(now: frozen_time, remaining: 2) + end + + it "enqueues one cycle for a claimable detail row under a granted core window" do + actor + detail_pending!(GithubActor) + + expect { dispatch.call(reason: "reconcile") } + .to have_enqueued_job(EnrichmentCycleJob).exactly(:once) + end + + it "enqueues nothing when the core allowance is spent too" do + actor + detail_pending!(GithubActor) + active_budget_window(now: frozen_time, enrichment_used: 4, reset_at: frozen_time + 600) expect { dispatch.call(reason: "reconcile") }.not_to have_enqueued_job + expect(dispatch.call(reason: "reconcile")) + .to include(blocked_by: [ :search_reserve_reached, :class_exhausted ]) end + end + + describe "when there is nothing claimable" do + it "enqueues nothing when every entity is decided, and names both empty lanes" do + actor(enrichment_status: "complete", enrichment_stage: "contract_complete", + fetched_at: frozen_time) + repository(enrichment_status: "permanent_failure", enrichment_stage: "terminal") - # An entity another worker is mid-way through is not pending work: the claim lease lives - # on next_retry_at, and every one of the selector's queries excludes it. + expect { dispatch.call(reason: "reconcile") }.not_to have_enqueued_job + expect(dispatch.call(reason: "reconcile")) + .to include(blocked_by: [ :no_batch_work, :no_detail_work ]) + end + + # An entity another worker is mid-way through is not pending work: the live lease + # excludes it from both claims until leased_until passes. it "enqueues nothing while the only candidate is leased by a live worker" do actor - GithubActor.update_all(next_retry_at: frozen_time + 600) + GithubActor.update_all(enrichment_stage: "batch_in_flight", + leased_until: frozen_time + 600) expect { dispatch.call(reason: "reconcile") }.not_to have_enqueued_job end @@ -87,77 +133,41 @@ def repository(**overrides) end end - # §9's effective_enrichment_time, minus the entity component this object is not choosing. - describe "when the ledger says enrichment cannot happen" do - it "enqueues nothing while a global block is in force, and names it" do - actor - active_budget_window(now: frozen_time, global_blocked_until: frozen_time + 300) - - expect { dispatch.call(reason: "reconcile") }.not_to have_enqueued_job - expect(dispatch.call(reason: "reconcile")).to include(blocked_by: :global_blocked_until) - end - - it "enqueues nothing once the enrichment class has spent its allowance" do - actor - active_budget_window(now: frozen_time, enrichment_used: 40, reset_at: frozen_time + 600) - - expect { dispatch.call(reason: "reconcile") }.not_to have_enqueued_job - expect(dispatch.call(reason: "reconcile")).to include(blocked_by: :enrichment_class_blocked_until) - end - - # §10: a share exhaustion is a *denial* relieved by borrowing, not a deferral. Refusing to - # enqueue on it would withhold work the ledger would have granted — the reason - # Github::EnrichmentSchedule leaves the share out too. - it "still enqueues when only one class's fairness share is spent" do + describe "when both ledgers deny" do + # blocked_by is always the pair [search-side, detail-side], so an operator reads one + # line and knows which lane to look at. + it "enqueues nothing while the search ledger is blocked and no core window exists" do actor - active_budget_window(now: frozen_time, actor_share_used: 20, enrichment_used: 20) - - expect { dispatch.call(reason: "reconcile") }.to have_enqueued_job(EnrichActorJob) - end - end - - # A clean checkout has no ledger row: the first enrichment reservation would create an - # uninitialized row and be denied until a poll supplies authoritative headers. Dispatch - # avoids enqueueing that known-no-op while remaining read-only. - describe "before the first window exists" do - it "does not enqueue or create the ledger row" do + detail_pending!(GithubActor) + active_search_window(now: frozen_time, blocked_until: frozen_time + 300) GithubApiBudget.delete_all - actor - expect { dispatch.call(reason: "ingestion") }.not_to have_enqueued_job - expect(GithubApiBudget.count).to eq(0) - expect(dispatch.call(reason: "reconcile")).to include(blocked_by: :window_uninitialized) + expect { dispatch.call(reason: "reconcile") }.not_to have_enqueued_job + expect(dispatch.call(reason: "reconcile")) + .to include(blocked_by: [ :search_blocked, :window_uninitialized ]) end - it "does not enqueue when an uninitialized ledger row already exists" do - GithubApiBudget.delete_all - Github::BudgetLedger.new.bootstrap!(now: frozen_time) + it "enqueues nothing once the search ceiling and the detail allowance are both spent" do actor + repository + detail_pending!(GithubRepository) + active_search_window(now: frozen_time, remaining: 5, used: 8) + active_budget_window(now: frozen_time, enrichment_used: 4, reset_at: frozen_time + 600) expect { dispatch.call(reason: "reconcile") }.not_to have_enqueued_job - expect(dispatch.call(reason: "reconcile")).to include(blocked_by: :window_uninitialized) - end - - [ 0, 40 ].each do |used| - it "does not enqueue after an old window elapses with #{used} enrichment attempts used" do - active_budget_window(now: frozen_time - 3600, reset_at: frozen_time - 1, - enrichment_used: used, - actor_share_used: used / 2, - repository_share_used: used / 2) - actor - - expect { dispatch.call(reason: "reconcile") }.not_to have_enqueued_job - expect(dispatch.call(reason: "reconcile")).to include(blocked_by: :window_elapsed) - end + expect(dispatch.call(reason: "reconcile")) + .to include(blocked_by: [ :search_ceiling_exhausted, :class_exhausted ]) end end it "makes no GitHub request and takes no lock" do actor expect(Github::RequestGate).not_to receive(:hold) + expect(Github::SourceLock).not_to receive(:acquire) dispatch.call(reason: "reconcile") expect(WebMock).not_to have_requested(:any, //) + expect(Github::LockOrder.held_keys).to be_empty end end diff --git a/spec/services/github/enrichment/end_to_end_spec.rb b/spec/services/github/enrichment/end_to_end_spec.rb index 5bf78d1..1ea4234 100644 --- a/spec/services/github/enrichment/end_to_end_spec.rb +++ b/spec/services/github/enrichment/end_to_end_spec.rb @@ -1,33 +1,49 @@ require "rails_helper" # §12: "GITHUB_MODE=fixture selects the FixtureEvents source and the Fixture transport — -# beneath both polling *and* enrichment, so the complete flow (poll → persist → stub → +# beneath both polling *and* enrichment, so the complete flow (poll → persist → stage → # enrich) runs with zero network." # -# The corpus already supports this with no new fixture authoring. Page 1 persists four push -# events (1, 2, 3 and 8; 4 is a WatchEvent and 5-7 quarantine, and the quarantine path -# writes no stubs), producing three actors and three repositories. Four resolve 200 and two -# resolve 404 — which is what fixtures/github/README.md documents event 8 for. +# The corpus supports the whole staged pipeline. Page 1 persists four push events +# (1, 2, 3 and 8; 4 is a WatchEvent and 5–7 quarantine), producing three actors and +# three repositories in FIFO order. One cycle then issues exactly four requests: one +# Search batch per class (octocat and monalisa resolve; ghostuser and deleted-org/gone +# are missing from the Search results), and one core detail fallback per ghost, both of +# which 404 — fixtures/github/README.md documents event 8 for exactly this. RSpec.describe "enrichment end to end", type: :integration do let(:now) { frozen_time } let(:transport) { fixture_transport } - let(:runner) { fixture_enrichment_runner(transport: transport, now: now) } + # Pacing is disabled so the second Search batch of the cycle needs no wall clock; + # every other knob keeps its pinned default. + let(:configuration) { configuration_with("GITHUB_MODE" => "fixture", "SEARCH_PACING_SECONDS" => "0") } # One transport for the whole flow, so the poll and the enrichment requests share a # scripted cursor exactly as one process would. - def ingest!(at: now, force: false) - fixture_runner(transport: transport, now: at) - .call(event_source: fixture_event_source, force: force) + def ingest!(at: now) + fixture_runner(transport: transport, now: at).call(event_source: fixture_event_source) end - def enrich!(cycles: 1, **arguments) - Array.new(cycles) { runner.call(**arguments) } + # The executor's clock travels with the cycle's: the ledgers roll their windows on + # the reservation clock, so a cycle asked to run "a minute later" must reserve a + # minute later too, exactly as one process would. + def run_cycle!(at: now) + executor = fixture_executor(transport: transport, ledger: ledger_for(configuration), + search_ledger: search_ledger_for(configuration), + clock: -> { at }) + + fixture_cycle_runner( + transport: transport, now: at, configuration: configuration, + batch_runner: fixture_batch_runner(transport: transport, now: at, + configuration: configuration, executor: executor), + detail_runner: fixture_detail_runner(transport: transport, now: at, + configuration: configuration, executor: executor) + ).call end - describe "the whole flow with no network at all" do + describe "one cycle over the freshly ingested corpus, with no network at all" do before do ingest! - enrich!(cycles: 6) + @cycle = run_cycle! end it "persists four push events and three entities of each class" do @@ -36,47 +52,82 @@ def enrich!(cycles: 1, **arguments) expect(GithubRepository.count).to eq(3) end - it "enriches both resolvable actors in the finite fixture corpus" do - expect(GithubActor.find_by(github_id: 583_231)) - .to have_attributes(enrichment_status: "complete", name: "The Octocat", fetched_at: now) + it "completes both resolvable actors from one Search batch" do + expect(GithubActor.find_by(github_id: 583_231)).to have_attributes( + enrichment_status: "complete", enrichment_stage: "contract_complete", + account_type: "User", fetched_at: now, batch_applied_at: now, + contract_completed_at: now, latest_observation_source: "search" + ) expect(GithubActor.find_by(github_id: 1_024_025)) - .to have_attributes(enrichment_status: "complete", name: "Mona Lisa Octocat") + .to have_attributes(enrichment_status: "complete", enrichment_stage: "contract_complete") end - it "writes §7's enrichment-owned columns for the fixture's resolved repository" do + it "completes both resolvable repositories with the staged contract columns" do expect(GithubRepository.find_by(github_id: 1_296_269)).to have_attributes( - enrichment_status: "complete", description: "My first repository on GitHub!", - language: "Ruby", owner_github_id: 583_231 + enrichment_status: "complete", enrichment_stage: "contract_complete", + description: "My first repository on GitHub!", language: "Ruby", + owner_github_id: 583_231, owner_login: "octocat", + fork: false, archived: false, default_branch: "main" + ) + expect(GithubRepository.find_by(github_id: 1_300_192)).to have_attributes( + enrichment_status: "complete", default_branch: "trunk", description: nil ) end # §10: "actor or repo URL returns 404/410 → entity permanent_failure; source stays - # enabled." - it "marks the two entities that no longer exist permanently failed" do - expect(GithubActor.find_by(github_id: 7_700_421).enrichment_status).to eq("permanent_failure") - expect(GithubRepository.find_by(github_id: 1_490_033).enrichment_status).to eq("permanent_failure") + # enabled" — reached through the staged path: missing from Search, admitted to the + # bounded detail lane, confirmed gone there. + it "sends both ghosts down the detail lane to an entity-specific terminal" do + expect(GithubActor.find_by(github_id: 7_700_421)).to have_attributes( + enrichment_status: "permanent_failure", enrichment_stage: "terminal", + terminal_at: now, last_error: "entity_gone_404" + ) + expect(GithubRepository.find_by(github_id: 1_490_033)).to have_attributes( + enrichment_status: "permanent_failure", enrichment_stage: "terminal" + ) end - it "never disables the event source over an entity that disappeared" do + it "never disables the event source over entities that disappeared" do expect(EventSource.sole).to have_attributes(status: "idle", enabled: true) end - it "spends exactly one enrichment request per entity, split evenly across the classes" do - expect(current_budget).to have_attributes(enrichment_used: 6, actor_share_used: 3, - repository_share_used: 3) + it "spends exactly two Search requests, split one per lane" do + expect(current_search_budget).to have_attributes(used: 2, actor_used: 1, repository_used: 1) + end + + it "spends exactly two core detail requests of the bounded fallback allowance" do + expect(current_budget).to have_attributes( + poll_used: 1, enrichment_used: 2, actor_share_used: 1, repository_share_used: 1 + ) + end + + it "retains one batch row per request, with the miss counted where it happened" do + search = EnrichmentBatch.where(request_kind: "search").order(:id) + detail = EnrichmentBatch.where(request_kind: "detail").order(:id) + + expect(search.pluck(:entity_kind, :status, :requested_count, :returned_count, + :valid_count, :missing_count)) + .to eq([ [ "actor", "succeeded", 3, 2, 2, 1 ], + [ "repository", "succeeded", 3, 2, 2, 1 ] ]) + expect(detail.pluck(:entity_kind, :status, :response_status)) + .to contain_exactly([ "actor", "failed", 404 ], [ "repository", "failed", 404 ]) end - it "stays well inside both fairness guarantees, so neither class starved the other" do - expect(current_budget.actor_share_used).to be <= 20 - expect(current_budget.repository_share_used).to be <= 20 + it "keeps the observation ledger complete: event pairs plus applied search items" do + expect(EnrichmentObservation.where(source: "event").count).to eq(8) + expect(EnrichmentObservation.where(source: "search").pluck(:validation_outcome).uniq) + .to eq([ "applied" ]) + expect(EnrichmentObservation.where(source: "search").pluck(:requested_identifier)) + .to contain_exactly("octocat", "monalisa", "octocat/Hello-World", "monalisa/Spoon-Knife") end - # §7 and ADR 0001: raw retention is semantic, not byte-exact — jsonb preserves neither - # whitespace nor key order. - it "retains each enriched document as jsonb, content-equivalent to the corpus body" do - body = JSON.parse(Rails.root.join("fixtures/github/bodies/users/octocat.json").read) + # §7 and ADR 0001: raw retention is semantic, not byte-exact — jsonb preserves + # neither whitespace nor key order. The staged pipeline retains the *Search item*. + it "retains each applied document as jsonb, content-equivalent to the corpus item" do + body = JSON.parse(Rails.root.join("fixtures/github/bodies/search/users-partial.json").read) + octocat_item = body.fetch("items").find { _1.fetch("id") == 583_231 } - expect(GithubActor.find_by(github_id: 583_231).raw_payload).to eq(body) + expect(GithubActor.find_by(github_id: 583_231).raw_payload).to eq(octocat_item) end # §7 is explicit that the envelope's repo.name is the qualified form and "is **not** @@ -86,33 +137,43 @@ def enrich!(cycles: 1, **arguments) .to have_attributes(name: "Hello-World", full_name: "octocat/Hello-World") end - it "makes no network request at any point" do + it "tells the whole story in one cycle's counters" do + expect(@cycle).to have_attributes( + batches_attempted: 2, batches_completed: 2, items_requested: 6, items_valid: 4, + fallbacks_admitted: 2, details_attempted: 2, details_completed: 0, + details_terminal: 2, batch_stop_reason: "no_batch_work", + detail_stop_reason: "no_detail_work" + ) + end + + it "issues exactly five offline requests, in staged order, and no network request" do + expect(transport.requests.map { _1.fetch(:key) }).to eq([ + "/events?per_page=100", + "/search/users?per_page=3&q=user%3Aoctocat+user%3Amonalisa+user%3Aghostuser", + "/search/repositories?per_page=3&q=repo%3Aoctocat%2FHello-World+repo%3Amonalisa%2FSpoon-Knife+repo%3Adeleted-org%2Fgone", + "/users/ghostuser", + "/repos/deleted-org/gone" + ]) expect(WebMock).not_to have_requested(:any, //) end end - describe "the freshness cache (plan §10, S3.5)" do + describe "the cycle after the cycle" do before do ingest! - enrich!(cycles: 6) - end - - it "has nothing left to do once every entity is decided" do - expect(runner.call).to have_attributes(status: "idle", deferral_reason: "no_candidate") - end - - it "spends no further budget on a fresh record" do - expect { runner.call }.not_to change { current_budget.enrichment_used }.from(6) + run_cycle! end - it "refreshes the most stale record once its TTL has passed" do - later = now + 86_401 - stale = fixture_enrichment_runner(transport: transport, now: later) - - result = stale.call + # Complete rows are fresh, terminal rows are decided: neither lane has claimable + # work, so the second cycle spends nothing at all. + it "finds nothing to do and spends nothing" do + second = run_cycle! - expect(result).to have_attributes(status: "enriched", pool: :refresh) - expect(GithubActor.find_by(github_id: result.github_id).fetched_at).to eq(later) + expect(second).to have_attributes(batches_attempted: 0, details_attempted: 0, + batch_stop_reason: "no_batch_work", + detail_stop_reason: "no_detail_work") + expect(current_search_budget.used).to eq(2) + expect(current_budget.enrichment_used).to eq(2) end end @@ -121,73 +182,56 @@ def enrich!(cycles: 1, **arguments) before { ingest! } - it "defers without touching the entity when the allowance is gone" do - active_budget_window(now: now, enrichment_used: 40) - before = actor.attributes - - expect(runner.call).to have_attributes(status: "deferred", deferral_reason: "class_exhausted") - expect(actor.reload.attributes).to eq(before) - end - - it "eventually enriches work that waited beyond one quota window" do - active_budget_window(now: now, enrichment_used: 40) - expect(runner.call.status).to eq("deferred") + # The admission pre-check is what stops the cycle here: a denied tick claims no + # rows, creates no batch row, and spends nothing — churn control, not just budget + # control. + it "defers without touching the staged rows when the search window is exhausted" do + active_search_window(now: now, used: 8) + before_rows = GithubActor.order(:id).map(&:attributes) - next_window = now + 7200 - active_budget_window(now: next_window, poll_used: 0, enrichment_used: 0, - actor_share_used: 0, repository_share_used: 0) - result = fixture_enrichment_runner(transport: transport, now: next_window).call + cycle = run_cycle! - expect(result).to have_attributes(status: "enriched", github_id: actor.github_id) - expect(actor.reload.enrichment_status).to eq("complete") + expect(cycle.batch_stop_reason).to eq("search_ceiling_exhausted") + expect(GithubActor.order(:id).map(&:attributes)).to eq(before_rows) + expect(EnrichmentBatch.count).to eq(0) end - end - # §12: "Class fairness: repository flood cannot starve actors (and vice versa)." - describe "class fairness under a one-sided backlog" do - before do - ingest! - active_budget_window(now: now, actor_share_used: 20, repository_share_used: 0, enrichment_used: 20) - end + it "eventually completes work that waited beyond one search window" do + active_search_window(now: now, used: 8) + expect(run_cycle!.batches_completed).to eq(0) - it "serves the class that has not spent its guarantee" do - expect(runner.call).to have_attributes(entity_type: :repository, borrow: false) - end - - it "borrows for the spent class only once the other has nothing eligible left" do - GithubRepository.update_all(enrichment_status: "permanent_failure") + # Sixty-one seconds later the ledger rolls the minute window on its next + # reservation, and the durable FIFO resumes exactly where it stopped. + cycle = run_cycle!(at: now + 61) - expect(runner.call).to have_attributes(entity_type: :actor, borrow: true) + expect(cycle.batches_completed).to eq(2) + expect(actor.reload).to have_attributes(enrichment_status: "complete", + enrichment_stage: "contract_complete") end end # §12: "Poll allowance protected from enrichment demand — and vice versa (class-blocking # isolation: one class exhausted, the other proceeds)." - describe "class isolation between polling and enrichment" do - # Polled past GitHub's own X-Poll-Interval floor, which the fixture corpus sets to 60 - # seconds and which --force deliberately does *not* bypass (§9). What is bypassed is - # only the configured cadence — so a poll that completes here completed *through* the - # class blocking that enrichment exhaustion would have caused if the two shared one - # timestamp. - it "still polls when enrichment has spent its whole allowance" do + describe "class isolation between polling and the staged lanes" do + it "still polls when the core detail allowance is spent" do ingest! - active_budget_window(now: now, enrichment_used: 40, poll_used: 1) + GithubApiBudget.where(id: GithubApiBudget::SINGLETON_ID).update_all(enrichment_used: 4) # A 304 is a poll that happened — §10 keeps its reservation debited — so the # assertion is that nothing deferred it, not which body came back. - expect(ingest!(at: now + 120, force: true)).not_to be_deferred + expect(fixture_runner(transport: transport, now: now + 120) + .call(event_source: EventSource.sole, force: true)).not_to be_deferred expect(current_budget.poll_used).to eq(2) end - # The attempt is what isolation means here. Which entity the fairness policy picked — - # and whether that one happens to 404 in the corpus — is beside the point; what matters - # is that a spent poll allowance did not defer it. - it "still enriches when polling has spent its whole allowance" do + it "still runs Search batches when polling has spent its whole allowance" do ingest! - active_budget_window(now: now, poll_used: 12) + GithubApiBudget.where(id: GithubApiBudget::SINGLETON_ID).update_all(poll_used: 12) + + cycle = run_cycle! - expect(runner.call).to be_attempted - expect(current_budget.enrichment_used).to eq(1) + expect(cycle.batches_completed).to eq(2) + expect(current_search_budget.used).to eq(2) end end end diff --git a/spec/services/github/enrichment/entity_state_spec.rb b/spec/services/github/enrichment/entity_state_spec.rb deleted file mode 100644 index 2a9c125..0000000 --- a/spec/services/github/enrichment/entity_state_spec.rb +++ /dev/null @@ -1,319 +0,0 @@ -require "rails_helper" - -RSpec.describe Github::Enrichment::EntityState do - # No jitter, so the backoff instant is exact and the example names a number rather than a - # range — the technique poll_state_spec.rb uses for the same reason. - subject(:state) do - described_class.new(backoff: Github::Enrichment::Backoff.new(random: instance_double(Random, rand: 0.0))) - end - - let(:now) { frozen_time } - let(:actor_type) { Github::Enrichment::EntityType.fetch(:actor) } - let(:github_id) { 583_231 } - let(:leased_until) { now + 600 } - - let!(:actor) do - create_actor(github_id: github_id, last_seen_at: now - 60, next_retry_at: leased_until) - end - - def lease(**overrides) - Github::Enrichment::Claim::Lease.new( - **{ entity_type: actor_type, pool: :pending, id: actor.id, github_id: github_id, - api_url: "https://api.github.com/users/octocat", enrichment_status: "pending", - enrichment_attempts: 0, fetched_at: nil, last_seen_at: now - 60, - previous_next_retry_at: nil, leased_until: leased_until }.merge(overrides) - ) - end - - def fetched(classification:, status: nil, error: nil, body: "", headers: {}) - request = Github::Request.new(url: "https://api.github.com/users/octocat", - request_class: :actor, origin: :payload) - return Github::FetchResult.from_error(request: request, error: error, classification: classification) if error - - Github::FetchResult.from_response(request: request, status: status, headers: headers, - body: body, duration_ms: 1.0) - end - - def record(classification:, status: nil, error: nil, body: "", headers: {}, - document: nil, decision: nil, **lease_overrides) - state.record!(lease: lease(**lease_overrides), - fetched: fetched(classification: classification, status: status, error: error, - body: body, headers: headers), - document: document, decision: decision, now: now) - end - - def document_for(body) = Github::Enrichment::ActorDocument.parse(body, github_id: github_id) - - let(:good_body) { JSON.generate("id" => github_id, "name" => "The Octocat") } - - describe "a document that parsed (§10's 200)" do - it "stores the document and marks the entity complete" do - written = record(classification: :ok, status: 200, body: good_body, document: document_for(good_body)) - - expect(written.outcome).to eq("enriched") - expect(actor.reload).to have_attributes(enrichment_status: "complete", name: "The Octocat", - fetched_at: now) - expect(actor.reload.raw_payload).to include("name" => "The Octocat") - end - - # enrichment_attempts counts attempts *since the last success*, following - # PollState#success's consecutive_failures: its only two consumers are the backoff - # exponent and the log line, and both want that number. - it "resets the attempt count, because it counts attempts since the last success" do - record(classification: :ok, status: 200, body: good_body, document: document_for(good_body), - enrichment_attempts: 4) - - expect(actor.reload.enrichment_attempts).to eq(0) - end - - # A stale error on a successful row is a permanent lie — PollState#success's clearing - # argument, applied to the entity. - it "clears failure state, which would otherwise outlive the failure" do - actor.update!(last_error: "boom") - - record(classification: :ok, status: 200, body: good_body, document: document_for(good_body)) - - expect(actor.reload.last_error).to be_nil - end - - # The next event for this row is a *refresh*, gated by fetched_at plus the TTL rather - # than by a retry instant. Leaving the lease would delay it by ten minutes and conflate - # two meanings on one column. - it "clears the retry instant, because the next event is a TTL refresh and not a retry" do - record(classification: :ok, status: 200, body: good_body, document: document_for(good_body)) - - expect(actor.reload.next_retry_at).to be_nil - end - end - - describe "a document it refuses (§10's malformed row)" do - let(:malformed) { document_for("") } - let(:mismatched) { document_for(JSON.generate("id" => 999)) } - - it "treats a malformed body as permanent, because it is a decided fact and not transport noise" do - written = record(classification: :ok, status: 200, body: "", document: malformed) - - expect(written.outcome).to eq("failed") - expect(actor.reload).to have_attributes(enrichment_status: "permanent_failure", next_retry_at: nil) - end - - it "refuses another entity's document under its own error code" do - written = record(classification: :ok, status: 200, document: mismatched) - - expect(written.error_code).to eq("identity_mismatch") - expect(actor.reload.enrichment_status).to eq("permanent_failure") - end - - # A bad refresh must not delete a good document. The payload columns are absent from - # every failing branch's attribute Hash, so this is structural rather than a check. - it "keeps the payload it already had, so a bad refresh cannot destroy a good document" do - actor.update!(enrichment_status: "complete", name: "The Octocat", - raw_payload: { "id" => github_id }, fetched_at: now - 90_000) - - record(classification: :ok, status: 200, document: malformed, enrichment_status: "complete") - - expect(actor.reload).to have_attributes(name: "The Octocat", raw_payload: { "id" => github_id }) - end - - it "refuses to record a 200 that was never parsed, which would be a caller bug" do - expect { record(classification: :ok, status: 200) }.to raise_error(ArgumentError, /parsed/) - end - end - - describe "outcomes that are facts about this entity" do - # §10: "actor or repo URL returns 404/410 → entity permanent_failure; source stays - # enabled." - it "marks a 404 permanently failed and schedules no retry" do - written = record(classification: :not_found, status: 404) - - expect(written.outcome).to eq("failed") - expect(actor.reload).to have_attributes(enrichment_status: "permanent_failure", next_retry_at: nil, - enrichment_attempts: 1) - expect(actor.reload.last_error).to include("404") - end - - # §10: "Never disable the event source because one enrichment target disappeared." The - # guarantee is structural — this class writes only entity_type.model and physically - # cannot reach event_sources. - it "leaves the event source untouched, because one dead entity is not a dead source" do - source = create_event_source - - expect { record(classification: :not_found, status: 404) } - .not_to change { source.reload.attributes } - end - - # 403 and 429 never land here: ResponseClassifier routes both to a limit - # classification. What does is 400, 422, 451 — all permanent for this row. - it "marks any other client error permanently failed" do - record(classification: :client_error, status: 422) - - expect(actor.reload.enrichment_status).to eq("permanent_failure") - end - - it "schedules a retry for a server error, counting the attempt against this entity" do - written = record(classification: :server_error, status: 500) - - expect(written.outcome).to eq("failed") - expect(actor.reload).to have_attributes(enrichment_status: "retryable_failure", - enrichment_attempts: 1, next_retry_at: now + 60) - end - - it "backs off further with each attempt since the last success" do - record(classification: :server_error, status: 500, enrichment_attempts: 2) - - expect(actor.reload).to have_attributes(enrichment_attempts: 3, next_retry_at: now + 240) - end - - it "schedules a retry for a transport failure the same way" do - record(classification: :transport_error, - error: Github::Errors::RequestTimeout.new("timed out")) - - expect(actor.reload).to have_attributes(enrichment_status: "retryable_failure", next_retry_at: now + 60) - end - - # §10: "Violations mark the entity permanent_failure." This covers a URL-policy - # violation, a TLS failure, and an exhausted redirect budget alike. - it "marks a URL-policy violation permanently failed and names every reason" do - violation = Github::Errors::UrlPolicyViolation.new("", [ :blank ]) - - record(classification: :permanent_error, error: violation) - - expect(actor.reload.enrichment_status).to eq("permanent_failure") - expect(actor.reload.last_error).to include("blank") - end - - # Enrichment sends no validator — §7's column list has no entity ETag, and §10's dated - # probe established that an unauthenticated 304 debits quota anyway. So a 304 means an - # assumption broke, and it is handled as an ordinary retryable failure with a WARN. - it "treats a 304 as retryable, because enrichment never sent a validator" do - allow(Rails.logger).to receive(:warn) - - record(classification: :not_modified, status: 304) - - expect(actor.reload).to have_attributes(enrichment_status: "retryable_failure", next_retry_at: now + 60) - expect(Rails.logger).to have_received(:warn).with(hash_including(event: "enrichment.unexpected_not_modified")) - end - - it "does not bump fetched_at on a 304, which is no evidence that what we hold is current" do - actor.update!(enrichment_status: "complete", fetched_at: now - 90_000) - - record(classification: :not_modified, status: 304, enrichment_status: "complete") - - expect(actor.reload.fetched_at).to eq(now - 90_000) - end - - # A transient 500 on a refresh would otherwise drop coverage for a network blip and - # jump the row into the high-priority pending pool ahead of never-enriched candidates. - # The status conveys nothing next_retry_at, last_error and the attempt count do not. - it "never downgrades a complete record on a retryable failure" do - actor.update!(enrichment_status: "complete", fetched_at: now - 90_000) - - record(classification: :server_error, status: 500, enrichment_status: "complete") - - expect(actor.reload).to have_attributes(enrichment_status: "complete", enrichment_attempts: 1, - next_retry_at: now + 60) - end - - it "does downgrade a complete record on a terminal outcome, which invalidates the document" do - actor.update!(enrichment_status: "complete", fetched_at: now - 90_000) - - record(classification: :not_found, status: 404, enrichment_status: "complete") - - expect(actor.reload.enrichment_status).to eq("permanent_failure") - end - end - - describe "outcomes that are facts about the IP rather than this entity" do - # PollState makes exactly this call for the source: "GitHub answered … but nothing is - # wrong with the source". Inflating an innocent entity's backoff for an IP-wide - # condition would, repeated, push it toward the hour-long cap. - it "does not blame the entity for a primary rate limit" do - before = actor.reload.attributes - - written = record(classification: :rate_limited, status: 403) - - expect(written.outcome).to eq("deferred") - expect(actor.reload.attributes).to eq(before) - end - - # §10 requires it: "also update the request-specific source or entity retry state". Not - # redundant with the global block — ROLL_WINDOW_SQL clears that at the window boundary - # while this component survives it. - it "defers this entity as well as the IP on a secondary limit" do - decision = Github::RateLimitPolicy::Decision.new(kind: :secondary_rate_limit, - blocked_until: now + 120, - source_retry_at: now + 120, window_status: nil) - - written = record(classification: :secondary_limited, status: 403, decision: decision) - - expect(written.outcome).to eq("deferred") - expect(actor.reload).to have_attributes(next_retry_at: now + 120, enrichment_attempts: 0, - enrichment_status: "pending") - end - - it "falls back to a plain release when a secondary limit named no instant" do - before = actor.reload.attributes - - record(classification: :secondary_limited, status: 403, decision: nil) - - expect(actor.reload.attributes).to eq(before) - end - - it "leaves the row exactly as it found it on a budget denial" do - before = actor.reload.attributes - - written = record(classification: :budget_denied, - error: Github::Errors::BudgetExhausted.new(:actor, :share_exhausted)) - - expect(written).to have_attributes(outcome: "deferred", lease_held: true) - expect(actor.reload.attributes).to eq(before) - end - - it "leaves the row exactly as it found it when the gate was held" do - before = actor.reload.attributes - - record(classification: :gate_unavailable, error: Github::Errors::GateUnavailable.new("busy")) - - expect(actor.reload.attributes).to eq(before) - end - end - - describe "a lease that expired mid-flight" do - # lease_seconds is the worst-case runtime by construction, so this is reachable rather - # than theoretical. Writing the outcome anyway would be the double-write the lease - # exists to prevent; reporting it is the graceful degradation. - it "refuses to write an outcome after another worker claimed the row" do - allow(Rails.logger).to receive(:warn) - actor.update!(next_retry_at: now + 9_999) - - written = record(classification: :ok, status: 200, body: good_body, document: document_for(good_body)) - - expect(written).to have_attributes(outcome: "lease_lost", lease_held: false) - expect(actor.reload.enrichment_status).to eq("pending") - expect(Rails.logger).to have_received(:warn).with(hash_including(event: "enrichment.lease_lost")) - end - end - - describe "the disposition table" do - # A frozen Hash with no default, so an unenumerated classification raises rather than - # silently taking a branch. - it "covers every classification a fetch can produce except the unreachable redirect" do - expect(described_class::DISPOSITIONS.keys) - .to match_array(Github::FetchResult::CLASSIFICATIONS - [ :redirect ]) - end - - # :redirect never escapes RequestExecutor#follow_redirects — it is either followed or - # converted into RedirectLimitExceeded, which classifies as :permanent_error. - it "raises on the classification the executor never returns, rather than guessing" do - expect { record(classification: :redirect, status: 301, headers: { "location" => "https://api.github.com/x" }) } - .to raise_error(ArgumentError, /redirect/) - end - - # FetchResult#successful? answers true for :not_modified, so dispatching on it would - # send a bodyless 304 down the "store the document" branch. - it "dispatches on the classification and never on whether the response was successful" do - expect(described_class::DISPOSITIONS.fetch(:not_modified)).to eq(:retryable) - expect(Github::ResponseClassifier.successful?(:not_modified)).to be(true) - end - end -end diff --git a/spec/services/github/enrichment/entity_type_spec.rb b/spec/services/github/enrichment/entity_type_spec.rb index ec87978..9ee36ac 100644 --- a/spec/services/github/enrichment/entity_type_spec.rb +++ b/spec/services/github/enrichment/entity_type_spec.rb @@ -2,8 +2,29 @@ RSpec.describe Github::Enrichment::EntityType do describe ".all" do - it "covers exactly the two classes the ledger has enrichment counters for" do - expect(described_class.keys).to eq(Github::Request::ENRICHMENT_CLASSES) + it "covers exactly the two entity classes, each with a detail and a search request class" do + expect(described_class.keys).to eq(Github::Request::DETAIL_CLASSES) + expect(described_class.all.map(&:request_class)).to eq(Github::Request::DETAIL_CLASSES) + expect(described_class.all.map(&:search_request_class)).to eq(Github::Request::SEARCH_CLASSES) + end + + # The two ledger-facing vocabularies must stay in lockstep: every request class the + # budget ledgers meter for enrichment is reachable from exactly one entity type, so a + # new class cannot be added on one side and silently go unmetered on the other. + it "spans the ledger's enrichment classes exactly" do + classes = described_class.all.flat_map do |type| + [ type.request_class, type.search_request_class ] + end + + expect(classes).to match_array(Github::Request::ENRICHMENT_CLASSES) + end + + # The batch runner reserves under :actor_search/:repository_search (the per-minute + # search ledger) while the detail fallback reserves under :actor/:repository (the + # bounded core allowance). The pairing is enumerated here, once. + it "pairs each key with its own search request class" do + expect(described_class.fetch(:actor).search_request_class).to eq(:actor_search) + expect(described_class.fetch(:repository).search_request_class).to eq(:repository_search) end it "maps each key to its own model, parser and log field" do diff --git a/spec/services/github/enrichment/event_native_derivation_spec.rb b/spec/services/github/enrichment/event_native_derivation_spec.rb new file mode 100644 index 0000000..ed4d0ef --- /dev/null +++ b/spec/services/github/enrichment/event_native_derivation_spec.rb @@ -0,0 +1,125 @@ +require "rails_helper" + +# Appendix G's first stage, which is not a stage at all but three instants: the accepted +# event's identity fragments become append-only observations, the locally derivable +# fields are derived, and the entity enters the batch FIFO — all inside the ingest +# transaction, with zero enrichment HTTP anywhere. +# +# Page 1 of the default corpus persists four push events (1, 2, 3 and 8) across three +# actors and three repositories, which is every count below. +RSpec.describe "event-native derivation at ingest", type: :integration do + let(:now) { frozen_time } + let(:transport) { fixture_transport } + + def ingest! + fixture_runner(transport: transport, now: now).call(event_source: fixture_event_source) + end + + describe "the observation ledger" do + before { ingest! } + + it "appends one actor and one repository observation per accepted event" do + observations = EnrichmentObservation.where(source: "event") + + expect(observations.count).to eq(8) + expect(observations.group(:entity_kind).count) + .to eq("actor" => 4, "repository" => 4) + expect(observations.pluck(:validation_outcome).uniq).to eq([ "event_native" ]) + end + + # The pair commits with its push_events row or not at all: every event observation + # names the accepted event it arrived on, and the quarantined envelopes of the same + # page produced none. + it "ties every observation to the accepted event that carried it" do + observations = EnrichmentObservation.where(source: "event") + + expect(observations.where(push_event_id: nil)).to be_empty + expect(observations.distinct.pluck(:push_event_id)).to match_array(PushEvent.pluck(:id)) + expect(observations.group(:push_event_id).count.values).to all(eq(2)) + end + + it "preserves the envelope fragment verbatim, fingerprinted" do + event = PushEvent.find_by!(github_event_id: "58000000001") + observation = EnrichmentObservation.find_by!(push_event_id: event.id, entity_kind: "actor") + + expect(observation.raw_payload).to eq(well_formed_envelope.fetch("actor")) + expect(observation.entity_github_id).to eq(IngestionHelpers::ACTOR_GITHUB_ID) + expect(observation.payload_fingerprint).to match(/\A\h{64}\z/) + expect(observation.observed_at).to eq(now) + end + end + + describe "the entity rows" do + before { ingest! } + + it "stamps every derivation instant and rests the row in the batch FIFO" do + (GithubActor.all + GithubRepository.all).each do |entity| + expect(entity).to have_attributes( + enrichment_status: "pending", enrichment_stage: "batch_pending", + event_native_at: now, derived_at: now, batch_pending_at: now + ) + end + end + + # The one derivable field no request is needed for: the envelope's qualified + # owner/repository form already carries the owner segment. + it "derives repository owner_login locally" do + expect(GithubRepository.order(:id).pluck(:full_name, :owner_login)).to eq([ + [ "octocat/Hello-World", "octocat" ], + [ "monalisa/Spoon-Knife", "monalisa" ], + [ "deleted-org/gone", "deleted-org" ] + ]) + end + end + + describe "the network boundary of derivation" do + it "issues the poll and nothing else — zero enrichment requests" do + ingest! + + expect(transport.requests.size).to eq(1) + expect(transport.requests.map { _1.fetch(:key) }).to all(start_with("/events")) + expect(WebMock).not_to have_requested(:any, //) + end + end + + # §8: repeated observation is expected, and a duplicate event may refresh identity but + # must never register activity — extended here to the observation ledger and the + # derivation instants, which are keep-first by construction (COALESCE, not overwrite). + describe "a duplicate replay of the same page" do + let(:page) { corpus_page("page-1.json") } + + before do + Github::Ingestion::PageWriter.new(clock: -> { now }) + .write(page, run_id: SecureRandom.uuid) + end + + def replay! + Github::Ingestion::PageWriter.new(clock: -> { now + 60 }) + .write(page, run_id: SecureRandom.uuid) + end + + it "appends no new observation" do + expect { replay! }.not_to change(EnrichmentObservation, :count).from(8) + end + + it "registers no new activity and restarts no pipeline clock" do + octocat = GithubActor.find_by!(github_id: IngestionHelpers::ACTOR_GITHUB_ID) + before_activity = [ octocat.first_seen_at, octocat.last_seen_at, octocat.latest_event_at ] + + replay! + + expect(octocat.reload).to have_attributes( + first_seen_at: before_activity[0], last_seen_at: before_activity[1], + latest_event_at: before_activity[2], + event_native_at: now, derived_at: now, batch_pending_at: now + ) + end + + it "reports the whole page as duplicates" do + tally = replay! + + expect(tally.events_created).to eq(0) + expect(tally.duplicates_skipped).to eq(4) + end + end +end diff --git a/spec/services/github/enrichment/fairness_spec.rb b/spec/services/github/enrichment/fairness_spec.rb deleted file mode 100644 index 1695db2..0000000 --- a/spec/services/github/enrichment/fairness_spec.rb +++ /dev/null @@ -1,243 +0,0 @@ -require "rails_helper" - -RSpec.describe Github::Enrichment::Fairness do - subject(:fairness) { described_class.new(configuration: configuration) } - - let(:configuration) { configuration_with } - let(:now) { frozen_time } - - def choose(**arguments) = fairness.choose(now: now, **arguments) - - def pending_actor(github_id: 1, **overrides) - create_actor(github_id: github_id, last_seen_at: now - 60, **overrides) - end - - def pending_repository(github_id: 2, **overrides) - create_repository(github_id: github_id, last_seen_at: now - 60, **overrides) - end - - describe "the conditions that stop all enrichment" do - it "chooses nothing while a global block is in force" do - pending_actor - active_budget_window(now: now, global_blocked_until: now + 60) - - expect(choose).to have_attributes(chosen?: false, reason: "globally_blocked") - end - - # §7: enrichment is ineligible until the first real poll initializes the window from - # authoritative headers. Asked here as well as in the ledger so a fresh install reports - # the honest reason instead of taking the global request gate to be told the same thing. - it "chooses nothing before the first poll has initialized the window" do - pending_actor - Github::BudgetLedger.new.bootstrap!(now: now) - - expect(choose).to have_attributes(chosen?: false, reason: "window_uninitialized") - end - - it "chooses nothing once the whole enrichment allowance is spent" do - pending_actor - active_budget_window(now: now, enrichment_used: 40) - - expect(choose).to have_attributes(chosen?: false, reason: "class_exhausted") - end - - # Nothing seeds the ledger row, so a clean checkout constrains nothing — the first - # reservation will create it, and the ledger will refuse there if it must. - it "still chooses work with no ledger row at all, which the ledger then rules on" do - pending_actor - - expect(choose).to have_attributes(chosen?: true, reason: "pending") - end - - it "reports having nothing to do separately from being refused" do - active_budget_window(now: now) - - expect(choose).to have_attributes(chosen?: false, reason: "no_candidate") - end - end - - describe "the pending pools" do - before { active_budget_window(now: now) } - - it "picks the class that has eligible work" do - pending_repository - - expect(choose).to have_attributes(entity_type: Github::Enrichment::EntityType.fetch(:repository), - pool: :pending, borrow: false) - end - - it "breaks a tie toward actors, which is the order EntityType declares" do - pending_actor - pending_repository - - expect(choose.entity_type.key).to eq(:actor) - end - - # §10's whole reason for the split: repository candidates alone exceed the hourly - # allowance, so a repository-first policy would starve actors to zero indefinitely. - it "moves to the other class once one has spent its guarantee" do - pending_actor - pending_repository - active_budget_window(now: now, actor_share_used: 20, enrichment_used: 20) - - expect(choose.entity_type.key).to eq(:repository) - end - - it "restricts to the class a caller named without bypassing anything else" do - pending_actor - pending_repository - - expect(choose(entity_class: GithubRepository).entity_type.key).to eq(:repository) - expect(choose(entity_class: :repository).entity_type.key).to eq(:repository) - end - - it "refuses a class that has no enrichment counters" do - expect { choose(entity_class: :organization) }.to raise_error(ArgumentError, /organization/) - end - end - - describe "borrowing (plan §10)" do - before { active_budget_window(now: now, actor_share_used: 20, enrichment_used: 20) } - - # §10: "a class may borrow the other's unused capacity only when the other class has no - # CURRENTLY ELIGIBLE candidate (not merely no rows)." - it "borrows when the other class has no eligible candidate" do - pending_actor - - expect(choose).to have_attributes(entity_type: Github::Enrichment::EntityType.fetch(:actor), - borrow: true, reason: "borrowed_pending") - end - - it "refuses to borrow while the other class still has eligible work" do - pending_actor - pending_repository - - expect(choose).to have_attributes(entity_type: Github::Enrichment::EntityType.fetch(:repository), - borrow: false) - end - - it "borrows when the other class's durable backlog is not currently due" do - pending_actor - pending_repository(github_id: 2, next_retry_at: now + 300) - - expect(choose).to have_attributes(borrow: true, reason: "borrowed_pending") - end - - it "does not borrow while the class is still inside its own guarantee" do - active_budget_window(now: now, actor_share_used: 19, enrichment_used: 19) - pending_actor - - expect(choose).to have_attributes(borrow: false, reason: "pending") - end - - it "gives a zero-guarantee class work only by borrowing" do - starved = described_class.new(configuration: configuration_with(ACTOR_ENRICHMENT_SHARE: "0.0")) - active_budget_window(now: now) - pending_actor - - expect(starved.choose(now: now)).to have_attributes(entity_type: Github::Enrichment::EntityType.fetch(:actor), - borrow: true) - end - end - - describe "TTL-stale refreshes" do - before { active_budget_window(now: now) } - - def stale_actor(github_id: 1) - create_actor(github_id: github_id, enrichment_status: "complete", fetched_at: now - 90_000, - last_seen_at: now - 60) - end - - # The durable first-time backlog always precedes TTL refreshes. - it "prefers a pending candidate over a stale refresh" do - stale_actor - pending_repository - - expect(choose).to have_attributes(entity_type: Github::Enrichment::EntityType.fetch(:repository), - pool: :pending) - end - - it "offers a refresh only when no class has a pending candidate" do - stale_actor - - expect(choose).to have_attributes(pool: :refresh, reason: "refresh") - end - - # §10 scopes the condition globally, so --class actor cannot promote an actor refresh - # above a repository still waiting to be enriched for the first time. - it "will not refresh one class while the other still has never-enriched work" do - stale_actor - pending_repository - - expect(choose(entity_class: :actor)).to have_attributes(chosen?: false, reason: "no_candidate") - end - - it "does not promote a refresh while the only never-enriched row is backed off" do - stale_actor - pending_repository(enrichment_status: "retryable_failure", - next_retry_at: now + 3600) - - expect(choose).to have_attributes(chosen?: false, reason: "no_candidate") - end - - # The other class has no refresh of its own to do, so nothing is starved by lending - # its idle capacity — the same condition #pending_choice borrows under. - it "borrows for a refresh once the class has spent its guarantee" do - active_budget_window(now: now, actor_share_used: 20, enrichment_used: 20) - stale_actor - - expect(choose).to have_attributes(pool: :refresh, borrow: true, reason: "borrowed_refresh") - end - - def stale_repository(github_id: 2) - create_repository(github_id: github_id, enrichment_status: "complete", fetched_at: now - 90_000, - last_seen_at: now - 60) - end - - # §10:898 scopes refreshes "within each class's share", and §10's whole reason for - # having shares is that "a naive repo-first policy would starve actor enrichment to - # zero indefinitely". Selecting the first refreshable class outright reproduced exactly - # that inside the refresh pool, with the order reversed: actor spent its own twenty and - # then borrowed the remaining twenty, while repository's untouched guarantee and - # eligible stale rows were never chosen. - describe "when both classes have stale rows" do - before do - active_budget_window(now: now, actor_share_used: 20, enrichment_used: 20) - stale_actor - stale_repository - end - - it "chooses the class that still has room, not the first in class order" do - expect(choose).to have_attributes(entity_type: Github::Enrichment::EntityType.fetch(:repository), - pool: :refresh, borrow: false, reason: "refresh") - end - - # The borrow condition is about the class this cycle did not pick, so a class with - # refresh work of its own is not idle capacity to lend. - it "refuses to borrow while the other class has a refresh of its own" do - expect(choose(entity_class: :actor)).to have_attributes(chosen?: false, reason: "no_candidate") - end - - # Once the other class runs dry the capacity genuinely is idle, and §10's borrowing - # rule applies to the refresh pool exactly as it does to the pending one. - it "borrows again once the other class has nothing left to refresh" do - GithubRepository.update_all(fetched_at: now) - - expect(choose).to have_attributes(entity_type: Github::Enrichment::EntityType.fetch(:actor), - pool: :refresh, borrow: true, reason: "borrowed_refresh") - end - end - - # The pending pool's borrow test stays pending-only. CandidateSelector documents why: - # counting refreshes there would let a refresh outrank a never-enriched candidate and - # invert §10's ladder. Only the refresh pool's own test changed. - it "still borrows for a pending candidate while the other class has a stale refresh" do - active_budget_window(now: now, actor_share_used: 20, enrichment_used: 20) - pending_actor - stale_repository - - expect(choose).to have_attributes(entity_type: Github::Enrichment::EntityType.fetch(:actor), - pool: :pending, borrow: true, reason: "borrowed_pending") - end - end -end diff --git a/spec/services/github/enrichment/fairness_stress_spec.rb b/spec/services/github/enrichment/fairness_stress_spec.rb deleted file mode 100644 index 23fb829..0000000 --- a/spec/services/github/enrichment/fairness_stress_spec.rb +++ /dev/null @@ -1,232 +0,0 @@ -require "rails_helper" - -# Story 3's ninth child issue asks for "starved-class enrichment" to be tested, and §10 says -# what starvation would look like: one observed live page held ~89 distinct actors and ~92 -# distinct repositories, so "repository candidates alone exceed the whole hourly allowance, -# and a repo-first policy — or any policy ordering purely by recency across a mixed pool — -# starves actor enrichment to zero indefinitely". -# -# That is a claim about how a *whole window* is distributed, and no single-choice example can -# express it. spec/services/github/enrichment/fairness_spec.rb asks for one choice at a time, -# and budget_ledger_spec.rb and enrichment/end_to_end_spec.rb check the boundaries against -# counters set with update_all. None of them ever spends a window, so none of them can -# observe the property Story 3 actually asks for: that after forty real reservations against -# a twenty-to-one flood, the starved class still got every request it had a candidate for. -# -# The drain drives Fairness, Claim and BudgetLedger directly rather than through -# Github::EnrichmentRunner, and deliberately not through the transport. The corpus resolves -# six entity URLs and its sticky tail would hand every flood row octocat's body, which -# RepositoryDocument.parse then rejects on identity — fifty-seven permanent failures with -# nothing to do with fairness, obscuring the sequence that is the whole point. The three -# objects below are the ones the property lives in, and they are the real ones. -RSpec.describe "enrichment fairness under a flood", type: :integration do - let(:configuration) { Github.configuration } - let(:selector) { Github::Enrichment::CandidateSelector.new(configuration: configuration) } - let(:fairness) { Github::Enrichment::Fairness.new(configuration: configuration, selector: selector) } - let(:claim) { Github::Enrichment::Claim.new(configuration: configuration, selector: selector) } - let(:ledger) { Github::BudgetLedger.new(configuration: configuration) } - - # [entity_type_key, borrowed] per granted request, in the order the window granted them. - let(:sequence) { [] } - - before { active_budget_window(now: frozen_time) } - - # One granted request: choose, lease so the row stops being eligible, debit. Returns the - # Choice so a caller can assert on a refusal. - def spend! - choice = fairness.choose(now: frozen_time) - return choice unless choice.chosen? - - claim.acquire(choice.entity_type, pool: choice.pool, now: frozen_time) - ledger.reserve!(choice.entity_type.request_class, now: frozen_time, borrow: choice.borrow) - sequence << [ choice.entity_type.key, choice.borrow ] - - choice - end - - def drain!(limit = 60) - limit.times { break unless spend!.chosen? } - end - - def flood_repositories(count) - count.times do |index| - create_repository(github_id: 300_000 + index, full_name: "flood/repo-#{index}", - name: "repo-#{index}", last_seen_at: frozen_time, - enrichment_status: "pending") - end - end - - def flood_actors(count) - count.times do |index| - create_actor(github_id: 400_000 + index, login: "flood-user-#{index}", - display_login: "flood-user-#{index}", last_seen_at: frozen_time, - enrichment_status: "pending") - end - end - - # At the pinned defaults the window is 40 requests split 20/20, so sixty repositories - # against three actors is §10's ratio with room to spare on one side and none on the other. - describe "a repository flood, twenty to one" do - before do - flood_repositories(60) - flood_actors(3) - drain! - end - - # Story 3's requirement, measured over a window rather than asserted about one choice. - it "serves all three actor candidates in this finite contention setup" do - expect(sequence.count { |(key, _)| key == :actor }).to eq(3) - end - - it "leases every actor it chose, so none of the three was chosen twice" do - leased = GithubActor.where.not(next_retry_at: nil).count - - expect(leased).to eq(3) - end - - # A sequence-level property: no per-choice example can state "only after", because the - # borrow condition is evaluated fresh each time and is legitimately true at the end. - it "borrows only after the other class has genuinely run dry" do - first_borrow = sequence.index { |(_, borrow)| borrow } - last_actor = sequence.rindex { |(key, _)| key == :actor } - - expect(first_borrow).to be > last_actor - end - - it "records all 40 debits with the two shares summing to the class counter" do - budget = current_budget - - expect(budget.enrichment_used).to eq(40) - expect(budget.actor_share_used).to eq(3) - expect(budget.repository_share_used).to eq(37) - expect(budget.actor_share_used + budget.repository_share_used).to eq(budget.enrichment_used) - end - - it "never lets either share exceed the class allowance" do - budget = current_budget - - expect(budget.actor_share_used).to be <= budget.enrichment_allowance - expect(budget.repository_share_used).to be <= budget.enrichment_allowance - end - - it "stops at the allowance rather than one request past it" do - expect(spend!.reason).to eq("class_exhausted") - expect(current_budget.enrichment_used).to eq(40) - end - - # A starved backlog must be *waiting*, not damaged: nothing was charged an attempt and - # nothing recorded an error, so the rows are exactly as eligible next window as they were - # this one. - it "leaves the unenriched backlog intact rather than corrupted" do - untouched = GithubRepository.where(next_retry_at: nil) - - expect(untouched.count).to eq(60 - 37) - expect(untouched.pluck(:enrichment_status).uniq).to eq([ "pending" ]) - expect(untouched.pluck(:enrichment_attempts).uniq).to eq([ 0 ]) - expect(untouched.pluck(:last_error).uniq).to eq([ nil ]) - end - end - - # §10 says "and vice versa", and a floor/remainder split is exactly the kind of arithmetic - # that works in one direction and is off by one in the other. - describe "an actor flood, twenty to one" do - before do - flood_actors(60) - flood_repositories(3) - drain! - end - - it "serves all three repository candidates in this finite contention setup" do - expect(sequence.count { |(key, _)| key == :repository }).to eq(3) - end - - it "holds the flooding class to its guarantee until the other class is quiet" do - within_guarantee = sequence.count { |(key, borrow)| key == :actor && !borrow } - - expect(within_guarantee).to eq(20) - end - - # The arithmetic of the drain, not a single boundary: the flooding class takes its - # twenty, the starved class takes the three it has candidates for, and the remainder is - # borrowed. - it "lets the flooding class borrow exactly the remainder" do - borrowed = sequence.count { |(_, borrow)| borrow } - - expect(borrowed).to eq(40 - 20 - 3) - end - - it "records all 40 debits with the mirrored share split" do - budget = current_budget - - expect(budget.enrichment_used).to eq(40) - expect(budget.actor_share_used).to eq(37) - expect(budget.repository_share_used).to eq(3) - end - end - - describe "class isolation under a real drain" do - let(:transport) { fixture_transport } - - before do - flood_repositories(60) - flood_actors(3) - drain! - end - - # The stress direction of Github::RateLimitPolicy's rule that only :reserve_reached, a - # primary limit and a secondary limit are global. Forty consecutive class-and-share - # denials wrote nothing global — which is unreachable from any spec that sets the - # counters with update_all, because there were no denials to write anything. - it "never writes a global block while denying a whole window of enrichment" do - spend! - spend! - - expect(current_budget.global_blocked_until).to be_nil - expect(current_budget.window_status).to eq("active") - end - - # The derived-not-stored isolation mechanism, observed after real spending. - it "blocks the enrichment class and leaves the poll class alone" do - budget = current_budget - - expect(budget.poll_used).to eq(0) - expect(budget.poll_class_blocked_until(now: frozen_time)).to be_nil - expect(budget.enrichment_class_blocked_until(now: frozen_time)).to eq(budget.reset_at) - end - - it "still polls after enrichment has spent its entire allowance" do - result = fixture_runner(transport: transport).call(event_source: fixture_event_source) - - expect(result).to be_completed - expect(current_budget.poll_used).to eq(1) - expect(PushEvent.count).to eq(4) - end - end - - describe "the backlog remains durable under a flood" do - let(:next_window) { frozen_time + 7200 } - - before do - flood_repositories(60) - flood_actors(3) - active_budget_window(now: frozen_time, enrichment_used: 40) - end - - it "keeps every entity pending after the exhausted window has passed" do - expect(GithubRepository.where(enrichment_status: "pending").count).to eq(60) - expect(GithubActor.where(enrichment_status: "pending").count).to eq(3) - end - - it "charges no entity attempt when quota exhaustion prevented every request" do - expect(GithubRepository.distinct.pluck(:enrichment_attempts)).to eq([ 0 ]) - expect(GithubActor.distinct.pluck(:enrichment_attempts)).to eq([ 0 ]) - end - - it "makes the old flood claimable when a later quota window opens" do - active_budget_window(now: next_window, poll_used: 0, enrichment_used: 0, - actor_share_used: 0, repository_share_used: 0) - - expect(fairness.choose(now: next_window)).to be_chosen - end - end -end diff --git a/spec/services/github/enrichment/lane_fairness_spec.rb b/spec/services/github/enrichment/lane_fairness_spec.rb new file mode 100644 index 0000000..0513610 --- /dev/null +++ b/spec/services/github/enrichment/lane_fairness_spec.rb @@ -0,0 +1,229 @@ +require "rails_helper" + +# Appendix G's fairness at the level that actually schedules work: CycleRunner's +# weighted LaneSchedule over real FOR UPDATE SKIP LOCKED claims and both real ledgers, +# with WebMock echoing every Search qualifier back as a validating item so the whole +# Faraday request stack runs offline. +# +# The property under test is the same one §12 stated for the per-entity design — a +# one-sided flood cannot starve the other class, and an idle lane's slots are borrowed +# rather than wasted — restated for batches and for the bounded core detail lane. +RSpec.describe "lane fairness across the weighted schedule", type: :integration do + let(:now) { frozen_time } + let(:clock) { -> { now } } + + def build_cycle_runner(configuration) + executor = Github::RequestExecutor.new( + transport: Github::Transports::Faraday.new, + ledger: ledger_for(configuration), + search_ledger: search_ledger_for(configuration), + mode: :live, sleeper: ->(_seconds) { }, clock: clock + ) + + Github::Enrichment::CycleRunner.new( + configuration: configuration, + batch_runner: Github::Enrichment::BatchRunner.new( + executor: executor, configuration: configuration, + claim: Github::Enrichment::BatchClaim.new(configuration: configuration), + search_ledger: search_ledger_for(configuration), + backoff: jitterless_backoff(configuration: configuration), clock: clock + ), + detail_runner: Github::Enrichment::DetailRunner.new( + executor: executor, configuration: configuration, + claim: Github::Enrichment::DetailClaim.new(configuration: configuration), + backoff: jitterless_backoff(configuration: configuration), clock: clock + ), + admission: Github::Enrichment::Admission.new(configuration: configuration), + batch_claim: Github::Enrichment::BatchClaim.new(configuration: configuration), + detail_claim: Github::Enrichment::DetailClaim.new(configuration: configuration), + clock: clock, sleeper: ->(_seconds) { } + ) + end + + # Echoes every requested qualifier back as a validating item, and records which lane + # each Search request served so the schedule's interleaving is assertable. + def stub_search_echo!(search_lanes) + stub_request(:get, %r{\Ahttps://api\.github\.com/search/(users|repositories)\?}) + .to_return do |request| + query = URI.decode_www_form(request.uri.query.to_s).to_h + identifiers = query.fetch("q").split(" ").map { |qualifier| qualifier.split(":", 2).last } + actor = request.uri.path.end_with?("/users") + search_lanes << (actor ? :actor : :repository) + + items = identifiers.map do |identifier| + github_id = identifier[/(\d+)\z/, 1].to_i + actor ? { "id" => github_id, "login" => identifier, "type" => "User" } + : repository_item(github_id, identifier) + end + + { + status: 200, + headers: { + "Content-Type" => "application/json", + "X-RateLimit-Resource" => "search", "X-RateLimit-Limit" => "10", + "X-RateLimit-Remaining" => (10 - current_search_budget.used).to_s, + "X-RateLimit-Reset" => (now + 60).to_i.to_s + }, + body: JSON.generate( + "total_count" => items.length, "incomplete_results" => false, "items" => items + ) + } + end + end + + def repository_item(github_id, full_name) + { + "id" => github_id, "full_name" => full_name, + "owner" => { "id" => github_id + 100_000, "login" => full_name.split("/").first }, + "fork" => false, "archived" => false, "default_branch" => "main", + "description" => "Fairness repository #{github_id}", "language" => "Ruby", + "created_at" => "2026-07-01T00:00:00Z" + } + end + + def create_flood_actor(github_id, index) + create_actor( + github_id: github_id, login: "fair-user-#{github_id}", + display_login: "fair-user-#{github_id}", + api_url: "https://api.github.com/users/fair-user-#{github_id}", + created_at: now - 1000 + index, updated_at: now + ) + end + + def create_flood_repository(github_id, index) + create_repository( + github_id: github_id, full_name: "fair/repo-#{github_id}", name: "repo-#{github_id}", + api_url: "https://api.github.com/repos/fair/repo-#{github_id}", + created_at: now - 1000 + index, updated_at: now + ) + end + + describe "a twenty-to-one repository flood against equal weights" do + let(:configuration) { configuration_with("SEARCH_PACING_SECONDS" => "0") } + let(:actor_ids) { [ 30_001 ] } + let(:repository_ids) { (40_001..40_020).to_a } + + before do + active_budget_window(now: now) + actor_ids.each_with_index { |github_id, index| create_flood_actor(github_id, index) } + repository_ids.each_with_index { |github_id, index| create_flood_repository(github_id, index) } + end + + it "serves the starved lane first, then hands its idle slots to the flood" do + search_lanes = [] + stub_search_echo!(search_lanes) + + cycle = build_cycle_runner(configuration).call + + # Rotation actor→repository at weight 1:1: the single actor rides slot one, the + # flood takes slot two, and slot three — scheduled for the now-empty actor + # lane — is borrowed by the repository backlog rather than wasted. + expect(search_lanes).to eq(%i[actor repository repository]) + expect(cycle).to have_attributes(batches_attempted: 3, batches_completed: 3, + items_requested: 21, items_valid: 21, + batch_stop_reason: "no_batch_work") + end + + it "records the split on the search ledger's per-lane counters" do + stub_search_echo!([]) + + build_cycle_runner(configuration).call + + expect(current_search_budget).to have_attributes(used: 3, actor_used: 1, repository_used: 2) + expect(EnrichmentBatch.where(request_kind: "search").group(:entity_kind).count) + .to eq("actor" => 1, "repository" => 2) + end + + it "completes every row in both classes — the flood starved nothing" do + stub_search_echo!([]) + + build_cycle_runner(configuration).call + + expect(GithubActor.distinct.pluck(:enrichment_stage)).to eq([ "contract_complete" ]) + expect(GithubRepository.distinct.pluck(:enrichment_stage)).to eq([ "contract_complete" ]) + end + end + + describe "a two-to-one repository weight over deep backlogs" do + # Ceiling 8 with the default reserve of 2 makes six spendable requests — two full + # a,r,r rotations — while the echoed x-ratelimit-remaining (10 - used = 4) stays + # above the reserve, so the ceiling and not a header block is what ends the window. + let(:configuration) do + configuration_with("SEARCH_PACING_SECONDS" => "0", "SEARCH_REQUEST_CEILING" => "8", + "REPOSITORY_ENRICHMENT_WEIGHT" => "2") + end + + before do + active_budget_window(now: now) + (50_001..50_030).each_with_index { |github_id, index| create_flood_actor(github_id, index) } + (60_001..60_050).each_with_index { |github_id, index| create_flood_repository(github_id, index) } + end + + # Both lanes stay claimable for the whole window, so the six spendable requests + # land exactly as the a,r,r rotation dictates — the weights, not the backlog + # depths, decide the split. + it "divides the window's requests by the configured weights" do + search_lanes = [] + stub_search_echo!(search_lanes) + + cycle = build_cycle_runner(configuration).call + + expect(search_lanes).to eq(%i[actor repository repository actor repository repository]) + expect(current_search_budget).to have_attributes(used: 6, actor_used: 2, repository_used: 4) + expect(cycle.batch_stop_reason).to eq("search_ceiling_exhausted") + end + + it "preserves the untaken remainder of both FIFOs for the next window" do + stub_search_echo!([]) + + build_cycle_runner(configuration).call + + expect(GithubActor.where(enrichment_stage: "batch_pending").order(:created_at, :id).pluck(:github_id)) + .to eq((50_021..50_030).to_a) + expect(GithubRepository.where(enrichment_stage: "batch_pending").count).to eq(10) + end + end + + describe "the bounded detail lane borrowing the idle class's slots" do + let(:configuration) { configuration_with("SEARCH_PACING_SECONDS" => "0") } + let(:repository_ids) { (70_001..70_003).to_a } + + before do + active_budget_window(now: now) + repository_ids.each_with_index { |github_id, index| create_flood_repository(github_id, index) } + GithubRepository.where(github_id: repository_ids).update_all( + enrichment_stage: "detail_pending", detail_pending_at: now, updated_at: now + ) + + stub_request(:get, %r{\Ahttps://api\.github\.com/repos/fair/repo-\d+\z}) + .to_return do |request| + github_id = request.uri.path[/-(\d+)\z/, 1].to_i + { + status: 200, + headers: { + "Content-Type" => "application/json", + "X-RateLimit-Resource" => "core", "X-RateLimit-Limit" => "60", + "X-RateLimit-Remaining" => "50", + "X-RateLimit-Reset" => current_budget.reset_at.to_i.to_s + }, + body: JSON.generate(repository_item(github_id, "fair/repo-#{github_id}")) + } + end + end + + # The repository guarantee is 2 of the 4-request core detail allowance. The third + # completion is only reachable because the scheduled actor slot had no candidate + # and the borrow flag travelled to the core ledger — §10's borrowing, exercised + # end to end through the CycleRunner rather than asserted on a unit. + it "lets the flooded class spend past its guarantee on the idle lane's slots" do + cycle = build_cycle_runner(configuration).call + + expect(cycle).to have_attributes(details_attempted: 3, details_completed: 3, + detail_stop_reason: "no_detail_work") + expect(current_budget).to have_attributes( + enrichment_used: 3, actor_share_used: 0, repository_share_used: 3 + ) + expect(GithubRepository.distinct.pluck(:enrichment_stage)).to eq([ "contract_complete" ]) + end + end +end diff --git a/spec/services/github/enrichment/multi_window_backlog_spec.rb b/spec/services/github/enrichment/multi_window_backlog_spec.rb deleted file mode 100644 index 6df121e..0000000 --- a/spec/services/github/enrichment/multi_window_backlog_spec.rb +++ /dev/null @@ -1,159 +0,0 @@ -require "rails_helper" - -# A backlog larger than one default enrichment allowance, driven through the same runner, -# request gate, ledger, response classification, document parser and entity state writes as -# production. WebMock prevents external connections while the Faraday request stack is still -# exercised. -RSpec.describe "a durable enrichment backlog spanning quota windows", type: :integration do - let(:now) { frozen_time } - let(:actor_ids) { (10_001..10_023).to_a } - let(:repository_ids) { (20_001..20_023).to_a } - let(:configuration) do - configuration_with( - POLL_INTERVAL_SECONDS: "300", MAX_PAGES_PER_POLL: "1", - ENABLED_LIVE_SOURCE_COUNT: "1", RATE_LIMIT_RESERVE: "8", - ACTOR_ENRICHMENT_SHARE: "0.50" - ) - end - - it "drains forty FIFO candidates, preserves every remainder, and finishes next window" do - request_order = [] - stub_backlog_documents!(request_order) - create_backlog! - - current_time = now - clock = -> { current_time } - executor = live_stubbed_executor(clock: clock, configuration: configuration) - runner = live_stubbed_runner(executor: executor, clock: clock, configuration: configuration) - open_window!(executor: executor, at: current_time) - - first_window = Array.new(40) { runner.call } - - expect(first_window).to all(be_enriched) - expect(request_order).to eq( - actor_ids.first(20).map { [ :actor, _1 ] } + - repository_ids.first(20).map { [ :repository, _1 ] } - ) - expect(current_budget).to have_attributes( - poll_used: 1, enrichment_used: 40, actor_share_used: 20, repository_share_used: 20 - ) - - expect(runner.call).to have_attributes(status: "deferred", deferral_reason: "class_exhausted") - expect(request_order.length).to eq(40) - expect_remaining_backlog(actor_ids.last(3), repository_ids.last(3)) - - current_time = now + 7200 - open_window!(executor: executor, at: current_time) - - second_window = Array.new(6) { runner.call } - - expect(second_window).to all(be_enriched) - expect(request_order.last(6)).to eq( - actor_ids.last(3).map { [ :actor, _1 ] } + - repository_ids.last(3).map { [ :repository, _1 ] } - ) - expect(current_budget).to have_attributes( - poll_used: 1, enrichment_used: 6, actor_share_used: 3, repository_share_used: 3 - ) - expect(GithubActor.where(github_id: actor_ids).distinct.pluck(:enrichment_status)) - .to eq([ "complete" ]) - expect(GithubRepository.where(github_id: repository_ids).distinct.pluck(:enrichment_status)) - .to eq([ "complete" ]) - expect(runner.call).to have_attributes(status: "idle", deferral_reason: "no_candidate") - expect(request_order.length).to eq(46) - expect(WebMock).to have_requested(:get, "https://api.github.com/events?per_page=1").twice - end - - private - - def create_backlog! - actor_ids.each_with_index do |github_id, index| - create_actor( - github_id: github_id, login: "backlog-user-#{github_id}", - display_login: "backlog-user-#{github_id}", - api_url: "https://api.github.com/users/backlog-user-#{github_id}", - created_at: now - 1000 + index - ) - end - - repository_ids.each_with_index do |github_id, index| - create_repository( - github_id: github_id, full_name: "backlog/repo-#{github_id}", name: "repo-#{github_id}", - api_url: "https://api.github.com/repos/backlog/repo-#{github_id}", - created_at: now - 1000 + index - ) - end - end - - def stub_backlog_documents!(request_order) - stub_request( - :get, - %r{\Ahttps://api\.github\.com/(?:users/backlog-user-\d+|repos/backlog/repo-\d+)\z} - ).to_return do |request| - github_id = request.uri.path[/-(\d+)\z/, 1].to_i - actor = request.uri.path.start_with?("/users/") - request_order << [ actor ? :actor : :repository, github_id ] - - body = if actor - { "id" => github_id, "name" => "Backlog User #{github_id}" } - else - { "id" => github_id, "description" => "Backlog repository #{github_id}", - "language" => "Ruby", "owner" => { "id" => github_id + 100_000 } } - end - - { status: 200, headers: { "Content-Type" => "application/json" }, body: JSON.generate(body) } - end - end - - def live_stubbed_executor(clock:, configuration:) - Github::RequestExecutor.new( - transport: Github::Transports::Faraday.new, - ledger: ledger_for(configuration), mode: :live, - sleeper: ->(_seconds) { }, clock: clock - ) - end - - def live_stubbed_runner(executor:, clock:, configuration:) - selector = Github::Enrichment::CandidateSelector.new(configuration: configuration) - Github::EnrichmentRunner.new( - executor: executor, configuration: configuration, clock: clock, - monotonic: -> { 0.0 }, selector: selector - ) - end - - def open_window!(executor:, at:) - stub_request(:get, "https://api.github.com/events?per_page=1").to_return( - status: 200, body: "[]", - headers: { - "X-RateLimit-Resource" => "core", "X-RateLimit-Limit" => "60", - "X-RateLimit-Remaining" => "59", "X-RateLimit-Used" => "1", - "X-RateLimit-Reset" => (at + 3600).to_i.to_s - } - ) - - result = executor.call( - Github::Request.new(url: "https://api.github.com/events?per_page=1", request_class: :poll) - ) - expect(result.classification).to eq(:ok) - expect(current_budget).to have_attributes( - window_status: "active", poll_used: 1, enrichment_used: 0, - actor_share_used: 0, repository_share_used: 0, enrichment_allowance: 40 - ) - end - - def expect_remaining_backlog(expected_actor_ids, expected_repository_ids) - actor_rows = GithubActor.where(github_id: actor_ids, - enrichment_status: Enrichable::CANDIDATE_STATUSES) - .order(:created_at, :id) - repository_rows = GithubRepository.where( - github_id: repository_ids, enrichment_status: Enrichable::CANDIDATE_STATUSES - ).order(:created_at, :id) - - expect(actor_rows.pluck(:github_id)).to eq(expected_actor_ids) - expect(repository_rows.pluck(:github_id)).to eq(expected_repository_ids) - expect(actor_rows.pluck(:enrichment_status, :enrichment_attempts, :next_retry_at, :last_error).uniq) - .to eq([ [ "pending", 0, nil, nil ] ]) - expect(repository_rows.pluck(:enrichment_status, :enrichment_attempts, :next_retry_at, :last_error).uniq) - .to eq([ [ "pending", 0, nil, nil ] ]) - end -end diff --git a/spec/services/github/enrichment/multi_window_batch_backlog_spec.rb b/spec/services/github/enrichment/multi_window_batch_backlog_spec.rb new file mode 100644 index 0000000..e87fb74 --- /dev/null +++ b/spec/services/github/enrichment/multi_window_batch_backlog_spec.rb @@ -0,0 +1,305 @@ +require "rails_helper" + +# A backlog larger than one Search window's spendable budget, driven through the same +# claims, request gate, both ledgers, response classification, document parsers and +# entity-state writes as production. WebMock prevents external connections while the +# Faraday request stack is still exercised. +# +# The ceiling is lowered to 4 with the default reserve of 2, so each minute window +# grants exactly two Search requests — small enough that draining fifty entities forces +# three window rolls, which is the durability property Appendix F states: quota +# exhaustion defers the FIFO, it never shortens it. +RSpec.describe "a durable staged backlog spanning quota windows", type: :integration do + let(:now) { frozen_time } + let(:actor_ids) { (10_001..10_025).to_a } + let(:repository_ids) { (20_001..20_025).to_a } + + # The five repositories the Search echo pretends not to know — one per position + # flavour (early, mid, late in their batches) — which is exactly the shape that + # admits them to the bounded core detail-fallback lane. + let(:missing_repository_ids) { [ 20_003, 20_007, 20_012, 20_018, 20_024 ].freeze } + + let(:configuration) do + configuration_with( + "SEARCH_REQUEST_CEILING" => "4", "SEARCH_SAFETY_RESERVE" => "2", + "SEARCH_BATCH_SIZE" => "10", "SEARCH_PACING_SECONDS" => "0", + "CORE_DETAIL_FALLBACK_ALLOWANCE" => "4" + ) + end + + it "drains the FIFO across search windows and finishes the fallbacks next core window" do + search_requests = [] + current_time = now + clock = -> { current_time } + + stub_search_echo!(search_requests, -> { current_time }) + stub_detail_documents!(-> { current_time }) + create_backlog! + + executor = live_stubbed_executor(clock: clock) + batch_runner = live_stubbed_batch_runner(executor: executor, clock: clock) + detail_runner = live_stubbed_detail_runner(executor: executor, clock: clock) + admission = Github::Enrichment::Admission.new(configuration: configuration) + open_window!(executor: executor, at: current_time) + + # ---- Act 1: the Search lane, two requests per minute window ------------------- + + # Window 1: the ceiling-minus-reserve pair of requests drains the oldest twenty + # actors, ten at a time, in strict created_at,id order. + 2.times do + expect(batch_runner.call(entity_class: GithubActor)) + .to have_attributes(status: "completed", requested_count: 10, valid_count: 10) + end + + # The third reservation is refused by the ledger under its row lock, the batch row + # records the reason, and the five leased rows come back bit-identical. + remaining_actors = GithubActor.where(github_id: actor_ids.last(5)).order(:id) + before_rows = remaining_actors.map(&:attributes) + + expect(admission.search(now: current_time).reason).to eq(:search_ceiling_exhausted) + expect(batch_runner.call(entity_class: GithubActor)) + .to have_attributes(status: "deferred", deferral_reason: "budget_denied") + + expect(remaining_actors.reload.map(&:attributes)).to eq(before_rows) + expect(EnrichmentBatch.order(:id).last) + .to have_attributes(status: "deferred") + expect(EnrichmentBatch.order(:id).last.last_error).to include("search_ceiling_exhausted") + expect(current_search_budget).to have_attributes(used: 2, actor_used: 2, repository_used: 0) + + # Window 2: sixty-one seconds later the ledger rolls the window on its next + # reservation — no sweeper, no reset job — and draining resumes where it stopped. + current_time = now + 61 + + expect(batch_runner.call(entity_class: GithubActor)) + .to have_attributes(status: "completed", requested_count: 5, valid_count: 5) + expect(batch_runner.call(entity_class: GithubRepository)) + .to have_attributes(status: "completed", requested_count: 10, valid_count: 8, + fallback_count: 2) + + # Window 3: the remaining repositories, again FIFO, again two requests. + current_time = now + 122 + + expect(batch_runner.call(entity_class: GithubRepository)) + .to have_attributes(status: "completed", requested_count: 10, valid_count: 8, + fallback_count: 2) + expect(batch_runner.call(entity_class: GithubRepository)) + .to have_attributes(status: "completed", requested_count: 5, valid_count: 4, + fallback_count: 1) + + # The exact identifiers of every Search request, across every window, in FIFO + # order — the whole point of the durable backlog. + expect(search_requests).to eq([ + [ :actor, actor_logins.first(10) ], + [ :actor, actor_logins[10, 10] ], + [ :actor, actor_logins.last(5) ], + [ :repository, repository_names.first(10) ], + [ :repository, repository_names[10, 10] ], + [ :repository, repository_names.last(5) ] + ]) + + expect(current_search_budget).to have_attributes( + used: 2, actor_used: 0, repository_used: 2, remaining: 8, blocked_until: nil + ) + + # ---- Act 2: the bounded core detail-fallback lane ----------------------------- + + fallback = GithubRepository.where(github_id: missing_repository_ids) + expect(fallback.pluck(:enrichment_stage).uniq).to eq([ "detail_pending" ]) + expect(fallback.order(:detail_pending_at, :id).pluck(:github_id)) + .to eq(missing_repository_ids) + + # Four fallbacks fit the CORE_DETAIL_FALLBACK_ALLOWANCE. The actor lane has no + # eligible candidate, so the third and fourth ride borrowed slots past the + # repository share guarantee — exactly the CycleRunner's borrow decision. + [ false, false, true, true ].each do |borrowed| + expect(detail_runner.call(entity_class: GithubRepository, borrow: borrowed)) + .to have_attributes(status: "completed") + end + + expect(admission.detail(now: current_time).reason).to eq(:class_exhausted) + expect(detail_runner.call(entity_class: GithubRepository, borrow: true)) + .to have_attributes(status: "deferred", reason: "budget_denied") + expect(EnrichmentBatch.order(:id).last.last_error).to include("class_allowance_exhausted") + expect(current_budget).to have_attributes(poll_used: 1, enrichment_used: 4) + + # The fifth waits out the core window as durable detail_pending work, and the next + # window — opened by the same bootstrap poll production uses — finishes it. + current_time = now + 7200 + open_window!(executor: executor, at: current_time) + + expect(detail_runner.call(entity_class: GithubRepository)) + .to have_attributes(status: "completed", github_id: missing_repository_ids.last) + + # Every one of the fifty rows completed the contract; quota pressure produced + # deferrals and window waits, never a terminal outcome. + expect(GithubActor.where(github_id: actor_ids).distinct.pluck(:enrichment_status, :enrichment_stage)) + .to eq([ [ "complete", "contract_complete" ] ]) + expect(GithubRepository.where(github_id: repository_ids) + .distinct.pluck(:enrichment_status, :enrichment_stage)) + .to eq([ [ "complete", "contract_complete" ] ]) + expect(GithubRepository.where(enrichment_stage: "terminal").count).to eq(0) + + expect(current_budget).to have_attributes( + poll_used: 1, enrichment_used: 1, actor_share_used: 0, repository_share_used: 1 + ) + expect(current_search_budget).to have_attributes(used: 2, actor_used: 0, repository_used: 2) + expect(WebMock).to have_requested(:get, "https://api.github.com/events?per_page=1").twice + end + + private + + def actor_logins + actor_ids.map { |github_id| "backlog-user-#{github_id}" } + end + + def repository_names + repository_ids.map { |github_id| "backlog/repo-#{github_id}" } + end + + # created_at staggers the FIFO; updated_at is pinned to the frozen clock so the + # deferred claim's release is provably a bit-identical restore. + def create_backlog! + actor_ids.each_with_index do |github_id, index| + create_actor( + github_id: github_id, login: "backlog-user-#{github_id}", + display_login: "backlog-user-#{github_id}", + api_url: "https://api.github.com/users/backlog-user-#{github_id}", + created_at: now - 1000 + index, updated_at: now + ) + end + + repository_ids.each_with_index do |github_id, index| + create_repository( + github_id: github_id, full_name: "backlog/repo-#{github_id}", name: "repo-#{github_id}", + api_url: "https://api.github.com/repos/backlog/repo-#{github_id}", + created_at: now - 1000 + index, updated_at: now + ) + end + end + + # Echoes every requested qualifier back as a validating Search item — except the five + # missing repositories — with authoritative per-minute Search headers computed from + # the ledger's own post-debit counter. + def stub_search_echo!(search_requests, current_time) + stub_request(:get, %r{\Ahttps://api\.github\.com/search/(users|repositories)\?}) + .to_return do |request| + query = URI.decode_www_form(request.uri.query.to_s).to_h + identifiers = query.fetch("q").split(" ").map { |qualifier| qualifier.split(":", 2).last } + actor = request.uri.path.end_with?("/users") + search_requests << [ actor ? :actor : :repository, identifiers ] + + items = identifiers.filter_map do |identifier| + github_id = identifier[/(\d+)\z/, 1].to_i + next if !actor && missing_repository_ids.include?(github_id) + + actor ? actor_item(github_id, identifier) : repository_item(github_id, identifier) + end + + { + status: 200, + headers: search_headers(current_time.call), + body: JSON.generate( + "total_count" => items.length, "incomplete_results" => false, "items" => items + ) + } + end + end + + def search_headers(at) + { + "Content-Type" => "application/json", + "X-RateLimit-Resource" => "search", + "X-RateLimit-Limit" => "10", + "X-RateLimit-Remaining" => (10 - current_search_budget.used).to_s, + "X-RateLimit-Used" => current_search_budget.used.to_s, + "X-RateLimit-Reset" => (at + 60).to_i.to_s + } + end + + def actor_item(github_id, login) + { "id" => github_id, "login" => login, "type" => "User" } + end + + def repository_item(github_id, full_name) + { + "id" => github_id, "full_name" => full_name, + "owner" => { "id" => github_id + 100_000, "login" => "backlog" }, + "fork" => false, "archived" => false, "default_branch" => "main", + "description" => "Backlog repository #{github_id}", "language" => "Ruby", + "created_at" => "2026-07-01T00:00:00Z" + } + end + + # The detail lane fetches the stored payload api_url through the CORE ledger; the + # body must satisfy the full repository contract, and the headers carry the same + # hourly core window the bootstrap poll opened. + def stub_detail_documents!(current_time) + stub_request(:get, %r{\Ahttps://api\.github\.com/repos/backlog/repo-\d+\z}) + .to_return do |request| + github_id = request.uri.path[/-(\d+)\z/, 1].to_i + + { + status: 200, + headers: { + "Content-Type" => "application/json", + "X-RateLimit-Resource" => "core", "X-RateLimit-Limit" => "60", + "X-RateLimit-Remaining" => "50", + "X-RateLimit-Reset" => (current_budget.reset_at || current_time.call + 3600).to_i.to_s + }, + body: JSON.generate(repository_item(github_id, "backlog/repo-#{github_id}")) + } + end + end + + def live_stubbed_executor(clock:) + Github::RequestExecutor.new( + transport: Github::Transports::Faraday.new, + ledger: ledger_for(configuration), + search_ledger: search_ledger_for(configuration), + mode: :live, sleeper: ->(_seconds) { }, clock: clock + ) + end + + def live_stubbed_batch_runner(executor:, clock:) + Github::Enrichment::BatchRunner.new( + executor: executor, configuration: configuration, + claim: Github::Enrichment::BatchClaim.new(configuration: configuration), + search_ledger: search_ledger_for(configuration), + backoff: jitterless_backoff(configuration: configuration), + clock: clock + ) + end + + def live_stubbed_detail_runner(executor:, clock:) + Github::Enrichment::DetailRunner.new( + executor: executor, configuration: configuration, + claim: Github::Enrichment::DetailClaim.new(configuration: configuration), + backoff: jitterless_backoff(configuration: configuration), + clock: clock + ) + end + + # The core window opens the way production opens it: one bootstrap poll whose + # authoritative headers initialize the hourly window (§7). Re-registered per call so + # the reset instant tracks the travelling clock. + def open_window!(executor:, at:) + stub_request(:get, "https://api.github.com/events?per_page=1").to_return( + status: 200, body: "[]", + headers: { + "Content-Type" => "application/json", + "X-RateLimit-Resource" => "core", "X-RateLimit-Limit" => "60", + "X-RateLimit-Remaining" => "59", "X-RateLimit-Used" => "1", + "X-RateLimit-Reset" => (at + 3600).to_i.to_s + } + ) + + result = executor.call( + Github::Request.new(url: "https://api.github.com/events?per_page=1", request_class: :poll) + ) + expect(result.classification).to eq(:ok) + expect(current_budget).to have_attributes( + window_status: "active", poll_used: 1, enrichment_used: 0, + actor_share_used: 0, repository_share_used: 0, enrichment_allowance: 4 + ) + end +end diff --git a/spec/services/github/enrichment/one_shot_spec.rb b/spec/services/github/enrichment/one_shot_spec.rb index b1e4535..d9b7fb3 100644 --- a/spec/services/github/enrichment/one_shot_spec.rb +++ b/spec/services/github/enrichment/one_shot_spec.rb @@ -1,56 +1,127 @@ require "rails_helper" RSpec.describe Github::Enrichment::OneShot do - # An instance_double, so the CLI contract is asserted independently of what enrichment - # actually does — the shape ingestion/one_shot_spec.rb uses for the same reason. - let(:runner) { instance_double(Github::EnrichmentRunner) } + # instance_doubles, so the CLI contract — lanes, the request limit, exit codes, the + # report — is asserted independently of what the runners actually do, the shape + # ingestion/one_shot_spec.rb uses for the same reason. + let(:batch_runner) { instance_double(Github::Enrichment::BatchRunner) } + let(:detail_runner) { instance_double(Github::Enrichment::DetailRunner) } + let(:admission) { instance_double(Github::Enrichment::Admission) } + let(:batch_claim) { instance_double(Github::Enrichment::BatchClaim) } + let(:detail_claim) { instance_double(Github::Enrichment::DetailClaim) } let(:output) { StringIO.new } let(:error_output) { StringIO.new } + let(:granted) { Github::Enrichment::Admission::GRANTED } + + def denial(reason) + Github::Enrichment::Admission::Verdict.new(reason: reason, retry_in_seconds: nil) + end + def one_shot(argv: []) - described_class.new(argv: argv, output: output, error_output: error_output, runner: runner) + described_class.new(argv: argv, output: output, error_output: error_output, + batch_runner: batch_runner, detail_runner: detail_runner, + admission: admission, batch_claim: batch_claim, + detail_claim: detail_claim) + end + + def batch_result(status:, **overrides) + defaults = { status: status, entity_type: :actor, batch_id: 41, requested_count: 3, + returned_count: 3, valid_count: 2, fallback_count: 1, deferral_reason: nil } + + Github::Enrichment::BatchRunner::Result.new(**defaults.merge(overrides)) + end + + def detail_result(status:, **overrides) + defaults = { status: status, entity_type: :actor, github_id: 583_231, batch_id: 42, + reason: nil } + + Github::Enrichment::DetailRunner::Result.new(**defaults.merge(overrides)) + end + + # Both lanes admissible and empty by default; each example states only its deviation. + before do + allow(admission).to receive(:search).and_return(granted) + allow(admission).to receive(:detail).and_return(granted) + allow(batch_claim).to receive(:claimable?).and_return(false) + allow(detail_claim).to receive(:claimable?).and_return(false) end - def result(status:, **overrides) - Github::EnrichmentRunner::Result.new(status: status, **overrides) + def batch_work! + allow(batch_claim).to receive(:claimable?).and_return(true) end - def returning(*results) - allow(runner).to receive(:call).and_return(*results) + def detail_work! + allow(detail_claim).to receive(:claimable?).and_return(true) end describe "exit codes" do - it "succeeds when it enriched something" do - returning(result(status: "enriched", entity_type: :actor, github_id: 1)) + it "succeeds when a batch applied its items" do + batch_work! + allow(batch_runner).to receive(:call).and_return(batch_result(status: "completed")) expect(one_shot.call.exit_code).to eq(described_class::SUCCESS) end - # §10 is explicit that a 404 on one enrichment target is not a failure of anything - # else. Without this rule a reviewer running the deterministic fixture scenario would - # get exit 1 for ghostuser, which is correct behaviour reported as breakage. - it "succeeds on a permanent failure, which is a decided and durable outcome" do - returning(result(status: "failed", entity_type: :actor, github_id: 1, - enrichment_status: "permanent_failure", last_error: "404")) + # §10 is explicit that a confirmed-gone entity is a decided, durable outcome. Without + # this rule a reviewer running the deterministic fixture scenario would get exit 1 for + # ghostuser — correct behaviour reported as breakage. + it "succeeds on a terminal detail outcome, which is decided and durable" do + detail_work! + allow(detail_runner).to receive(:call) + .and_return(detail_result(status: "terminal", reason: "entity_gone_404")) - expect(one_shot.call.exit_code).to eq(described_class::SUCCESS) + expect(one_shot(argv: %w[ --stage detail ]).call.exit_code).to eq(described_class::SUCCESS) end - it "fails on a retryable failure, which is the one case where work is genuinely unfinished" do - returning(result(status: "failed", entity_type: :actor, github_id: 1, - enrichment_status: "retryable_failure", last_error: "500")) + it "fails when a batch failed, which schedules retries for every claimed item" do + batch_work! + allow(batch_runner).to receive(:call) + .and_return(batch_result(status: "failed", deferral_reason: "search_http_500")) expect(one_shot.call.exit_code).to eq(described_class::FAILURE) end + it "fails when a detail attempt is scheduled to be retried" do + detail_work! + allow(detail_runner).to receive(:call) + .and_return(detail_result(status: "retry_scheduled", reason: "502")) + + expect(one_shot(argv: %w[ --stage detail ]).call.exit_code).to eq(described_class::FAILURE) + end + # §9's rule for the ingestion command, and the same reasoning: the request never # happened, so a reviewer's command must not look broken. - it "succeeds on every deferral, because nothing failed" do - %w[ idle deferred lease_lost ].each do |status| - returning(result(status: status, deferral_reason: "class_exhausted")) + it "succeeds on a ledger deferral, because nothing failed" do + batch_work! + allow(batch_runner).to receive(:call) + .and_return(batch_result(status: "deferred", deferral_reason: "search_reserve_reached")) - expect(one_shot.call.exit_code).to eq(described_class::SUCCESS), "expected #{status} to exit 0" - end + expect(one_shot.call.exit_code).to eq(described_class::SUCCESS) + end + + # A lease lost to a concurrent worker means the entity was decided elsewhere — work + # happened, just not here. + it "succeeds on a lost lease and an empty backlog alike" do + expect(one_shot.call.exit_code).to eq(described_class::SUCCESS) + + detail_work! + allow(detail_runner).to receive(:call).and_return(detail_result(status: "lease_lost")) + + expect(one_shot(argv: %w[ --stage detail ]).call.exit_code).to eq(described_class::SUCCESS) + end + + # There is no pacing sleep here: the always-on cycle waits pacing out, a one-shot + # reports the truth and exits 0. The unstubbed batch_runner double is itself the + # proof that no request was attempted — a call would raise. + it "reports a pacing denial and stops without a request" do + batch_work! + allow(admission).to receive(:search).and_return(denial(:search_pacing)) + + result = one_shot(argv: %w[ --stage batch ]).call + + expect(result.exit_code).to eq(described_class::SUCCESS) + expect(output.string).to include("Search deferred — search_pacing") end it "refuses a bad option rather than running with a guess" do @@ -58,8 +129,19 @@ def returning(*results) expect(error_output.string).to include("bin/enrich") end + it "refuses a configuration the initializer would have refused" do + batch_work! + allow(batch_runner).to receive(:call) + .and_raise(Github::Errors::ConfigurationError, "SEARCH_BATCH_SIZE must be positive") + + expect(one_shot.call.exit_code).to eq(described_class::REFUSED) + expect(error_output.string).to include("Configuration error") + end + it "refuses a corpus gap, which is an authoring bug that recurs until it is fixed" do - allow(runner).to receive(:call).and_raise(Github::Errors::FixtureMiss.new("/users/nobody")) + batch_work! + allow(batch_runner).to receive(:call) + .and_raise(Github::Errors::FixtureMiss.new("/search/users?q=user%3Anobody")) expect(one_shot.call.exit_code).to eq(described_class::REFUSED) expect(error_output.string).to include("Fixture corpus error") @@ -67,40 +149,63 @@ def returning(*results) end describe "--limit" do - it "runs one cycle by default, which is the unit PR 8's jobs wrap" do - expect(runner).to receive(:call).once.and_return(result(status: "enriched", entity_type: :actor, github_id: 1)) + it "attempts one request by default" do + batch_work! + detail_work! + expect(batch_runner).to receive(:call).once.and_return(batch_result(status: "completed")) one_shot.call end - it "runs up to the number of cycles it was given" do - expect(runner).to receive(:call).exactly(3).times - .and_return(result(status: "enriched", entity_type: :actor, github_id: 1)) + # The limit counts REQUESTS across both lanes, not entities and not lanes — a batch of + # ten entities is one request against it. + it "spends the request budget across both lanes in order" do + allow(batch_claim).to receive(:claimable?).and_return(true, false, false) + detail_work! + expect(batch_runner).to receive(:call).once.with(entity_class: :actor) + .and_return(batch_result(status: "completed")) + expect(detail_runner).to receive(:call).once.with(entity_class: :actor, borrow: false) + .and_return(detail_result(status: "completed")) + + expect(one_shot(argv: %w[ --limit 2 ]).call.exit_code).to eq(described_class::SUCCESS) + end + + it "attempts up to the number of requests it was given" do + batch_work! + expect(batch_runner).to receive(:call).exactly(3).times + .and_return(batch_result(status: "completed")) one_shot(argv: %w[ --limit 3 ]).call end - it "stops early when nothing is eligible, rather than asking again for no reason" do - expect(runner).to receive(:call).once.and_return(result(status: "idle")) + # Two consecutive idle claims end a lane — the same race guard the cycle uses — so an + # emptied backlog cannot spin the remaining limit away on claims. + it "stops a lane after two consecutive idle claims" do + batch_work! + expect(batch_runner).to receive(:call).twice.and_return(batch_result(status: "idle")) - one_shot(argv: %w[ --limit 5 ]).call + one_shot(argv: %w[ --limit 5 --stage batch ]).call end - it "stops early on a deferral, because the next cycle would be refused identically" do - expect(runner).to receive(:call).once - .and_return(result(status: "deferred", deferral_reason: "class_exhausted")) + it "stops on a deferral, because the next request would be refused identically" do + batch_work! + expect(batch_runner).to receive(:call).once + .and_return(batch_result(status: "deferred", deferral_reason: "search_reserve_reached")) - one_shot(argv: %w[ --limit 5 ]).call + one_shot(argv: %w[ --limit 5 --stage batch ]).call + + expect(output.string).to include("Search batch deferred — search_reserve_reached") end # §16 requires malformed data not to terminate the batch, and the same reasoning - # applies to an entity that 404s halfway through a run. - it "does not stop on a failed entity, which must not terminate the batch" do - expect(runner).to receive(:call).exactly(3).times.and_return( - result(status: "failed", entity_type: :actor, github_id: 1, enrichment_status: "permanent_failure") - ) - - one_shot(argv: %w[ --limit 3 ]).call + # applies to a failed request halfway through a run. + it "does not stop on a failed batch, which must not terminate the run" do + batch_work! + expect(batch_runner).to receive(:call).twice + .and_return(batch_result(status: "failed", deferral_reason: "search_http_500")) + + expect(one_shot(argv: %w[ --limit 2 --stage batch ]).call.exit_code) + .to eq(described_class::FAILURE) end it "refuses a non-positive limit" do @@ -108,21 +213,46 @@ def returning(*results) end end - describe "--class" do - it "passes the class through to the runner" do - expect(runner).to receive(:call).with(entity_class: "actor") - .and_return(result(status: "idle")) + describe "--stage" do + it "runs only the batch lane when asked, never consulting the detail side" do + batch_work! + detail_work! + allow(batch_runner).to receive(:call).and_return(batch_result(status: "completed")) + + one_shot(argv: %w[ --stage batch ]).call - one_shot(argv: %w[ --class actor ]).call + expect(admission).not_to have_received(:detail) end - it "asks the runner to choose when no class was named" do - expect(runner).to receive(:call).with(entity_class: nil).and_return(result(status: "idle")) + it "runs only the detail lane when asked, never consulting the search side" do + batch_work! + detail_work! + allow(detail_runner).to receive(:call).and_return(detail_result(status: "completed")) - one_shot.call + one_shot(argv: %w[ --stage detail ]).call + + expect(admission).not_to have_received(:search) + end + + it "refuses a stage it does not have" do + expect(one_shot(argv: %w[ --stage hourly ]).call.exit_code).to eq(described_class::REFUSED) + expect(error_output.string).to include("hourly") + end + end + + describe "--class" do + it "narrows selection to the named class without asking about the other" do + batch_work! + allow(batch_runner).to receive(:call).and_return(batch_result(status: "completed")) + + one_shot(argv: %w[ --class actor --stage batch ]).call + + expect(batch_runner).to have_received(:call).with(entity_class: :actor) + expect(batch_claim).not_to have_received(:claimable?) + .with(Github::Enrichment::EntityType.fetch(:repository)) end - it "rejects a class that has no enrichment counters" do + it "rejects a class that has no enrichment lanes" do expect(one_shot(argv: %w[ --class organization ]).call.exit_code).to eq(described_class::REFUSED) expect(error_output.string).to include("organization") end @@ -139,29 +269,26 @@ def returning(*results) # §8 step 1: "Enrichment jobs skip this step — they take only the request gate." There # is no busy path to contract, and that absence is the guarantee made visible. it "never waits for a source lock, which enrichment does not take" do - expect(runner).to receive(:call).with(entity_class: nil).and_return(result(status: "idle")) + batch_work! + allow(batch_runner).to receive(:call).and_return(batch_result(status: "completed")) + expect(Github::SourceLock).not_to receive(:acquire) one_shot.call + + expect(Github::LockOrder.held_keys).to be_empty end end describe "the report" do - before { returning(result(status: "idle")) } - # §9's rule for the ingestion command applies here too: stdout always proves system # state, even when nothing happened. - it "prints the state blocks even when nothing was enriched" do - one_shot.call - - expect(output.string).to include("Nothing to enrich", "Actor backlog", - "Enrichment backlog budget", "Persisted push events") - end - - it "prints the enrichment counters for the invocation" do + it "prints the tally and both persisted-state blocks even when nothing was attempted" do one_shot.call - expect(output.string).to include("Enrichment cycles", "Cycles with nothing eligible") - expect(output.string).not_to include("Candidates skipped") + expect(output.string).to include( + "Nothing to enrich", "Requests attempted", "Actor contract backlog", + "Search budget", "Detail fallback budget", "Persisted push events" + ) end it "writes the whole block in one call, so a JSON log line cannot split it" do @@ -170,20 +297,42 @@ def returning(*results) one_shot.call end - it "names the entity and the outcome in the headline" do - returning(result(status: "enriched", entity_type: :actor, github_id: 583_231)) + it "names the batch outcome in the headline" do + batch_work! + allow(batch_runner).to receive(:call).and_return(batch_result(status: "completed")) one_shot.call - expect(output.string).to include("Enriched actor 583231 — complete") + expect(output.string).to include("Search batch actor #41 — requested 3, valid 2, fallback 1") + end + + it "names the detail outcome in the headline" do + detail_work! + allow(detail_runner).to receive(:call).and_return(detail_result(status: "completed")) + + one_shot(argv: %w[ --stage detail ]).call + + expect(output.string).to include("Detail actor 583231 — complete") + end + + it "names the retry reason rather than reporting a bare failure" do + detail_work! + allow(detail_runner).to receive(:call) + .and_return(detail_result(status: "retry_scheduled", reason: "502")) + + one_shot(argv: %w[ --stage detail ]).call + + expect(output.string).to include("Detail actor 583231 — retry scheduled: 502") end - it "names the deferral reason rather than reporting a bare failure" do - returning(result(status: "deferred", deferral_reason: "class_exhausted")) + it "still prints the persisted state when it refused to run" do + batch_work! + allow(batch_runner).to receive(:call) + .and_raise(Github::Errors::ConfigurationError, "SEARCH_BATCH_SIZE must be positive") one_shot.call - expect(output.string).to include("Enrichment deferred — class_exhausted") + expect(output.string).to include("Actor contract backlog", "Persisted push events") end # A summary that cannot be read must not mask the outcome that was already decided. diff --git a/spec/services/github/enrichment/redirect_boundary_spec.rb b/spec/services/github/enrichment/redirect_boundary_spec.rb index 9adbc8b..3f1506c 100644 --- a/spec/services/github/enrichment/redirect_boundary_spec.rb +++ b/spec/services/github/enrichment/redirect_boundary_spec.rb @@ -1,126 +1,166 @@ require "rails_helper" -# The corpus has carried `redirecting_repository` and `hostile_redirect` since PR 4, and -# until now nothing consumed them: fixtures/github/README.md said "the redirect ones wait for -# PR 11", and the only reference anywhere was a spec asserting the scenario *names* exist. -# §16's final-review gate forbids dead infrastructure, and corpus that exists only to be -# enumerated is exactly that. +# The corpus has carried `redirecting_repository` and `hostile_redirect` since PR 4; the +# staged pipeline reroutes them through the one path that still follows a stored URL: +# the Search batch reports octocat/Hello-World missing, the row is admitted to the +# bounded core detail-fallback lane, and the detail fetch of its retained payload +# api_url meets the redirect. # -# spec/services/github/request_executor_spec.rb already covers the redirect machinery at the +# spec/services/github/request_executor_spec.rb covers the redirect machinery at the # executor level — following a validated target, debiting each hop, refusing an off-host # Location, stopping at MAX_REDIRECTS. What it cannot show is the consequence for an -# *entity*, because the executor has no entity: whether a rename lands as `complete`, whether -# a hostile hop costs the entity its record, and — the one §10 cares about most — whether -# either of them can take the event source out of service. That is this file. -# -# Both scenarios are single-hop, so MAX_REDIRECTS stays at its default of 2 and nothing is -# added to the compose anchor to run them. -RSpec.describe "enrichment across a redirect", type: :integration do - # The corpus body for this id is repos/octocat_hello-world.json, whose own `id` is +# *entity*: whether a rename lands as `complete`, whether a hostile hop can cost the +# entity its record, and — the one §10 cares about most — whether either of them can +# take the event source out of service. That is this file. +RSpec.describe "staged enrichment across a redirect", type: :integration do + let(:now) { frozen_time } + let(:configuration) { configuration_with("GITHUB_MODE" => "fixture", "SEARCH_PACING_SECONDS" => "0") } + + # The corpus Search key carries exactly this trio in this order, and the corpus body + # for the redirect target is repos/octocat_hello-world.json, whose own `id` is # 1296269 — RepositoryDocument.parse checks identity, so the row and the body have to # agree or a rename would be indistinguishable from a mis-served body. let!(:repository) do - create_repository(github_id: IngestionHelpers::REPOSITORY_GITHUB_ID, - full_name: "octocat/Hello-World", name: "Hello-World", + create_repository(github_id: 1_296_269, full_name: "octocat/Hello-World", + name: "Hello-World", api_url: "https://api.github.com/repos/octocat/Hello-World", - last_seen_at: frozen_time, enrichment_status: "pending") + last_seen_at: now, created_at: now - 3, updated_at: now) + end + let!(:second_repository) do + create_repository(github_id: 1_300_192, full_name: "monalisa/Spoon-Knife", + name: "Spoon-Knife", + api_url: "https://api.github.com/repos/monalisa/Spoon-Knife", + last_seen_at: now, created_at: now - 2, updated_at: now) + end + let!(:third_repository) do + create_repository(github_id: 1_490_033, full_name: "deleted-org/gone", name: "gone", + api_url: "https://api.github.com/repos/deleted-org/gone", + last_seen_at: now, created_at: now - 1, updated_at: now) end let!(:event_source) { fixture_event_source } - before { active_budget_window(now: frozen_time) } + before { active_budget_window(now: now) } - # --class repository, because fairness picks actor before repository as its tie-break and - # an unscoped cycle would not reliably reach this row. Narrowing the class bypasses no - # budget rule — the allowance, the share, the reserve and every global block still bind. - def enrich(scenario) - transport = fixture_transport(scenario: scenario) - result = fixture_enrichment_runner(transport: transport).call(entity_class: GithubRepository) + # The staged route to the redirect: one Search batch (both scenarios' Search body is + # missing Hello-World, so it and deleted-org/gone fall back), then one detail claim, + # which FIFO order hands to Hello-World first. + def admit_and_fetch_detail(transport) + batch = fixture_batch_runner(transport: transport, configuration: configuration) + .call(entity_class: GithubRepository) + expect(batch).to have_attributes(status: "completed", fallback_count: 2) + expect(repository.reload.enrichment_stage).to eq("detail_pending") - [ result, transport ] + fixture_detail_runner(transport: transport, configuration: configuration) + .call(entity_class: GithubRepository) end describe "redirecting_repository — a rename the URL policy accepts" do + let(:transport) { fixture_transport(scenario: "redirecting_repository") } + it "follows the hop and completes the entity" do - result, = enrich("redirecting_repository") + result = admit_and_fetch_detail(transport) - expect(result).to be_enriched - expect(repository.reload.enrichment_status).to eq("complete") + expect(result).to have_attributes(status: "completed", github_id: 1_296_269) + expect(repository.reload).to have_attributes( + enrichment_status: "complete", enrichment_stage: "contract_complete", + latest_observation_source: "detail" + ) end it "stores the document the second hop returned" do - enrich("redirecting_repository") + admit_and_fetch_detail(transport) - expect(repository.reload.full_name).to eq("octocat/Hello-World") - expect(repository.reload.fetched_at).to be_present + body = JSON.parse(Rails.root.join("fixtures/github/bodies/repos/octocat_hello-world.json").read) + expect(repository.reload.raw_payload).to eq(body) + expect(repository.reload.fetched_at).to eq(now) end - # §7's "failures stay spent" generalizes to hops: each one is a real outbound request - # and the ledger debits every attempt. Two requests for one entity is the honest cost of - # a rename, and an operator reading the per-class counter should see it. - it "charges the repository share for both hops, not one" do - _, transport = enrich("redirecting_repository") + # §7's "failures stay spent" generalizes to hops: each one is a real outbound + # request and the core ledger debits every attempt. Two requests for one entity is + # the honest cost of a rename, and both land on the detail lane's class counter — + # the Search request that admitted the row spent the *search* ledger, not this one. + it "charges the core detail lane for both hops, not one" do + admit_and_fetch_detail(transport) - expect(transport.requests.size).to eq(2) - expect(current_budget.repository_share_used).to eq(2) - expect(current_budget.enrichment_used).to eq(2) + expect(current_budget).to have_attributes(enrichment_used: 2, repository_share_used: 2) + expect(current_search_budget).to have_attributes(used: 1, repository_used: 1) end it "leaves no lease behind on the completed entity" do - enrich("redirecting_repository") + admit_and_fetch_detail(transport) - expect(repository.reload.next_retry_at).to be_nil + expect(repository.reload).to have_attributes( + lease_token: nil, leased_until: nil, current_enrichment_batch_id: nil, next_retry_at: nil + ) end end describe "hostile_redirect — a Location pointing off-host" do - it "refuses the entity rather than following it" do - result, = enrich("hostile_redirect") + let(:transport) { fixture_transport(scenario: "hostile_redirect") } + + # §10: "Violations mark the entity permanent_failure." A refused redirect target is + # a property of the stored URL, not a transient condition, so no number of retries + # could change the answer — and the bounded core detail allowance is too scarce to + # spend re-refusing the same hop. The refusal terminates on sight. + def enrich_through_hostile_hop + admit_and_fetch_detail(transport) + end + + it "refuses the hop and terminates the entity on sight" do + result = enrich_through_hostile_hop - expect(result).to be_failed - expect(repository.reload.enrichment_status).to eq("permanent_failure") + expect(result.status).to eq("terminal") + expect(repository.reload).to have_attributes( + enrichment_status: "permanent_failure", enrichment_stage: "terminal", + terminal_at: now, detail_attempts: 1, next_retry_at: nil + ) end - # The assertion that makes this an SSRF test rather than an error-handling test: the - # second hop was never sent. The URL policy runs *before* the request gate, so the - # refusal happens outside any reservation and the socket is never opened. + # The assertion that makes this an SSRF test rather than an error-handling test: + # the hostile Location was never fetched. The URL policy runs *before* the request + # gate, so the refusal happens outside any reservation and the socket never opens. it "never sends the second request" do - _, transport = enrich("hostile_redirect") + enrich_through_hostile_hop - expect(transport.requests.size).to eq(1) - expect(transport.requests.map(&:to_s)).to all(include("api.github.com")) + hops = transport.requests.map { _1.fetch(:url) } + expect(hops).to all(include("api.github.com")) + expect(hops.grep(/evil/)).to be_empty end - it "spends one debit — the hop that did happen — and no more" do - enrich("hostile_redirect") + it "spends one core debit — the hop that did happen — and no more" do + enrich_through_hostile_hop - expect(current_budget.repository_share_used).to eq(1) - expect(current_budget.enrichment_used).to eq(1) + # One Search request on the search ledger; one detail attempt, a single debited + # hop. The refused second hop reserved nothing, because the URL policy runs + # before the gate. + expect(current_budget).to have_attributes(enrichment_used: 1, repository_share_used: 1) + expect(current_search_budget.used).to eq(1) end it "records the policy violation on the entity so an operator sees the reason" do - enrich("hostile_redirect") + enrich_through_hostile_hop expect(repository.reload.last_error).to be_present end # §10's rule, and the one a hostile payload would otherwise be able to exploit: a - # redirect target this application refuses is a fact about one entity, never about the - # feed. Taking the source out of service on it would let one crafted repository URL stop - # ingestion for everything. + # redirect target this application refuses is a fact about one entity, never about + # the feed. Taking the source out of service on it would let one crafted repository + # URL stop ingestion for everything. it "never takes the event source out of service" do - enrich("hostile_redirect") + enrich_through_hostile_hop - expect(event_source.reload.status).to eq("idle") - expect(event_source.reload.enabled).to be(true) - expect(event_source.reload.consecutive_failures).to eq(0) + expect(event_source.reload).to have_attributes(status: "idle", enabled: true, + consecutive_failures: 0) end - it "writes no global block, because one bad target is not a rate limit" do - enrich("hostile_redirect") + it "writes no global block on either ledger, because one bad target is not a rate limit" do + enrich_through_hostile_hop expect(current_budget.global_blocked_until).to be_nil expect(current_budget.window_status).to eq("active") + expect(current_search_budget.blocked_until).to be_nil end end end diff --git a/spec/services/github/enrichment/repository_document_spec.rb b/spec/services/github/enrichment/repository_document_spec.rb index 1ccf6df..17971e9 100644 --- a/spec/services/github/enrichment/repository_document_spec.rb +++ b/spec/services/github/enrichment/repository_document_spec.rb @@ -4,18 +4,35 @@ let(:body) { File.read(Rails.root.join("fixtures/github/bodies/repos/octocat_hello-world.json")) } let(:github_id) { 1_296_269 } + # A minimal document satisfying every contract field, so each refusal example states + # only its one deviation. + def contract_fields + { + "id" => github_id, + "owner" => { "id" => 583_231, "login" => "octocat" }, + "fork" => false, + "archived" => false, + "default_branch" => "main", + "created_at" => "2011-01-26T19:01:12Z" + } + end + def parse(document, id: github_id) described_class.parse(JSON.generate(document), github_id: id) end - describe "the §7 mapping" do - it "populates description, language, owner_github_id and the whole document" do + describe "the useful-data contract" do + it "projects the contract fields and the whole document from the corpus body" do document = described_class.parse(body, github_id: github_id) expect(document).to be_ok expect(document.attributes).to include( - description: "My first repository on GitHub!", language: "Ruby", owner_github_id: 583_231 + description: "My first repository on GitHub!", language: "Ruby", + owner_github_id: 583_231, owner_login: "octocat", + fork: false, archived: false, default_branch: "main", + github_created_at: Time.utc(2011, 1, 26, 19, 1, 12) ) + expect(document.attributes[:raw_payload]).to include("full_name" => "octocat/Hello-World") end # §7 is most explicit about this one: the envelope's repo.name is the qualified @@ -25,48 +42,108 @@ def parse(document, id: github_id) it "never writes name or full_name, which are envelope-derived" do document = described_class.parse(body, github_id: github_id) - expect(document.attributes.keys) - .to contain_exactly(:description, :language, :owner_github_id, :raw_payload) + expect(document.attributes.keys).to contain_exactly( + :description, :language, :owner_github_id, :owner_login, :fork, :archived, + :default_branch, :github_created_at, :raw_payload + ) + end + + # Parsed to a UTC instant rather than stored as text: the projection lands in a + # timestamptz column, and the parse is also the validation. + it "parses github_created_at to UTC from any ISO-8601 offset" do + document = parse(contract_fields.merge("created_at" => "2011-01-26T20:01:12+01:00")) + + expect(document).to be_ok + expect(document.attributes[:github_created_at]).to eq(Time.utc(2011, 1, 26, 19, 1, 12)) end end - describe "tolerance for optional fields" do + # §7's tolerant-parser doctrine, narrowed by the contract: description and language are + # genuinely nullable facts about a repository, so NULL is an accepted projection — + # while the fields the contract names are strict. + describe "tolerance for nullable fields" do it "stores a null description and language rather than refusing the document" do - document = parse({ "id" => github_id, "description" => nil, "language" => nil }) + document = parse(contract_fields.merge("description" => nil, "language" => nil)) expect(document).to be_ok expect(document.attributes).to include(description: nil, language: nil) end - # owner_github_id is nullable with no foreign key, so a missing owner object is a - # column that stays NULL rather than a document that is thrown away. - it "stores a null owner id when the owner object is absent" do - expect(parse({ "id" => github_id }).attributes[:owner_github_id]).to be_nil + it "treats a blank description as absent rather than storing an empty string" do + expect(parse(contract_fields.merge("description" => "")).attributes[:description]).to be_nil + end + + # owner_login is optional inside a required owner object: the id is the identity the + # contract needs, the login is a convenience projection. + it "stores a null owner_login when the owner object omits it" do + document = parse(contract_fields.merge("owner" => { "id" => 583_231 })) + + expect(document).to be_ok + expect(document.attributes).to include(owner_github_id: 583_231, owner_login: nil) + end + + it "degrades a non-String owner_login to null rather than coercing it" do + document = parse(contract_fields.merge("owner" => { "id" => 583_231, "login" => 42 })) + + expect(document).to be_ok + expect(document.attributes[:owner_login]).to be_nil + end + end + + describe "the contract fields it refuses" do + it "refuses a missing or non-object owner, whose id the contract requires" do + expect(parse(contract_fields.except("owner")).error_code).to eq("invalid_contract_field") + expect(parse(contract_fields.merge("owner" => "octocat")).error_code).to eq("invalid_contract_field") + end + + it "refuses an owner whose id is not an integer" do + document = parse(contract_fields.merge("owner" => { "id" => "583231" })) + + expect(document.error_code).to eq("invalid_contract_field") + expect(document.error_message).to include("owner.id") + end + + it "refuses a non-boolean fork flag" do + expect(parse(contract_fields.except("fork")).error_code).to eq("invalid_contract_field") + expect(parse(contract_fields.merge("fork" => "false")).error_code).to eq("invalid_contract_field") + end + + it "refuses a non-boolean archived flag" do + expect(parse(contract_fields.except("archived")).error_code).to eq("invalid_contract_field") + expect(parse(contract_fields.merge("archived" => 0)).error_code).to eq("invalid_contract_field") + end + + it "refuses a missing or blank default_branch" do + expect(parse(contract_fields.except("default_branch")).error_code).to eq("invalid_contract_field") + expect(parse(contract_fields.merge("default_branch" => " ")).error_code).to eq("invalid_contract_field") end - it "stores a null owner id when the owner is not an object" do - expect(parse({ "id" => github_id, "owner" => "octocat" }).attributes[:owner_github_id]).to be_nil + it "refuses a created_at that is absent or not ISO-8601" do + expect(parse(contract_fields.except("created_at")).error_code).to eq("invalid_contract_field") + expect(parse(contract_fields.merge("created_at" => "yesterday")).error_code).to eq("invalid_contract_field") + expect(parse(contract_fields.merge("created_at" => 1_296_069_672)).error_code).to eq("invalid_contract_field") end - it "stores a null owner id when the owner carries no integer id" do - expect(parse({ "id" => github_id, "owner" => { "id" => "583231" } }).attributes[:owner_github_id]).to be_nil + it "refuses a non-String, non-null description rather than guessing at it" do + expect(parse(contract_fields.merge("description" => 42)).error_code).to eq("invalid_contract_field") + expect(parse(contract_fields.merge("language" => [ "Ruby" ])).error_code).to eq("invalid_contract_field") end end - describe "documents it refuses" do + describe "documents it refuses outright" do it "refuses a body that is not JSON at all" do expect(described_class.parse("not json", github_id: github_id).error_code).to eq("unparsable_document") end it "refuses a document with no integer id" do - expect(parse({ "full_name" => "octocat/Hello-World" }).error_code).to eq("missing_identity") + expect(parse(contract_fields.except("id")).error_code).to eq("missing_identity") end # A repository rename is safe — GitHub 301s to the new path and the id is stable, so # the redirect the executor follows produces a match. A genuine mismatch means the URL # points at a different repository and must never be trusted for this row again. it "refuses another repository's document" do - expect(parse({ "id" => 999 }).kind).to eq(:identity_mismatch) + expect(parse(contract_fields.merge("id" => 999)).kind).to eq(:identity_mismatch) end end end diff --git a/spec/services/github/enrichment/search_query_spec.rb b/spec/services/github/enrichment/search_query_spec.rb new file mode 100644 index 0000000..fa1f6e7 --- /dev/null +++ b/spec/services/github/enrichment/search_query_spec.rb @@ -0,0 +1,103 @@ +require "rails_helper" + +RSpec.describe Github::Enrichment::SearchQuery do + let(:actors) { Github::Enrichment::EntityType.fetch(:actor) } + let(:repositories) { Github::Enrichment::EntityType.fetch(:repository) } + + def query_params(url) + URI.decode_www_form(URI.parse(url).query).to_h + end + + describe "the batch qualifier syntax" do + # The live probe behind issue #45: `user:a OR user:b` answers HTTP 422, while + # space-joined repeated qualifiers are the documented AND-of-qualifiers form that + # matches each exact login. The decoded q is asserted, and the absence of OR — + # ever — is the regression guard. + it "joins repeated exact qualifiers with spaces, never OR" do + url = described_class.build(actors, %w[ octocat monalisa hubot ], mode: :live) + + expect(query_params(url)["q"]).to eq("user:octocat user:monalisa user:hubot") + expect(url).not_to include("OR") + end + + it "uses the repo qualifier for repositories" do + url = described_class.build(repositories, %w[ octocat/Hello-World rails/rails ], mode: :live) + + expect(query_params(url)["q"]).to eq("repo:octocat/Hello-World repo:rails/rails") + end + + # per_page equals the batch size so the requested and returned sets are comparable + # one-to-one — a short page is then evidence, not pagination. + it "sets per_page to exactly the batch size" do + url = described_class.build(actors, %w[ octocat monalisa ], mode: :live) + + expect(query_params(url)["per_page"]).to eq("2") + end + + it "sets per_page to one for a single-entity batch" do + url = described_class.build(actors, %w[ octocat ], mode: :live) + + expect(query_params(url)["per_page"]).to eq("1") + end + end + + describe "encoding" do + # URI.encode_www_form's application/x-www-form-urlencoded rules: the qualifier + # colon and the owner/name slash are percent-encoded, and the joining spaces + # become `+`. What matters downstream is that the raw query component contains no + # literal colon, slash, or space for a proxy or log line to mis-split on. + it "percent-encodes the qualifier colon and the repository slash, and joins with +" do + url = described_class.build(repositories, %w[ octocat/Hello-World rails/rails ], mode: :live) + query = URI.parse(url).query + + expect(query).to eq("q=repo%3Aoctocat%2FHello-World+repo%3Arails%2Frails&per_page=2") + expect(query).not_to include(":", "/", " ") + end + end + + describe "endpoints" do + it "targets /search/users for actors" do + url = described_class.build(actors, %w[ octocat ], mode: :live) + + expect(url).to start_with("https://api.github.com/search/users?") + end + + it "targets /search/repositories for repositories" do + url = described_class.build(repositories, %w[ octocat/Hello-World ], mode: :live) + + expect(url).to start_with("https://api.github.com/search/repositories?") + end + end + + # Mode-aware for the same reason Github::EventSources splits PublicEvents from + # FixtureEvents: these are application-origin URLs, and Github::UrlPolicy accepts + # only the fixture scheme in fixture mode — fail closed, never a live fallback. + describe "mode awareness" do + it "builds a fixture-scheme origin in fixture mode" do + url = described_class.build(actors, %w[ octocat ], mode: :fixture) + + expect(url).to start_with("fixture://api.github.com/search/users?") + end + + # Github::Configuration#mode is a String, and the default mode argument comes from + # it — so the string spelling must resolve exactly like the symbol. + it "accepts the configuration's string spelling of a mode" do + url = described_class.build(actors, %w[ octocat ], mode: "fixture") + + expect(url).to start_with("fixture://api.github.com/") + end + + it "refuses an unknown mode rather than defaulting to live" do + expect { described_class.build(actors, %w[ octocat ], mode: :staging) } + .to raise_error(ArgumentError, /staging/) + end + end + + # An empty batch would build `q=&per_page=0` — a request that spends a search + # reservation to ask GitHub for nothing. The claim never produces one, so reaching + # here with no identifiers is a programming error. + it "refuses an empty batch" do + expect { described_class.build(actors, [], mode: :live) } + .to raise_error(ArgumentError, /empty/) + end +end diff --git a/spec/services/github/enrichment/search_response_spec.rb b/spec/services/github/enrichment/search_response_spec.rb new file mode 100644 index 0000000..1a22590 --- /dev/null +++ b/spec/services/github/enrichment/search_response_spec.rb @@ -0,0 +1,111 @@ +require "rails_helper" + +RSpec.describe Github::Enrichment::SearchResponse do + # The envelope GitHub's Search API documents: total_count, incomplete_results, items. + def envelope(**overrides) + { + "total_count" => 2, + "incomplete_results" => false, + "items" => [ + { "id" => 583_231, "login" => "octocat" }, + { "id" => 583_232, "login" => "monalisa" } + ] + }.merge(overrides.transform_keys(&:to_s)) + end + + describe "a well-formed envelope" do + it "parses the raw bytes into the three fields the runner applies" do + response = described_class.parse(JSON.generate(envelope)) + + expect(response).to have_attributes(ok: true, total_count: 2, + incomplete_results: false, error_message: nil) + expect(response).to be_ok + expect(response.items.map { |item| item["login"] }).to eq(%w[ octocat monalisa ]) + end + + # The runner reads the items from inside its projection transaction; freezing the + # array makes an accidental in-place mutation a loud TypeError instead of a + # corrupted batch. + it "freezes the items" do + expect(described_class.parse(JSON.generate(envelope)).items).to be_frozen + end + + it "accepts an already-decoded Hash body without re-parsing" do + response = described_class.parse(envelope) + + expect(response).to be_ok + expect(response.total_count).to eq(2) + end + + it "keeps GitHub's own truncation flag visible" do + response = described_class.parse(JSON.generate(envelope(incomplete_results: true))) + + expect(response.incomplete_results).to be(true) + end + + it "parses an empty result set, which is a valid answer and not a failure" do + response = described_class.parse(JSON.generate(envelope(total_count: 0, items: []))) + + expect(response).to be_ok + expect(response.items).to eq([]) + end + end + + # Every malformed shape becomes a failure value rather than an exception: the batch + # runner records the reason on the batch row and schedules a retry, and a parser that + # raised would turn a bad response body into a crashed cycle. + describe "a malformed envelope" do + def failure_for(body) + response = described_class.parse(body) + expect(response).not_to be_ok + response + end + + it "rejects a body that is valid JSON but not an object" do + response = failure_for("[]") + + expect(response.error_message).to eq("Search response is not an object") + end + + it "rejects items that are not an array" do + response = failure_for(JSON.generate(envelope(items: { "id" => 583_231 }))) + + expect(response.error_message).to eq("Search response items is not an array") + end + + it "rejects a missing items key the same way" do + body = envelope.except("items") + + expect(failure_for(JSON.generate(body)).error_message) + .to eq("Search response items is not an array") + end + + # "2" would compare and sort like a count right up until arithmetic on it didn't; + # the contract check refuses the envelope before any item is applied. + it "rejects a total_count that is not an integer" do + response = failure_for(JSON.generate(envelope(total_count: "2"))) + + expect(response.error_message).to eq("Search response metadata is malformed") + end + + it "rejects an incomplete_results that is not a boolean" do + response = failure_for(JSON.generate(envelope(incomplete_results: "false"))) + + expect(response.error_message).to eq("Search response metadata is malformed") + end + + it "turns unparseable bytes into a failure carrying the parser's own message" do + response = failure_for("{\"total_count\": ") + + expect(response.error_message).to be_present + expect(response).to have_attributes(total_count: nil, incomplete_results: nil) + end + + it "leaves a failure with an empty, frozen item list" do + response = failure_for("not json at all") + + expect(response.items).to eq([]) + expect(response.items).to be_frozen + end + end +end diff --git a/spec/services/github/enrichment/summary_spec.rb b/spec/services/github/enrichment/summary_spec.rb index 765ff56..c25f6f7 100644 --- a/spec/services/github/enrichment/summary_spec.rb +++ b/spec/services/github/enrichment/summary_spec.rb @@ -3,267 +3,297 @@ RSpec.describe Github::Enrichment::Summary do let(:now) { frozen_time } + def capture(**overrides) + described_class.capture(now: now, **overrides) + end + describe ".capture" do - it "reports raw statuses and the durable backlog separately" do - create_actor(github_id: 1, enrichment_status: "complete", fetched_at: now) - create_actor(github_id: 2, enrichment_status: "retryable_failure", - next_retry_at: now + 3600, created_at: now - 600) - create_repository(github_id: 3, created_at: now - 300) - - summary = described_class.capture(now: now) - - expect(summary.actor_counts).to eq("complete" => 1, "retryable_failure" => 1) - expect(summary.repository_counts).to eq("pending" => 1) - expect(summary).to have_attributes( - actor_backlog_count: 1, repository_backlog_count: 1, - actor_oldest_pending_at: now - 600, - repository_oldest_pending_at: now - 300, - actor_oldest_pending_age_seconds: 600, - repository_oldest_pending_age_seconds: 300 + it "carries each class's backlog entry from the shared aggregate" do + create_actor(github_id: 1, created_at: now - 600) + create_repository(github_id: 2, created_at: now - 300) + + summary = capture + + expect(summary.actor).to have_attributes( + status_counts: { "pending" => 1 }, contract_backlog_count: 1, + oldest_pending_at: now - 600, oldest_pending_age_seconds: 600 + ) + expect(summary.repository).to have_attributes( + contract_backlog_count: 1, oldest_pending_at: now - 300 + ) + end + + it "reports the detail-fallback budget against the guarantees the ledger enforces" do + active_budget_window(now: now, enrichment_used: 4, + actor_share_used: 3, repository_share_used: 1) + + expect(capture).to have_attributes( + detail_used: 4, detail_allowance: 4, + actor_share_used: 3, repository_share_used: 1, + actor_guarantee: 2, repository_guarantee: 2, + window_status: "active", window_ready: true ) end - it "reports the per-class share usage against the guarantees the ledger enforces" do - active_budget_window(now: now, actor_share_used: 6, repository_share_used: 5, enrichment_used: 11) + it "projects the search ledger's spend against its spendable ceiling" do + active_search_window(now: now, used: 3) - expect(described_class.capture(now: now)).to have_attributes( - actor_share_used: 6, repository_share_used: 5, enrichment_used: 11, - actor_guarantee: 20, repository_guarantee: 20, enrichment_allowance: 40 + expect(capture).to have_attributes( + search_present: true, search_used: 3, search_spendable: 8, + search_remaining: 9, search_blocked_until: nil, next_search_at: nil ) end - # Nothing seeds the ledger row — only a reservation creates it — so a clean checkout is - # the ordinary state rather than an error. + # Nothing seeds either ledger row — only a reservation creates one — so a clean + # checkout is the ordinary state rather than an error. it "reports no ledger rather than a fabricated zero on a clean checkout" do - expect(described_class.capture(now: now).actor_share_used).to be_nil - expect(described_class.capture(now: now).to_s).to include(described_class::NO_LEDGER) + summary = capture + + expect(summary).to have_attributes(detail_used: nil, detail_allowance: nil, + search_present: false, search_used: nil) + expect(summary.to_s).to include(described_class::NO_LEDGER) end - # The same structural guarantee Github::Ingestion::StateSummary carries, and §11 places - # on /status: no executor, no transport, no ledger. + # The same structural guarantee Github::Ingestion::StateSummary carries, and §11 + # places on /status: no executor, no transport, no ledger writer. it "never initiates a GitHub request" do transport = fixture_transport allow(Github).to receive(:transport).and_return(transport) - described_class.capture(now: now) + capture expect(transport.requests).to be_empty end - it "does not create the ledger row it reads, which only a reservation may" do - expect { described_class.capture(now: now) }.not_to change(GithubApiBudget, :count).from(0) + it "does not create the ledger rows it reads, which only a reservation may" do + expect { capture }.not_to change(GithubApiBudget, :count).from(0) + expect { capture }.not_to change(GithubSearchBudget, :count).from(0) end - it "does not claim enrichment can run before a poll initializes the window" do - create_actor(github_id: 1) + # /status reads each singleton once and passes both down, so its blocks cannot + # straddle a committing reservation and disagree about one instant. + it "projects the rows it was handed rather than reading its own" do + active_budget_window(now: now, enrichment_used: 3) + active_search_window(now: now, used: 5) + budget = current_budget + search_budget = current_search_budget + allow(GithubApiBudget).to receive(:find_by) + allow(GithubSearchBudget).to receive(:find_by) - summary = described_class.capture(now: now) + summary = capture(budget: budget, search_budget: search_budget) - expect(summary).to have_attributes(claimable_now: false, next_enrichment_at: nil) - expect(summary.to_s).to include(described_class::WAITING_FOR_WINDOW) + expect(summary).to have_attributes(detail_used: 3, search_used: 5) end + end - it "still says nothing is waiting on an empty clean checkout" do - summary = described_class.capture(now: now) + describe "#claimable_now" do + # The batch lane needs no core window: the Search ledger self-bootstraps from + # configuration, so a missing row is a grant and never-enriched work is claimable + # on a fresh checkout. + it "is true when search is admissible and batch backlog exists" do + create_actor(github_id: 1) - expect(summary).to have_attributes(work_waiting: false, claimable_now: false, - next_enrichment_at: nil) - expect(summary.to_s).to include(described_class::NOTHING_WAITING) - expect(summary.to_s).not_to include(described_class::WAITING_FOR_WINDOW) + expect(capture).to have_attributes(claimable_now: true, next_enrichment_at: nil) + expect(capture.to_s).to include(described_class::DUE_NOW) end - it "treats an existing uninitialized ledger the same way" do - Github::BudgetLedger.new.bootstrap!(now: now) + # Pacing is a wait, not a refusal — a cycle sleeps through it, so paced work is + # still "due now" rather than deferred to an instant. + it "is true while search pacing holds, because a cycle waits pacing out" do + active_search_window(now: now, last_request_at: now - 2) create_actor(github_id: 1) - expect(described_class.capture(now: now)) - .to have_attributes(window_status: "uninitialized", claimable_now: false, - next_enrichment_at: nil) + expect(capture).to have_attributes(claimable_now: true, next_enrichment_at: nil) end - [ 0, 40 ].each do |used| - it "waits for a poll after an old window elapses with #{used} enrichment attempts used" do - active_budget_window(now: now - 3600, reset_at: now - 1, - enrichment_used: used, - actor_share_used: used / 2, - repository_share_used: used / 2) - create_actor(github_id: 1) + it "is true via the detail lane when the core window grants and fallback work waits" do + active_budget_window(now: now) + create_actor(github_id: 1, enrichment_stage: "detail_pending", + detail_pending_at: now - 60) - summary = described_class.capture(now: now) + expect(capture).to have_attributes(claimable_now: true, next_enrichment_at: nil) + end - expect(summary).to have_attributes(window_ready: false, work_waiting: true, - claimable_now: false, next_enrichment_at: nil) - expect(summary.to_s).to include(described_class::WAITING_FOR_WINDOW) - end + it "is false under a search block, however much batch work is waiting" do + active_search_window(now: now, blocked_until: now + 30) + create_actor(github_id: 1) + + expect(capture).to have_attributes(claimable_now: false, + next_enrichment_at: now + 30) end - end - describe "#next_enrichment_at" do - it "is nil when something is claimable right now" do - active_budget_window(now: now) - create_actor(github_id: 1, last_seen_at: now - 60) + it "is false when pacing holds but there is no work at all" do + active_search_window(now: now, last_request_at: now - 2) + + summary = capture - expect(described_class.capture(now: now).next_enrichment_at).to be_nil + expect(summary).to have_attributes(work_waiting: false, claimable_now: false, + next_enrichment_at: nil) + expect(summary.to_s).to include(described_class::NOTHING_WAITING) end - it "names the soonest instant at which a deferred candidate becomes claimable" do - active_budget_window(now: now) - create_actor(github_id: 1, last_seen_at: now - 60, next_retry_at: now + 120) + # Detail work cannot ride the search grant: the fallback lane spends the core + # ledger, and the core ledger's bootstrap discipline is no window, no enrichment. + it "is false when only detail work waits and no poll has initialized the core window" do + create_actor(github_id: 1, enrichment_stage: "detail_pending", + detail_pending_at: now - 60) - expect(described_class.capture(now: now).next_enrichment_at).to eq(now + 120) + summary = capture + + expect(summary).to have_attributes(claimable_now: false, window_ready: false, + next_enrichment_at: nil) + expect(summary.to_s).to include(described_class::WAITING_FOR_WINDOW) end - # The state the deterministic fixture run actually reaches, and the one that made this - # line contradict the command it is printed beside: with every entity enriched and - # inside its TTL, the next legal action is a refresh, not an enrichment now. - it "names the refresh instant when the whole backlog is enriched and fresh" do - active_budget_window(now: now) - create_actor(github_id: 1, enrichment_status: "complete", fetched_at: now, last_seen_at: now - 60) + it "is false when the detail-fallback allowance is spent, deferring to the reset" do + active_budget_window(now: now, enrichment_used: 4) + create_actor(github_id: 1, enrichment_stage: "detail_pending", + detail_pending_at: now - 60) - expect(described_class.capture(now: now).next_enrichment_at).to eq(now + 86_400) + expect(capture).to have_attributes(claimable_now: false, + next_enrichment_at: now + 3600) end + end - # A retryable failure on a refresh keeps the row complete, so it is invisible to the - # pending pool while still being genuinely deferred. - it "names the backoff of a refresh that failed, which the pending pool cannot see" do - active_budget_window(now: now) - create_actor(github_id: 1, enrichment_status: "complete", fetched_at: now - 90_000, - next_retry_at: now + 300, last_seen_at: now - 60) + describe "#next_enrichment_at" do + it "names the pacing resume when paced search is not the lane the work needs" do + active_search_window(now: now, last_request_at: now - 2) + create_actor(github_id: 1, enrichment_stage: "detail_pending", + detail_pending_at: now - 60) - expect(described_class.capture(now: now).next_enrichment_at).to eq(now + 300) + expect(capture).to have_attributes(claimable_now: false, + next_enrichment_at: now + 4) end - # Deriving "claimable now" from "no deferred pending row" got this backwards: work was - # available, and the report named an instant instead of saying so. - it "is nil when one candidate is due even though another is deferred" do - active_budget_window(now: now) - create_actor(github_id: 1, last_seen_at: now - 60) - create_actor(github_id: 2, last_seen_at: now - 60, next_retry_at: now + 300) + it "names the backoff instant of a scheduled batch retry" do + create_actor(github_id: 1, enrichment_status: "retryable_failure", + enrichment_stage: "retry_scheduled", next_retry_at: now + 120) - expect(described_class.capture(now: now).next_enrichment_at).to be_nil + expect(capture).to have_attributes(claimable_now: false, + next_enrichment_at: now + 120) end - it "is nil when a stale refresh is the work that is waiting" do - active_budget_window(now: now) - create_actor(github_id: 1, enrichment_status: "complete", fetched_at: now - 90_000) + it "names the expiry of a live lease, after which the rows are reclaimable" do + create_actor(github_id: 1, enrichment_stage: "batch_in_flight", + lease_token: SecureRandom.uuid, leased_until: now + 300) - expect(described_class.capture(now: now).next_enrichment_at).to be_nil + expect(capture).to have_attributes(claimable_now: false, + next_enrichment_at: now + 300) end - it "does not report a refresh due while first-time backlog is backed off" do - active_budget_window(now: now) - create_actor(github_id: 1, enrichment_status: "complete", fetched_at: now - 90_000) - create_repository(github_id: 2, enrichment_status: "retryable_failure", - next_retry_at: now + 300) + it "names the instant the oldest fresh completed row crosses its refresh TTL" do + create_actor(github_id: 1, enrichment_status: "complete", + enrichment_stage: "contract_complete", + fetched_at: now - 1000, last_seen_at: now) - expect(described_class.capture(now: now)) - .to have_attributes(claimable_now: false, next_enrichment_at: now + 300) + expect(capture.next_enrichment_at).to eq(now + 85_400) end - it "takes the earliest across both classes" do - active_budget_window(now: now) - create_actor(github_id: 1, enrichment_status: "complete", fetched_at: now) - create_repository(github_id: 2, last_seen_at: now - 60, next_retry_at: now + 90) + it "is nil when a stale refresh is claimable right now" do + create_actor(github_id: 1, enrichment_status: "complete", + enrichment_stage: "contract_complete", + fetched_at: now - 90_000, last_seen_at: now) - expect(described_class.capture(now: now).next_enrichment_at).to eq(now + 90) + expect(capture).to have_attributes(claimable_now: true, next_enrichment_at: nil) end - # §9's effective_enrichment_time, answered for the pool: a global block or an exhausted - # class outranks any individual entity's retry. - it "names the window reset when the class allowance is spent" do - active_budget_window(now: now, enrichment_used: 40) - create_actor(github_id: 1, last_seen_at: now - 60) + it "takes the earliest instant across both classes and both mechanisms" do + create_actor(github_id: 1, enrichment_status: "retryable_failure", + enrichment_stage: "retry_scheduled", next_retry_at: now + 120) + create_repository(github_id: 2, enrichment_stage: "batch_in_flight", + lease_token: SecureRandom.uuid, leased_until: now + 90) - expect(described_class.capture(now: now).next_enrichment_at).to eq(now + 3600) + expect(capture.next_enrichment_at).to eq(now + 90) end - it "names a global block when one outlasts everything else" do - active_budget_window(now: now, global_blocked_until: now + 7200) + # A terminal row is finished: whatever timestamps it retains, it must never + # schedule anything, and with nothing else in the table nothing is waiting. + it "ignores a terminal row entirely" do + create_actor(github_id: 1, enrichment_status: "permanent_failure", + enrichment_stage: "terminal", terminal_at: now - 60, + next_retry_at: now + 60) - expect(described_class.capture(now: now).next_enrichment_at).to eq(now + 7200) + summary = capture + + expect(summary).to have_attributes(work_waiting: false, claimable_now: false, + next_enrichment_at: nil) + expect(summary.to_s).to include(described_class::NOTHING_WAITING) end end describe "#to_s" do # Every value lands in the same column as the block bin/ingest already prints, so a - # reviewer reads the two as one report. A label that fills LABEL_WIDTH exactly gets no - # padding, and its value runs straight into the colon — which is what this catches. + # reviewer reads the two as one report. A label that fills LABEL_WIDTH exactly gets + # no padding, and its value runs straight into the colon — which is what this catches. it "column-aligns with the block bin/ingest already prints, so both read as one report" do active_budget_window(now: now) width = Github::Ingestion::Report::LABEL_WIDTH - described_class.capture(now: now).to_s.lines.each do |line| + capture.to_s.lines.each do |line| expect(line[width - 1]).to eq(" "), "expected #{line.inspect} to pad its label to #{width}" expect(line[width]).not_to eq(" ") end end - it "prints each class's backlog and oldest wait" do + it "prints each class's contract backlog and oldest wait" do create_actor(github_id: 1, created_at: now - 300) - create_actor(github_id: 2, enrichment_status: "complete", fetched_at: now) - active_budget_window(now: now) + create_actor(github_id: 2, enrichment_status: "complete", + enrichment_stage: "contract_complete", fetched_at: now) - expect(described_class.capture(now: now).to_s).to include( - "Actor backlog:", "1", "Oldest actor pending:", "300s old" + expect(capture.to_s).to include( + "Actor contract backlog:", "1", "Oldest actor pending:", "300s old" ) end - it "says due now when a candidate is actually claimable" do - create_actor(github_id: 1) - active_budget_window(now: now) + # The search row is configuration-born rather than poll-born, so "not yet + # initialized" simply means no search request has ever been attempted. + it "says the search budget is not yet initialized before any search request" do + expect(capture.to_s) + .to include("Search budget:", described_class::NO_LEDGER) + end + + it "prints search spend as used-of-spendable, appending the pacing resume" do + active_search_window(now: now, used: 3, last_request_at: now - 2) - expect(described_class.capture(now: now).to_s).to include(described_class::DUE_NOW) + expect(capture.to_s) + .to include("3 of 8 spendable used", "next request #{(now + 4).utc.iso8601}") end # The reason claimable_now exists. A nil next_enrichment_at means "no deferral - # applies", which is equally true of a claimable candidate and of an empty backlog — - # and this line used to say "due now" to a reviewer whose next line of output was - # "nothing to enrich". + # applies", which is equally true of a claimable candidate and of an empty + # backlog — and those are opposite reports to an operator. it "says nothing waiting rather than due now on an empty backlog" do - active_budget_window(now: now) - - expect(described_class.capture(now: now).to_s) - .to include(described_class::NOTHING_WAITING) + expect(capture.to_s).to include(described_class::NOTHING_WAITING) + expect(capture.to_s).not_to include(described_class::DUE_NOW) end end - describe "#claimable_now" do - it "is false when nothing is enrichable, so a nil instant is never ambiguous" do - active_budget_window(now: now) - - expect(described_class.capture(now: now)) - .to have_attributes(claimable_now: false, next_enrichment_at: nil) - end - - it "is true when a candidate could be claimed this second" do - create_actor(github_id: 1) - active_budget_window(now: now) - - expect(described_class.capture(now: now)) - .to have_attributes(claimable_now: true, next_enrichment_at: nil) - end + describe "#to_log" do + it "publishes the staged-pipeline projection under stable keys" do + active_budget_window(now: now, enrichment_used: 2, actor_share_used: 2) + active_search_window(now: now, used: 3) + create_actor(github_id: 1, created_at: now - 300) - # A ledger block outranks every per-entity instant: the candidate is there, but no - # request may be issued for it, so it is not claimable. - it "is false under a global block, however many candidates are waiting" do - create_actor(github_id: 1) - active_budget_window(now: now, global_blocked_until: now + 300) + log = capture.to_log - expect(described_class.capture(now: now)) - .to have_attributes(claimable_now: false, next_enrichment_at: now + 300) + expect(log).to include( + actor_counts: { "pending" => 1 }, + actor_contract_backlog_count: 1, repository_contract_backlog_count: 0, + actor_oldest_pending_at: (now - 300).utc.iso8601, + detail_used: 2, detail_allowance: 4, actor_share_used: 2, + search_used: 3, search_spendable: 8, search_remaining: 9, + window_status: "active", claimable_now: true + ) + expect(log[:actor_stage_counts]).to include("batch_pending" => 1, "terminal" => 0) end - end - describe "the ledger row it reports on" do - # /status reads the singleton once and passes it down, so its poll block and its - # ledger block cannot straddle a committing reservation and disagree. - it "uses the row it was handed instead of reading its own" do - active_budget_window(now: now, enrichment_used: 7) - budget = current_budget - allow(GithubApiBudget).to receive(:find_by) + # .compact keeps the log line honest on a clean checkout: an absent ledger produces + # absent keys, never fabricated zeros. + it "omits the budget keys no ledger row exists to answer" do + log = capture.to_log - expect(described_class.capture(now: now, budget: budget).enrichment_used).to eq(7) - expect(GithubApiBudget).not_to have_received(:find_by) + expect(log).not_to include(:detail_used, :search_used, :next_enrichment_at) + expect(log).to include(claimable_now: false) end end end diff --git a/spec/services/github/enrichment/tally_spec.rb b/spec/services/github/enrichment/tally_spec.rb index c767aea..a631f11 100644 --- a/spec/services/github/enrichment/tally_spec.rb +++ b/spec/services/github/enrichment/tally_spec.rb @@ -1,21 +1,95 @@ require "rails_helper" RSpec.describe Github::Enrichment::Tally do - def result(status:) - Github::EnrichmentRunner::Result.new(status: status) + def batch_result(status:, requested: 0, valid: 0, fallback: 0) + Github::Enrichment::BatchRunner::Result.new( + status: status, entity_type: :actor, batch_id: 41, requested_count: requested, + returned_count: valid, valid_count: valid, fallback_count: fallback, + deferral_reason: nil + ) + end + + def detail_result(status:) + Github::Enrichment::DetailRunner::Result.new( + status: status, entity_type: :actor, github_id: 583_231, batch_id: 42, reason: nil + ) end it "starts at zero on every counter" do expect(described_class.empty.to_h.values).to all(eq(0)) end - it "counts each outcome under its own name" do - tally = described_class.empty - .record(result(status: "enriched")) - .record(result(status: "failed")) - .record(result(status: "deferred")) + describe "#record_batch" do + it "counts a completed batch with its item outcomes" do + tally = described_class.empty + .record_batch(batch_result(status: "completed", requested: 10, + valid: 8, fallback: 2)) + + expect(tally).to have_attributes(requests: 1, batches_completed: 1, batches_failed: 0, + items_requested: 10, items_valid: 8, fallbacks_admitted: 2) + end + + # A failed batch still spent the request and still asked for its items — that is what + # the fill-ratio arithmetic downstream needs — but applied nothing. + it "counts a failed batch's request and items without inventing applied ones" do + tally = described_class.empty.record_batch(batch_result(status: "failed", requested: 10)) + + expect(tally).to have_attributes(requests: 1, batches_failed: 1, batches_completed: 0, + items_requested: 10, items_valid: 0, fallbacks_admitted: 0) + end - expect(tally).to have_attributes(cycles: 3, enriched: 1, failed: 1, deferred: 1, idle: 0) + it "counts a deferred batch as a spent request that decided nothing" do + tally = described_class.empty.record_batch(batch_result(status: "deferred")) + + expect(tally).to have_attributes(requests: 1, deferred: 1, batches_completed: 0) + end + + # An idle claim locked nothing and asked GitHub for nothing — counting it as a request + # would make --limit stop short of the requests the operator asked for. + it "counts an idle claim without counting a request" do + tally = described_class.empty.record_batch(batch_result(status: "idle")) + + expect(tally).to have_attributes(requests: 0, idle: 1) + end + + it "refuses an unknown status rather than dropping it" do + expect { described_class.empty.record_batch(batch_result(status: "invented")) } + .to raise_error(ArgumentError, /invented/) + end + end + + describe "#record_detail" do + # Every decided detail outcome is one spent request under its own name; a terminal 404 + # is a decision, not a failure, which is why it has its own counter and exit-code rule. + it "counts each decided outcome under its own name" do + tally = described_class.empty + .record_detail(detail_result(status: "completed")) + .record_detail(detail_result(status: "terminal")) + .record_detail(detail_result(status: "retry_scheduled")) + + expect(tally).to have_attributes(requests: 3, details_completed: 1, details_terminal: 1, + details_retrying: 1) + end + + it "counts a deferral and a lost lease as spent requests that decided nothing" do + tally = described_class.empty + .record_detail(detail_result(status: "deferred")) + .record_detail(detail_result(status: "lease_lost")) + + expect(tally).to have_attributes(requests: 2, deferred: 1, lease_lost: 1, + details_completed: 0) + end + + it "counts an idle claim without counting a request" do + tally = described_class.empty.record_detail(detail_result(status: "idle")) + + expect(tally).to have_attributes(requests: 0, idle: 1) + end + + it "refuses an unknown status rather than dropping it" do + expect { described_class.empty.record_detail(detail_result(status: "invented")) } + .to raise_error(ArgumentError, /invented/) + end end # Immutable, like Github::Ingestion::Tally: a partially accumulated count can never be @@ -23,23 +97,35 @@ def result(status:) it "returns a new value rather than mutating, so no caller sees a partial count" do empty = described_class.empty - expect { empty.record(result(status: "enriched")) }.not_to change { empty.cycles }.from(0) + expect { empty.record_batch(batch_result(status: "completed")) } + .not_to change { empty.requests }.from(0) end - # Keyed by the runner's own statuses, so a new outcome cannot be silently uncounted. - it "covers every status the runner can return" do - expect(described_class::COUNTERS.keys).to match_array(Github::EnrichmentRunner::Result::STATUSES) - end + it "accumulates both lanes into one request count, which is what --limit bounds" do + tally = described_class.empty + .record_batch(batch_result(status: "completed", requested: 10, valid: 10)) + .record_detail(detail_result(status: "completed")) - it "refuses an unknown status rather than dropping it" do - expect { described_class.empty.record(Struct.new(:status).new("invented")) } - .to raise_error(ArgumentError, /invented/) + expect(tally.requests).to eq(2) end - it "prints cycle outcomes without a discarded-work counter" do - rendered = described_class.empty.record(result(status: "enriched")).to_s + describe "#to_s" do + it "prints the staged-pipeline counters an operator reads after a run" do + rendered = described_class.empty + .record_batch(batch_result(status: "completed", requested: 10, + valid: 8, fallback: 2)) + .record_detail(detail_result(status: "terminal")) + .to_s + + expect(rendered).to include( + "Requests attempted", "Search batches completed", "Batch items requested", + "Batch items applied", "Fallbacks admitted", "Detail completions", + "Detail terminal outcomes", "Requests deferred", "Claims with nothing eligible" + ) + end - expect(rendered).to include("Entities enriched", "Cycles deferred", "Cycles with nothing eligible") - expect(rendered).not_to include("skipped") + it "carries no discarded-work counter, because deferral is not discard" do + expect(described_class.empty.to_s).not_to include("skipped") + end end end diff --git a/spec/services/github/enrichment/throughput_spec.rb b/spec/services/github/enrichment/throughput_spec.rb new file mode 100644 index 0000000..d6a0a32 --- /dev/null +++ b/spec/services/github/enrichment/throughput_spec.rb @@ -0,0 +1,158 @@ +require "rails_helper" + +RSpec.describe Github::Enrichment::Throughput do + let(:now) { frozen_time } + + def from(configuration: Github.configuration) + backlog = Github::Enrichment::BacklogMetrics.capture(now: now, + configuration: configuration) + described_class.from(backlog, now: now, configuration: configuration) + end + + describe "the sample window" do + # No entities means no observed history at all: there is nothing to divide by, so + # the rates are null and the verdict refuses to claim anything either way. + it "reports no sample and no rates on a fresh database" do + throughput = from + + expect(throughput).to have_attributes(sample_started_at: nil, sample_seconds: nil, + catch_up_state: "insufficient_sample") + expect(throughput.combined).to have_attributes( + arrivals: 0, completions: 0, terminals: 0, exits: 0, + arrival_rate_per_hour: nil, completion_rate_per_hour: nil, backlog_delta: 0 + ) + expect(throughput.to_s).to eq("Keeping up: insufficient sample (0s < 900s)") + end + + # A table younger than the window must not divide by the full hour — that would + # understate every rate. The sample starts at the oldest row the table has held. + it "truncates the sample to the earliest created_at inside the window" do + create_actor(github_id: 1, created_at: now - 1200) + + throughput = from + + expect(throughput).to have_attributes(sample_started_at: now - 1200, + sample_seconds: 1200) + expect(throughput.actor.arrival_rate_per_hour).to eq(3.0) + end + + it "spans at most the metrics window however old the table is" do + create_actor(github_id: 1, enrichment_status: "complete", + enrichment_stage: "contract_complete", created_at: now - 90_000, + contract_completed_at: now - 90_000) + + expect(from).to have_attributes(sample_started_at: now - 3600, + sample_seconds: 3600) + end + end + + describe "the catch-up verdict" do + it "is insufficient_sample until the history spans the configured minimum" do + create_actor(github_id: 1, created_at: now - 100) + + expect(from).to have_attributes(sample_seconds: 100, + catch_up_state: "insufficient_sample") + end + + it "is keeping_up when the backlog shrank over the window" do + create_actor(github_id: 1, enrichment_status: "complete", + enrichment_stage: "contract_complete", created_at: now - 5000, + contract_completed_at: now - 60) + + throughput = from + + expect(throughput.combined).to have_attributes(arrivals: 0, exits: 1, + backlog_delta: -1) + expect(throughput.catch_up_state).to eq("keeping_up") + expect(throughput.to_s).to start_with("Keeping up: yes") + end + + it "is keeping_up on a level window only with zero contract debt" do + create_actor(github_id: 1, enrichment_status: "complete", + enrichment_stage: "contract_complete", created_at: now - 5000, + contract_completed_at: now - 5000) + + throughput = from + + expect(throughput.combined.backlog_delta).to eq(0) + expect(throughput.catch_up_state).to eq("keeping_up") + end + + # The honest verdict issue #45 asks for: a backlog that holds level while contract + # debt exists is not being caught up on, even though nothing got worse. + it "is not_keeping_up on a flat nonzero backlog" do + create_actor(github_id: 1, created_at: now - 1000) + create_repository(github_id: 2, enrichment_status: "complete", + enrichment_stage: "contract_complete", created_at: now - 5000, + contract_completed_at: now - 30) + + throughput = from + + expect(throughput.combined).to have_attributes(arrivals: 1, exits: 1, + backlog_delta: 0) + expect(throughput.catch_up_state).to eq("not_keeping_up") + expect(throughput.to_s).to start_with("Keeping up: NO") + end + + it "reads the minimum sample from the configuration it was given" do + create_actor(github_id: 1, created_at: now - 100) + configuration = configuration_with(CATCH_UP_MIN_SAMPLE_SECONDS: "60") + + throughput = from(configuration: configuration) + + expect(throughput).to have_attributes(min_sample_seconds: 60, + catch_up_state: "not_keeping_up") + end + end + + describe "the lane arithmetic" do + # backlog_delta is arrivals minus exits — a counted slope, not a fit or a forecast — + # and a terminal outcome is an exit exactly as a completion is. + it "computes exits and the backlog delta from arrivals, completions and terminals" do + create_actor(github_id: 1, created_at: now - 1000) + create_actor(github_id: 2, created_at: now - 900) + create_actor(github_id: 3, enrichment_status: "complete", + enrichment_stage: "contract_complete", created_at: now - 7200, + contract_completed_at: now - 20) + create_actor(github_id: 4, enrichment_status: "permanent_failure", + enrichment_stage: "terminal", created_at: now - 7200, + terminal_at: now - 10) + + expect(from.actor).to have_attributes( + arrivals: 2, completions: 1, terminals: 1, exits: 2, backlog_delta: 0 + ) + end + + it "combines both lanes and rates the combined counts over the shared sample" do + create_actor(github_id: 1, created_at: now - 1800) + create_repository(github_id: 2, created_at: now - 900) + + throughput = from + + expect(throughput.combined).to have_attributes(arrivals: 2, backlog_delta: 2) + # Two arrivals over the 1800-second sample the older row anchors: 2 * 3600 / 1800. + expect(throughput.combined.arrival_rate_per_hour).to eq(4.0) + expect(throughput.actor.arrival_rate_per_hour).to eq(2.0) + end + end + + describe "#payload" do + it "publishes a fixed key set with the verdict beside its threshold" do + create_actor(github_id: 1, created_at: now - 600) + + payload = from.payload + + expect(payload.keys).to eq(%i[window_seconds window_start sample_started_at + sample_seconds actors repositories combined catch_up]) + expect(payload).to include(window_seconds: 3600, + window_start: (now - 3600).utc.iso8601, + sample_started_at: (now - 600).utc.iso8601, + sample_seconds: 600) + expect(payload[:actors].keys).to eq(%i[arrivals completions terminals exits + arrival_rate_per_hour + completion_rate_per_hour backlog_delta]) + expect(payload[:catch_up]).to eq(state: "insufficient_sample", + min_sample_seconds: 900) + end + end +end diff --git a/spec/services/github/enrichment_runner_spec.rb b/spec/services/github/enrichment_runner_spec.rb deleted file mode 100644 index 2a0a46e..0000000 --- a/spec/services/github/enrichment_runner_spec.rb +++ /dev/null @@ -1,307 +0,0 @@ -require "rails_helper" - -RSpec.describe Github::EnrichmentRunner do - subject(:runner) { fixture_enrichment_runner(transport: transport) } - - let(:transport) { fixture_transport } - let(:now) { frozen_time } - - # The corpus entities page 1 actually produces, so a spec here is a spec about the real - # payload shape. octocat and Hello-World resolve 200; ghostuser and deleted-org/gone - # resolve 404, which is what fixtures/github/README.md documents event 8 for. - def octocat(**overrides) - create_actor(github_id: 583_231, api_url: "https://api.github.com/users/octocat", - last_seen_at: now - 60, **overrides) - end - - def hello_world(**overrides) - create_repository(github_id: 1_296_269, full_name: "octocat/Hello-World", name: "Hello-World", - api_url: "https://api.github.com/repos/octocat/Hello-World", - last_seen_at: now - 60, **overrides) - end - - def ghostuser(**overrides) - create_actor(github_id: 7_700_421, login: "ghostuser", - api_url: "https://api.github.com/users/ghostuser", last_seen_at: now - 60, **overrides) - end - - describe "one cycle" do - before { active_budget_window(now: now) } - - it "enriches one entity and returns one Result describing it" do - actor = octocat - - result = runner.call - - expect(result).to have_attributes(status: "enriched", entity_type: :actor, - github_id: 583_231, pool: :pending, classification: :ok) - expect(actor.reload).to have_attributes(enrichment_status: "complete", name: "The Octocat") - end - - # §5 names EnrichActorJob and EnrichRepositoryJob, so one entity is the unit PR 8 wraps. - # Batching is the caller's loop, which is what Github::Enrichment::OneShot is. - it "enriches at most one entity, whatever the backlog" do - octocat - hello_world - - runner.call - - expect(GithubActor.complete.count + GithubRepository.complete.count).to eq(1) - end - - it "debits the class and the share for the entity it chose" do - octocat - - expect { runner.call } - .to change { current_budget.actor_share_used }.from(0).to(1) - .and change { current_budget.enrichment_used }.from(0).to(1) - end - - it "reports having nothing to do rather than pretending it was refused" do - expect(runner.call).to have_attributes(status: "idle", deferral_reason: "no_candidate") - end - end - - describe "deferrals" do - it "returns deferred without touching the entity when the class allowance is gone" do - actor = octocat - active_budget_window(now: now, enrichment_used: 40) - before = actor.reload.attributes - - expect(runner.call).to have_attributes(status: "deferred", deferral_reason: "class_exhausted") - expect(actor.reload.attributes).to eq(before) - end - - it "returns deferred while a global block is in force" do - octocat - active_budget_window(now: now, global_blocked_until: now + 60) - - expect(runner.call).to have_attributes(status: "deferred", deferral_reason: "globally_blocked") - end - - # §7: enrichment is ineligible until the first real poll initializes the window from - # authoritative headers — "never assume 60 remaining". - it "returns deferred before any poll has initialized the window" do - octocat - Github::BudgetLedger.new.bootstrap!(now: now) - - expect(runner.call).to have_attributes(status: "deferred", deferral_reason: "window_uninitialized") - end - - it "spends nothing on a deferred cycle" do - octocat - active_budget_window(now: now, enrichment_used: 40) - - expect { runner.call }.not_to change { current_budget.enrichment_used }.from(40) - end - - it "keeps old work durable across quota windows and enriches it when capacity returns" do - actor = octocat(last_seen_at: now - 100_000, created_at: now - 100_000) - active_budget_window(now: now, enrichment_used: 40) - - expect(runner.call).to have_attributes(status: "deferred", deferral_reason: "class_exhausted") - expect(actor.reload.enrichment_status).to eq("pending") - - next_window = now + 3601 - active_budget_window(now: next_window, poll_used: 0, enrichment_used: 0, - actor_share_used: 0, repository_share_used: 0) - result = fixture_enrichment_runner(transport: transport, now: next_window).call - - expect(result).to have_attributes(status: "enriched", github_id: actor.github_id) - expect(actor.reload.enrichment_status).to eq("complete") - end - end - - describe "the guarantees §5 and §8 make structurally" do - before { active_budget_window(now: now) } - - # §8 step 1: "Enrichment jobs skip this step — they take only the request gate." - it "never acquires a source lock, because enrichment belongs to no event source" do - octocat - expect(Github::SourceLock).not_to receive(:acquire) - - runner.call - end - - # BudgetLedger#assert_committable! raises inside a joinable application transaction, and - # every enrichment attempt goes through the executor — so this passing at all is the - # assertion that nothing wrapped the fetch. - it "opens no transaction across the GitHub request" do - octocat - - expect(runner.call.status).to eq("enriched") - end - - # §10: "Never disable the event source because one enrichment target disappeared." - it "records a permanent failure without disabling the event source" do - ghostuser - source = fixture_event_source - - result = runner.call - - expect(result).to have_attributes(status: "failed", classification: :not_found) - expect(GithubActor.find_by(github_id: 7_700_421).enrichment_status).to eq("permanent_failure") - expect(source.reload).to have_attributes(status: "idle", enabled: true) - end - - # The executor validates before the gate specifically so an SSRF-violating enrichment - # URL never debits budget — §10's "violations mark the entity permanent_failure", - # satisfied by the chain that already exists rather than by a special case here. - it "spends no budget on a URL-policy violation, and still marks the entity permanently failed" do - actor = octocat(api_url: "https://evil.example.com/users/octocat") - - result = runner.call - - expect(result).to have_attributes(status: "failed", classification: :permanent_error) - expect(actor.reload.enrichment_status).to eq("permanent_failure") - expect(current_budget.enrichment_used).to eq(0) - end - - it "treats an entity with no API URL the same way, with no special case anywhere" do - actor = octocat(api_url: nil) - - runner.call - - expect(actor.reload.enrichment_status).to eq("permanent_failure") - expect(current_budget.enrichment_used).to eq(0) - end - end - - describe "the borrow it asserts to the ledger" do - it "asks to borrow when the other class has no eligible candidate" do - octocat - active_budget_window(now: now, actor_share_used: 20, enrichment_used: 20) - - expect(runner.call).to have_attributes(status: "enriched", borrow: true) - expect(current_budget.actor_share_used).to eq(21) - end - - it "does not ask to borrow while the other class still has work" do - octocat - hello_world - active_budget_window(now: now, actor_share_used: 20, enrichment_used: 20) - - expect(runner.call).to have_attributes(entity_type: :repository, borrow: false) - end - end - - describe "restricting to one class" do - before { active_budget_window(now: now) } - - it "enriches only the class it was given" do - octocat - hello_world - - expect(runner.call(entity_class: GithubRepository).entity_type).to eq(:repository) - end - - # --class narrows selection; §10's constraints are all still in force. - it "still obeys the budget, so restricting a class cannot spend an allowance that is gone" do - octocat - active_budget_window(now: now, enrichment_used: 40) - - expect(runner.call(entity_class: GithubActor).status).to eq("deferred") - end - end - - describe "errors it refuses to launder" do - before { active_budget_window(now: now) } - - # §6: "if a URL is not present in the corpus, a fixture error is raised". A corpus gap - # is an authoring bug, and turning it into a plausible-looking entity failure would let - # a deterministic demo report the wrong thing. - it "releases the lease and re-raises on a corpus gap, costing the entity no attempt" do - actor = octocat(api_url: "https://api.github.com/users/not-in-the-corpus") - before = actor.reload.attributes - - expect { runner.call }.to raise_error(Github::Errors::FixtureMiss) - expect(actor.reload.attributes).to eq(before) - end - end - - describe "logging (plan §11)" do - before { active_budget_window(now: now) } - - it "logs a completed enrichment at INFO with the entity and its classification" do - octocat - allow(Rails.logger).to receive(:info) - - runner.call - - expect(Rails.logger).to have_received(:info).with( - hash_including(event: "enrichment.completed", entity_type: :actor, github_id: 583_231, - classification: :ok, entity_status: "complete") - ) - end - - it "logs a failure at INFO with the error and the scheduled retry" do - ghostuser - allow(Rails.logger).to receive(:info) - - runner.call - - expect(Rails.logger).to have_received(:info) - .with(hash_including(event: "enrichment.failed", entity_status: "permanent_failure")) - end - - # Under PR 8's recurring task an exhausted window would otherwise emit a line a minute - # for the rest of the hour — the volume argument BudgetLedger#log_class_exhausted makes. - it "keeps a deferral at DEBUG, so an exhausted window does not bury the stream" do - octocat - active_budget_window(now: now, enrichment_used: 40) - allow(Rails.logger).to receive(:info) - allow(Rails.logger).to receive(:debug) - - runner.call - - expect(Rails.logger).to have_received(:debug).with(hash_including(event: "enrichment.deferred")) - expect(Rails.logger).not_to have_received(:info).with(hash_including(event: "enrichment.deferred")) - end - - # §11's common-field list includes the attempt number, and this is the line §11 actually - # asks reviewers to read. lease.to_log already carried it onto the DEBUG request line, - # so it was present exactly where nobody was looking and absent where they were. - it "carries §11's attempt number onto the outcome line, not only onto the request line" do - ghostuser(enrichment_attempts: 1) - allow(Rails.logger).to receive(:info) - - runner.call - - expect(Rails.logger).to have_received(:info).with( - hash_including(event: "enrichment.failed", enrichment_attempt: 2, - entity_status: "permanent_failure") - ) - end - - it "counts the attempt from zero on an entity that has never been fetched" do - octocat - allow(Rails.logger).to receive(:info) - - runner.call - - expect(Rails.logger).to have_received(:info) - .with(hash_including(event: "enrichment.completed", enrichment_attempt: 1)) - end - - # Result::EVENTS owns "enrichment.failed" for the ordinary outcome §11 lists — INFO, - # with an entity status and a scheduled retry. An escaped exception has a released - # lease, an error pair and no entity outcome at all, so sharing the name would make one - # alert match two structurally different records. - it "reports an escaped exception under its own name, not the outcome's" do - octocat - allow(Rails.logger).to receive(:error) - exploding = instance_double(Github::Enrichment::EntityState) - allow(exploding).to receive(:record!).and_raise(RuntimeError, "boom") - - expect { fixture_enrichment_runner(transport: transport, entity_state: exploding).call } - .to raise_error(RuntimeError, "boom") - - expect(Rails.logger).to have_received(:error).with( - hash_including(event: "enrichment.cycle_failed", entity_type: :actor, - error_class: "RuntimeError", error_message: "boom") - ) - expect(Rails.logger).not_to have_received(:error) - .with(hash_including(event: "enrichment.failed")) - end - end -end diff --git a/spec/services/github/ingestion/page_writer_spec.rb b/spec/services/github/ingestion/page_writer_spec.rb index 8020872..e79088b 100644 --- a/spec/services/github/ingestion/page_writer_spec.rb +++ b/spec/services/github/ingestion/page_writer_spec.rb @@ -66,6 +66,94 @@ def write(envelopes, at: nil) end end + # §8 steps 6–8 as one durability boundary: the accepted event and both event-native + # identity fragments commit together. The entity row coalesces future demand by stable + # GitHub id; the observation rows preserve the distinct raw evidence each accepted event + # supplied. + describe "event-native observations" do + it "appends one observation per entity, carrying the raw fragment and its fingerprint" do + write(well_formed_envelope) + + event = PushEvent.sole + actor_observation = EnrichmentObservation.find_by(entity_kind: "actor") + repository_observation = EnrichmentObservation.find_by(entity_kind: "repository") + + expect(EnrichmentObservation.count).to eq(2) + expect(actor_observation).to have_attributes( + entity_github_id: 583_231, source: "event", validation_outcome: "event_native", + push_event_id: event.id, observed_at: received_at + ) + expect(actor_observation.raw_payload).to eq(well_formed_envelope.fetch("actor")) + expect(actor_observation.payload_fingerprint) + .to eq(Github::Events::PayloadFingerprint.fingerprint(well_formed_envelope.fetch("actor"))) + expect(repository_observation).to have_attributes( + entity_github_id: 1_296_269, source: "event", validation_outcome: "event_native", + push_event_id: event.id + ) + expect(repository_observation.raw_payload).to eq(well_formed_envelope.fetch("repo")) + end + + # Forcing the transaction's last insert to fail is the proof of the boundary: nothing + # about the envelope survives, not even the push_events row that had already inserted + # or the stubs upserted before it. + it "commits the observations with the event, or commits nothing at all" do + allow(EnrichmentObservation).to receive(:insert_all!) + .and_raise(ActiveRecord::StatementInvalid, "forced by the example") + + tally = write(well_formed_envelope) + + expect(tally.to_h).to include(events_created: 0, events_failed: 1) + expect(PushEvent.count).to eq(0) + expect(EnrichmentObservation.count).to eq(0) + expect(GithubActor.count).to eq(0) + expect(GithubRepository.count).to eq(0) + end + + # §8's duplicate rule extended to evidence: a replay registered no activity, so it + # also supplies no new observation rows. + it "adds no observation for a duplicate replay" do + write(well_formed_envelope) + write(well_formed_envelope, at: frozen_time + 60) + + expect(EnrichmentObservation.count).to eq(2) + end + + it "stamps the pipeline entry instants when the entity is first observed" do + write(well_formed_envelope) + + expect(GithubActor.sole).to have_attributes( + event_native_at: received_at, derived_at: received_at, batch_pending_at: received_at + ) + expect(GithubRepository.sole).to have_attributes( + event_native_at: received_at, derived_at: received_at, batch_pending_at: received_at + ) + end + + # COALESCE keeps the first-ever instant: a repeat event for a known entity adds its + # evidence but does not restart the entity's pipeline clock, so the FIFO position a + # backlog row earned on first observation is never lost to later popularity. + it "keeps the first pipeline instants when a second event references the same entity" do + write(well_formed_envelope) + write(well_formed_envelope("id" => "58000000099", + "payload" => { "push_id" => 27_500_000_099 }), + at: frozen_time + 60) + + expect(EnrichmentObservation.count).to eq(4) + expect(GithubActor.sole).to have_attributes( + event_native_at: frozen_time, derived_at: frozen_time, batch_pending_at: frozen_time + ) + end + + # Only accepted events supply evidence: a quarantined envelope wrote no entity rows to + # attach evidence to, and an ignored one wrote nothing at all. + it "writes no observation for a quarantined or ignored envelope" do + write([ well_formed_envelope("type" => "WatchEvent"), + well_formed_envelope("payload" => { "head" => nil }) ]) + + expect(EnrichmentObservation.count).to eq(0) + end + end + # GitHub's clock and ours are different clocks, and the documented 30s–6h latency means # occurred_at can sit either side of the observation. Nothing may clamp it. it "lets latest_event_at exceed last_seen_at when GitHub's timestamp is later" do diff --git a/spec/services/github/ingestion_runner_spec.rb b/spec/services/github/ingestion_runner_spec.rb index c5b9576..a14f7f0 100644 --- a/spec/services/github/ingestion_runner_spec.rb +++ b/spec/services/github/ingestion_runner_spec.rb @@ -455,9 +455,10 @@ def ingest(runner = fixture_runner, **options) # so what these examples pin is *when* the hint is emitted — after every row is durable, # outside the source lock, and only when the run created something. describe "enqueueing enrichment after commit" do - it "enqueues one cycle per class when the run created events" do - expect { ingest }.to have_enqueued_job(EnrichActorJob).exactly(:once) - .and have_enqueued_job(EnrichRepositoryJob).exactly(:once) + it "enqueues one wake-up cycle when the run created events" do + expect { ingest }.to have_enqueued_job(EnrichmentCycleJob).exactly(:once) + + expect(ActiveJob::Base.queue_adapter.enqueued_jobs.map { _1[:job] }).to eq([ EnrichmentCycleJob ]) end # A replay refreshes identity but creates no event row, so there is no new work hint to @@ -483,7 +484,7 @@ def ingest(runner = fixture_runner, **options) # a write to another database). it "enqueues only once the rows are durable, the run row is closed, and the lock is released" do observed = [] - allow(EnrichActorJob).to receive(:perform_later) do + allow(EnrichmentCycleJob).to receive(:perform_later) do transaction = ActiveRecord::Base.lease_connection.current_transaction observed << { diff --git a/spec/services/github/request_executor_spec.rb b/spec/services/github/request_executor_spec.rb index 0a7f168..d83c1ce 100644 --- a/spec/services/github/request_executor_spec.rb +++ b/spec/services/github/request_executor_spec.rb @@ -134,6 +134,93 @@ def executor(transport, **overrides) end end + # Appendix F: the executor routes on Request#search?. A search-class request reserves + # and reconciles against the per-minute search ledger, and a core-class request against + # the hourly core ledger — the same chain, two ledgers, with no cross-talk in either + # direction. + describe "the two ledgers (Appendix F)" do + let(:search_request) do + Github::Request.new(url: "https://api.github.com/search/users?q=user%3Aoctocat&per_page=10", + request_class: :actor_search) + end + + def search_rate_limit_headers(remaining: 8) + { + "x-ratelimit-resource" => "search", "x-ratelimit-limit" => "10", + "x-ratelimit-remaining" => remaining.to_s, + "x-ratelimit-reset" => (frozen_time + 60).to_i.to_s + } + end + + def search_executor(transport) + executor(transport, search_ledger: Github::SearchBudgetLedger.new(configuration: Github.configuration)) + end + + it "reserves a search request on the search ledger before the transport is called" do + active_search_window + observed = nil + transport = recording_transport do + observed = { search_used: current_search_budget.used, gate_held: Github::RequestGate.held? } + response(status: 200, headers: search_rate_limit_headers, body: "{}") + end + + result = search_executor(transport).call(search_request) + + expect(result).to be_ok + expect(observed).to eq(search_used: 1, gate_held: true) + expect(current_search_budget).to have_attributes(actor_used: 1, repository_used: 0, + last_request_at: frozen_time) + end + + # The reconciliation lands on the search row — monotonic against the local estimate, + # exactly as the core ledger treats its own headers. + it "reconciles the search response onto the search row" do + active_search_window(remaining: 9) + transport = recording_transport { response(status: 200, headers: search_rate_limit_headers(remaining: 7), body: "{}") } + + search_executor(transport).call(search_request) + + expect(current_search_budget).to have_attributes(remaining: 7, reset_at: frozen_time + 60) + end + + # The strongest form of "no cross-talk": the core singleton is created only by a core + # reservation, so a search request that touched it at all would have left a row behind. + it "leaves the core ledger row untouched by a search request" do + active_search_window + transport = recording_transport { response(status: 200, headers: search_rate_limit_headers, body: "{}") } + + search_executor(transport).call(search_request) + + expect(GithubApiBudget.count).to eq(0) + end + + # And the mirror image: a core-class request never creates or spends the search row. + it "touches only the core ledger for a core-class request" do + active_budget_window + transport = recording_transport { response(status: 200, headers: rate_limit_headers) } + + executor(transport).call(poll_request) + + expect(current_budget.poll_used).to eq(1) + expect(GithubSearchBudget.count).to eq(0) + end + + # A search-ledger denial takes the same shape as a core one: nothing spent, nothing + # sent, and a deferral the caller can read the reason off. + it "surfaces a search-ledger denial as a budget denial that spends nothing" do + active_search_window(used: 8) + transport = recording_transport { response(status: 200, headers: search_rate_limit_headers, body: "{}") } + + result = search_executor(transport).call(search_request) + + expect(transport.calls).to be_empty + expect(result.classification).to eq(:budget_denied) + expect(result).to be_deferred + expect(result.error).to be_a(Github::Errors::BudgetExhausted) + expect(result.error.reason).to eq(:search_ceiling_exhausted) + end + end + describe "retries (plan §10)" do # "Retry up to MAX_HTTP_RETRIES through the same gate and ledger — each attempt is a # reservation." Collapsing that to one debit per logical fetch would silently @@ -383,8 +470,11 @@ def always(status) expect(transport.calls.size).to eq(1) end + # An explicit forty-attempt window: three hops are three repository reservations, + # which the default two-attempt repository guarantee would deny before the redirect + # limit — and this example is about the limit, not the budget. it "stops after MAX_REDIRECTS rather than looping" do - active_budget_window + active_budget_window(enrichment_allowance: 40) transport = recording_transport do |call| response(status: 301, headers: rate_limit_headers.merge( "location" => "https://api.github.com/repos/octocat/hop-#{call}" diff --git a/spec/services/github/request_spec.rb b/spec/services/github/request_spec.rb index 9af9e36..3248b17 100644 --- a/spec/services/github/request_spec.rb +++ b/spec/services/github/request_spec.rb @@ -13,22 +13,51 @@ def request(**overrides) expect { described_class.new(url: "https://api.github.com/events") }.to raise_error(ArgumentError) end - it "accepts exactly the three classes the ledger has counters for" do + it "names exactly the five classes the two ledgers have counters for" do + expect(described_class::CLASSES) + .to eq(%i[ poll actor repository actor_search repository_search ]) + end + + it "accepts every named class" do described_class::CLASSES.each do |request_class| expect(request(request_class: request_class).request_class).to eq(request_class) end end + # :search in particular: the search *pair* is spelled :actor_search and + # :repository_search, and the bare word must stay an error rather than silently + # spending some default allowance. it "rejects an unknown class rather than silently spending the wrong allowance" do expect { request(request_class: :search) } .to raise_error(ArgumentError, /request_class/) end - it "treats actor and repository requests as enrichment, and polls as not" do - expect(request(request_class: :actor)).to be_enrichment - expect(request(request_class: :repository)).to be_enrichment + # Appendix F splits enrichment across two ledgers: the search pair debits the + # per-minute search ledger, the detail pair debits the core detail-fallback + # allowance, and every one of the four is enrichment as opposed to polling. + it "counts the search pair and the detail pair as enrichment, and polls as not" do + expect(described_class::ENRICHMENT_CLASSES) + .to match_array(%i[ actor repository actor_search repository_search ]) + described_class::ENRICHMENT_CLASSES.each do |request_class| + expect(request(request_class: request_class)).to be_enrichment + end expect(request(request_class: :poll)).not_to be_enrichment end + + it "keeps the detail and search pairs disjoint, so no class debits both ledgers" do + expect(described_class::DETAIL_CLASSES).to eq(%i[ actor repository ]) + expect(described_class::SEARCH_CLASSES).to eq(%i[ actor_search repository_search ]) + expect(described_class::DETAIL_CLASSES & described_class::SEARCH_CLASSES).to be_empty + end + + # The predicate Github::RequestExecutor routes ledgers on: a search-class request + # reserves and reconciles against the search ledger, everything else the core one. + it "answers #search? for the search pair alone" do + expect(request(request_class: :actor_search)).to be_search + expect(request(request_class: :repository_search)).to be_search + expect(request(request_class: :actor)).not_to be_search + expect(request(request_class: :poll)).not_to be_search + end end describe "the fairness borrow (plan §10)" do @@ -36,11 +65,27 @@ def request(**overrides) expect(request(request_class: :actor).borrow).to be(false) end - it "refuses a borrowing poll, because borrowing is a concept between the two enrichment classes" do + it "refuses a borrowing poll, because borrowing is a concept between the two detail classes" do expect { request(request_class: :poll, borrow: true) } .to raise_error(ArgumentError, /borrow/) end + # The search ledger has no shares to borrow between: fairness on that lane is the + # cycle's weighted rotation, not a per-class counter, so a borrowing search request + # is a programming error rather than a wider authorization. + it "refuses a borrowing search request, which has no share to spend past" do + described_class::SEARCH_CLASSES.each do |request_class| + expect { request(request_class: request_class, borrow: true) } + .to raise_error(ArgumentError, /borrow/) + end + end + + it "permits the borrow on both detail classes, which own the core fallback shares" do + described_class::DETAIL_CLASSES.each do |request_class| + expect(request(request_class: request_class, borrow: true).borrow).to be(true) + end + end + # The reason the flag rides on the request rather than on a RequestExecutor argument: # a redirect hop reserves again, and re-reserving a borrowed request under the # guarantee cap would deny mid-chain after the first hop was already spent. diff --git a/spec/services/github/search_budget_ledger_spec.rb b/spec/services/github/search_budget_ledger_spec.rb new file mode 100644 index 0000000..bbf53e8 --- /dev/null +++ b/spec/services/github/search_budget_ledger_spec.rb @@ -0,0 +1,507 @@ +require "rails_helper" + +RSpec.describe Github::SearchBudgetLedger do + subject(:ledger) { described_class.new } + + # GitHub's Search window is one minute, not core's hour — the whole reason this + # ledger exists as a separate row (Appendix F). + let(:window_reset) { frozen_time + 60 } + + # Both defined in spec/support/budget_helpers.rb, so "what an active search window + # looks like" is stated once and the executor's specs use the same definition. + def budget = current_search_budget + def active_window(**overrides) = active_search_window(**overrides) + + def snapshot(**overrides) + Github::RateLimitSnapshot.from_headers({ + "x-ratelimit-resource" => "search", "x-ratelimit-limit" => "10", + "x-ratelimit-remaining" => "9", "x-ratelimit-reset" => window_reset.to_i.to_s + }.merge(overrides.transform_keys(&:to_s)), observed_at: frozen_time) + end + + describe "#bootstrap!" do + # Unlike the core ledger, the search row self-initializes from configuration: the + # ceiling/reserve pair is a configured budget, not something a response teaches us. + it "creates the singleton row from the configured ceiling and reserve" do + expect { ledger.bootstrap!(now: frozen_time) }.to change(GithubSearchBudget, :count).from(0).to(1) + + expect(budget).to have_attributes( + request_ceiling: 10, reserve: 2, used: 0, actor_used: 0, repository_used: 0, + limit: nil, remaining: nil, reset_at: nil, blocked_until: nil, last_request_at: nil + ) + end + + # Two processes starting cold race here; insert_all's unique_by makes the loser a + # no-op rather than a RecordNotUnique that would poison a transaction. + it "tolerates a row another process created first" do + ledger.bootstrap!(now: frozen_time) + + expect { ledger.bootstrap!(now: frozen_time) }.not_to change(GithubSearchBudget, :count) + end + + it "never overwrites a live row's counters, because bootstrap is not a reset" do + active_window(used: 5, actor_used: 5) + + ledger.bootstrap!(now: frozen_time) + + expect(budget).to have_attributes(used: 5, actor_used: 5) + end + end + + describe "#reserve!" do + it "debits the shared counter and the actor lane, and stamps the pacing instant" do + active_window + + ledger.reserve!(:actor_search, now: frozen_time) + + expect(budget).to have_attributes(used: 1, actor_used: 1, repository_used: 0, + last_request_at: frozen_time) + end + + it "debits the repository lane for its own class" do + active_window + + ledger.reserve!(:repository_search, now: frozen_time) + + expect(budget).to have_attributes(used: 1, actor_used: 0, repository_used: 1) + end + + it "decrements the local remaining estimate, so a failure is spent against it too" do + active_window(remaining: 9) + + expect { ledger.reserve!(:actor_search, now: frozen_time) } + .to change { budget.remaining }.from(9).to(8) + end + + it "leaves an unobserved remaining null rather than collapsing it to zero" do + active_window(remaining: nil) + + ledger.reserve!(:actor_search, now: frozen_time) + + expect(budget.remaining).to be_nil + end + + it "creates the ledger row on first use, so a cold start needs no seeding step" do + expect { ledger.reserve!(:actor_search, now: frozen_time) } + .to change(GithubSearchBudget, :count).from(0).to(1) + end + + # Borrowing is a core-ledger fairness concept: the search lanes share one ceiling + # and the LaneSchedule rotates them, so "spend past your guarantee" has no meaning + # here. Accepting the flag silently would let a caller believe it was honoured. + it "refuses a borrow as a programming error, not a denial" do + active_window + + expect { ledger.reserve!(:actor_search, now: frozen_time, borrow: true) } + .to raise_error(ArgumentError, /borrow/) + end + + it "refuses a non-search request class rather than debiting nothing silently" do + active_window + + %i[ poll actor repository ].each do |request_class| + expect { ledger.reserve!(request_class, now: frozen_time) } + .to raise_error(ArgumentError, /#{request_class}/) + end + end + end + + # The order is diagnostic, not arbitrary: each reason names the condition an operator + # should look at first. A blocked ledger must say "blocked" even when it is also + # paced, past its reserve, and out of ceiling. + describe "the denial order" do + def reason_for_reservation + ledger.reserve!(:actor_search, now: frozen_time) + nil + rescue Github::Errors::BudgetExhausted => error + error.reason + end + + it "names the block first, whatever else is also true" do + active_window(blocked_until: frozen_time + 30, last_request_at: frozen_time, + remaining: 2, used: 8) + + expect(reason_for_reservation).to eq(:search_blocked) + end + + it "names pacing once the block has passed" do + active_window(blocked_until: frozen_time - 1, last_request_at: frozen_time - 1, + remaining: 2, used: 8) + + expect(reason_for_reservation).to eq(:search_pacing) + end + + it "names the reserve once pacing has elapsed" do + active_window(last_request_at: frozen_time - 60, remaining: 2, used: 8) + + expect(reason_for_reservation).to eq(:search_reserve_reached) + end + + it "names the ceiling when only the local counter is exhausted" do + active_window(remaining: nil, used: 8) + + expect(reason_for_reservation).to eq(:search_ceiling_exhausted) + end + + it "raises Errors::BudgetExhausted carrying the class and the reason" do + active_window(blocked_until: frozen_time + 30) + + expect { ledger.reserve!(:repository_search, now: frozen_time) } + .to raise_error(Github::Errors::BudgetExhausted) do |error| + expect(error.request_class).to eq(:repository_search) + expect(error.reason).to eq(:search_blocked) + end + end + + it "spends nothing when it denies, so a refused reservation costs no quota" do + active_window(remaining: nil, used: 8) + + expect { suppress(Github::Errors::BudgetExhausted) { ledger.reserve!(:actor_search, now: frozen_time) } } + .not_to change { budget.used }.from(8) + end + + # The spendable boundary: ceiling 10 minus reserve 2 leaves 8, so the eighth + # request is granted and the ninth is not. + it "grants right up to ceiling minus reserve and refuses the next" do + active_window(remaining: nil, used: 7) + + expect { ledger.reserve!(:actor_search, now: frozen_time) }.not_to raise_error + expect { ledger.reserve!(:actor_search, now: frozen_time + 30) } + .to raise_error(Github::Errors::BudgetExhausted, /search_ceiling_exhausted/) + end + + it "grants while the observed remaining still clears the reserve" do + active_window(remaining: 3) + + expect { ledger.reserve!(:actor_search, now: frozen_time) }.not_to raise_error + end + + it "names only the documented denial reasons" do + expect(described_class::DENIAL_REASONS) + .to contain_exactly(:search_blocked, :search_pacing, :search_reserve_reached, + :search_ceiling_exhausted) + end + end + + describe "pacing" do + it "defers a reservation inside the pacing interval" do + active_window(last_request_at: frozen_time - 3) + + expect { ledger.reserve!(:actor_search, now: frozen_time) } + .to raise_error(Github::Errors::BudgetExhausted, /search_pacing/) + end + + it "grants again exactly one pacing interval after the last request" do + active_window(last_request_at: frozen_time - 6) + + expect { ledger.reserve!(:actor_search, now: frozen_time) }.not_to raise_error + end + + # SEARCH_PACING_SECONDS=0 is a documented operating point (the offline fixture + # walkthrough runs both lanes back to back), so zero must mean "no pacing" and not + # "deny everything". + it "disables pacing entirely at SEARCH_PACING_SECONDS=0" do + unpaced = described_class.new(configuration: configuration_with(SEARCH_PACING_SECONDS: "0")) + active_window + + unpaced.reserve!(:actor_search, now: frozen_time) + + expect { unpaced.reserve!(:actor_search, now: frozen_time) }.not_to raise_error + expect(budget.used).to eq(2) + end + end + + describe "window rollover" do + it "resets the counters and header state when GitHub's reset instant has passed" do + active_window(used: 8, actor_used: 5, repository_used: 3, remaining: 2, + blocked_until: window_reset - 10) + + ledger.reserve!(:actor_search, now: window_reset + 1) + + expect(budget).to have_attributes(used: 1, actor_used: 1, repository_used: 0, + remaining: nil, reset_at: nil, blocked_until: nil) + end + + # A dead window's remaining of 2 against a reserve of 2 would otherwise deny every + # search forever — the search analogue of the core ledger's stale-remaining deadlock. + it "clears a stale remaining that would otherwise pin the ledger at its reserve" do + active_window(remaining: 2) + + expect { ledger.reserve!(:actor_search, now: window_reset + 1) }.not_to raise_error + expect(budget.remaining).to be_nil + end + + # No response ever supplied reset_at — a run of header-less transport failures — + # so the fallback horizon is one full search window of silence. Without it, `used` + # would sit at the ceiling forever with nothing able to roll it. + it "rolls a header-less window once a full search window has passed in silence" do + ledger.bootstrap!(now: frozen_time) + GithubSearchBudget.where(id: described_class::SINGLETON_ID) + .update_all(reset_at: nil, last_request_at: frozen_time - 61, + used: 8, actor_used: 8) + + ledger.reserve!(:actor_search, now: frozen_time) + + expect(budget).to have_attributes(used: 1, actor_used: 1, repository_used: 0) + end + + it "does not roll a header-less window while the last attempt is still recent" do + ledger.bootstrap!(now: frozen_time) + GithubSearchBudget.where(id: described_class::SINGLETON_ID) + .update_all(reset_at: nil, last_request_at: frozen_time - 30, used: 8) + + expect { ledger.reserve!(:actor_search, now: frozen_time) } + .to raise_error(Github::Errors::BudgetExhausted, /search_ceiling_exhausted/) + expect(budget.used).to eq(8) + end + + # last_request_at deliberately survives the roll: the pacing contract is "no two + # searches closer than the interval", and a window boundary between them does not + # make two requests further apart in time. + it "keeps pacing continuous across the roll, because the wire does not reset" do + active_window(used: 8, last_request_at: window_reset - 2) + + expect { ledger.reserve!(:actor_search, now: window_reset + 1) } + .to raise_error(Github::Errors::BudgetExhausted, /search_pacing/) + expect(budget.last_request_at).to eq(window_reset - 2) + end + + # The rollover genuinely happened, so it must survive even when the reservation + # that discovered it is then refused. + it "commits the reset even when the reservation is then denied" do + active_window(used: 8, last_request_at: window_reset - 2) + + suppress(Github::Errors::BudgetExhausted) { ledger.reserve!(:actor_search, now: window_reset + 1) } + + expect(budget.used).to eq(0) + end + end + + describe "#reconcile!" do + it "does nothing when a transport failure produced no snapshot at all" do + active_window + + expect(ledger.reconcile!(nil, now: frozen_time)).to eq(:no_headers) + end + + # Folding core's hourly numbers into a per-minute row would import a 3600-second + # reset as this window's boundary — the mirror of the core ledger refusing a + # "search" snapshot. + it "refuses a snapshot from the core resource and leaves the row untouched" do + active_window(remaining: 9) + + result = ledger.reconcile!( + snapshot("x-ratelimit-resource" => "core", "x-ratelimit-remaining" => "1"), now: frozen_time + ) + + expect(result).to eq(:resource_mismatch) + expect(budget).to have_attributes(remaining: 9, reset_at: window_reset) + end + + # A response with no reservation behind it would mean a request was made without + # reserving, which is the invariant the whole class exists to hold. + it "never creates the ledger row, because only a reservation may" do + expect(ledger.reconcile!(snapshot, now: frozen_time)).to eq(:no_ledger) + expect(GithubSearchBudget.count).to eq(0) + end + + it "does nothing with a snapshot that lacks the quantitative trio" do + active_window(remaining: 9) + + result = ledger.reconcile!( + Github::RateLimitSnapshot.from_headers({ "x-ratelimit-resource" => "search" }, + observed_at: frozen_time), + now: frozen_time + ) + + expect(result).to eq(:partial_headers) + expect(budget.remaining).to eq(9) + end + + # Regression: an earlier draft's transaction block ended on an assignment, so a + # successful reconcile reported the assigned value rather than :updated and the + # executor logged every good response as unreconciled. + it "reports :updated for a good snapshot it applied" do + active_window + + expect(ledger.reconcile!(snapshot, now: frozen_time)).to eq(:updated) + end + + it "adopts the observed limit, reset and observation instant" do + active_window(limit: nil, observed_at: nil) + + ledger.reconcile!(snapshot, now: frozen_time) + + expect(budget).to have_attributes(limit: 10, reset_at: window_reset, observed_at: frozen_time) + end + + # Within one window remaining only ever moves down — headers may arrive out of + # order, and the lower number is the one GitHub will actually enforce. + it "takes the lower of the local estimate and the observed value" do + active_window(remaining: 3) + + ledger.reconcile!(snapshot("x-ratelimit-remaining" => "9"), now: frozen_time) + + expect(budget.remaining).to eq(3) + end + + it "adopts an observed value lower than the local estimate" do + active_window(remaining: 9) + + ledger.reconcile!(snapshot("x-ratelimit-remaining" => "4"), now: frozen_time) + + expect(budget.remaining).to eq(4) + end + + it "adopts the observed remaining outright when no local estimate exists" do + active_window(remaining: nil) + + ledger.reconcile!(snapshot("x-ratelimit-remaining" => "7"), now: frozen_time) + + expect(budget.remaining).to eq(7) + end + + # The read-side block: once the observed remaining is inside the reserve, the next + # reservation would be denied anyway, so the row says until when. + it "blocks until the reset once the observed remaining is inside the reserve" do + active_window + + ledger.reconcile!(snapshot("x-ratelimit-remaining" => "2"), now: frozen_time) + + expect(budget.blocked_until).to eq(window_reset) + end + + it "sets no block while the observed remaining still clears the reserve" do + active_window + + ledger.reconcile!(snapshot("x-ratelimit-remaining" => "3"), now: frozen_time) + + expect(budget.blocked_until).to be_nil + end + + # A superseding reset_at is GitHub saying it counted the in-flight request in the + # new minute. Zeroing the counters outright would let this window issue one more + # request than GitHub honours, so the debit rides across into its own lane. + describe "a window that moves on while a request is in flight" do + def superseding(remaining: "9") + snapshot("x-ratelimit-reset" => (window_reset + 60).to_i.to_s, + "x-ratelimit-remaining" => remaining) + end + + it "carries the in-flight debit into the new window as one used" do + active_window(used: 5, actor_used: 3, repository_used: 2, remaining: nil) + + ledger.reconcile!(superseding, request_class: :actor_search, now: window_reset + 1) + + expect(budget).to have_attributes(used: 1, actor_used: 1, repository_used: 0, + reset_at: window_reset + 60, remaining: 9) + end + + it "carries a repository request into its own lane" do + active_window(used: 5, actor_used: 3, repository_used: 2, remaining: nil) + + ledger.reconcile!(superseding, request_class: :repository_search, now: window_reset + 1) + + expect(budget).to have_attributes(used: 1, actor_used: 0, repository_used: 1) + end + end + end + + describe "#block_from!" do + def limited(status: 403, **headers) + request = Github::Request.new(url: "https://api.github.com/search/users?q=user%3Aoctocat&per_page=1", + request_class: :actor_search) + Github::FetchResult.from_response(request: request, status: status, + headers: headers.transform_keys(&:to_s), + body: "", duration_ms: 1.0) + end + + # 403 with a non-zero remaining classifies as :secondary_limited; with "0" it is + # :rate_limited — the same discriminator Github::ResponseClassifier applies to core. + it "prefers Retry-After, the server's explicit instruction" do + active_window + + ledger.block_from!(limited("retry-after" => "120", + "x-ratelimit-reset" => window_reset.to_i.to_s), + now: frozen_time) + + expect(budget.blocked_until).to eq(frozen_time + 120) + end + + it "falls back to the reset header when no Retry-After was sent" do + active_window + + ledger.block_from!(limited("x-ratelimit-reset" => window_reset.to_i.to_s), now: frozen_time) + + expect(budget.blocked_until).to eq(window_reset) + end + + it "falls back to one search window when the response named no instant at all" do + active_window + + ledger.block_from!(limited, now: frozen_time) + + expect(budget.blocked_until).to eq(frozen_time + described_class::SEARCH_WINDOW_SECONDS) + end + + it "blocks on a primary exhaustion as well as a secondary limit" do + active_window + + ledger.block_from!(limited("x-ratelimit-remaining" => "0", "retry-after" => "90"), + now: frozen_time) + + expect(budget.blocked_until).to eq(frozen_time + 90) + end + + # GREATEST ignores NULL, so a block only ever moves later — a short block landing + # after a long one must not resume searching into an exhausted quota. + it "only ever moves a block later" do + active_window + + ledger.block_from!(limited("retry-after" => "300"), now: frozen_time) + ledger.block_from!(limited("retry-after" => "60"), now: frozen_time) + expect(budget.blocked_until).to eq(frozen_time + 300) + + ledger.block_from!(limited("retry-after" => "600"), now: frozen_time) + expect(budget.blocked_until).to eq(frozen_time + 600) + end + + it "ignores every classification that is not a rate limit" do + active_window + + ledger.block_from!(limited(status: 500), now: frozen_time) + ledger.block_from!(limited(status: 200), now: frozen_time) + + expect(budget.blocked_until).to be_nil + end + end + + # The row-lock property the single-threaded examples cannot prove: two genuinely + # separate PostgreSQL sessions reserving at once must serialise on the row, with no + # lost debit and no deadlock. Transactional tests are off for the reason + # spec/support/concurrency_helpers.rb documents, paid for with explicit cleanup. + describe "under concurrency" do + self.use_transactional_tests = false + + around do |example| + example.run + ensure + ActiveRecord::Base.connection.execute("DELETE FROM github_search_budget") + restore_connection_pool! + end + + it "serialises two concurrent reservations into exactly two debits" do + unpaced = described_class.new(configuration: configuration_with(SEARCH_PACING_SECONDS: "0")) + now = Time.current + + outcomes = reservation_outcomes( + in_parallel(2, threads: 2) { unpaced.reserve!(:actor_search, now: now) } + ) + + expect(outcomes[:unexpected]).to be_empty + expect(outcomes).to include(granted: 2, denied: 0) + expect(GithubSearchBudget.uncached { GithubSearchBudget.find(described_class::SINGLETON_ID) }) + .to have_attributes(used: 2, actor_used: 2, repository_used: 0) + end + end +end diff --git a/spec/services/github/status/ledger_state_spec.rb b/spec/services/github/status/ledger_state_spec.rb new file mode 100644 index 0000000..37659df --- /dev/null +++ b/spec/services/github/status/ledger_state_spec.rb @@ -0,0 +1,91 @@ +require "rails_helper" + +RSpec.describe Github::Status::LedgerState do + let(:now) { frozen_time } + + describe "an absent row" do + # Nothing seeds github_api_budget; only a reservation creates it. "No ledger row at + # all" and "a window whose remaining is genuinely 0" are different facts an operator + # acts on differently, and present is the boolean that separates them. + it "reports present false and every other field null" do + state = described_class.from(nil) + + expect(state.present).to be(false) + expect(state.to_h.except(:present).values).to all(be_nil) + end + end + + describe "a present row" do + it "projects every per-class counter §11 names" do + active_budget_window(now: now, poll_used: 7, enrichment_used: 3, + actor_share_used: 2, repository_share_used: 1) + + payload = described_class.from(current_budget).payload + + expect(payload).to include( + present: true, resource: "core", window_status: "active", + limit: 60, remaining: 55, reserve: 8, + reset_at: (now + 3600).utc.iso8601, + poll: { used: 7, allowance: 12 }, + detail_fallback: { used: 3, allowance: 4 }, + actor_requests: { used: 2, guarantee: 2, available: 0 }, + repository_requests: { used: 1, guarantee: 2, available: 1 } + ) + end + + # Appendix G renames the block for what the number now is: the explicit core + # detail-fallback allowance, not "everything after poll + reserve". The Search + # budget is a different rate-limit resource and lives in the search_ledger block. + it "publishes the allowance as detail_fallback, never as enrichment" do + active_budget_window(now: now) + + expect(described_class.from(current_budget).payload.keys) + .not_to include(:enrichment) + expect(described_class.from(current_budget).payload[:detail_fallback]) + .to eq(used: 0, allowance: 4) + end + + # available is a floor, not a ceiling: §10 lets one class borrow the other's unspent + # capacity, which is what takes share_used past the guarantee — a negative number + # here would read as an accounting error rather than as the borrow it actually is. + it "floors available at zero, because borrowing takes a class past its guarantee" do + active_budget_window(now: now, enrichment_used: 4, actor_share_used: 3, + repository_share_used: 1) + + expect(described_class.from(current_budget).payload.dig(:actor_requests, :available)) + .to eq(0) + end + + # Derived from the *stored* enrichment_allowance, never a fresh Allowances.derive: + # the allowances are fixed at window initialization, and a guarantee recomputed + # mid-window from a different total could report headroom the ledger would refuse. + it "splits the guarantees from the stored allowance the ledger enforces" do + active_budget_window(now: now, enrichment_allowance: 5) + + payload = described_class.from(current_budget).payload + + expect(payload.dig(:actor_requests, :guarantee)).to eq(2) + expect(payload.dig(:repository_requests, :guarantee)).to eq(3) + end + + it "keeps the two shares summing to the class counter they were split from" do + active_budget_window(now: now, enrichment_used: 4, + actor_share_used: 3, repository_share_used: 1) + payload = described_class.from(current_budget).payload + + expect(payload.dig(:actor_requests, :used) + payload.dig(:repository_requests, :used)) + .to eq(payload.dig(:detail_fallback, :used)) + end + end + + describe "consistency" do + # Snapshot reads the singleton once and hands the row down — this projection must + # never issue a query of its own, or two blocks could describe different instants. + it "issues no query of its own" do + active_budget_window(now: now) + budget = current_budget + + expect(capture_sql { described_class.from(budget).payload }).to be_empty + end + end +end diff --git a/spec/services/github/status/scheduler_settings_spec.rb b/spec/services/github/status/scheduler_settings_spec.rb new file mode 100644 index 0000000..2707c75 --- /dev/null +++ b/spec/services/github/status/scheduler_settings_spec.rb @@ -0,0 +1,70 @@ +require "rails_helper" + +RSpec.describe Github::Status::SchedulerSettings do + describe "#payload" do + # The block exists so an operator can read the numbers the ledgers and workers are + # actually enforcing without a shell. Full-hash equality, so a knob added to the + # configuration cannot be forgotten here silently and a dropped key cannot hide. + it "publishes every staged-enrichment knob from the configuration it was given" do + configuration = configuration_with( + SEARCH_REQUEST_CEILING: "9", SEARCH_SAFETY_RESERVE: "3", SEARCH_BATCH_SIZE: "5", + SEARCH_PACING_SECONDS: "4", ACTOR_ENRICHMENT_WEIGHT: "2", + REPOSITORY_ENRICHMENT_WEIGHT: "3", ACTOR_ENRICHMENT_SHARE: "0.25", + CORE_DETAIL_FALLBACK_ALLOWANCE: "6", RATE_LIMIT_RESERVE: "5", + ENRICHMENT_RETRY_BASE_SECONDS: "30", ENRICHMENT_RETRY_MAX_SECONDS: "1800", + DETAIL_FALLBACK_MAX_ATTEMPTS: "2", ENRICHMENT_LEASE_SECONDS: "700", + ENRICHMENT_CYCLE_BUDGET_SECONDS: "50", ACTOR_REFRESH_TTL_SECONDS: "43200", + REPOSITORY_REFRESH_TTL_SECONDS: "86400", + REFRESH_ACTIVE_WITHIN_SECONDS: "302400", + ENRICHMENT_METRICS_WINDOW_SECONDS: "1800", CATCH_UP_MIN_SAMPLE_SECONDS: "600" + ) + + expect(described_class.from(configuration).payload).to eq( + search: { + request_ceiling: 9, safety_reserve: 3, batch_size: 5, + pacing_seconds: 4, worker_concurrency: 1 + }, + fairness: { actor_weight: 2, repository_weight: 3, actor_enrichment_share: 0.25 }, + core: { detail_fallback_allowance: 6, rate_limit_reserve: 5 }, + retry: { + base_seconds: 30, max_seconds: 1800, detail_fallback_max_attempts: 2, + lease_seconds: 700, cycle_budget_seconds: 50 + }, + refresh: { + actor_ttl_seconds: 43_200, repository_ttl_seconds: 86_400, + active_within_seconds: 302_400 + }, + metrics: { window_seconds: 1800, catch_up_min_sample_seconds: 600 } + ) + end + + it "publishes the pinned defaults for an untouched environment" do + expect(described_class.from(configuration_with).payload).to include( + search: { + request_ceiling: 10, safety_reserve: 2, batch_size: 10, + pacing_seconds: 6, worker_concurrency: 1 + }, + core: { detail_fallback_allowance: 4, rate_limit_reserve: 8 }, + metrics: { window_seconds: 3600, catch_up_min_sample_seconds: 900 } + ) + end + + # The configuration stores the share as a Rational so the guarantee floor is exact; + # a JSON payload wants the plain decimal an operator typed, as a Float. + it "renders the actor enrichment share as a Float" do + share = described_class.from(configuration_with).payload + .dig(:fairness, :actor_enrichment_share) + + expect(share).to be_a(Float) + expect(share).to eq(0.5) + end + + # A pure projection of validated configuration: zero reads, zero writes, so the + # scheduler block can never disagree with the environment the process booted with. + it "touches the database not at all" do + settings = described_class.from(configuration_with) + + expect(capture_sql { settings.payload }).to be_empty + end + end +end diff --git a/spec/services/github/status/search_ledger_state_spec.rb b/spec/services/github/status/search_ledger_state_spec.rb new file mode 100644 index 0000000..b1f34af --- /dev/null +++ b/spec/services/github/status/search_ledger_state_spec.rb @@ -0,0 +1,118 @@ +require "rails_helper" + +RSpec.describe Github::Status::SearchLedgerState do + let(:now) { frozen_time } + + describe "an absent row" do + # Unlike core, the search row is configuration-born: it appears with the first + # search reservation, so a clean checkout legitimately has none. present is the + # one boolean that separates "no row" from "a row with unknown counters". + it "reports present false and every other field null" do + state = described_class.from(nil, now: now) + + expect(state.present).to be(false) + expect(state.to_h.except(:present).values).to all(be_nil) + end + + it "keeps the payload's fixed key set with nulls, never missing keys" do + payload = described_class.from(nil, now: now).payload + + expect(payload.keys).to eq(%i[present resource limit remaining reset_at + observed_at request_ceiling reserve spendable used + actor_used repository_used available blocked_until + last_request_at next_request_earliest_at]) + expect(payload).to include(present: false, spendable: nil, + next_request_earliest_at: nil) + end + end + + describe "a present row" do + it "projects every column plus the derived spendable ceiling" do + active_search_window(now: now, used: 3, actor_used: 2, repository_used: 1) + + state = described_class.from(current_search_budget, now: now) + + expect(state).to have_attributes( + present: true, resource: "search", limit: 10, remaining: 9, + reset_at: now + 60, observed_at: now, request_ceiling: 10, reserve: 2, + spendable: 8, used: 3, actor_used: 2, repository_used: 1, + blocked_until: nil, last_request_at: nil, next_request_earliest_at: nil + ) + expect(state.available).to eq(current_search_budget.available) + end + + # spendable is the fixed ceiling-minus-reserve pair, not the headroom left — the + # headroom is `available` beside it, and publishing both makes the spend checkable. + it "keeps spendable constant however much has been spent" do + active_search_window(now: now, used: 9) + + expect(described_class.from(current_search_budget, now: now).spendable).to eq(8) + end + + it "renders every timestamp the way the printed report does" do + active_search_window(now: now, last_request_at: now - 2, blocked_until: now + 30) + + payload = described_class.from(current_search_budget, now: now).payload + + expect(payload).to include( + reset_at: (now + 60).utc.iso8601, + observed_at: now.utc.iso8601, + last_request_at: (now - 2).utc.iso8601, + blocked_until: (now + 30).utc.iso8601 + ) + end + end + + describe "#next_request_earliest_at" do + it "names the pacing resume after a recent request" do + active_search_window(now: now, last_request_at: now - 2) + + expect(described_class.from(current_search_budget, now: now) + .next_request_earliest_at).to eq(now + 4) + end + + it "lets a block outlast pacing when it ends later" do + active_search_window(now: now, last_request_at: now - 2, blocked_until: now + 30) + + expect(described_class.from(current_search_budget, now: now) + .next_request_earliest_at).to eq(now + 30) + end + + it "lets pacing outlast a block about to expire" do + active_search_window(now: now, last_request_at: now - 2, blocked_until: now + 1) + + expect(described_class.from(current_search_budget, now: now) + .next_request_earliest_at).to eq(now + 4) + end + + # null means "a request is permitted right now" — an instant in the past would + # force every consumer to compare against its own clock to learn the same fact. + it "is null once pacing and any block have both cleared" do + active_search_window(now: now, last_request_at: now - 10, blocked_until: now - 5) + + expect(described_class.from(current_search_budget, now: now) + .next_request_earliest_at).to be_nil + end + + it "paces from the configuration it was given" do + active_search_window(now: now, last_request_at: now - 2) + configuration = configuration_with(SEARCH_PACING_SECONDS: "10") + + state = described_class.from(current_search_budget, configuration: configuration, + now: now) + + expect(state.next_request_earliest_at).to eq(now + 8) + end + end + + describe "consistency" do + # Snapshot reads the singleton once and hands the row down — this projection must + # never issue a query of its own, or two blocks could describe different instants. + it "issues no query of its own" do + active_search_window(now: now) + budget = current_search_budget + + expect(capture_sql { described_class.from(budget, now: now).payload }).to be_empty + end + end +end diff --git a/spec/services/github/status/snapshot_spec.rb b/spec/services/github/status/snapshot_spec.rb index e96ca14..4f9a859 100644 --- a/spec/services/github/status/snapshot_spec.rb +++ b/spec/services/github/status/snapshot_spec.rb @@ -7,9 +7,10 @@ def payload described_class.capture(now: now).payload end - describe "the response shape (plan §11)" do - it "names every block §11 asks for, in §11's order" do - expect(payload.keys).to eq(%i[captured_at sources ledger enrichment coverage]) + describe "the response shape (plan §11, Appendix G)" do + it "names every block in the amended order" do + expect(payload.keys).to eq(%i[captured_at sources ledger search_ledger scheduler + enrichment batches throughput coverage]) end it "answers on a clean checkout without inventing anything" do @@ -17,39 +18,76 @@ def payload expect(body[:sources]).to eq([]) expect(body[:ledger][:present]).to be(false) + expect(body[:search_ledger][:present]).to be(false) expect(body[:coverage][:actor_coverage_pct]).to be_nil expect(body[:coverage][:event_count]).to eq(0) + expect(body.dig(:throughput, :catch_up, :state)).to eq("insufficient_sample") + end + + # The scheduler block is pure configuration, so it is the one block that is fully + # populated even on a clean checkout — an operator can always read the knobs. + it "publishes the full scheduler block on a clean checkout" do + scheduler = payload[:scheduler] + + expect(scheduler.keys).to eq(%i[search fairness core retry refresh metrics]) + expect(scheduler[:search]).to eq(request_ceiling: 10, safety_reserve: 2, + batch_size: 10, pacing_seconds: 6, + worker_concurrency: 1) + expect(scheduler[:core]).to eq(detail_fallback_allowance: 4, rate_limit_reserve: 8) end end - describe "the enrichment counts (plan §11)" do - it "reports raw statuses and the durable backlog under distinct names" do + describe "the enrichment counts (plan §11, Appendix G)" do + it "reports raw statuses, both backlog notions, and the oldest wait" do create_actor(github_id: 1, created_at: now - 600) create_actor(github_id: 2, login: "two", enrichment_status: "retryable_failure", - next_retry_at: now + 3600, created_at: now - 300) + enrichment_stage: "retry_scheduled", next_retry_at: now + 3600, + created_at: now - 300) actors = payload.dig(:enrichment, :actors) expect(actors).to include( - pending: 1, retryable_failure: 1, backlog_count: 2, + pending: 1, retryable_failure: 1, + backlog_count: 2, contract_backlog_count: 2, oldest_pending_at: (now - 600).utc.iso8601, oldest_pending_age_seconds: 600 ) expect(GithubActor.enrichment_candidates.count).to eq(2) - expect(Github::Ingestion::StateSummary.capture(now: now).pending_actor_count).to eq(2) end it "names every status including the ones with no rows" do expected = Enrichable::ENRICHMENT_STATUSES.map(&:to_sym) + - %i[backlog_count oldest_pending_at oldest_pending_age_seconds] + %i[backlog_count contract_backlog_count + oldest_pending_at oldest_pending_age_seconds stages] expect(payload.dig(:enrichment, :actors).keys).to eq(expected) expect(payload.dig(:enrichment, :repositories).keys).to eq(expected) end + # The staged-pipeline view: all seven stages, each with its count and its oldest + # FIFO instant, zeros and nulls kept so a consumer never handles two shapes. + it "publishes all seven stages with count, oldest instant and age" do + create_actor(github_id: 1, created_at: now - 900) + create_actor(github_id: 2, login: "two", enrichment_stage: "detail_pending", + detail_pending_at: now - 60, created_at: now - 300) + + stages = payload.dig(:enrichment, :actors, :stages) + + expect(stages.keys).to eq(Enrichable::ENRICHMENT_STAGES.map(&:to_sym)) + expect(stages.each_value.map(&:keys)) + .to all(eq(%i[count oldest_created_at oldest_age_seconds])) + expect(stages[:batch_pending]).to eq(count: 1, + oldest_created_at: (now - 900).utc.iso8601, + oldest_age_seconds: 900) + expect(stages[:detail_pending]).to include(count: 1) + expect(stages[:terminal]).to eq(count: 0, oldest_created_at: nil, + oldest_age_seconds: nil) + end + it "counts each class separately, so one cannot mask the other" do create_actor(github_id: 1) - create_repository(github_id: 2, enrichment_status: "permanent_failure") + create_repository(github_id: 2, enrichment_status: "permanent_failure", + enrichment_stage: "terminal") expect(payload.dig(:enrichment, :actors)).to include(pending: 1, permanent_failure: 0, backlog_count: 1) @@ -58,8 +96,8 @@ def payload end end - describe "the ledger block (plan §11)" do - it "distinguishes an absent row from an exhausted budget" do + describe "the ledger blocks (plan §11, Appendix G)" do + it "distinguishes an absent core row from an exhausted budget" do expect(payload[:ledger]).to include(present: false, remaining: nil, reserve: nil) active_budget_window(now: now, remaining: 0) @@ -67,36 +105,71 @@ def payload expect(payload[:ledger]).to include(present: true, remaining: 0, reserve: 8) end - it "reports every per-class counter §11 names" do - active_budget_window(now: now, poll_used: 7, enrichment_used: 23, - actor_share_used: 9, repository_share_used: 14) + # Appendix G's rename: the core block publishes the explicit detail-fallback + # allowance; the Search spend is a different resource in its own block beside it. + it "reports the core detail-fallback budget under its renamed key" do + active_budget_window(now: now, poll_used: 7, enrichment_used: 3, + actor_share_used: 2, repository_share_used: 1) expect(payload[:ledger]).to include( window_status: "active", resource: "core", limit: 60, remaining: 55, poll: { used: 7, allowance: 12 }, - enrichment: { used: 23, allowance: 40 }, - actor_requests: { used: 9, guarantee: 20, available: 11 }, - repository_requests: { used: 14, guarantee: 20, available: 6 } + detail_fallback: { used: 3, allowance: 4 }, + actor_requests: { used: 2, guarantee: 2, available: 0 }, + repository_requests: { used: 1, guarantee: 2, available: 1 } ) + expect(payload[:ledger].keys).not_to include(:enrichment) end - # available is the headroom inside a guarantee, and §10 lets a class borrow past it - # when the other has no eligible candidate. A negative number here would read as an - # accounting error rather than as the borrow it actually is. - it "floors available at zero, because borrowing takes a class past its guarantee" do - active_budget_window(now: now, actor_share_used: 26) + it "projects the search ledger row beside the core one" do + active_search_window(now: now, used: 3, actor_used: 2, repository_used: 1, + last_request_at: now - 2) - expect(payload.dig(:ledger, :actor_requests)) - .to eq(used: 26, guarantee: 20, available: 0) + expect(payload[:search_ledger]).to include( + present: true, resource: "search", limit: 10, remaining: 9, + request_ceiling: 10, reserve: 2, spendable: 8, used: 3, + actor_used: 2, repository_used: 1, + next_request_earliest_at: (now + 4).utc.iso8601 + ) end + end - it "keeps the two shares summing to the class counter it was split from" do - active_budget_window(now: now, enrichment_used: 23, - actor_share_used: 9, repository_share_used: 14) - ledger = payload[:ledger] + describe "the batches and throughput blocks (issue #45)" do + it "publishes all four batch groups with counted zeros before any batch ran" do + batches = payload[:batches] - expect(ledger.dig(:actor_requests, :used) + ledger.dig(:repository_requests, :used)) - .to eq(ledger.dig(:enrichment, :used)) + expect(batches.keys).to eq(%i[window_seconds window_start search detail]) + expect(batches.dig(:search, :actors)).to include(attempts: 0, succeeded: 0, + fill_ratio: nil) + expect(batches.dig(:detail, :repositories)).to include(attempts: 0) + end + + it "aggregates enrichment_batches inside the metrics window" do + EnrichmentBatch.create!(request_kind: "search", entity_kind: "actor", + status: "succeeded", correlation_id: SecureRandom.uuid, + started_at: now - 60, requested_count: 10, + returned_count: 9, valid_count: 8, missing_count: 1, + invalid_count: 1, incomplete_results: false) + + expect(payload.dig(:batches, :search, :actors)).to include( + attempts: 1, succeeded: 1, requested_items: 10, returned_items: 9, + valid_items: 8, missing_items: 1, invalid_items: 1, fill_ratio: 0.9, + incomplete_results_count: 0 + ) + end + + # Throughput is built from the same aggregate the enrichment block reads, so the + # two can never disagree about the counts they publish side by side. + it "publishes the catch-up verdict from the shared backlog aggregate" do + create_actor(github_id: 1, enrichment_status: "complete", + enrichment_stage: "contract_complete", created_at: now - 7200, + contract_completed_at: now - 60) + + throughput = payload[:throughput] + + expect(throughput[:combined]).to include(arrivals: 0, completions: 1, + backlog_delta: -1) + expect(throughput.dig(:catch_up, :state)).to eq("keeping_up") end end @@ -192,26 +265,25 @@ def payload end describe "consistency of the snapshot" do - # The reason this is one aggregate rather than StateSummary and Summary side by side. - # Three independent reads could straddle a committing reservation and produce a body - # whose poll block contradicts its ledger block. - it "performs one ledger-row read for a snapshot" do - active_budget_window(now: now) - allow(GithubApiBudget).to receive(:find_by).and_call_original - - described_class.capture(now: now).payload - - expect(GithubApiBudget).to have_received(:find_by).once - end - it "renders every timestamp the way the printed report does" do create_event_source(cadence_due_at: now + 300) active_budget_window(now: now) + active_search_window(now: now) body = payload expect(body[:captured_at]).to eq(now.utc.iso8601) expect(body.dig(:ledger, :reset_at)).to eq((now + 3600).utc.iso8601) + expect(body.dig(:search_ledger, :reset_at)).to eq((now + 60).utc.iso8601) expect(body[:sources].first[:next_poll_at]).to eq((now + 300).utc.iso8601) end + + it "reads persisted state without writing anything" do + create_event_source + create_actor(github_id: 1) + active_budget_window(now: now) + active_search_window(now: now) + + expect(write_statements { payload }).to be_empty + end end end diff --git a/spec/support/budget_helpers.rb b/spec/support/budget_helpers.rb index 3cf1349..f8276a6 100644 --- a/spec/support/budget_helpers.rb +++ b/spec/support/budget_helpers.rb @@ -1,8 +1,9 @@ -# A ledger row in the state most reservations actually happen in: a window already -# initialized from response headers, with the formula's allowances and nothing spent. +# Ledger rows in the state most reservations actually happen in. # -# Shared between the ledger's own specs and the executor's, so "what an active window -# looks like" is defined once. Values match the plan's defaults: 60 - 8 - 12 = 40. +# Shared between the ledgers' own specs and the executors', so "what an active window +# looks like" is defined once. Core values match the plan's Appendix G defaults: +# 12 poll + 4 detail fallback + 8 reserve on the hourly core resource; the search +# window is the per-minute 10-ceiling / 2-reserve pair. module BudgetHelpers def active_budget_window(now: frozen_time, **overrides) Github::BudgetLedger.new.bootstrap!(now: now) @@ -10,7 +11,7 @@ def active_budget_window(now: frozen_time, **overrides) GithubApiBudget.where(id: GithubApiBudget::SINGLETON_ID).update_all({ window_status: "active", window_initialized_at: now, limit: 60, remaining: 55, reset_at: now + 3600, observed_at: now, - poll_allowance: 12, enrichment_allowance: 40, reserve: 8 + poll_allowance: 12, enrichment_allowance: 4, reserve: 8 }.merge(overrides)) current_budget @@ -19,6 +20,24 @@ def active_budget_window(now: frozen_time, **overrides) def current_budget GithubApiBudget.find(GithubApiBudget::SINGLETON_ID) end + + # A search-ledger row mid-window: bootstrapped from configuration defaults with the + # first response's headers already observed. Overrides let a spec spend it down, + # block it, or start pacing from a chosen instant. + def active_search_window(now: frozen_time, **overrides) + Github::SearchBudgetLedger.new(configuration: Github.configuration).bootstrap!(now: now) + + GithubSearchBudget.where(id: GithubSearchBudget::SINGLETON_ID).update_all({ + limit: 10, remaining: 9, reset_at: now + 60, observed_at: now, + request_ceiling: 10, reserve: 2 + }.merge(overrides)) + + current_search_budget + end + + def current_search_budget + GithubSearchBudget.find(GithubSearchBudget::SINGLETON_ID) + end end RSpec.configure do |config| diff --git a/spec/support/ingestion_helpers.rb b/spec/support/ingestion_helpers.rb index 000aeb7..706df47 100644 --- a/spec/support/ingestion_helpers.rb +++ b/spec/support/ingestion_helpers.rb @@ -99,28 +99,73 @@ def fixture_event_source Github::Ingestion::SourceProvisioner.ensure!(mode: :fixture, now: frozen_time) end - # The enrichment counterpart of #fixture_runner: the real ledger, gate and URL policy - # over the offline transport, with only the clocks and the sleeper replaced. The same - # ledger_for caveat applies — a custom configuration has to reach the ledger, or the - # runner and the ledger would disagree in specs while agreeing in production. - def fixture_enrichment_runner(transport: fixture_transport, now: frozen_time, - configuration: nil, executor: nil, **overrides) + # No jitter, so a scheduled retry is an instant a spec can name rather than a range. + def jitterless_backoff(configuration: Github.configuration) + Github::Enrichment::Backoff.new( + random: Struct.new(:value) { def rand(*) = 0.0 }.new, + configuration: configuration + ) + end + + # The enrichment counterparts of #fixture_runner: the real ledgers, gate and URL + # policy over the offline transport, with only the clocks and the sleeper replaced. + # The same ledger_for caveat applies — a custom configuration has to reach both + # ledgers, or the runners and the ledgers would disagree in specs while agreeing in + # production. + def fixture_batch_runner(transport: fixture_transport, now: frozen_time, + configuration: nil, executor: nil, **overrides) + configuration ||= Github.configuration + + Github::Enrichment::BatchRunner.new( + executor: executor || fixture_executor(transport: transport, ledger: ledger_for(configuration), + search_ledger: search_ledger_for(configuration)), + configuration: configuration, + claim: Github::Enrichment::BatchClaim.new(configuration: configuration), + search_ledger: search_ledger_for(configuration), + backoff: jitterless_backoff(configuration: configuration), + clock: -> { now }, + **overrides + ) + end + + def fixture_detail_runner(transport: fixture_transport, now: frozen_time, + configuration: nil, executor: nil, **overrides) configuration ||= Github.configuration - selector = Github::Enrichment::CandidateSelector.new(configuration: configuration) - Github::EnrichmentRunner.new( - executor: executor || fixture_executor(transport: transport, ledger: ledger_for(configuration)), + Github::Enrichment::DetailRunner.new( + executor: executor || fixture_executor(transport: transport, ledger: ledger_for(configuration), + search_ledger: search_ledger_for(configuration)), configuration: configuration, + claim: Github::Enrichment::DetailClaim.new(configuration: configuration), + backoff: jitterless_backoff(configuration: configuration), clock: -> { now }, - monotonic: -> { 0.0 }, - selector: selector, - # No jitter, so a scheduled retry is an instant a spec can name rather than a range. - entity_state: Github::Enrichment::EntityState.new( - backoff: Github::Enrichment::Backoff.new(random: Struct.new(:value) { def rand(*) = 0.0 }.new) - ), **overrides ) end + + # A full staged cycle over the offline transport: batches until the search ledger + # denies, then detail fallbacks until the core allowance denies. The sleeper is a + # no-op, so pacing waits cost no wall clock. + def fixture_cycle_runner(transport: fixture_transport, now: frozen_time, + configuration: nil, **overrides) + configuration ||= Github.configuration + + Github::Enrichment::CycleRunner.new( + configuration: configuration, + batch_runner: fixture_batch_runner(transport: transport, now: now, configuration: configuration), + detail_runner: fixture_detail_runner(transport: transport, now: now, configuration: configuration), + admission: Github::Enrichment::Admission.new(configuration: configuration), + batch_claim: Github::Enrichment::BatchClaim.new(configuration: configuration), + detail_claim: Github::Enrichment::DetailClaim.new(configuration: configuration), + clock: -> { now }, + sleeper: ->(_seconds) { }, + **overrides + ) + end + + def search_ledger_for(configuration) + Github::SearchBudgetLedger.new(configuration: configuration) + end end RSpec.configure do |config| diff --git a/spec/support/shared_examples/enrichable_entity.rb b/spec/support/shared_examples/enrichable_entity.rb index 93ea9e6..52aee12 100644 --- a/spec/support/shared_examples/enrichable_entity.rb +++ b/spec/support/shared_examples/enrichable_entity.rb @@ -2,7 +2,8 @@ # (IMPLEMENTATION_PLAN.md §7), so its data-level guarantees are asserted once here. # # Scope note: activity is the transition the ingest path owns. Fetch outcomes belong to -# Github::Enrichment::EntityState, and leasing belongs to Github::Enrichment::Claim. +# Github::Enrichment::BatchRunner and DetailRunner, and leasing belongs to +# Github::Enrichment::BatchClaim and DetailClaim. # # The host group must provide `valid_attributes`. RSpec.shared_examples "an enrichable entity" do @@ -14,6 +15,10 @@ "index_#{described_class.table_name}_on_enrichment_refresh" end + let(:stage_fifo_index_name) do + "index_#{described_class.table_name}_on_stage_fifo" + end + describe "enrichment status column" do it "starts pending so a newly observed entity is enrichment-eligible" do expect(described_class.create!(valid_attributes).enrichment_status).to eq("pending") @@ -56,6 +61,44 @@ end end + # The staged pipeline's resting positions (§7, Appendix F). enrichment_status stays the + # business outcome; the stage is where the row currently sits between the batch and + # detail lanes, and the CHECK constraint is what keeps a torn write from inventing an + # eighth position no claim would ever select. + describe "enrichment stage column" do + it "starts at batch_pending, the staged pipeline's entry position" do + expect(described_class.create!(valid_attributes).enrichment_stage).to eq("batch_pending") + end + + it "accepts every resting stage the plan documents" do + Enrichable::ENRICHMENT_STAGES.each_with_index do |stage, index| + record = described_class.create!( + valid_attributes.merge(github_id: 93_000 + index, enrichment_stage: stage) + ) + expect(record.reload.enrichment_stage).to eq(stage) + end + end + + it "rejects an undocumented stage at the database level" do + record = described_class.create!(valid_attributes) + + expect_violation(ActiveRecord::CheckViolation) do + described_class.where(id: record.id).update_all(enrichment_stage: "invented") + end + end + + # Both claims select FIFO *within* a stage set — batch by created_at, id over the + # backlog stages — so the leading stage column is what lets one index answer either + # claim's ordered scan without touching the other's rows. + it "is backed by the stage-FIFO index the claims select through" do + index = described_class.connection.indexes(described_class.table_name) + .find { |i| i.name == stage_fifo_index_name } + + expect(index).not_to be_nil + expect(index.columns).to eq(%w[enrichment_stage created_at id]) + end + end + describe ".enrichment_candidates" do it "returns only pending and retryable_failure rows" do records = Enrichable::ENRICHMENT_STATUSES.each_with_index.to_h do |status, index| diff --git a/spec/support/shared_examples/enrichment_job.rb b/spec/support/shared_examples/enrichment_job.rb deleted file mode 100644 index e9dc561..0000000 --- a/spec/support/shared_examples/enrichment_job.rb +++ /dev/null @@ -1,70 +0,0 @@ -# EnrichActorJob and EnrichRepositoryJob are the same job over §7's identical state machine, -# so their contract is written once — the precedent spec/support/shared_examples/ -# enrichable_entity.rb set for the models themselves. -# -# What is actually being asserted is the job's *boundary*: which class it asks for, that one -# call is one entity, that it never reaches for a source lock, and that an outcome nobody has -# to act on is not an error. -RSpec.shared_examples "an enrichment job" do |entity_class:, entity_type:, log_key:| - let(:runner) { instance_double(Github::EnrichmentRunner) } - let(:enriched) do - Github::EnrichmentRunner::Result.new(status: "enriched", entity_type: entity_type, github_id: 4_242) - end - - before { allow(Github::EnrichmentRunner).to receive(:new).and_return(runner) } - - it "runs on the dedicated enrichment queue" do - expect(described_class.new.queue_name).to eq("enrichment") - end - - it "runs exactly one cycle, narrowed to its own class" do - expect(runner).to receive(:call).with(entity_class: entity_class).once.and_return(enriched) - - described_class.new.perform_now - end - - it "joins the cycle to the job on one line" do - allow(runner).to receive(:call).and_return(enriched) - allow(Rails.logger).to receive(:info) - - job = described_class.new - job.perform_now - - expect(Rails.logger).to have_received(:info).with( - hash_including(event: "job.completed", job_id: job.job_id, job_class: described_class.name, - log_key => 4_242, enrichment_outcome: "enriched") - ) - end - - # §8 step 1: "Enrichment jobs skip this step — they take only the request gate." Asserted at - # the job boundary as well as inside the runner, because this is where a future "just lock - # the source while we enrich it" would be written. - it "never takes a source lock" do - allow(runner).to receive(:call).and_return(enriched) - expect(Github::SourceLock).not_to receive(:acquire) - - described_class.new.perform_now - - expect(Github::LockOrder.held_keys).to be_empty - end - - # Nothing eligible, or a ledger that refused: ordinary outcomes of a system whose budget is - # 40 requests an hour. Failing the job would fill solid_queue_failed_executions with the - # steady state. - %w[idle deferred].each do |status| - it "treats a #{status} cycle as a completed job" do - allow(runner).to receive(:call) - .and_return(Github::EnrichmentRunner::Result.new(status: status, deferral_reason: "no_candidate")) - - expect { described_class.new.perform_now }.not_to raise_error - end - end - - # §6 requires a corpus gap to be raised rather than laundered into a failed fetch, and the - # runner has already put the lease back by the time it arrives here. - it "lets a fixture corpus gap fail the job" do - allow(runner).to receive(:call).and_raise(Github::Errors::FixtureMiss, "no such body") - - expect { described_class.new.perform_now }.to raise_error(Github::Errors::FixtureMiss) - end -end From b4b25bd892398a5718757a70092b617e61258ef9 Mon Sep 17 00:00:00 2001 From: Umang Date: Sun, 2 Aug 2026 15:40:05 -0500 Subject: [PATCH 10/12] Document the staged batch design and its measured gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Appendix G records the decision, what it supersedes from Appendix F, the refresh composition rule, and the capacity hypothesis kept explicitly labeled as a hypothesis until measured. Sections 5, 7, 10, 11, 12, and 16 are amended in place, including the two-origin SSRF wording: Search URLs are application-origin constants built from stored identifiers and still validated in-chain, while detail URLs remain payload-origin under the full boundary — no identifier is ever turned into a constructed detail URL after a Search miss. Section 16 gains a forbidden claim to match the honesty rule this change introduces: no guaranteed catch-up, only a dated measured comparison. ADR 0013 records the decision and its rejected alternatives; ADRs 0004, 0007, 0008, and 0010 carry amendment lines in the established format. README, .env.example, CLAUDE.md, the design brief, and the submission checklist follow the same numbers throughout. Co-Authored-By: Claude Fable 5 --- .env.example | 126 +++- CLAUDE.md | 28 +- IMPLEMENTATION_PLAN.md | 475 +++++++++++-- README.md | 659 ++++++++++++------ app/services/github/enrichment_runner.rb | 223 ------ docs/DESIGN_BRIEF.md | 83 ++- docs/SUBMISSION_CHECKLIST.md | 36 +- docs/adr/0004-class-aware-budget-ledger.md | 16 +- ...nrichment-fairness-shares-and-borrowing.md | 17 +- ...nqueue-and-entity-scoped-reconciliation.md | 16 +- ...it-escalation-and-refresh-pool-fairness.md | 17 +- ...erivation-first-staged-batch-enrichment.md | 120 ++++ 12 files changed, 1234 insertions(+), 582 deletions(-) delete mode 100644 app/services/github/enrichment_runner.rb create mode 100644 docs/adr/0013-derivation-first-staged-batch-enrichment.md diff --git a/.env.example b/.env.example index 10949b1..0bcdacf 100644 --- a/.env.example +++ b/.env.example @@ -64,22 +64,26 @@ GITHUB_MODE=live # The always-on poller attempts once and defers to its next cycle regardless. # SOURCE_LOCK_WAIT_SECONDS=30 -# Inputs to the one authoritative allowance formula (§10): +# Inputs to the one authoritative core allowance formula (§10, as amended by +# Appendix G): # # poll_attempt_allowance = ceil(3600 / POLL_INTERVAL_SECONDS) # x MAX_PAGES_PER_POLL x ENABLED_LIVE_SOURCE_COUNT -# enrichment_allowance = rate_limit - RATE_LIMIT_RESERVE - poll_attempt_allowance +# feasible iff poll_attempt_allowance + RATE_LIMIT_RESERVE +# + CORE_DETAIL_FALLBACK_ALLOWANCE <= rate_limit # # With these defaults: ceil(3600/300) x 1 x 1 = 12 poll attempts/hour, and -# 60 - 8 - 12 = 40 enrichment attempts/hour. Startup fails if -# poll_attempt_allowance + RATE_LIMIT_RESERVE reaches the limit, which would -# leave no capacity for the enrichment Story 3 requires. +# 12 poll + 4 detail fallback + 8 reserve = 24 of 60 — the remaining 36 core +# requests are deliberately unspent headroom. Normal-path enrichment runs on the +# separate per-minute *search* budget below, not on core. Startup fails when the +# three core lanes together exceed the limit. # # Both are formula inputs and both are obeyed at runtime. POLL_INTERVAL_SECONDS is # the cadence: a source polled at T is due again at T + this, and an unforced run # before then is deferred rather than made. MAX_PAGES_PER_POLL caps how many -# `Link`-followed pages one poll may fetch — raising it trades enrichment -# allowance for capture depth, so at 3 the split moves from 12/40 to 36/16. +# `Link`-followed pages one poll may fetch — raising it spends the unspent core +# headroom on capture depth (at 5 pages the formula is infeasible and boot +# refuses); it no longer cuts enrichment, which lives on the search budget. # # The poll allowance is enforced on top of the cadence: every # `docker compose run --rm ingest` spends one poll attempt per page, so at these @@ -97,43 +101,121 @@ GITHUB_MODE=live # ENABLED_LIVE_SOURCE_COUNT=1 # RATE_LIMIT_RESERVE=8 -# How the enrichment allowance is split between the two entity classes (§10): +# How the core detail-fallback allowance is split between the two entity classes +# (§10; since Appendix G this share governs the detail lane only — Search batch +# lanes rotate by the weights in the staged section below): # # actor_guarantee = floor(enrichment_allowance x ACTOR_ENRICHMENT_SHARE) # repository_guarantee = enrichment_allowance - actor_guarantee # -# With these defaults: 40 x 0.50 = 20 actor and 20 repository attempts an hour. The -# remainder always goes to repository, because the formula floors one side and -# subtracts for the other — so the two always add up to the whole allowance. -# -# The split exists because one observed live page held ~89 distinct actors against -# ~92 distinct repositories: repository candidates alone exceed the entire hourly -# allowance, so a queue ordered purely by recency would starve actor enrichment to -# zero indefinitely. +# With these defaults: 4 x 0.50 = 2 actor and 2 repository detail-fallback +# attempts an hour. The remainder always goes to repository, because the formula +# floors one side and subtracts for the other — so the two always add up to the +# whole allowance. # # It is a guarantee, not a cap. A class may borrow the other's unused capacity when -# the other class has no *currently claimable* backlog candidate — not merely no rows — so a +# the other class has no *currently claimable* detail candidate — not merely no rows — so a # quiet hour for repositories is spent on actors rather than wasted. Both ends of # the range are legal: 0.0 means "repositories first, actors during the quiet # periods". Below 0 or above 1 is rejected at startup, because either would put one # guarantee above the class allowance. # ACTOR_ENRICHMENT_SHARE=0.50 -# How long before an enriched entity becomes refresh-eligible (§10): +# How long before an enriched entity becomes refresh-eligible (§10, Appendix G): # # ACTOR_REFRESH_TTL_SECONDS / REPOSITORY_REFRESH_TTL_SECONDS - the minimum time an # enriched entity is reused before it may be re-fetched. Never-enriched entity rows # are a durable FIFO backlog: quota exhaustion defers them to a later window and -# never terminates them. A selection that observes never-enriched work does not choose -# a refresh, so raising these values affects freshness only after the durable backlog -# drains. Ingestion can commit one new row between that read and a request debit; the -# runner's one-request cycle bounds the window and the next selection suppresses refresh. +# never terminates them. Refreshes ride the same Search batch path: a batch fills +# from its own class's backlog first, tops up spare slots with TTL-stale refresh +# candidates only when its own backlog is exhausted and the other class has none +# claimable, and a refresh-only batch runs only when neither class has backlog — +# so raising these values affects freshness only after the backlog drains. # # Both must be greater than zero. A zero TTL turns the freshness cache off entirely. # "Never refresh" is a large number, not zero. # ACTOR_REFRESH_TTL_SECONDS=86400 # REPOSITORY_REFRESH_TTL_SECONDS=86400 +# --------------------------------------------------------------------------- +# Staged batch enrichment (plan Appendix G) +# --------------------------------------------------------------------------- + +# The bounded core lane for individual detail fetches. Only batch items that came +# back missing, renamed, identity-mismatched, or contract-invalid reach it, at the +# stored payload-provided URL. It is the third term of the core feasibility rule +# above and can never take the polling allocation. Zero is legal and turns the +# fallback off. +# CORE_DETAIL_FALLBACK_ALLOWANCE=4 + +# The per-minute Search budget — a separate GitHub rate-limit resource from core. +# The ceiling is the observed unauthenticated header limit (10/minute); the +# reserve is never spent, leaving 8 spendable Search requests a minute. Windows +# are 60 seconds and roll from response headers, or without headers after a +# minute of inactivity. +# SEARCH_REQUEST_CEILING=10 +# SEARCH_SAFETY_RESERVE=2 + +# How many repeated exact user:/repo: qualifiers one Search request carries. +# Capped at 10 — the probe-verified batching form; qualifiers are never joined +# with OR, which GitHub answers with a 422. +# SEARCH_BATCH_SIZE=10 + +# Minimum seconds between Search requests, spread inside the per-minute window +# rather than burst at its top. 0 disables pacing — the fixture walkthrough uses +# that so batches run back to back. Must stay below +# ENRICHMENT_CYCLE_BUDGET_SECONDS. +# SEARCH_PACING_SECONDS=6 + +# Must be exactly 1 while the global request gate serializes every outbound +# request; the variable exists so the constraint is stated rather than implied. +# SEARCH_WORKER_CONCURRENCY=1 + +# How the batch lanes rotate. Weights, not shares: with 1/1 the cycle alternates +# actor and repository batches, and a lane with no claimable work yields its slot +# to the other. +# ACTOR_ENRICHMENT_WEIGHT=1 +# REPOSITORY_ENRICHMENT_WEIGHT=1 + +# The detail-fallback retry ladder. A retryable detail failure re-rests in +# detail_pending with backoff until this many attempts, then the entity goes +# terminal; a 404/410 goes terminal immediately. Batch retries are unbounded — +# a batch failure says nothing entity-specific. +# DETAIL_FALLBACK_MAX_ATTEMPTS=3 + +# The durable claim lease. A batch or detail claim owns its rows until released +# or until this many seconds pass; a crashed worker's lease expires by +# arithmetic and the rows are reclaimed. Must exceed the worst-case single-fetch +# time — (retries+1) x (redirects+1) x (45s gate wait + open + read) = 585s at +# the HTTP defaults — or a slow live fetch would be reclaimed while in flight. +# ENRICHMENT_LEASE_SECONDS=600 + +# Enrichment retry backoff, jittered: base doubles per attempt and is capped at +# max. Base must not exceed max. +# ENRICHMENT_RETRY_BASE_SECONDS=60 +# ENRICHMENT_RETRY_MAX_SECONDS=3600 + +# One enrichment cycle's time budget. The cycle stops claiming new work at this +# boundary so it finishes inside the 60-second dispatch tick; must be below 60 +# and above SEARCH_PACING_SECONDS. +# ENRICHMENT_CYCLE_BUDGET_SECONDS=55 + +# Refresh eligibility beyond the TTLs: only entities seen pushing within this +# window are refreshed at all, so quota is never spent re-fetching an entity +# nothing references any more. Default is seven days. +# REFRESH_ACTIVE_WITHIN_SECONDS=604800 + +# The window behind /status's throughput and batch-quality blocks — arrivals, +# completions, fill ratios, and backlog slope are measured over this many +# seconds. Reporting only, like the coverage window below. +# ENRICHMENT_METRICS_WINDOW_SECONDS=3600 + +# How much sample the catch-up verdict requires before it will say anything. +# Below this, /status reports catch_up.state = insufficient_sample rather than +# guessing; at or above it, the verdict is keeping_up or not_keeping_up from +# measured rates. There is deliberately no ETA anywhere. +# CATCH_UP_MIN_SAMPLE_SECONDS=900 + # How far back GET /status looks when it computes §11's three coverage # percentages. Unlike every other knob in this file it changes only what the # system *reports* — nothing schedules, reserves, defers or skips on it, so two diff --git a/CLAUDE.md b/CLAUDE.md index 548ced6..cffc7b2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,8 +8,10 @@ plan wins. 1. Read `IMPLEMENTATION_PLAN.md` (repository root). It is the frozen execution plan; its pre-implementation revision history lives in Git and in its - Appendices A–D, Appendix E records how the build diverged from it, and Appendix F - supersedes the enrichment load-shedding policy with a durable backlog. + Appendices A–D, Appendix E records how the build diverged from it, Appendix F + supersedes the enrichment load-shedding policy with a durable backlog, and + Appendix G supersedes Appendix F's per-entity service model with derivation-first + staged batch enrichment. 2. Read `docs/DESIGN_BRIEF.md` and the ADRs under `docs/adr/`. 3. Do not change architectural direction, add infrastructure, or add dependencies without first updating the plan and stating the tradeoff. @@ -77,13 +79,21 @@ Enrichment fetches only validated URLs: HTTPS, host exactly `api.github.com`, no userinfo, no non-default port, no IP literals, bounded re-validated redirects. Fixture mode fails closed — never a live fallback. -### Durable enrichment backlog (plan §10, Appendix F) - -Never-enriched entity rows remain actionable across quota windows. Select them FIFO by -`created_at ASC, id ASC`; quota or fairness denial defers rather than terminates. The -default hourly split is 12 polling requests, 40 backlog-enrichment requests, and 8 safety -reserve requests, with 20/20 actor/repository guarantees and borrowing. Do not schedule a -refresh while either class has never-enriched work. +### Durable staged enrichment backlog (plan §10, Appendices F–G) + +Entity rows are durable work, coalesced by stable GitHub ID, selected FIFO by +`created_at ASC, id ASC`. The normal path is GitHub Search batches of up to +`SEARCH_BATCH_SIZE` repeated exact `user:`/`repo:` qualifiers — never joined with `OR` — +on the minute-scoped search ledger (ceiling 10, reserve 2, 6-second pacing). The +payload-URL detail fallback serves only missing/renamed/mismatched/contract-invalid batch +items and is bounded by `CORE_DETAIL_FALLBACK_ALLOWANCE` (4/hour); it never takes the +polling allocation. Quota, pacing, reserve, and fairness denials defer work — they never +terminate an entity. Batch results apply only on a stable-ID match. Observations are +append-only; a refresh repoints the projection and never overwrites retained evidence. +Refresh composition: a batch fills from its own class's never-enriched backlog first, +tops up spare slots with TTL-stale refresh candidates only when its own backlog is +exhausted and the other class has none claimable, and refresh-only batches run only when +neither class has backlog. ## Database changes diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md index 3ec4412..812094e 100644 --- a/IMPLEMENTATION_PLAN.md +++ b/IMPLEMENTATION_PLAN.md @@ -1,6 +1,6 @@ # GitHub Push Event Ingestion Service — Implementation Plan -> **This plan was finalized through four pre-implementation review rounds**: an adversarial multi-lens design review with live probes of the unauthenticated GitHub API (**Appendix A**), an independent validation pass against official GitHub, PostgreSQL, and Rails/Solid Queue documentation (**Appendix B**), an implementation-readiness re-check that corrected locking, scheduling, Compose, and PR-ordering defects (**Appendix C**), and a freeze-readiness pass that corrected lock scoping, class-level blocking, bootstrap, restart, and entity-activity semantics (**Appendix D**). The initial plan (V1) and the full revision trail are preserved in Git history; the appendices record what changed and why. Appendix F supersedes the original enrichment load-shedding policy with the durable-backlog design adopted on 2026-08-02. The one section added during revision is numbered **2A** to keep the original numbering stable. +> **This plan was finalized through four pre-implementation review rounds**: an adversarial multi-lens design review with live probes of the unauthenticated GitHub API (**Appendix A**), an independent validation pass against official GitHub, PostgreSQL, and Rails/Solid Queue documentation (**Appendix B**), an implementation-readiness re-check that corrected locking, scheduling, Compose, and PR-ordering defects (**Appendix C**), and a freeze-readiness pass that corrected lock scoping, class-level blocking, bootstrap, restart, and entity-activity semantics (**Appendix D**). The initial plan (V1) and the full revision trail are preserved in Git history; the appendices record what changed and why. Appendix F supersedes the original enrichment load-shedding policy with the durable-backlog design adopted on 2026-08-02, and **Appendix G** supersedes Appendix F's per-entity service model with derivation-first staged batch enrichment, adopted the same day. The one section added during revision is numbered **2A** to keep the original numbering stable. > > File locations: this plan lives at the repository root. `DESIGN_BRIEF.md` and the ADRs live under `docs/`. @@ -32,7 +32,7 @@ The delivered implementation will satisfy the required public-events source whil - `PushEvent` filtering and processing - Raw event payload retention (semantic retention via `jsonb` — see Section 7) - Structured push-event fields -- Actor and repository enrichment (**durable, quota-paced FIFO backlog with per-class fairness** — see Section 10) +- Actor and repository enrichment (**durable, staged, batch-served FIFO backlog with per-class fairness** — see Section 10 and Appendix G) - Duplicate-safe `push_events` persistence - Pagination via the `Link` response header - ETag and `304 Not Modified` handling (bandwidth/correctness measure, scoped to the canonical first-page request — see Sections 9–10) @@ -279,14 +279,14 @@ Child issues: Two request paths share the executor; only polling takes a source lock: ```text -POLLING PATH ENRICHMENT PATH +POLLING PATH ENRICHMENT PATH (Appendix G) -IngestionRunner EnrichActorJob / EnrichRepositoryJob - → SourceLock(event_source_id) │ +IngestionRunner EnrichmentCycleJob → CycleRunner + → SourceLock(event_source_id) │ batch lanes, then detail lanes → EventSource (PublicEvents|Fixture) │ → RequestExecutor ◄─────────────────────┘ → RequestGate (global, serial) - → BudgetLedger (class-aware, formula-derived, per-window) + → BudgetLedger (core) | SearchBudgetLedger (per-minute search) → UrlPolicy → Transport: Faraday (live) | Fixture | @@ -294,6 +294,7 @@ IngestionRunner EnrichActorJob / EnrichRepositoryJob PostgreSQL Transaction - raw payload; structured fields - stub actor/repo upserts (merge rules) +- event-source enrichment observations (append-only) - INSERT … ON CONFLICT DO NOTHING RETURNING id - activity updates ONLY when a row returned - malformed-event quarantine (fingerprints) @@ -301,17 +302,21 @@ PostgreSQL Transaction v Solid Queue (PostgreSQL-backed) | - +-----+------+ - | | - v v -Actor Job Repository Job - (fair-share (fair-share - budgeted) budgeted) - | | - +-----+------+ + v +EnrichmentCycleJob (one queued cycle at a time) + | + +-----+---------------------+ + | | + v v +Search batches Detail fallbacks + (BatchClaim/BatchRunner, (DetailClaim/DetailRunner, + search ledger, ≤ 10 core detail-fallback + exact qualifiers) allowance, payload URLs) + | | + +-----+---------------------+ | v -PostgreSQL +PostgreSQL (observations + projections + batch envelopes) ``` **Lock-order invariant:** source lock → request gate, never the reverse. @@ -324,7 +329,8 @@ PostgreSQL - `Github::RequestExecutor` - `Github::SourceLock` (session advisory lock, `(SOURCE_LOCK, event_source_id)`; owned by `IngestionRunner`, polling only) - `Github::RequestGate` (global session advisory lock, `(REQUEST_GATE, 1)`) -- `Github::BudgetLedger` (class-aware; allowance formula; per-window bootstrap) +- `Github::BudgetLedger` (core resource; class-aware; allowance formula; per-window bootstrap) +- `Github::SearchBudgetLedger` (search resource; per-minute window — Appendix G) - `Github::RateLimitPolicy` - `Github::RetryPolicy` - `Github::UrlPolicy` (enrichment URL validation) @@ -338,11 +344,18 @@ PostgreSQL - `Github::Events::ProcessorRegistry` - `Github::Events::PushEventProcessor` - `Github::IngestionRunner` +- `Github::Enrichment::SearchQuery` / `Github::Enrichment::SearchResponse` (batch query construction and envelope parsing) +- `Github::Enrichment::BatchClaim` / `Github::Enrichment::BatchRunner` (Search-batch lane: FIFO claim under lease, apply-by-stable-ID) +- `Github::Enrichment::DetailClaim` / `Github::Enrichment::DetailRunner` (payload-URL detail-fallback lane) +- `Github::Enrichment::Admission` (read-only dual-ledger admission verdicts) +- `Github::Enrichment::CycleRunner` (one enrichment cycle: batch lanes, then detail lanes, inside a time budget) +- `Github::Enrichment::ObservationRecorder` (append-only observation writes) - `PollEventSourceJob` -- `EnrichActorJob` -- `EnrichRepositoryJob` +- `EnrichmentCycleJob` - `ReconcilePendingEnrichmentsJob` +(The per-entity `EnrichActorJob`/`EnrichRepositoryJob`, `EnrichmentRunner`, `Enrichment::Fairness`, `CandidateSelector`, `Claim`, and `EntityState` from the pre-Appendix-G design are deleted; `Github::EnrichmentSchedule` remains as the core admission value object.) + ## 6. Event-Source Design The required delivered source will call: @@ -417,7 +430,9 @@ Single-row global ledger (constrained singleton), through which **every** outbou - `window_status` — `uninitialized | active | globally_blocked` - `window_initialized_at` - `poll_allowance`, `poll_used` -- `enrichment_allowance`, `enrichment_used` +- `enrichment_allowance`, `enrichment_used` — since Appendix G these budget the + **detail-fallback** lane only (`CORE_DETAIL_FALLBACK_ALLOWANCE`, default 4); the + batch normal path spends the separate search ledger - `actor_share_used`, `repository_share_used` (fairness accounting — Section 10) - `reserve` - `observed_at` @@ -598,6 +613,90 @@ event.repo.url → api_url (enrichment populates description, language, owner_github_id, raw_payload) ``` +`owner_login` and `name` are **derived locally** from `full_name` at ingest (Appendix G) — +no network call is spent on a field the stored payload already determines. + +### Staged enrichment columns on both entity tables (new in Appendix G) + +Both entity tables carry the staged pipeline alongside the unchanged +`enrichment_status` business outcome: + +- `enrichment_stage` — check-constrained to the seven **resting** stages: + `batch_pending | batch_in_flight | detail_pending | detail_in_flight | + retry_scheduled | contract_complete | terminal`. Event-native persistence, local + derivation, and batch application are instants, not stages — they are recorded by + the timestamp columns below because no row ever rests in them. +- Instant timestamps, `COALESCE`-keep-first where first observation matters: + `event_native_at`, `derived_at`, `batch_pending_at`, `batch_applied_at`, + `detail_pending_at`, `retry_scheduled_at`, `contract_completed_at`, `terminal_at` +- `detail_attempts` — the detail-fallback retry ladder, bounded by + `DETAIL_FALLBACK_MAX_ATTEMPTS` (default 3) +- `lease_token` (uuid) + `leased_until` — the durable batch/detail claim lease + (`ENRICHMENT_LEASE_SECONDS`, default 600); expired leases are reclaimed, and + projection writes are guarded by token + batch id +- `latest_observation_id` / `latest_observation_source` / `latest_observed_at` — + the projection's pointer into the append-only observation history +- `current_enrichment_batch_id` — the in-flight request attempt owning this row +- Contract columns: actors add `account_type`; repositories add `fork`, `archived`, + `default_branch`, `github_created_at`, and the locally derived `owner_login` — + joining the existing `description`, `language`, and `owner_github_id` +- Index `(enrichment_stage, created_at, id)` — the per-stage FIFO scan; plus an + index on `leased_until` for stale-lease reclaim + +### `enrichment_batches` (new in Appendix G) + +One row per enrichment **request attempt** — search batch or detail fallback — so +batch quality is measurable from durable state: + +- `id`, `correlation_id` (uuid, unique) +- `request_kind` — `search | detail`; `entity_kind` — `actor | repository` +- `status` — `in_flight | succeeded | failed | deferred | stale_lease` +- `requested_github_ids`, `requested_identifiers` (`jsonb`) — the claimed membership +- `request_url`, `response_status`; `response_body` stored **only** for non-OK + responses, truncated +- `total_count`, `incomplete_results` — the Search envelope facts +- `requested_count`, `returned_count`, `valid_count`, `missing_count`, + `invalid_count` — non-negative check-constrained +- `started_at`, `completed_at`, `last_error` +- Observed rate-limit headers: `rate_limit_resource`, `rate_limit_limit`, + `rate_limit_remaining`, `rate_limit_used`, `rate_limit_reset_at` +- timestamps; indexes on `correlation_id`, `(request_kind, entity_kind, started_at)`, + and `started_at` + +### `enrichment_observations` (new in Appendix G) + +Append-only evidence — the model is read-only after persist, and refresh repoints the +projection rather than overwriting retained raw responses: + +- `id`, `entity_kind` (`actor | repository`), `entity_github_id` (stable GitHub ID) +- `source` — `event | search | detail` (check-constrained) +- `observed_at`, `raw_payload` (`jsonb`, the complete raw item) +- `payload_fingerprint` — the same canonical SHA-256 algorithm as quarantine +- `enrichment_batch_id`, `push_event_id`, `request_correlation_id`, + `requested_identifier` — provenance +- `validation_outcome` — including `unrequested_result` for items nobody asked for +- timestamps; indexed by `(entity_kind, entity_github_id, observed_at)` and by + fingerprint + +Event-source observations are written **inside the ingest transaction** with the push +event, so evidence commits with the event it came from. + +### `github_search_budget` (new in Appendix G) + +Singleton (`CHECK (id = 1)`) ledger for GitHub's **search** rate-limit resource — a +separate per-minute budget the core ledger must not conflate: + +- `resource` — `"search"` (verified against `x-ratelimit-resource`) +- `limit`, `remaining`, `reset_at`, `observed_at` — header-reconciled, monotonic + within a window +- `request_ceiling` (default 10), `reserve` (default 2), `used`, `actor_used`, + `repository_used` — non-negative check-constrained +- `blocked_until`, `last_request_at`, `lock_version`, timestamps + +Windows are 60 seconds. The window rolls at `reset_at`, or — because a denied minute +may observe no headers at all — when `last_request_at` is at least 60 seconds old and +`used` is positive (the header-less roll). + ## 8. Durability and Crash Recovery ### Durability boundary @@ -683,6 +782,9 @@ effective_enrichment_time = max( ) ``` +(Since Appendix G this rule governs the core **detail-fallback** lane; Search batches are +admitted against the independent per-minute search ledger instead.) + `reset_at` is informational; it never participates in scheduling directly. With defaults: configured cadence 300s, observed floor 60s → effective normal cadence 300s. ### Pagination @@ -752,7 +854,7 @@ Documented constraints (official GitHub docs, verified during the review rounds) **Unauthenticated `304` accounting.** The endpoint documentation contains a general statement that `304` responses do not affect the rate limit, while GitHub’s REST best-practices documentation limits that exemption to correctly authorized requests. Two dated unauthenticated probes (review-supplied evidence, 2026-07-28) showed `x-ratelimit-used` increasing across a `304` (one transcript: 200 → used 4, remaining 56; immediate conditional replay → 304, used 5, remaining 55). This implementation therefore budgets every unauthenticated request — including `304`s — as one request. ETag remains a bandwidth/correctness measure, never a quota saver, in this configuration. **PR 6 re-runs and commits a dated probe transcript as a required validation gate** — normal `GET`, explicit `X-GitHub-Api-Version: 2022-11-28`, exact UTC timestamps, ETag, and complete before/after rate-limit headers — so the design brief cites first-party evidence. -**Allowance formula**, computed at startup and on configuration change: +**Allowance formula** (amended by Appendix G), computed at startup and on configuration change: ```text poll_attempt_allowance = @@ -760,25 +862,58 @@ poll_attempt_allowance = × MAX_PAGES_PER_POLL × ENABLED_LIVE_SOURCE_COUNT -enrichment_allowance = - rate_limit − RATE_LIMIT_RESERVE − poll_attempt_allowance +detail_fallback_allowance = CORE_DETAIL_FALLBACK_ALLOWANCE # a bounded reserve, not the remainder + +feasible ⇔ poll_attempt_allowance + RATE_LIMIT_RESERVE + detail_fallback_allowance ≤ rate_limit ``` With defaults: ```text ceil(3600 / 300) × 1 × 1 = 12 poll attempts/hour -60 − 8 − 12 = 40 enrichment attempts/hour +12 poll + 4 detail fallback + 8 reserve = 24 ≤ 60 ``` | Allocation | Default | |---|---:| | Scheduled polling | 12 request-attempts/hour | -| Enrichment | up to 40 request-attempts/hour | +| Detail fallback (bounded core lane — Appendix G) | up to 4 request-attempts/hour | | Intentionally unspent reserve | 8 requests/hour | +| **Deliberately unspent remainder** | 36 requests/hour | | **Total** | **60 requests/hour** | -Startup validation **rejects** any configuration where `poll_attempt_allowance + reserve >= effective_limit` — that would leave no capacity for required Story 3 enrichment. +The remainder is deliberately unspent: since Appendix G, normal-path enrichment runs on +the **search** rate-limit resource, not on core, so leaving core headroom costs enrichment +nothing and protects polling from co-tenant pressure. The stored +`enrichment_allowance`/`enrichment_used` pair now budgets the detail-fallback lane. + +Startup validation **rejects** any configuration where +`poll_attempt_allowance + RATE_LIMIT_RESERVE + CORE_DETAIL_FALLBACK_ALLOWANCE` exceeds the +effective limit — that would over-commit the core budget the moment every lane ran hot. + +### Search budget (per-minute resource — Appendix G) + +GitHub's Search endpoints report their own rate-limit resource (`x-ratelimit-resource: +search`, observed limit 10 per minute unauthenticated). The `github_search_budget` +singleton ledger accounts for it independently of core: + +```text +SEARCH_REQUEST_CEILING = 10 per minute (observed header limit) +SEARCH_SAFETY_RESERVE = 2 per minute (never spent) +spendable = 8 search requests per minute +SEARCH_PACING_SECONDS = 6 seconds between search requests (0 disables) +SEARCH_BATCH_SIZE ≤ 10 exact qualifiers per request +``` + +Windows are 60 seconds, rolled from response headers or — because a fully denied minute +observes no headers — by the header-less rule: `last_request_at` at least 60 seconds old +with `used` positive. Denials are ordered `search_blocked` → `search_pacing` → +`search_reserve_reached` → `search_ceiling_exhausted`, and every denial **defers** the +batch; none terminates an entity. Search rate-limit and secondary-limit responses set +`blocked_until` from `Retry-After`, then the reset instant, then now + 60 seconds, +whichever is latest. Reconciliation is monotonic and discards headers whose resource is +not `search`. `SEARCH_WORKER_CONCURRENCY` is pinned to 1 while the global request gate +serializes all outbound calls. **Global vs class blocking — one timestamp cannot serve both.** The stored `global_blocked_until` covers only conditions that must stop *all* live requests: @@ -793,7 +928,7 @@ poll_class_blocked_until = poll_used >= poll_allowance ? reset_at : nil enrichment_class_blocked_until = enrichment_used >= enrichment_allowance ? reset_at : nil ``` -So enrichment exhausting its 40 attempts never stops polling, and polling exhausting its 12 never stops enrichment. Actor/repository share exhaustion lives inside `BudgetLedger.reserve!(:actor | :repository)` and never touches the global block. A routine future `X-RateLimit-Reset` on a successful response never defers anything. +So the detail-fallback lane exhausting its 4 attempts never stops polling, and polling exhausting its 12 never stops enrichment — batch enrichment does not even spend core. Actor/repository share exhaustion lives inside `BudgetLedger.reserve!(:actor | :repository)` and never touches the global block; search-window exhaustion lives in the search ledger and blocks only search. A routine future `X-RateLimit-Reset` on a successful response never defers anything. **Secondary rate limits are global.** They are IP-scoped, and they can arise on *any* live request — including enrichment, which has no source row. On any secondary-limit response: set `global_blocked_until` from `Retry-After` (or ≥ 1 minute with exponential backoff when the header is absent), also update the request-specific source or entity retry state, and stop all live requests until the block expires. @@ -802,48 +937,61 @@ So enrichment exhausting its 40 attempts never stops polling, and polling exhaus Consequence, stated honestly: one observed live page of `/events` held ~92–95 PushEvents with ~89 distinct actors and ~92 distinct repositories: ```text -89 actors + 92 repositories = 181 entity requests/page (cold) -181 × 12 polls/hour ≈ 2,172 requests/hour of cold demand -40 available enrichments/hour ≈ 1.8% same-hour service ratio under the all-cold assumption +89 actors + 92 repositories = 181 entity references/page (cold) +181 × 12 polls/hour ≈ 2,172 references/hour of cold demand ``` This is a cold-demand pressure scenario, not a measured arrival rate: it assumes every poll contains entirely new identities, while entity rows deduplicate repeated actors and -repositories across events and overlapping pages. The actual unique arrival rate must be -measured against the 40-attempt service rate. - -**Enrichment is a durable, quota-paced backlog**, and it must be **fair across classes**: -repository candidates alone exceed the entire hourly allowance, so a naive repo-first -policy would starve actor enrichment indefinitely — violating Story 3, which requires both. -The default ledger reserves 12 attempts for polling, 40 for draining never-enriched work, -and 8 as a safety reserve. Fairness policy with explicit rounding: +repositories across events and overlapping pages. Under the pre-Appendix-G one-request- +per-entity model, 40 core attempts/hour could not plausibly meet it; Appendix G's staged +batch path raises the theoretical service ceiling to 4,800 items/hour (a labeled +hypothesis, not proof), and `/status` publishes the **measured** arrival and completion +rates so the comparison is data rather than arithmetic. + +**Enrichment is a durable, staged, batch-served backlog** (Appendix G), and it must be +**fair across classes**. Never-enriched work is served FIFO by `created_at ASC, id ASC` +within each class. The normal path claims up to `SEARCH_BATCH_SIZE` (10) entities per +Search request under the per-minute search ledger; actor and repository batch lanes +rotate by weight (`ACTOR_ENRICHMENT_WEIGHT` / `REPOSITORY_ENRICHMENT_WEIGHT`, defaults +1/1), and a lane with no claimable work yields its slot to the other. Only items a batch +could not settle — missing, renamed, identity-mismatched, or contract-invalid — enter the +detail-fallback lane, which is bounded by the core detail allowance and split by the +fairness shares with explicit rounding: ```text actor_guarantee = floor(enrichment_allowance × ACTOR_ENRICHMENT_SHARE) repository_guarantee = enrichment_allowance − actor_guarantee -Defaults: ACTOR_ENRICHMENT_SHARE = 0.50 → 20 actor / 20 repository +Defaults: 4 × 0.50 → 2 actor / 2 repository detail-fallback attempts/hour -Borrowing: a class may borrow the other’s unused capacity only when the -other class has no CURRENTLY CLAIMABLE backlog candidate (not merely no rows). +Borrowing: a class may borrow the other’s unused detail capacity only when the +other class has no CURRENTLY CLAIMABLE detail candidate (not merely no rows). ``` -Within each class, never-enriched candidates are served FIFO by `created_at ASC, id ASC`. -Their entity rows remain durable across quota exhaustion, -window rollover, process restart, and lost enqueue hints. A denied reservation defers the -candidate; it never terminates it. A TTL-stale refresh receives no request while either -class has any never-enriched backlog work. If unique arrivals continuously exceed 40 -attempts per hour, the backlog can grow without a bounded completion estimate; `/status` -therefore reports backlog size, oldest pending timestamp/age, and reserved allowance usage. -It does not report a drain ETA because no durable outcome history exists from which to -derive an honest service rate. +Entity rows remain durable across quota exhaustion, window rollover, process restart, and +lost enqueue hints. A denied reservation — core or search, ceiling, reserve, or pacing — +defers the work; it never terminates it. **There is no quota-based terminal outcome.** +TTL-stale refreshes ride the same batch path under Appendix G's composition rule: a +batch fills from its own class's never-enriched backlog first, tops up spare slots with +refresh candidates only when its own backlog is exhausted **and** the other class has no +claimable backlog, and a refresh-only batch runs only when neither class has backlog. If +unique arrivals continuously exceed the measured service rate, the backlog grows; +`/status` reports backlog size, per-stage counts and oldest ages, measured arrival and +completion rates, and a tri-state catch-up verdict. It still publishes no drain forecast. Timing configuration (pinned defaults; tunable): ```text ACTOR_REFRESH_TTL_SECONDS = 86400 REPOSITORY_REFRESH_TTL_SECONDS = 86400 +REFRESH_ACTIVE_WITHIN_SECONDS = 604800 (refresh only recently active entities) ENRICHMENT_COVERAGE_WINDOW_SECONDS = 86400 +ENRICHMENT_METRICS_WINDOW_SECONDS = 3600 (throughput/batch-quality window) +CATCH_UP_MIN_SAMPLE_SECONDS = 900 (below this, catch-up is insufficient_sample) +ENRICHMENT_LEASE_SECONDS = 600 +ENRICHMENT_RETRY_BASE_SECONDS = 60, ENRICHMENT_RETRY_MAX_SECONDS = 3600 +ENRICHMENT_CYCLE_BUDGET_SECONDS = 55 (one cycle stays inside the dispatch tick) ``` ### Enrichment URL validation (SSRF boundary) @@ -857,6 +1005,14 @@ Enrichment follows URLs supplied inside event payloads, so a strict trust bounda - No arbitrary external URL is ever fetched - Violations mark the entity `permanent_failure` +Appendix G splits enrichment URLs into **two origins**. Search URLs are +**application-origin constants**: `Github::Enrichment::SearchQuery` builds them from a +constant host and path plus URL-encoded exact qualifiers derived from stored identity +fields, and they still pass through the same in-chain validation as every other request. +Detail URLs remain **payload-origin** and clear the full boundary above. The two never +mix: after a Search miss, the fallback fetches only the entity's stored payload-provided +`api_url` — no identifier is ever turned into a constructed detail URL. + ### Headers to process - `ETag` @@ -913,13 +1069,20 @@ actor/repo response malformed → entity permanent_failure or retryable_f Never disable the event source because one enrichment target disappeared. -### Request prioritization +### Request prioritization (amended by Appendix G) + +On the **core** resource: 1. Polling for new events (from `poll_attempt_allowance`) -2. Actor and repository enrichment (from `enrichment_allowance`, under the fairness guarantees — neither class can starve the other) -3. Refreshing stale enrichment (within each class’s share, and only when neither entity class has any never-enriched backlog work) +2. Detail fallback, only within its explicit `CORE_DETAIL_FALLBACK_ALLOWANCE` (4/hour), + under the fairness guarantees — it can never take the polling allocation +3. Nothing else spends core; the remainder is deliberately unspent headroom -Polling receives priority because raw-event capture is more time-sensitive than enrichment — and the enrichment slice is guaranteed by its own allowance rather than starved by priority alone. +The **search** resource is independent: batch enrichment — backlog first, then refresh +under Appendix G's composition rule — spends the per-minute search ledger and competes +with nothing on core. Polling receives core priority because raw-event capture is +time-sensitive; enrichment throughput lives on its own resource, so neither starves the +other by construction. ## 11. Observability @@ -938,9 +1101,31 @@ Common fields: timestamp, level, service, environment, event name, `run_id`, job - `GET /health/live` — process is running. **Never calls GitHub, never consumes budget.** - `GET /health/ready` — primary database reachable and required schema present. Same guarantee. -- `GET /status` — reports persisted state only; **never initiates a GitHub request**: +- `GET /status` — reports persisted state only; **never initiates a GitHub request**. + Top-level blocks (as amended by Appendix G): `captured_at`, `sources`, `ledger`, + `search_ledger`, `scheduler`, `enrichment`, `batches`, `throughput`, `coverage`: - poll state (scheduling components, last run) - - ledger state: window status, per-class used/allowance (`actor_requests_used/available`, `repository_requests_used/available`, poll used/allowance), `remaining`, `reset_at`, `global_blocked_until`, reserve + - core ledger state: window status, `poll` used/allowance, `detail_fallback` + used/allowance (renamed from `enrichment`), `actor_requests`/`repository_requests` + used/available, `remaining`, `reset_at`, `global_blocked_until`, reserve + - search ledger state: presence, observed limit/remaining/reset, ceiling, reserve, + spendable, used (total and per lane), `blocked_until`, `last_request_at`, and the + pacing-derived `next_request_earliest_at` + - scheduler settings, published so every tunable is visible: search + ceiling/reserve/batch size/pacing/concurrency, fairness weights and share, core + detail-fallback allowance and reserve, retry/backoff/lease/cycle timings, refresh + TTLs and activity window, metrics windows + - per-class enrichment state: the four status counts, backlog and contract-backlog + counts, oldest pending timestamp/age, and per-stage counts with oldest ages across + all seven stages; plus `claimable_now` and `next_enrichment_at` + - batch quality over the metrics window: attempts, in-flight, succeeded, failed, + deferred, stale-lease, requested/returned/valid/missing/invalid item counts, fill + ratio, and `incomplete_results` count — for all four request-kind × entity-kind + groups + - throughput: measured arrivals, completions, terminals, exits, hourly rates, and + backlog delta per lane and combined, plus a tri-state `catch_up.state` + (`keeping_up | not_keeping_up | insufficient_sample`) gated by + `CATCH_UP_MIN_SAMPLE_SECONDS`. Measured rates, never a drain forecast - enrichment coverage, computed over `ENRICHMENT_COVERAGE_WINDOW_SECONDS` with **defined formulas**: ```text @@ -956,7 +1141,9 @@ events_with_both_entities_enriched_pct = ``` - per-class backlog size and oldest pending timestamp/age, alongside reserved allowance - usage; no drain ETA without durable outcome history + usage. Appendix G supersedes the original no-drain-ETA wording: durable outcome + history now exists, so `/status` publishes measured completion and arrival rates — + but still no forecast, because a measured past rate does not bound future arrivals - `GET /api/push_events` - `GET /api/push_events/:id` @@ -984,6 +1171,11 @@ Testing focuses on correctness boundaries rather than exhaustive framework behav - Pagination stop logic (`Link`-header driven; cap / allowance / no-next / empty) - Enrichment state machine transitions, including durable quota deferral, FIFO ordering, and duplicate replay without new activity +- Staged-batch units (Appendix G): `SearchQuery` construction (repeated exact + qualifiers, never `OR`; encoding; batch-size cap), `SearchResponse` envelope parsing, + stable-ID validation including rename and identity-mismatch refusal, search-ledger + accounting (ceiling, reserve, pacing, header-less window roll, blocked_until + precedence), admission verdicts, and configuration validation for every new variable ### Persistence tests @@ -1007,6 +1199,17 @@ Testing focuses on correctness boundaries rather than exhaustive framework behav - FIFO selection by `created_at ASC, id ASC` under sustained arrivals - Refresh suppression while any never-enriched actor or repository work remains - Class fairness: repository flood cannot starve actors (and vice versa); borrowing only when the other class has no claimable backlog candidate +- Batch response matrix (Appendix G, driven by the search fixture corpus): complete, + partial, empty, malformed-envelope, renamed-repository, unrequested-result, and + `incomplete_results` responses — ID-valid items applied, everything else observed and + routed to detail fallback or retry, all in one transaction per response +- Dual-window deferral: core hourly exhaustion and search per-minute exhaustion defer + independently, and neither produces a terminal entity outcome +- Catch-up metrics: throughput window arithmetic, `insufficient_sample` below the + minimum sample, `keeping_up`/`not_keeping_up` from backlog delta and contract backlog +- Migration interplay: legacy `enrichment_status` rows map onto the staged stage + machine (`complete → contract_complete`, `retryable_failure → retry_scheduled`, + candidates → `batch_pending`) without losing durable work - Poll allowance protected from enrichment demand — and vice versa (class-blocking isolation: one class exhausted, the other proceeds) - Rate-limit exhaustion (`403` + headers → `global_blocked_until`); routine `reset_at` never defers; secondary limit blocks globally including enrichment - Per-window bootstrap: new window → counters reset → enrichment ineligible until the first poll initializes it @@ -1088,8 +1291,8 @@ File locations: `IMPLEMENTATION_PLAN.md` at the repository root; `DESIGN_BRIEF.m Must include: - Pointer to `IMPLEMENTATION_PLAN.md`, noting pre-implementation history in Git and - Appendices A–D, execution deltas in Appendix E, and the durable-backlog correction in - Appendix F + Appendices A–D, execution deltas in Appendix E, the durable-backlog correction in + Appendix F, and the staged-batch enrichment design in Appendix G - Problem overview - Architecture summary - Requirements @@ -1126,9 +1329,9 @@ Keep within one to two pages — the brief is the reviewer’s primary architect - Data model - Durability boundary - **The request-budget formula and table, and the unauthenticated `304` finding** — worded precisely: the endpoint documentation contains a general statement that `304` responses do not affect the rate limit, while the REST best-practices documentation limits that exemption to correctly authorized requests; dated unauthenticated probes showed `x-ratelimit-used` increasing across a `304`; this implementation therefore budgets unauthenticated conditional requests as one request -- **Enrichment as a durable FIFO backlog with per-class fairness; quota exhaustion defers - without terminating, refresh waits for the never-enriched pool to empty, and unbounded - growth is reported rather than hidden** +- **Enrichment as a durable, staged, batch-served FIFO backlog with per-class fairness; + quota exhaustion defers without terminating, refresh rides the same batch path after + backlog, and catch-up is measured and published rather than promised** - Duplicate-safe event persistence and restart recovery (advisory-lock ownership; outbox-style recovery; Docker restart policies) - Enrichment strategy and the SSRF boundary - Tradeoffs and assumptions (including `jsonb` semantic retention) @@ -1141,8 +1344,9 @@ Keep within one to two pages — the brief is the reviewer’s primary architect The plan’s pre-implementation revision history is preserved in Git history and summarized in Appendices A–D — the review-driven revision rounds are themselves submission-worthy -evidence of process. Appendix E records execution deltas and Appendix F the durable-backlog -correction. +evidence of process. Appendix E records execution deltas, Appendix F the durable-backlog +correction, and Appendix G the staged-batch enrichment design that supersedes Appendix F's +service model. ### Architecture Decision Records (`docs/adr/`) @@ -1218,17 +1422,20 @@ evidence for the other. - Reconciliation recovers missing enrichment scheduling - Never-enriched entity rows remain durable across quota exhaustion and window rollover; FIFO selection prevents newer arrivals from starving older work -- A selection that observes either class has never-enriched backlog work does not choose a - refresh; a concurrent insert after that read can cross at most one one-request runner cycle +- A batch claim that observes claimable never-enriched work in either class takes no + refresh candidate (Appendix G's composition rule); a concurrent insert after that read + can cross at most one already-claimed batch before the next claim suppresses refresh ### Operability - Logs are readable through `docker compose logs -f` at the default level - Correlation fields (`run_id`, job ID) are present - `/health/live` and `/health/ready` are meaningful and never consume budget -- `/status` reports window status, poll state, per-class ledger state, backlog size, oldest - pending timestamp/age, reserved allowance usage, and coverage percentages computed by the - defined formulas — without initiating GitHub requests or fabricating a drain ETA +- `/status` reports window status, poll state, both ledgers' state, backlog size and + per-stage counts, oldest pending timestamp/age, reserved allowance usage, batch quality, + measured throughput with the tri-state catch-up verdict, and coverage percentages + computed by the defined formulas — without initiating GitHub requests or fabricating a + drain ETA - Retry behavior is visible - Failures contain actionable context @@ -1253,6 +1460,8 @@ evidence for the other. - No misleading guarantee of complete upstream event capture - No claim of exactly-once execution - No claim that enrichment coverage is complete +- No claim of guaranteed catch-up — only a dated, measured comparison of completion and + arrival rates, with `/status` reporting `not_keeping_up` when the comparison fails - No failing or flaky tests ## 17. Delivery Principle @@ -1410,3 +1619,133 @@ The arithmetic in Appendix A still matters, but its conclusion changes: if uniqu continue above the 40-attempt service rate, backlog size and oldest pending age can grow without a bounded completion estimate. That is an operational fact to expose and capacity to revisit, not permission to discard durable work. + +## Appendix G — Derivation-first staged batch enrichment (2026-08-02) + +Appendix F made the backlog durable; it did not make it drain. Its service model — one +core request per entity, 40 attempts per hour — cannot plausibly meet the observed cold +demand, and authentication remains out of scope. This appendix supersedes Appendix F's +per-entity capacity assumption while preserving its durability guarantees, and amends +Sections 5, 7, 10, 11, 12, and 16 in place. A live unauthenticated probe supplied the +enabling facts: repeated exact `user:`/`repo:` Search qualifiers returned 5/5 requested +users and 9/10 requested repositories with `incomplete_results: false`; the miss was +`facebook/react` redirecting to `react/react` — a rename, which is why stable-ID +validation and a fallback exist; both responses reported the `search` rate-limit resource +with a limit of 10; and joining exact qualifiers with `OR` produced HTTP 422. + +### The decision + +- **Derivation first.** Ingestion persists event-native identity and appends an + event-source observation transactionally with each push event, derives every locally + computable field (repository `owner_login` and short `name` from `full_name`), and + stamps the event-native/derived/batch-pending instants. No network request is spent on + a fact the stored payload already determines. Duplicate demand coalesces by stable + GitHub entity ID. +- **Search batches are the normal path.** Up to `SEARCH_BATCH_SIZE` (10) repeated exact + `user:` / `repo:` qualifiers per request — joined by spaces, **never `OR`** — on the + separate per-minute search ledger (ceiling 10, reserve 2, pacing 6 seconds, 60-second + windows with a header-less roll). Results are mapped by stable integer ID, never by + result order or mutable login/name alone. +- **Payload-URL detail fallback, bounded.** Only items a batch could not settle — + missing, renamed, identity-mismatched, or contract-invalid — fetch their stored + payload-provided `api_url` through the core ledger's + `CORE_DETAIL_FALLBACK_ALLOWANCE` (4/hour). The fallback never constructs a URL from an + identifier and never touches the polling allocation. +- **Dual ledgers.** `github_api_budget` (core: 12 poll + 4 detail fallback + 8 reserve + ≤ 60, remainder deliberately unspent) and `github_search_budget` (per-minute search) + are reconciled independently against their own `x-ratelimit-resource` headers. The + global request gate still serializes all outbound requests. +- **A useful-data completion contract per entity.** Completion is an explicit, queryable + contract, not "every field GitHub can return": actors require account type plus the + complete raw search item on top of event-native identity; repositories require + description, primary language, owner GitHub ID, fork status, archived status, default + branch, and GitHub creation time plus the raw item, with short name and owner login + derived locally. Nullable fields are valid as nulls; "complete" means a valid response + was durably observed and the contract evaluated. Actor profile name, company, + location, bio, and follower counts are deliberately outside the contract. +- **Append-only observations plus batch envelopes.** Every raw item — event, search, or + detail — is an append-only `enrichment_observations` row with fingerprint, provenance, + and validation outcome; every request attempt is an `enrichment_batches` envelope with + counts, envelope facts, and observed rate-limit headers. Entity tables remain the + latest queryable projection and point at their latest successful observation. A + refresh repoints the projection; it never overwrites retained evidence. +- **A stage machine with seven resting stages.** `batch_pending`, `batch_in_flight`, + `detail_pending`, `detail_in_flight`, `retry_scheduled`, `contract_complete`, + `terminal` — check-constrained, leased (`ENRICHMENT_LEASE_SECONDS` = 600), with + instant timestamps for the conditions no row rests in (event-native, derived, batch + applied). `enrichment_status` stays the business outcome. `retry_scheduled` belongs to + the batch path; detail retries rest in `detail_pending` with `next_retry_at` — the + claimable stage sets are provably disjoint. Detail retries are bounded by + `DETAIL_FALLBACK_MAX_ATTEMPTS` (3); a 404/410 on a detail URL is an immediate + entity-specific terminal outcome that retains events, observations, reason, and + timestamps. +- **No quota-based terminal outcome, ever.** Ceiling, reserve, pacing, and fairness + denials defer work — batch rows to `deferred`, entity rows released to their resting + stage. `skipped_budget` and every trace of it are removed. Terminal outcomes exist + only for entity-specific facts. +- **Refresh through the same batch path.** There is no separate refresh request shape; + TTL-stale, recently active (`REFRESH_ACTIVE_WITHIN_SECONDS`) complete rows re-enter + the same Search batches under the composition rule below. + +### The refresh composition rule + +A batch claim fills from its own class's never-enriched backlog first, FIFO by +`created_at ASC, id ASC`. TTL-stale refresh candidates (oldest `fetched_at` first) may +top up spare slots only when the claiming class's own backlog is exhausted **and** the +other class has no claimable backlog either. A refresh-only batch runs only when neither +class has any claimable backlog. Never-enriched work therefore always outranks freshness, +in both lanes, without a separate scheduling mechanism. + +### The capacity hypothesis, and the measured gate + +The arithmetic that motivated this design is a **hypothesis, not proof**: 8 spendable +search requests per minute × batches of 10 → a theoretical ceiling of 4,800 returned +items per hour, against the 2,172–2,280 cold entities per hour of Appendix A's short +pressure sample — before misses, fallback, retries, and pacing. The acceptance gate is +therefore **measured, not derived**: `/status` publishes a `throughput` block (arrivals, +completions, terminals, hourly rates, backlog delta, per lane and combined) and a +tri-state `catch_up.state` — `keeping_up`, `not_keeping_up`, or `insufficient_sample` +below `CATCH_UP_MIN_SAMPLE_SECONDS` — and the service reports `not_keeping_up` rather +than claiming eventual catch-up when completions do not exceed arrivals. This supersedes +Appendix F's no-drain-ETA wording: durable outcome history now exists, so measured rates +are published — still no forecast, because a measured past rate does not bound future +arrivals. + +### What the live verification changed + +Three defects survived design review and the offline corpus, and were found only by +running the staged path against the real unauthenticated API +([dated transcript](docs/evidence/2026-08-02-live-staged-batch-enrichment.md)): + +1. **A payload URL that cannot be parsed is a permanent outcome, not a retryable one.** + The actor `github-actions[bot]` supplies a URL containing brackets, which + `Github::UrlPolicy` refuses before the gate. Retrying it would spend three of the four + hourly core detail requests re-refusing the same stored string. Detail fallback now + terminates on `not_found`, `client_error`, and `permanent_error` alike, matching §10's + classification table. +2. **Search answers `422` — not an empty result set — when every requested identifier is + unsearchable.** A batch of ten silently omits its unsearchable members, which is why + the exploratory probe never saw this; a batch whose members are *all* renamed or + deleted gets a validation failure instead. Read as a generic client error, the members + retried on the one lane that can never resolve them. A `422` carrying that signature is + now treated as "every requested identifier is missing", and its members take the same + fallback route an omitted item takes. +3. **A multi-queue Solid Queue worker must be declared as a YAML list.** + `queues: polling,control` names one queue literally called `"polling,control"`, because + `SolidQueue::QueueSelector` wraps its input in `Array()`. That worker registered, + heartbeated, and polled forever while claiming nothing, so the always-on container + never polled or reconciled. This predates the staged-enrichment work; the suite missed + it because the spec split the configured string itself and so asserted intent rather + than runtime behavior. Both queue specs now assert through the selector. + +The general lesson, recorded because it is the reason this gate exists: each of the three +is a case where the code and its tests agreed with each other and disagreed with the +world. + +What supersedes what: Appendix F's durable-backlog invariants (rows survive quota +windows; denial defers; FIFO by `created_at, id`; no quota terminal state) carry forward +unchanged. Its one-request-per-entity service model, its "40 backlog-enrichment requests" +core split, its refresh-suppression phrasing, and its refusal to publish any service rate +are superseded by the staged batch path, the 12 + 4 + 8 core formula with the search +budget beside it, the refresh composition rule, and the measured catch-up block. ADR 0013 +records the decision and its rejected alternatives. diff --git a/README.md b/README.md index ae9084e..7ce5876 100644 --- a/README.md +++ b/README.md @@ -10,15 +10,18 @@ collect → save → enrich → inspect flow in a few minutes. ```bash GITHUB_MODE=fixture docker compose up --build -d db setup web GITHUB_MODE=fixture docker compose run --rm ingest -GITHUB_MODE=fixture docker compose run --rm enrich --limit 6 +GITHUB_MODE=fixture docker compose run --rm -e SEARCH_PACING_SECONDS=0 enrich --limit 6 curl -s http://localhost:3000/api/push_events ``` On a fresh project database, ingestion saves **4 valid push events**, quarantines **3 -malformed events**, and ignores **1 event that is not a push**. Enrichment then attempts all -**6 actor/repository backlog rows**: **4 complete successfully**, while **2 intentional -`404` outcomes** become permanent failures for a deleted user and repository. Nothing in -this walkthrough contacts GitHub or consumes real API quota. +malformed events**, and ignores **1 event that is not a push**. Enrichment then serves all +**6 actor/repository backlog rows in 4 requests**: two Search batches enrich **4 +entities**, and the two entities the batches could not answer fall back to their detail +URLs, meet **intentional `404`s**, and become permanent failures for a deleted user and +repository. (`SEARCH_PACING_SECONDS=0` lets the second batch run immediately instead of +waiting out the default 6-second search pacing.) Nothing in this walkthrough contacts +GitHub or consumes real API quota. Already used this project and seeing different totals or a deferred poll? The database is preserved between runs. Follow [Deterministic fixture verification](#deterministic-fixture-verification) @@ -68,19 +71,22 @@ What it does, running: processes every fetched page in full. - **Persists** each `PushEvent` with typed columns and its raw `jsonb` payload, quarantining malformed envelopes durably instead of dropping them or failing the batch. -- **Enriches** actors and repositories through the same budget-governed request chain, - under per-class fairness guarantees. +- **Enriches** actors and repositories through the same gated request chain — batched + Search requests as the normal path, bounded detail fallbacks for what batches cannot + settle — with per-class fairness in both lanes. - **Recovers** on its own: advisory locks die with their session, entity leases expire by arithmetic, and a 60-second reconciler rebuilds pending work from committed rows. -**Enrichment is a durable, quota-paced backlog.** With the defaults, the hourly ledger -reserves 12 requests for polling, 40 for the enrichment class, and 8 as a safety reserve. -Durable backlog work has priority over refreshes within those 40 attempts. Actors and -repositories receive 20/20 guarantees with borrowing. Entity rows remain actionable until -enrichment succeeds or an entity-specific terminal outcome is established; quota exhaustion -only defers them to a later window. Selection remains FIFO, oldest-first within each class, -and delay never becomes a terminal outcome. See [Known -limitations](#known-limitations) for what happens when arrivals outpace that service rate. +**Enrichment is a durable, staged, batch-served backlog.** The normal path resolves up to +ten entities per GitHub Search request on Search's own per-minute budget (10 ceiling, 2 +reserved, 6-second pacing); only items a batch could not settle fall back to individual +payload-URL fetches inside a bounded core allowance of 4 per hour. The core ledger keeps +12 requests for polling and 8 in reserve, leaving the rest deliberately unspent. Entity +rows remain actionable until enrichment satisfies the useful-data contract or an +entity-specific terminal outcome is established; quota exhaustion, pacing, and reserve +denials only defer them. Selection remains FIFO, oldest-first within each class, and delay +never becomes a terminal outcome. `/status` measures arrival and completion rates and says +whether the service is keeping up — see [Known limitations](#known-limitations). **With the default `GITHUB_MODE=live`, `docker compose up` starts spending real unauthenticated quota** — twelve poll requests an hour at the default cadence, plus @@ -97,8 +103,9 @@ counters, budget use, and logs may repeat. This system does not claim exactly-on best place to start. The authoritative execution plan is [`IMPLEMENTATION_PLAN.md`](IMPLEMENTATION_PLAN.md) — its pre-implementation revision history lives in Git and in its Appendices A–D, and Appendix E records how the build diverged from -it. Appendix F records the durable-enrichment-backlog correction. Delivery is tracked on the -[GitHub Push Ingestor Delivery project](https://github.com/users/batbrainy/projects/1). +it. Appendix F records the durable-enrichment-backlog correction, and Appendix G the +derivation-first staged batch enrichment design built on top of it. Delivery is tracked on +the [GitHub Push Ingestor Delivery project](https://github.com/users/batbrainy/projects/1). ## Requirements @@ -253,7 +260,7 @@ grep -E 'Push events created:[[:space:]]+4' "$fixture_ingest_output" grep -E 'Events quarantined:[[:space:]]+3' "$fixture_ingest_output" grep -E 'Non-push events ignored:[[:space:]]+1' "$fixture_ingest_output" -GITHUB_MODE=fixture docker compose run --rm enrich --limit 6 +GITHUB_MODE=fixture docker compose run --rm -e SEARCH_PACING_SECONDS=0 enrich --limit 6 docker compose exec db psql -U postgres -d github_push_ingestor_development -c " SELECT (SELECT COUNT(*) FROM push_events) AS push_events, (SELECT COUNT(*) FROM github_actors) AS actors, @@ -480,15 +487,29 @@ budget use, and logs may still change. See ### Enrichment, offline The same corpus resolves every entity the page referenced, so the whole flow — poll, -persist, stub, enrich — runs with no network: +persist, stub, batch, fall back — runs with no network: ```bash -GITHUB_MODE=fixture docker compose run --rm enrich --limit 6 +GITHUB_MODE=fixture docker compose run --rm -e SEARCH_PACING_SECONDS=0 enrich --limit 6 ``` -Four of the six entities resolve, and two are gone. That is deliberate: corpus event -`58000000008` references an actor and a repository that both `404`, because §10 -requires a dead enrichment target to fail the *entity* and leave the source running. +```bash +docker compose run --rm enrich --limit 3 # up to 3 requests, batches first +docker compose run --rm enrich --stage batch # Search batches only +docker compose run --rm enrich --stage detail # detail fallbacks only +docker compose run --rm enrich --class actor --limit 2 # one lane, still both ledgers +docker compose run --rm enrich --help # options and exit codes +``` + +`--limit` counts **requests**, not entities, and `--stage batch` / `--stage detail` +restrict the run to one lane. Here the six backlog rows are served in four requests: an +actor Search batch and a repository Search batch each answer two of their three entities +by stable ID, and the two missing ones — corpus event `58000000008` references an actor +and a repository that are gone — fall back to their stored detail URLs, meet `404`s, and +go terminal, because §10 requires a dead enrichment target to fail the *entity* and leave +the source running. The pacing override matters only for back-to-back batches: at the +default 6 seconds the second batch would report `Search deferred — search_pacing` instead +of running immediately. ```bash docker compose exec db psql -U postgres -d github_push_ingestor_development -c " @@ -502,21 +523,28 @@ docker compose exec db psql -U postgres -d github_push_ingestor_development -c " # repository | permanent_failure | 1 ``` -Six requests, split evenly, and the event source untouched by either `404`: +Two search requests on the per-minute search ledger, two detail fallbacks on the core +ledger, and the event source untouched by either `404`: ```bash docker compose exec db psql -U postgres -d github_push_ingestor_development -c " + SELECT used, actor_used, repository_used FROM github_search_budget; SELECT enrichment_used, actor_share_used, repository_share_used FROM github_api_budget; SELECT status, enabled FROM event_sources;" +# used | actor_used | repository_used +# 2 | 1 | 1 # enrichment_used | actor_share_used | repository_share_used -# 6 | 3 | 3 +# 2 | 1 | 1 # status | enabled # idle | t ``` Run `enrich` again and it reports `Nothing to enrich — no eligible candidate`: every -entity is either enriched and inside its refresh TTL, or permanently failed. That is -the freshness cache, and it costs nothing. +entity is either at its useful-data contract and inside its refresh TTL, or terminal. +That is the freshness cache, and it costs nothing. The evidence trail persists too — +`enrichment_batches` holds the four request envelopes with their counts and observed +rate-limit headers, and `enrichment_observations` the append-only raw items behind each +projection. ### Pagination, offline @@ -542,9 +570,11 @@ plan §9's rule that every fetched page is processed in full and `github_event_i uniqueness absorbs the overlap — there is no stop-on-known-event, because documented event latency is 30 seconds to 6 hours and a delayed event can surface beside one already seen. And **raising the cap is not free**: at -`MAX_PAGES_PER_POLL=3` the poll allowance becomes 12 × 3 = 36 attempts an hour and -enrichment drops from 40 to 60 − 8 − 36 = 16. That is plan §9's "raising it trades -enrichment allowance for capture depth", as arithmetic. +`MAX_PAGES_PER_POLL=3` the poll allowance becomes 12 × 3 = 36 attempts an hour, and the +core headroom left deliberately unspent shrinks from 36 to 60 − 36 − 8 − 4 = 12; one more +page and it is zero, and at 5 the formula is infeasible and boot refuses. Since plan +Appendix G, capture depth spends headroom rather than enrichment capacity — batch +enrichment lives on the separate search budget. At `MAX_PAGES_PER_POLL=2` the counts are identical except `Pages fetched: 2` — page 3 is empty — and the stop reason on the `ingestion.pagination_stopped` debug line @@ -560,13 +590,18 @@ bypass, so allow a minute between successive ingestion scenarios. |---|---|---|---| | `default` | `GITHUB_MODE=fixture docker compose run --rm ingest` | 4 push events, 3 actors, 3 repositories, 3 quarantined | none | | `default` (replay) | `… run --rm ingest --force` after 60s | `duplicates_skipped > 0`, occurrence counts climb, entity activity does not move | none | -| `default` (enrich) | `… run --rm enrich --limit 6` | both classes enrich within their fairness shares | none | +| `default` (enrich) | `… run --rm -e SEARCH_PACING_SECONDS=0 enrich --limit 6` | two Search batches enrich four entities; two misses fall back to detail `404` terminals | none | | `paginated` | `GITHUB_FIXTURE_SCENARIO=paginated MAX_PAGES_PER_POLL=3 … ingest` | `Link`-driven walk over 3 pages | none | | `paginated_final_page` | `GITHUB_FIXTURE_SCENARIO=paginated_final_page … ingest` | the walk stops when no `next` link exists | none | | `transient_failure` | `GITHUB_FIXTURE_SCENARIO=transient_failure … ingest` | one `500`, then success on retry | none | | `transient_failure_exhausted` | `GITHUB_FIXTURE_SCENARIO=transient_failure_exhausted … ingest` | retries exhausted; the source backs off | source backoff | -| `redirecting_repository` | `GITHUB_FIXTURE_SCENARIO=redirecting_repository … enrich --class repository` | a renamed repository followed across one validated hop, debited twice | none | -| `hostile_redirect` | `GITHUB_FIXTURE_SCENARIO=hostile_redirect … enrich --class repository` | an off-host `Location` refused by the URL policy; the second hop is never sent and the event source stays in service | none | +| `search_complete` | `GITHUB_FIXTURE_SCENARIO=search_complete … -e SEARCH_PACING_SECONDS=0 enrich --limit 2` | both batches return every requested entity — the whole backlog completes in two requests, no fallback | none | +| `search_renamed_repository` | `GITHUB_FIXTURE_SCENARIO=search_renamed_repository … enrich --stage batch` | an ID-intact rename is refused by validation (`renamed_repository`) and admitted to detail fallback rather than applied | none | +| `search_incomplete_results` | `GITHUB_FIXTURE_SCENARIO=search_incomplete_results … enrich --stage batch` | `incomplete_results: true` is an envelope fact — ID-valid items still apply, the missing one still falls back | none | +| `search_malformed` | `GITHUB_FIXTURE_SCENARIO=search_malformed … enrich --stage batch` | a malformed Search envelope fails the batch and reschedules every member; nothing is applied | batch retry backoff | +| `search_rate_limited` | `GITHUB_FIXTURE_SCENARIO=search_rate_limited … enrich --stage batch` | search-window exhaustion defers the batch; the core ledger and polling are untouched | search block (≤ 1 minute) | +| `redirecting_repository` | `GITHUB_FIXTURE_SCENARIO=redirecting_repository … enrich --class repository` | a Search miss falls back to detail and meets a `301`; the renamed repository is followed across one validated hop, debited twice | none | +| `hostile_redirect` | `GITHUB_FIXTURE_SCENARIO=hostile_redirect … enrich --class repository` | the same Search miss, but the `Location` points off-host and the URL policy refuses it; the second hop is never sent and the event source stays in service | none | | `secondary_rate_limited` | `GITHUB_FIXTURE_SCENARIO=secondary_rate_limited … ingest` | a secondary limit blocks globally | global block | | `rate_limited` | `GITHUB_FIXTURE_SCENARIO=rate_limited … ingest` | primary exhaustion → `global_blocked_until` | **a real one-hour global block** | @@ -588,39 +623,73 @@ curl -s http://localhost:3000/api/push_events/ | jq .data.raw_p ### `GET /status` -Reports persisted state only: the poll schedule per event source (all five of §9's -components, plus which one is binding), the per-class ledger state, §11's three -enrichment coverage percentages, the per-status entity counts, and durable-backlog -telemetry: per-class size, oldest pending timestamp/age, and reserved allowance usage. +Reports persisted state only, as nine top-level blocks in a fixed order: `captured_at`, +`sources`, `ledger`, `search_ledger`, `scheduler`, `enrichment`, `batches`, `throughput`, +`coverage`. + +- **`sources`** — the poll schedule per event source: all five of §9's components, plus + which one is binding. +- **`ledger`** — the core hourly ledger: window status, `poll` used/allowance, + **`detail_fallback`** used/allowance (renamed from `enrichment` — it budgets the + bounded detail lane, default 4/hour), and the `actor_requests`/`repository_requests` + share pairs. +- **`search_ledger`** — the per-minute Search ledger: `present`, the observed + resource/limit/remaining/reset, the configured `request_ceiling` and `reserve`, the + derived `spendable`, `used` with per-lane `actor_used`/`repository_used`, + `blocked_until`, `last_request_at`, and the pacing-derived + `next_request_earliest_at`. +- **`scheduler`** — every staged tunable, published so a reading needs no shell access: + the search ceiling/reserve/batch size/pacing/concurrency, the fairness weights and + share, the core detail-fallback allowance and rate-limit reserve, the + retry/backoff/lease/cycle timings, the refresh TTLs and activity window, and the + metrics windows. +- **`enrichment`** — per entity class: the four `enrichment_status` counts, + `backlog_count`, `contract_backlog_count` (rows not yet at the useful-data contract or + a terminal outcome), oldest pending timestamp/age, and a `stages` map over all seven + resting stages, each with its count, oldest `created_at`, and oldest age. Plus + `claimable_now` and `next_enrichment_at` across both classes. +- **`batches`** — batch quality over the metrics window, for all four request-kind × + entity-kind groups (search/detail × actors/repositories, zeros counted): attempts, + in-flight, succeeded, failed, deferred, stale-lease, requested/returned/valid/missing/ + invalid item counts, the fill ratio (`null` when nothing was requested), and how many + envelopes carried `incomplete_results`. +- **`throughput`** — measured rates over the same window: arrivals, completions, + terminals, exits, hourly rates, and backlog delta, per lane and combined, with the + sample bounds published. `catch_up.state` is tri-state — `keeping_up`, + `not_keeping_up`, or `insufficient_sample` until the sample reaches + `CATCH_UP_MIN_SAMPLE_SECONDS`. There is deliberately no ETA anywhere: the rates are + measured facts, and a forecast from them would not be. +- **`coverage`** — §11's three coverage percentages, unchanged. Three conventions in the response body are worth knowing before reading one: - **`null` is never a zero.** A counted zero prints `0`; a number that does not exist prints `null`. Where `null` would be ambiguous the disambiguating fact gets - its own field — `ledger.present` separates "no ledger row yet" from "remaining is - genuinely 0", `due_now` separates "no constraint applies" from "unknown", and - `claimable_now` says whether work is eligible under persisted backlog and ledger state; + its own field — `ledger.present` and `search_ledger.present` separate "no ledger row + yet" from "remaining is genuinely 0", `due_now` separates "no constraint applies" from + "unknown", `claimable_now` says whether work is eligible under persisted backlog and + ledger state, and `catch_up.state` says `insufficient_sample` rather than guessing; `backlog_count`, `next_enrichment_at`, and the ledger window/block fields distinguish deferred work from an empty backlog. An empty coverage window reports `null` - percentages, not `0.0`: the ratio is undefined, not zero, and every denominator is - published beside its ratio so you can check it. + percentages, not `0.0`, and a batch group that requested nothing reports a `null` fill + ratio: the ratio is undefined, not zero, and every denominator is published beside its + ratio so you can check it. - **The coverage window is measured on `created_at`** — when *this application* persisted the event, not GitHub's `occurred_at`. Coverage grades this - application's enrichment pipeline and uses the same local clock as backlog and - freshness reporting. The basis is published as + application's enrichment pipeline and uses the same local clock as backlog, batch, and + throughput reporting. The basis is published as `coverage.basis` so the choice is visible rather than assumed. Widen `ENRICHMENT_COVERAGE_WINDOW_SECONDS` when reviewing a fixture corpus that has aged. - **`actor_requests.available` is a floor, not a ceiling.** §10 lets one class - borrow the other's unspent capacity when the other has no claimable backlog candidate, so a - class does not stop at zero available. The real ceiling is the `enrichment` pair - beside it. + borrow the other's unspent detail capacity when the other has no claimable detail + candidate, so a class does not stop at zero available. The real ceiling is the + `detail_fallback` pair beside it. `pending` in the entity counts means `enrichment_status = 'pending'` exactly. The `backlog_count` beside it is pending **plus** `retryable_failure` — the "how much work is -left" number the command summary prints. That work has no age cutoff. `/status` -deliberately does not publish a drain ETA: the schema has no durable outcome history from -which to calculate an honest service rate, and sustained arrivals can make the backlog grow -indefinitely. +left" number the command summary prints — and `contract_backlog_count` is the staged +version of the same question, counted by stage rather than status. That work has no age +cutoff. ### `GET /api/push_events` and `GET /api/push_events/:id` @@ -683,6 +752,27 @@ SELECT window_status, "limit", remaining, reserve, FROM github_api_budget; ``` +(`enrichment_used`/`enrichment_allowance` budget the detail-fallback lane — plan +Appendix G.) + +**The search budget ledger** — the per-minute resource beside it + +```sql +SELECT "limit", remaining, request_ceiling, reserve, used, + actor_used, repository_used, blocked_until, last_request_at, reset_at + FROM github_search_budget; +``` + +**Batch quality and evidence** — what each enrichment request asked and got + +```sql +SELECT request_kind, entity_kind, status, requested_count, returned_count, + valid_count, missing_count, invalid_count, incomplete_results, started_at + FROM enrichment_batches ORDER BY started_at DESC LIMIT 10; +SELECT entity_kind, entity_github_id, source, validation_outcome, observed_at + FROM enrichment_observations ORDER BY observed_at DESC LIMIT 10; +``` + **The last five runs and what they did** ```sql @@ -718,13 +808,16 @@ docker compose exec db psql -U postgres -d github_push_ingestor_queue_developmen ## The data model -Seven business tables in three groups: +Ten business tables in four groups: - **Source and run state** — `event_sources` (what to poll and when), `ingestion_runs` (what each attempt did). - **Business records** — `push_events`, `github_actors`, `github_repositories`, `quarantined_events`. -- **The global ledger** — `github_api_budget`, one row. +- **Enrichment evidence** — `enrichment_observations` (append-only raw items with + provenance) and `enrichment_batches` (one envelope per enrichment request attempt). +- **The two ledgers** — `github_api_budget` (core, hourly) and `github_search_budget` + (search, per-minute), one row each. A second database, `github_push_ingestor_queue_development`, holds Solid Queue's tables. Queued enrichment dispatches are hints, not the pending-enrichment source of truth: removing @@ -738,10 +831,13 @@ not claimed to be reconstructible. See | `event_sources` | one pollable feed and its schedule | `source_type` | `Ingestion::PollState`, `SourceProvisioner` | | `ingestion_runs` | one poll attempt that reached GitHub | `run_id` (uuid, unique) | `Ingestion::RunRecorder` | | `push_events` | one accepted GitHub `PushEvent` | `github_event_id` (unique) | `Ingestion::PageWriter` | -| `github_actors` | one GitHub user seen pushing | `github_id` (unique) | `PageWriter` (stub), `Enrichment::EntityState` | -| `github_repositories` | one GitHub repository seen receiving a push | `github_id` (unique) | `PageWriter` (stub), `Enrichment::EntityState` | +| `github_actors` | one GitHub user seen pushing — the latest projection | `github_id` (unique) | `PageWriter` (stub + derivation), `Enrichment::BatchRunner`, `Enrichment::DetailRunner` | +| `github_repositories` | one GitHub repository seen receiving a push — the latest projection | `github_id` (unique) | `PageWriter` (stub + derivation), `Enrichment::BatchRunner`, `Enrichment::DetailRunner` | | `quarantined_events` | one distinct malformed payload | `payload_fingerprint` (unique) | `Ingestion::PageWriter` | -| `github_api_budget` | the hourly request budget | `id = 1` (check constraint) | `Github::BudgetLedger` | +| `enrichment_observations` | one raw observed item — append-only, read-only after persist | `id`; fingerprint + `observed_at` describe content | `Ingestion::PageWriter` (event source), `Enrichment::ObservationRecorder` (search/detail) | +| `enrichment_batches` | one enrichment request attempt (search batch or detail fallback) | `correlation_id` (uuid, unique) | `Enrichment::BatchClaim`, both runners | +| `github_api_budget` | the hourly core request budget | `id = 1` (check constraint) | `Github::BudgetLedger` | +| `github_search_budget` | the per-minute Search request budget | `id = 1` (check constraint) | `Github::SearchBudgetLedger` | ### Columns that carry a rule @@ -758,16 +854,22 @@ not claimed to be reconstructible. See | `occurred_at` | `timestamp` `NOT NULL`, indexed | GitHub's `created_at`. Distinct from this row's `created_at`, which is when *this* application persisted it | | `raw_payload` | `jsonb` `NOT NULL` | Retention is **semantic, not byte-exact** ([ADR 0001](docs/adr/0001-jsonb-semantic-retention.md)) | -**`github_actors` / `github_repositories`** — identical enrichment state machines. +**`github_actors` / `github_repositories`** — identical staged enrichment machines. | Column | Type | The rule it encodes | |---|---|---| -| `github_id` | `bigint` **unique** | Identity. The stub upsert conflicts on this | -| `enrichment_status` | `text`, check-constrained | Four legal values; see below | +| `github_id` | `bigint` **unique** | Identity. The stub upsert conflicts on this, and batch results apply only when the returned item's ID matches it | +| `enrichment_status` | `text`, check-constrained | Four legal business outcomes; see below | +| `enrichment_stage` | `text`, check-constrained | The staged pipeline's seven **resting** stages; see below | | `first_seen_at`, `last_seen_at`, `latest_event_at` | `timestamp` | Activity. Updated **only** when a `push_events` insert actually returned a row | -| `next_retry_at` | `timestamp` | Double duty: the backoff instant *and* the enrichment lease. One column, one predicate, so candidate selection and claiming cannot drift apart | +| `next_retry_at` | `timestamp` | The backoff instant — batch retries and detail retries alike wait on it | +| `lease_token`, `leased_until` | `uuid`, `timestamp` | The durable claim lease. Expires by arithmetic (`ENRICHMENT_LEASE_SECONDS`, 600); projection writes are guarded by token + batch id, so a stale worker cannot double-apply | +| `event_native_at` … `terminal_at` | `timestamp` | Keep-first instants for every observable condition — event-native persisted, derived, batch pending/applied, detail pending, retry scheduled, contract complete, terminal | | `fetched_at` | `timestamp` | When enrichment last succeeded — the input to the refresh TTL | -| `raw_payload` | `jsonb` nullable | The enrichment response. Null until a fetch succeeds | +| `latest_observation_id`, `latest_observation_source`, `latest_observed_at` | FK, `text`, `timestamp` | The projection's pointer into the append-only observation history — a refresh repoints it, never overwrites evidence | +| `detail_attempts` | `integer` | The detail-fallback ladder, terminal at `DETAIL_FALLBACK_MAX_ATTEMPTS` (3) | +| `raw_payload` | `jsonb` nullable | The latest applied enrichment item. Null until one applies | +| contract columns | actors: `account_type`; repositories: `owner_login` (derived), `owner_github_id`, `fork`, `archived`, `default_branch`, `github_created_at` | The useful-data completion contract. Nullable values returned by GitHub stay valid nulls | **`event_sources`** — five independent scheduling columns, never one collapsed timestamp: `cadence_due_at`, `poll_floor_until`, `retry_not_before_at`, plus the ledger's @@ -788,19 +890,46 @@ non-negative; `consecutive_secondary_limits` survives window rollover because se limits are IP-scoped rather than window-scoped ([ADR 0010](docs/adr/0010-secondary-limit-escalation-and-refresh-pool-fairness.md)). -### The enrichment state machine +### The staged enrichment pipeline -| Status | Entered when | Left when | Spends budget | -|---|---|---|---| -| `pending` | Ingestion creates a stub row | Success, a retryable outcome, or a permanent outcome | When a request is admitted; not for a URL-policy rejection | -| `complete` | An enrichment fetch succeeds | Stays complete while fresh; after TTL it may refresh once the backlog is observed empty. Transient refresh failures keep it complete; terminal outcomes may make it `permanent_failure` | When a refresh request is admitted | -| `retryable_failure` | A retryable entity or transport outcome occurs | Retried after backoff; leaves on success or a permanent outcome | When a request is admitted | -| `permanent_failure` | A permanent response, document, redirect/policy, or transport outcome occurs | Never, automatically | Every admitted outbound attempt; a pre-gate URL rejection spends none | +`enrichment_status` stays the business outcome — `pending`, `complete`, +`retryable_failure`, `permanent_failure` — and `enrichment_stage` says where in the +pipeline the row rests: -`pending` and `retryable_failure` rows are the durable backlog. Within each entity class -they are selected FIFO, oldest first, and quota exhaustion only defers them to a later -window. A duplicate replay may refresh identity fields but does not register new activity -or change backlog priority. +```text +batch_pending ──► batch_in_flight ──┬──► contract_complete + ▲ │ + └── retry_scheduled ◄─────────┤ (batch failed — backoff, then re-batch) + │ + └──► detail_pending ──► detail_in_flight + ▲ │ + └── retry backoff ◄─┼──► contract_complete + └──► terminal +``` + +- **`batch_pending`** — durable backlog, claimed FIFO (`created_at, id`) into Search + batches of up to `SEARCH_BATCH_SIZE`. +- **`batch_in_flight` / `detail_in_flight`** — leased to one request attempt; a crashed + worker's lease expires by arithmetic and the rows are reclaimed + (`enrichment.stale_lease_reclaimed`). +- **`detail_pending`** — admitted to the fallback lane because the batch came back + missing, renamed, identity-mismatched, or contract-invalid for this row. Detail + retries rest here with `next_retry_at`; **`retry_scheduled` belongs to the batch path + only**, so the two lanes' claimable sets are provably disjoint. +- **`contract_complete`** — the useful-data contract evaluated against a durably + observed, ID-validated item. Nullable fields count as evaluated, not missing. +- **`terminal`** — an entity-specific fact only: a `404`/`410` immediately, or a detail + ladder exhausted at `DETAIL_FALLBACK_MAX_ATTEMPTS` (3). The events, observations, + reason, and timestamps all remain. **No stage and no status is ever entered because + budget ran out** — every ledger denial defers and releases. + +Refreshes ride the same stages: a TTL-stale, recently active `complete` row re-enters a +Search batch (so `complete` legally pairs with `batch_in_flight` and, on fallback, the +detail stages) only under the composition rule — a batch fills from its own class's +never-enriched backlog first, tops up spare slots with refresh candidates only when its +own backlog is exhausted and the other class has none claimable, and a refresh-only batch +runs only when neither class has backlog. A duplicate event replay may refresh identity +fields but does not register new activity or change backlog priority. ### Replay behavior by table @@ -809,6 +938,7 @@ or change backlog priority. | `push_events` | `INSERT … ON CONFLICT (github_event_id) DO NOTHING RETURNING id` | No-op; the accepted raw event is never mutated | | `github_actors`, `github_repositories` | `INSERT … ON CONFLICT (github_id) DO UPDATE` on identity fields only | Identity refreshed; enrichment payload untouched | | `quarantined_events` | `INSERT … ON CONFLICT (payload_fingerprint) DO UPDATE` | `occurrence_count` increments; the first classification is permanent | +| `enrichment_observations` | append-only `INSERT`, read-only after persist | A re-observation is a new row with its own `observed_at`; nothing is overwritten | | Entity activity fields | Gated on the `push_events` insert returning a row | No activity registered | Transactions are **one per event**, not one per page, so a single malformed envelope can @@ -829,7 +959,11 @@ planner scans only rows that could possibly qualify: - `index_github_{actors,repositories}_on_enrichment_refresh` on `(fetched_at, next_retry_at)` `WHERE enrichment_status = 'complete'` — the TTL refresh pool. -[`db/schema.rb`](db/schema.rb) is authoritative (version `2026_08_02_000000`). For live +Two more serve the staged pipeline: `index_github_{actors,repositories}_on_stage_fifo` on +`(enrichment_stage, created_at, id)` — the per-stage FIFO claim scan — and a plain index +on `leased_until` for stale-lease reclaim. + +[`db/schema.rb`](db/schema.rb) is authoritative (version `2026_08_02_010000`). For live truth: ```bash @@ -891,12 +1025,12 @@ and the JSON stream, so a leading brace is exactly what separates them. Boot, then a poll that created four events and quarantined three: ```text -{"timestamp":"2026-07-31T15:50:49.677Z","level":"info","service":"github-push-ingestor","environment":"development","event":"config.budget_resolved","mode":"fixture","poll_interval_seconds":300,"max_pages_per_poll":1,"enabled_live_source_count":1,"worst_case_reservations_per_poll":9,"limit":60,"reserve":8,"poll_allowance":12,"enrichment_allowance":40,"actor_guarantee":20,"repository_guarantee":20} +{"timestamp":"2026-07-31T15:50:49.677Z","level":"info","service":"github-push-ingestor","environment":"development","event":"config.budget_resolved","mode":"fixture","poll_interval_seconds":300,"max_pages_per_poll":1,"enabled_live_source_count":1,"worst_case_reservations_per_poll":9,"limit":60,"reserve":8,"poll_allowance":12,"enrichment_allowance":4,"actor_guarantee":2,"repository_guarantee":2} {"timestamp":"2026-07-31T15:50:49.896Z","level":"info","service":"github-push-ingestor","environment":"development","event":"ingestion.run_started","run_id":"099d562d-1261-488d-9003-cb0c443cdb55","event_source_id":1,"source_type":"github_fixture_events","github_mode":"fixture","forced":false,"lock_wait_ms":2.2} -{"timestamp":"2026-07-31T15:50:49.924Z","level":"info","service":"github-push-ingestor","environment":"development","event":"budget.window_initialized","limit":60,"reserve":8,"poll_allowance":12,"enrichment_allowance":40,"actor_guarantee":20,"repository_guarantee":20,"rate_limit_resource":"core","rate_limit_limit":60,"rate_limit_remaining":59,"rate_limit_used":1,"rate_limit_reset_at":"2026-07-31T16:50:49Z","poll_used":1} +{"timestamp":"2026-07-31T15:50:49.924Z","level":"info","service":"github-push-ingestor","environment":"development","event":"budget.window_initialized","limit":60,"reserve":8,"poll_allowance":12,"enrichment_allowance":4,"actor_guarantee":2,"repository_guarantee":2,"rate_limit_resource":"core","rate_limit_limit":60,"rate_limit_remaining":59,"rate_limit_used":1,"rate_limit_reset_at":"2026-07-31T16:50:49Z","poll_used":1} {"timestamp":"2026-07-31T15:50:49.965Z","level":"info","service":"github-push-ingestor","environment":"development","event":"ingestion.event_quarantined","run_id":"099d562d-1261-488d-9003-cb0c443cdb55","github_event_id":"58000000006","event_type":"PushEvent","error_code":"invalid_field_format","error_message":"payload.head is \"not-a-valid-object-name\", not 40 or 64 hexadecimal characters","payload_fingerprint":"a8ad67ca97a4c48049f5fa447d5d88ae10c58c514e0129546e18b5ff22368020"} {"timestamp":"2026-07-31T15:50:49.973Z","level":"info","service":"github-push-ingestor","environment":"development","event":"ingestion.run_completed","run_id":"099d562d-1261-488d-9003-cb0c443cdb55","event_source_id":1,"duration_ms":100.7,"next_poll_at":"2026-07-31T15:55:49Z","consecutive_failures":0,"run_status":"completed","classification":"ok","stop_reason":"no_next_link","pages_fetched":1,"events_received":8,"push_events_seen":6,"events_created":4,"duplicates_skipped":0,"events_quarantined":3,"events_ignored":1,"events_failed":0} -{"timestamp":"2026-07-31T15:50:50.024Z","level":"info","service":"github-push-ingestor","environment":"development","event":"enrichment.dispatched","actor_enqueued":1,"repository_enqueued":1,"reason":"ingestion","actor_counts":{"pending":3},"repository_counts":{"pending":3},"actor_backlog_count":3,"repository_backlog_count":3,"actor_oldest_pending_age_seconds":0,"repository_oldest_pending_age_seconds":0,"actor_share_used":0,"repository_share_used":0,"actor_guarantee":20,"repository_guarantee":20,"enrichment_used":0,"enrichment_allowance":40,"window_status":"active","claimable_now":true} +{"timestamp":"2026-07-31T15:50:50.024Z","level":"info","service":"github-push-ingestor","environment":"development","event":"enrichment.dispatched","cycle_enqueued":1,"reason":"ingestion"} ``` A second run inside the cadence window makes no request at all, and names the component @@ -906,13 +1040,14 @@ holding it: {"timestamp":"2026-07-31T15:51:00.142Z","level":"info","service":"github-push-ingestor","environment":"development","event":"ingestion.not_due","event_source_id":1,"forced":false,"deferral_reason":"cadence_due_at","next_poll_at":"2026-07-31T15:55:49Z","cadence_due_at":"2026-07-31T15:55:49Z","poll_floor_until":"2026-07-31T15:51:49Z"} ``` -From the worker — a job, its enrichment outcome, and the deliberate `404` the corpus plants -so a dead target fails the *entity* rather than the source: +From the worker — a Search batch that applied two of its three members and admitted the +third to detail fallback, and the deliberate `404` the corpus plants so a dead target +fails the *entity* rather than the source: ```text -{"timestamp":"2026-07-31T15:50:50.780Z","level":"info","service":"github-push-ingestor","environment":"development","event":"job.completed","job_id":"58a1e78d-f473-4440-aee0-fe0bbb22027f","job_class":"EnrichActorJob","queue":"enrichment","attempt":1,"duration_ms":74.6,"entity_type":"actor","github_actor_id":7700421,"enrichment_outcome":"failed"} -{"timestamp":"2026-07-31T15:50:50.780Z","level":"info","service":"github-push-ingestor","environment":"development","enrichment_outcome":"failed","entity_type":"actor","github_id":7700421,"pool":"pending","classification":"not_found","entity_status":"permanent_failure","enrichment_attempt":1,"error_message":"GitHub returned 404 (not_found)","duration_ms":56.6,"event":"enrichment.failed"} -{"timestamp":"2026-07-31T15:51:00.949Z","level":"info","service":"github-push-ingestor","environment":"development","enrichment_outcome":"enriched","entity_type":"actor","github_id":1024025,"pool":"pending","classification":"ok","entity_status":"complete","enrichment_attempt":1,"duration_ms":27.3,"event":"enrichment.completed"} +{"timestamp":"2026-07-31T15:50:50.780Z","level":"info","service":"github-push-ingestor","environment":"development","event":"enrichment.batch_completed","status":"completed","entity_type":"actor","batch_id":1,"requested_count":3,"returned_count":2,"valid_count":2,"fallback_count":1,"deferral_reason":null,"incomplete_results":false} +{"timestamp":"2026-07-31T15:50:50.781Z","level":"info","service":"github-push-ingestor","environment":"development","event":"enrichment.fallback_admitted","entity_type":"actor","github_actor_id":7700421,"reason":"missing_search_result","enrichment_batch_id":1} +{"timestamp":"2026-07-31T15:51:00.949Z","level":"warn","service":"github-push-ingestor","environment":"development","event":"enrichment.detail_terminal","entity_type":"actor","github_actor_id":7700421,"detail_attempts":1,"reason":"entity_gone_404"} ``` A retry ladder, from the `transient_failure` scenario — the failed request at `warn`, the @@ -955,13 +1090,17 @@ means this container's clock runs ahead of GitHub's and enrichment will stay ine until it does not; and `config.amplification` at boot when retries and redirects multiply to more request attempts per poll than the whole poll allowance. -Enrichment adds `enrichment.completed` and `enrichment.failed`, each carrying the -entity type, its GitHub id, the response classification, the resulting entity status -and the attempt number; budget/class-exhaustion and reconciliation summaries show when -durable backlog work is waiting for a later quota window; `enrichment.lease_lost` appears -at warning level when an outcome arrived after another worker had claimed the row; and -`enrichment.cycle_failed` at error level when an exception escapes a cycle entirely, -carrying the lease it released and the error class. +Enrichment speaks the staged vocabulary. Per batch: `enrichment.batch_completed` with its +requested/returned/valid/fallback counts and the `incomplete_results` flag, +`enrichment.batch_deferred` naming the ledger reason, and `enrichment.batch_failed` at +warning with its reason — every member is then rescheduled with backoff. Per item: `enrichment.fallback_admitted` with the +reason a batch could not settle it, then `enrichment.detail_completed`, +`enrichment.detail_retry_scheduled`, `enrichment.detail_deferred`, or — at warning — +`enrichment.detail_terminal` with the entity-specific reason. `enrichment.cycle_completed` +summarizes each cycle, `enrichment.stale_lease_reclaimed` at warning names rows recovered +from a crashed claim, and the search ledger contributes `search_budget.window_rolled`, +`search_budget.blocked`, and `search_budget.reserve_reached` at info with +`search_budget.pacing_deferred` at debug. **Every failure and every retry reaches the default level.** `github.request` is raised to warning when the request failed — a 5xx, a network timeout, a 404 on an @@ -1023,46 +1162,87 @@ docker compose logs | grep '"event":"budget\.' ## Rate limits and the request budget -### The allowance formula +### Two resources, two ledgers + +Since plan Appendix G, this service accounts for **two independent GitHub rate-limit +resources**. The `core` resource (60/hour unauthenticated) funds polling and a small +detail-fallback lane through `github_api_budget`; the `search` resource (10/minute +unauthenticated) funds normal-path batch enrichment through `github_search_budget`. Each +ledger reconciles only against response headers naming its own `x-ratelimit-resource`, +both sit behind the same global request gate, and a denial from either defers work — it +never terminates an entity. + +### The core allowance formula -`POLL_INTERVAL_SECONDS`, `MAX_PAGES_PER_POLL`, `ENABLED_LIVE_SOURCE_COUNT` and -`RATE_LIMIT_RESERVE` feed the one authoritative allowance formula (plan §10): +`POLL_INTERVAL_SECONDS`, `MAX_PAGES_PER_POLL`, `ENABLED_LIVE_SOURCE_COUNT`, +`RATE_LIMIT_RESERVE` and `CORE_DETAIL_FALLBACK_ALLOWANCE` feed the one authoritative +core formula (plan §10): ```text poll_attempt_allowance = ceil(3600 / POLL_INTERVAL_SECONDS) x MAX_PAGES_PER_POLL x ENABLED_LIVE_SOURCE_COUNT -enrichment_allowance = rate_limit - RATE_LIMIT_RESERVE - poll_attempt_allowance +feasible ⇔ poll_attempt_allowance + RATE_LIMIT_RESERVE + + CORE_DETAIL_FALLBACK_ALLOWANCE <= rate_limit ``` -With the defaults: 12 poll attempts and 40 enrichment attempts an hour, against -GitHub's unauthenticated limit of 60. **The process refuses to boot** if the -polling requirement leaves no capacity for enrichment. +With the defaults: 12 poll attempts, 4 detail-fallback attempts, and 8 in reserve — +24 of GitHub's unauthenticated 60, with **36 core requests an hour deliberately +unspent**. Enrichment's normal path does not appear in this formula because it does not +spend core at all. **The process refuses to boot** if the three core lanes together +exceed the limit. -The enrichment allowance is then split by `ACTOR_ENRICHMENT_SHARE` (plan §10): +The detail-fallback allowance is then split by `ACTOR_ENRICHMENT_SHARE` (plan §10): ```text actor_guarantee = floor(enrichment_allowance x ACTOR_ENRICHMENT_SHARE) repository_guarantee = enrichment_allowance - actor_guarantee ``` -With the defaults: 20 actor and 20 repository attempts an hour. The remainder +With the defaults: 2 actor and 2 repository detail attempts an hour. The remainder always goes to repositories, because the formula floors one side and subtracts for -the other — so the two always add up to the whole allowance. +the other — so the two always add up to the whole allowance. A class may borrow the +other's unused detail capacity when the other has no claimable detail candidate. -### The budget table +### The core budget table -At limit 60, reserve 8, cadence 300s, one source — the only variable is page depth: +At limit 60, reserve 8, detail allowance 4, cadence 300s, one source — the only variable +is page depth: -| `MAX_PAGES_PER_POLL` | Poll allowance | Enrichment allowance | Actor guarantee | Repository guarantee | +| `MAX_PAGES_PER_POLL` | Poll allowance | Detail fallback | Reserve | Deliberately unspent | |---|---|---|---|---| -| 1 (default) | 12 | 40 | 20 | 20 | -| 2 | 24 | 28 | 14 | 14 | -| 3 | 36 | 16 | 8 | 8 | -| 4 | 48 | 4 | 2 | 2 | +| 1 (default) | 12 | 4 | 8 | 36 | +| 2 | 24 | 4 | 8 | 24 | +| 3 | 36 | 4 | 8 | 12 | +| 4 | 48 | 4 | 8 | 0 | | 5 | 60 | — | — | **refuses to boot** | -Capture depth is bought with backlog-drain capacity, at a fixed exchange rate, and the -formula tells you the price before you pay it. +Capture depth now spends the unspent headroom rather than enrichment capacity: batch +enrichment lives on the search budget, so deeper polls no longer starve it — the formula +simply becomes infeasible at 5 pages. + +### The Search budget + +The search resource is a **per-minute** window, and the ledger treats every one of its +numbers as tunable and published (`/status`'s `scheduler` and `search_ledger` blocks): + +```text +SEARCH_REQUEST_CEILING = 10 the observed unauthenticated header limit +SEARCH_SAFETY_RESERVE = 2 never spent +spendable = 8 search requests a minute +SEARCH_BATCH_SIZE <= 10 exact user:/repo: qualifiers per request, never OR-joined +SEARCH_PACING_SECONDS = 6 minimum spacing between search requests (0 disables) +``` + +Pacing spreads the eight spendable requests across the minute instead of bursting them at +its top — `search_budget.pacing_deferred` at debug marks a request asked to wait. Windows +roll from response headers (`search_budget.window_rolled`), or, because a fully denied +minute observes no headers, after sixty header-less seconds of inactivity. Search +rate-limit and secondary-limit responses set the search ledger's own `blocked_until` +(`search_budget.blocked`) without touching the core ledger, and vice versa — which is +what makes "the poller ran out", "the detail lane ran out", and "search is pacing" three +separately answerable questions. At 8 spendable requests × batches of 10, the theoretical +ceiling is 4,800 returned items an hour; that is a capacity hypothesis the throughput +block of `/status` exists to check against measured rates, not a promise. ### Global blocks versus class exhaustion @@ -1073,8 +1253,9 @@ Two different things that both stop requests, deliberately kept apart: limit set it; it stops **everything**. `budget.global_block_set` and `budget.global_block_cleared` mark the edges. - **Class exhaustion** is *derived* from the counters and writes nothing. When polling has - spent its twelve, enrichment carries on; when enrichment has spent its forty, polling - carries on. `budget.class_exhausted` fires once per class per window, and + spent its twelve, the detail lane carries on; when the detail lane has spent its four, + polling carries on — and batch enrichment, on its own resource, notices neither. + `budget.class_exhausted` fires once per class per window, and `budget.share_exhausted` once per fairness share. So "the poller ran out" never means "the system stopped", and a reviewer can tell which @@ -1085,7 +1266,8 @@ happened from one log line. The ledger is never seeded by a discovery request. The **first canonical page-one poll of each rate-limit window** initialises it from that response's authoritative headers — it is a normal, counted, event-processing poll that happens to also establish the window. Until -the window is `active`, enrichment is ineligible. `budget.window_initialized` marks it, +the window is `active`, the core detail-fallback lane is ineligible (the search ledger +bootstraps itself from configuration). `budget.window_initialized` marks it, `budget.window_rolled` marks the hour turning over, and `budget.window_reset_in_past` warns that this container's clock runs ahead of GitHub's ([ADR 0004](docs/adr/0004-class-aware-budget-ledger.md)). @@ -1141,13 +1323,29 @@ is [`.env.example`](.env.example). | `MAX_REDIRECTS` | `2` | Redirect hops followed per request, each re-validated and separately reserved | | `SOURCE_LOCK_WAIT_SECONDS` | `30` | How long the one-shot waits for a busy source lock; the poller attempts once (plan §9) | | `POLL_INTERVAL_SECONDS` | `300` | The poll cadence, and an allowance-formula input. A source polled at T is due again at T + this; an unforced run before then is deferred rather than made. The worker's 60-second tick checks the schedule; it does not replace it (plan §9, §10) | -| `MAX_PAGES_PER_POLL` | `1` | How many `Link`-followed pages one poll may fetch, and an allowance-formula input. Raising it trades enrichment allowance for capture depth — see [the budget table](#the-budget-table) (plan §9, §10) | +| `MAX_PAGES_PER_POLL` | `1` | How many `Link`-followed pages one poll may fetch, and an allowance-formula input. Raising it spends the deliberately unspent core headroom on capture depth — see [the core budget table](#the-core-budget-table) (plan §9, §10) | | `ENABLED_LIVE_SOURCE_COUNT` | `1` | Allowance-formula input: live sources sharing one per-IP budget. **A fallback rather than the authority** — at window initialization and rollover the formula counts the enabled, in-service `event_sources` rows of the running mode and uses that instead, falling back to this value only when there are none yet. A disagreement is logged as `budget.source_allocation_drift` (plan §10, [ADR 0009](docs/adr/0009-runtime-source-allocation-and-shared-ip-observability.md)) | -| `RATE_LIMIT_RESERVE` | `8` | Requests per hour left deliberately unspent (plan §10) | -| `ACTOR_ENRICHMENT_SHARE` | `0.50` | How the enrichment allowance splits between actors and repositories: `floor(allowance x this)` guarantees actors, the remainder goes to repositories. A guarantee, not a cap — either class may borrow the other's unused capacity when the other has no *currently claimable* backlog candidate. Both ends of `[0, 1]` are legal (plan §10) | -| `ACTOR_REFRESH_TTL_SECONDS` | `86400` | Minimum reuse time before an enriched actor may be re-fetched. At each selection decision, refreshes are suppressed when either class has never-enriched backlog work (plan §10); see limitation 7 for the bounded concurrent-insert window | +| `RATE_LIMIT_RESERVE` | `8` | Core requests per hour left deliberately unspent (plan §10) | +| `CORE_DETAIL_FALLBACK_ALLOWANCE` | `4` | The bounded core lane for individual detail fetches — only batch items that came back missing, renamed, mismatched, or contract-invalid reach it. Third term of the core feasibility rule; can never take the polling allocation (plan §10, Appendix G) | +| `SEARCH_REQUEST_CEILING` | `10` | Per-minute Search request ceiling — the observed unauthenticated header limit (Appendix G) | +| `SEARCH_SAFETY_RESERVE` | `2` | Search requests per minute never spent, leaving 8 spendable | +| `SEARCH_BATCH_SIZE` | `10` | Exact `user:`/`repo:` qualifiers per Search request, capped at 10 and never `OR`-joined | +| `SEARCH_PACING_SECONDS` | `6` | Minimum spacing between Search requests, spread inside the minute; `0` disables (the fixture walkthrough uses that) | +| `SEARCH_WORKER_CONCURRENCY` | `1` | Must be exactly 1 while the global request gate serializes outbound calls — stated rather than implied | +| `ACTOR_ENRICHMENT_WEIGHT` | `1` | Batch-lane rotation weight for actors; with 1/1 the cycle alternates lanes, and an empty lane yields its slot | +| `REPOSITORY_ENRICHMENT_WEIGHT` | `1` | The same, for repositories | +| `ACTOR_ENRICHMENT_SHARE` | `0.50` | How the **detail-fallback** allowance splits between the classes: `floor(allowance x this)` guarantees actors, the remainder goes to repositories. A guarantee, not a cap — either class may borrow the other's unused detail capacity when the other has no *currently claimable* detail candidate. Both ends of `[0, 1]` are legal (plan §10) | +| `DETAIL_FALLBACK_MAX_ATTEMPTS` | `3` | The detail retry ladder — a retryable detail failure backs off until this many attempts, then the entity goes terminal; a `404`/`410` goes terminal immediately | +| `ENRICHMENT_LEASE_SECONDS` | `600` | The durable claim lease; expires by arithmetic and must exceed the worst-case single fetch (585s at the HTTP defaults) | +| `ENRICHMENT_RETRY_BASE_SECONDS` | `60` | Enrichment backoff base, jittered and doubling per attempt | +| `ENRICHMENT_RETRY_MAX_SECONDS` | `3600` | Enrichment backoff cap; base must not exceed it | +| `ENRICHMENT_CYCLE_BUDGET_SECONDS` | `55` | One enrichment cycle's time budget — below the 60-second dispatch tick, above the pacing interval | +| `ACTOR_REFRESH_TTL_SECONDS` | `86400` | Minimum reuse time before an enriched actor may be re-fetched. Refreshes ride the batch path under the composition rule — backlog always fills first (plan §10, Appendix G) | | `REPOSITORY_REFRESH_TTL_SECONDS` | `86400` | The same, for repositories | -| `ENRICHMENT_COVERAGE_WINDOW_SECONDS` | `86400` | How far back `GET /status` looks when computing §11's three coverage percentages, measured on `push_events.created_at`. The only knob here that changes what the system *reports* rather than what it *does* | +| `REFRESH_ACTIVE_WITHIN_SECONDS` | `604800` | Refresh eligibility beyond the TTLs: only entities seen pushing within this window are refreshed at all | +| `ENRICHMENT_METRICS_WINDOW_SECONDS` | `3600` | The window behind `/status`'s `batches` and `throughput` blocks. Reporting only | +| `CATCH_UP_MIN_SAMPLE_SECONDS` | `900` | Sample required before the catch-up verdict says anything; below it, `catch_up.state` is `insufficient_sample` | +| `ENRICHMENT_COVERAGE_WINDOW_SECONDS` | `86400` | How far back `GET /status` looks when computing §11's three coverage percentages, measured on `push_events.created_at`. Like the two knobs above it, it changes what the system *reports*, never what it *does* | Database connection settings (`POSTGRES_HOST`, `POSTGRES_PORT`, `POSTGRES_USER`, `POSTGRES_PASSWORD`) are managed by the compose topology @@ -1170,21 +1368,24 @@ one-shot — takes one chain, and nothing outside it calls GitHub IngestionRunner ──► SourceLock ──► PollSchedule (due? — five components, §9) │ ├──► PageLoop ──► RequestExecutor ──► RequestGate -EnrichmentRunner ────────────────┘ │ ▲ ──► BudgetLedger.reserve! - FIFO backlog → class, borrow │ │ ──► UrlPolicy - Refresh after backlog observed empty │ └── LinkHeader.next_url ──► Transport (Faraday | Fixture) - Claim → lease on next_retry_at │ │ - (never a SourceLock, §8 step 1) │ ▼ +CycleRunner ─────────────────────┘ │ ▲ ──► BudgetLedger.reserve! + batch lanes (BatchClaim: FIFO │ │ | SearchBudgetLedger.reserve! + batches under lease), then detail │ │ ──► UrlPolicy + lanes (DetailClaim, one row each) │ └── LinkHeader.next_url ──► Transport (Faraday | Fixture) + (never a SourceLock, §8 step 1) │ │ + │ ▼ │ PageWriter: one transaction per event - │ stub upserts → INSERT … ON CONFLICT - │ DO NOTHING RETURNING id → entity - │ activity updates only when - │ a row returned + │ stub upserts + local derivation + + │ event observations → INSERT … ON + │ CONFLICT DO NOTHING RETURNING id → + │ activity updates only when a row + │ returned ▼ RateLimitPolicy ──► BudgetLedger#block_globally! - │ + │ (search responses: SearchBudgetLedger#block_from!) ├──► PollState: the event_sources write - └──► EntityState: the one entity write, lease-guarded + └──► BatchRunner / DetailRunner: observations appended, + projection applied lease-guarded, batch row finalized ``` - **`Github::IngestionRunner`** owns one polling operation: it holds the source lock @@ -1237,43 +1438,50 @@ EnrichmentRunner ────────────────┘ │ the source lock. Its rule: the scheduling components move only when a poll attempt actually happened, so a budget denial or a held gate can neither advance the cadence nor burn a healthy source's failure count. -- **`Github::EnrichmentRunner`** owns one enrichment cycle and enriches at most one - entity: it asks fairness which class works next, claims the oldest row in that class, - fetches through the same chain, and writes one outcome. A denied reservation leaves the - row durable for a later window. It never takes a source lock and opens no transaction - across the request. -- **`Github::Enrichment::Fairness`** applies §10's ladder: a never-enriched candidate in a - class still inside its guarantee, then borrowing when the other class has no - *currently claimable* backlog candidate, then a TTL-stale refresh only when the selection - observes no never-enriched work in either class. Once refresh is allowed, it uses the same - two steps — prefer a class with room, then borrow only from a class with nothing to - refresh — so one refresh class cannot starve the other. Fairness decides; the ledger - enforces, so a wrong answer produces a refused reservation rather than an overspend - ([ADR 0007](docs/adr/0007-enrichment-fairness-shares-and-borrowing.md), - [ADR 0010](docs/adr/0010-secondary-limit-escalation-and-refresh-pool-fairness.md)). -- **`Github::Enrichment::Claim`** prevents two workers enriching one entity by leasing the - row — a conditional `UPDATE` that pushes `next_retry_at` forward. One column, one - meaning: the same predicate excludes leases, backoffs and secondary-limit deferrals from - candidate selection, so the queries cannot drift apart. A crashed worker leaves nothing - to clean up; the lease expires. -- **`Github::Enrichment::EntityState`** is §10's response behaviour resolved onto one - entity row, and `PollState`'s twin. Its rules: an attempt is counted only for an outcome - that says something about *this entity* — a rate limit is a fact about the IP — and a - retryable failure never downgrades an already-enriched record, so a transient `500` on a - refresh cannot drop coverage. -- **`Github::EnrichmentSchedule`** is §9's enrichment rule as a value: the maximum of the - entity's `next_retry_at`, `global_blocked_until` and a derived - `enrichment_class_blocked_until`. Three components, no `--force` — §9 licenses that - against a poll cadence, and enrichment has none — and no per-class share, because a share - exhaustion is a denial rather than a deferral. +- **`Github::SearchBudgetLedger`** is the second ledger, over the per-minute search + resource: reservation before execution, pacing, a never-spent reserve, monotonic + header reconciliation that discards non-`search` resources, and its own + `blocked_until` for search-scoped limits. Core and search cannot teach each other + their numbers. +- **`Github::Enrichment::CycleRunner`** owns one enrichment cycle inside + `ENRICHMENT_CYCLE_BUDGET_SECONDS`: batch lanes first — rotating actor and repository + by weight, an empty lane yielding its slot — then detail lanes. Every claim it makes + is admission-checked first, so a denied ledger produces a deferral log rather than a + spent request. It never takes a source lock and opens no transaction across a request. +- **`Github::Enrichment::BatchClaim` / `BatchRunner`** assemble and serve the normal + path: claim up to `SEARCH_BATCH_SIZE` never-enriched rows FIFO (`created_at, id`) + under a `FOR UPDATE SKIP LOCKED` lease, top up with TTL-stale refresh candidates only + under the composition rule, build the repeated-exact-qualifier query, then apply + results **by stable ID only** — renamed, mismatched, contract-invalid, and missing + items become observations plus detail-fallback admissions, never applications. All of + one response commits in one transaction; the batch envelope records what happened + ([ADR 0013](docs/adr/0013-derivation-first-staged-batch-enrichment.md)). +- **`Github::Enrichment::DetailClaim` / `DetailRunner`** serve the fallback lane one row + at a time from the stored payload `api_url`, on the bounded core allowance with the + fairness shares and borrowing ([ADR 0007](docs/adr/0007-enrichment-fairness-shares-and-borrowing.md)). + A `404`/`410` is an immediate entity-specific terminal; other failures climb a bounded + ladder. A crashed worker leaves nothing to clean up; the lease expires by arithmetic + and the claim is reclaimed. +- **`Github::Enrichment::Admission`** answers "may a search / detail request proceed + right now" from persisted ledger state, read-only — the honest deferral reason without + taking the request gate to learn it. (`Github::EnrichmentSchedule` remains the core + admission value object behind the detail verdict.) +- **`Github::Enrichment::ObservationRecorder`** appends the evidence rows; projections + are applied lease-guarded by the runners, so a stale worker's late outcome cannot + double-apply. ### The SSRF boundary -Enrichment follows URLs that arrive **inside GitHub payloads**, `Link` headers and -`Location` headers — data this application did not construct and an attacker could -influence. So `Github::UrlPolicy` is a trust boundary, not a formatter. Every URL is -rebuilt from validated components, and a `:payload`-origin URL always clears the full -*live* policy first, whatever mode the process is in: +Enrichment URLs have two origins. Search URLs are **application-origin constants**: +`Github::Enrichment::SearchQuery` builds them from a constant host and path plus +URL-encoded exact qualifiers over stored identity fields, and they still pass the +in-chain validation like every other request. Detail URLs arrive **inside GitHub +payloads**, `Link` headers and `Location` headers — data this application did not +construct and an attacker could influence — and a Search miss never turns an identifier +into a constructed detail URL; only the stored payload `api_url` is fetched. So +`Github::UrlPolicy` is a trust boundary, not a formatter. Every URL is rebuilt from +validated components, and a `:payload`-origin URL always clears the full *live* policy +first, whatever mode the process is in: - HTTPS only - Host exactly `api.github.com` @@ -1310,15 +1518,18 @@ repeated execution with duplicate-safe event writes (0005), decomposed poll defe (0006), enrichment fairness shares and borrowing (0007), post-commit enqueue with entity-scoped reconciliation (0008), runtime source allocation with shared-IP observability (0009), secondary-limit escalation with refresh-pool fairness (0010), the -pinned API version (0011), and Solid Queue over Kafka (0012). +pinned API version (0011), Solid Queue over Kafka (0012), and derivation-first staged +batch enrichment (0013). ## Continuous ingestion The `worker` container runs one Solid Queue supervisor: a dispatcher, a scheduler running [`config/recurring.yml`](config/recurring.yml), and two single-thread worker processes from -[`config/queue.yml`](config/queue.yml). One process serves `polling,control`; the other is -dedicated to `enrichment`, so polling and control work never queue behind the enrichment -workload. Outbound polling and enrichment attempts still serialize through `RequestGate`. +[`config/queue.yml`](config/queue.yml). One process serves the `polling` and `control` +queues; the other is dedicated to `enrichment`, so polling and control work never queue +behind the enrichment workload. Multi-queue workers are declared as a YAML list — Solid +Queue reads the value through `Array()`, so a comma-joined string would name one queue +that no job is ever enqueued into. Outbound polling and enrichment attempts still serialize through `RequestGate`. Two tasks fire every 60 seconds. **A tick is not a poll.** `PollEventSourceJob` selects the sources whose cached @@ -1332,22 +1543,23 @@ A source another process is polling is reported at INFO and left alone; the tick does not retry it in the same execution; another tick is nominally scheduled 60 seconds later. **Enrichment is scheduled twice over, deliberately.** A run that created events -schedules one cycle per class as soon as its rows are committed and its advisory +dispatches one staged cycle as soon as its rows are committed and its advisory lock is released. That enqueue is a *hint*: the durable record of pending work is the entity rows themselves, so `ReconcilePendingEnrichmentsJob` sweeps them every 60 -seconds and schedules a cycle for any class that has claimable work and is not -blocked by the ledger. If a crash loses an enrichment-dispatch hint, the committed +seconds and dispatches a cycle whenever admission and claimability say work could +proceed. If a crash loses an enrichment-dispatch hint, the committed entity state remains discoverable on a later successful scheduled tick without a special cleanup job or queue inspection (plan §8, [ADR 0008](docs/adr/0008-post-commit-enqueue-and-entity-scoped-reconciliation.md)). -Each dispatch call enqueues at most one job per class regardless of backlog depth; entity -rows, not queued jobs, are the backlog. Each cycle enriches at most one entity, chosen by -§10's fairness policy under a lease. -Steady state at the defaults: twelve polls an hour, and at most forty enrichment -requests an hour split into 20/20 actor/repository guarantees with borrowing. Within each -class the oldest never-enriched entity is selected first; a selection considers refresh -work only after observing the entire never-enriched backlog empty. +Each dispatch call enqueues at most one `EnrichmentCycleJob` regardless of backlog depth; +entity rows, not queued jobs, are the backlog. Each cycle runs batch lanes then detail +lanes inside its 55-second budget, every claim under a lease. +Steady state at the defaults: twelve polls an hour on core, up to eight paced Search +batches a minute serving as many as ten entities each, and at most four detail fallbacks +an hour split 2/2 with borrowing. Within each class the oldest never-enriched entity is +batched first; refresh candidates ride along only under the composition rule — own +backlog exhausted, none claimable in the other class. ## Processing guarantees @@ -1369,8 +1581,9 @@ boundary. activity update are replay-safe; execution records, counters, budget use, and logs may repeat. - **Not complete upstream capture.** The feed is a sliding window with hours of latency; see [Known limitations](#known-limitations). -- **Not bounded-time enrichment completion.** Durable work is not discarded, but sustained - arrivals can exceed the 40-request hourly service rate; see [Known +- **Not bounded-time enrichment completion, and not guaranteed catch-up.** Durable work is + not discarded, but sustained arrivals can exceed the measured batch service rate; + `/status` reports `not_keeping_up` when they do — see [Known limitations](#known-limitations). ### The four crash cases @@ -1379,8 +1592,8 @@ boundary. |---|---|---| | Before the event commits | Nothing from this page | A later overlapping poll can re-fetch it only while it remains in GitHub's feed window; there is no stop-on-known-event | | After the event commits, before enrichment is enqueued | The `push_events` row and its stub entities | `ReconcilePendingEnrichmentsJob`, from committed entity rows, on a later successful tick (scheduled every 60s) | -| Worker dies before the enrichment commit | The entity row, still `pending`, its lease expiring by arithmetic | The next reconcile tick after `next_retry_at` passes | -| Worker dies after the enrichment commit, before acknowledgement | The enriched entity row | Solid Queue may re-run the job; the freshness check leaves that row unchanged | +| Worker dies before the enrichment commit | The entity rows, still `pending`, their claim lease expiring by arithmetic | The next claim reclaims the leased rows (`enrichment.stale_lease_reclaimed`) once `leased_until` passes | +| Worker dies after the enrichment commit, before acknowledgement | The enriched entity rows, their observations, and the batch envelope | Solid Queue may re-run the cycle; the lease-token guard and freshness checks leave applied rows unchanged | ### The mechanisms @@ -1389,8 +1602,9 @@ boundary. register new activity or change FIFO backlog age. - `payload_fingerprint` uniqueness on quarantine — one row per distinct malformed payload, occurrence-counted. -- The entity lease is a `next_retry_at` timestamp that **expires by arithmetic**, so a - crashed worker leaves nothing to release. +- The claim lease is a `lease_token` plus a `leased_until` timestamp that **expires by + arithmetic**, so a crashed worker leaves nothing to release, and projection writes are + guarded by token + batch id so a stale worker cannot double-apply. - Session advisory locks die with their PostgreSQL session, so a killed poller releases its source the moment its connection drops. - `enqueue_after_transaction_commit` plus an entity-scoped reconciler — the enqueue is a @@ -1452,11 +1666,14 @@ into `pending`, empty the queue (the crash), and start it again: ```bash docker compose stop worker docker compose exec db psql -U postgres -d github_push_ingestor_development \ - -c "UPDATE github_actors SET enrichment_status = 'pending', fetched_at = NULL, next_retry_at = NULL;" + -c "UPDATE github_actors + SET enrichment_status = 'pending', enrichment_stage = 'batch_pending', + fetched_at = NULL, next_retry_at = NULL, + lease_token = NULL, leased_until = NULL;" docker compose exec db psql -U postgres -d github_push_ingestor_queue_development \ -c "TRUNCATE solid_queue_jobs CASCADE;" docker compose start worker -docker compose logs -f worker # enrichment.dispatched, then enrichment.completed +docker compose logs -f worker # enrichment.dispatched, then enrichment.batch_completed ``` In this scenario, emptying the queue does not discard pending enrichment state: that work is @@ -1476,7 +1693,7 @@ state can be reconstructed. | Worker logs nothing | `setup` did not complete, so `worker` never started | `docker compose logs setup` | | `/health/ready` fails while `/health/live` is fine | Database unreachable or schema not loaded. `web`'s healthcheck curls `/health/live`, so `web` stays green through a `db` outage | `docker compose ps`, `docker compose logs db` | | `up` fails with `Bind for 0.0.0.0:3000 failed: port is already allocated` | Another process owns host port 3000 | Free port 3000, or edit `web`'s `ports:` mapping in `docker-compose.yml` | -| Enrichment stuck at `pending` | The allowance is spent for this window, or the window is not `active` yet | `curl -s localhost:3000/status \| jq .ledger` | +| Enrichment stuck at `pending` | The search window is blocked or pacing, the detail allowance is spent, or the core window is not `active` yet | `curl -s localhost:3000/status \| jq '.ledger, .search_ledger'` | ### When a source goes out of service @@ -1538,14 +1755,17 @@ script/verify_recovery.sh --confirm --phase=cleanup ```bash docker compose exec db psql -U postgres -d github_push_ingestor_development -c " TRUNCATE push_events, quarantined_events, github_actors, github_repositories, - ingestion_runs, event_sources, github_api_budget RESTART IDENTITY CASCADE;" + enrichment_observations, enrichment_batches, ingestion_runs, + event_sources, github_api_budget, github_search_budget + RESTART IDENTITY CASCADE;" ``` Safe because **nothing is seeded**: `Github::Ingestion::SourceProvisioner.ensure!` -recreates the `event_sources` row lazily at the point of use, and -`Github::BudgetLedger#bootstrap!` recreates the ledger row from the next window's response -headers. The next `ingest` behaves exactly like a first run — that is expected, not a -broken system. +recreates the `event_sources` row lazily at the point of use, +`Github::BudgetLedger#bootstrap!` recreates the core ledger row from the next window's +response headers, and `Github::SearchBudgetLedger#bootstrap!` recreates the search row +from configuration. The next `ingest` behaves exactly like a first run — that is +expected, not a broken system. **Level 3 — destroy everything, including the volume.** @@ -1578,16 +1798,18 @@ samples the public feed rather than mirroring it.** The nine limitations that follow are consequences of that, of the 60-request hourly ceiling, and of deliberate scope decisions. Each is a stated operational boundary. -**1. Enrichment has no bounded completion time.** One observed live page held ~92–95 -`PushEvent` records with ~89 distinct actors and ~92 distinct repositories — 181 cold -entity requests per page, or ~2,172 an hour if every poll contained entirely new entities, -against 40 available. That extrapolation is a pressure scenario, not the measured -deduplicated arrival rate: repeated actors, repositories, and overlapping pages collapse to -shared rows. Entity rows nevertheless remain durable backlog work, FIFO within each entity -class; quota exhaustion defers them to later windows and never terminates them. If measured -unique arrivals remain above 40 attempts per hour, backlog size and oldest pending age grow -and no finite drain estimate is honest. `/status` publishes backlog count, oldest pending age, -and the reserved allowance's usage; it does not fabricate an ETA from incomplete history. +**1. Catch-up is measured, never guaranteed.** One observed live page held ~92–95 +`PushEvent` records with ~89 distinct actors and ~92 distinct repositories — ~2,172 +cold entity references an hour if every poll contained entirely new entities. That +extrapolation is a pressure scenario, not the measured deduplicated arrival rate. The +staged batch path's theoretical ceiling of 4,800 items an hour (8 spendable Search +requests a minute × batches of 10) is likewise a hypothesis, eroded by misses, fallback, +retries, and pacing. So `/status` measures both sides — arrivals, completions, backlog +delta — and publishes a tri-state `catch_up.state`: the claim is valid only while +measured arrivals stay under the measured service rate, and when they do not, the service +reports `not_keeping_up` rather than promising eventual catch-up. Entity rows remain +durable backlog work throughout, FIFO within each class; quota exhaustion defers them and +never terminates them, and there is still no drain ETA anywhere. **2. There is no guarantee of complete upstream capture.** Pagination deepens a single poll within the budget; it does not backfill. Events that rolled out of the feed's window @@ -1614,15 +1836,15 @@ constraint is deliberate, not an oversight. See the scaling path in [`docs/DESIGN_BRIEF.md`](docs/DESIGN_BRIEF.md). **7. Enriched entities can remain stale while backlog exists.** -`ACTOR_REFRESH_TTL_SECONDS` and `REPOSITORY_REFRESH_TTL_SECONDS` default to 86400, but a -selection that observes never-enriched work does not choose a refresh. Under sustained -backlog pressure, staleness can therefore exceed the TTL; the TTL is an earliest refresh -time, not a deadline. There is one bounded concurrency window: ingestion can commit a new -candidate after fairness has observed both tables empty but before the chosen refresh is -debited. Because a runner cycle issues at most one entity request, at most one refresh can -cross that boundary; the next selection sees the backlog and suppresses refreshes. The race -cannot discard or terminally mark backlog work, and the ledger still enforces the 40-request -enrichment cap. +`ACTOR_REFRESH_TTL_SECONDS` and `REPOSITORY_REFRESH_TTL_SECONDS` default to 86400, but +refreshes only top up Search batches under the composition rule — own-class backlog +exhausted and none claimable in the other class — and only for entities active within +`REFRESH_ACTIVE_WITHIN_SECONDS` (seven days). Under sustained backlog pressure, staleness +can therefore exceed the TTL; the TTL is an earliest refresh time, not a deadline, and an +entity nothing has referenced for a week is not refreshed at all. Ingestion can commit a +new candidate between a claim's emptiness read and its request; the claim already made +proceeds, the next claim sees the new backlog, and the race can neither discard backlog +work nor exceed either ledger's cap. **8. Extension C (object storage) was deliberately not attempted.** A decision with a stated reason, not an omission — the remaining budget went to rate-limit correctness, @@ -1637,14 +1859,16 @@ verification](#crash-recovery-verification); Extension D (testing strategy) is described there. **9. Business tables and the enrichment backlog can grow without bound.** `push_events`, -`ingestion_runs`, and `quarantined_events` are append-only, and actor and repository rows -remain actionable until enrichment succeeds or establishes an entity-specific terminal -outcome — this service is the system of record, -and retention, pruning, and archival were deliberately not built. Twelve poll attempts an -hour at up to ~100 events each can add roughly 1,200 event rows and as many as 2,400 entity -references an hour before deduplication, while only 40 enrichment request attempts per hour -are available to work that backlog. The only shipped pruning is Solid Queue's finished-job -cleanup in the queue database, which holds no business data. +`ingestion_runs`, `quarantined_events`, `enrichment_observations`, and +`enrichment_batches` are append-only, and actor and repository rows remain actionable +until enrichment satisfies the contract or establishes an entity-specific terminal +outcome — this service is the system of record, and retention, pruning, and archival were +deliberately not built. Twelve poll attempts an hour at up to ~100 events each can add +roughly 1,200 event rows and as many as 2,400 entity references an hour before +deduplication, against a batch service ceiling of 4,800 items an hour that is +hypothesis, not measurement — and every observation and batch envelope adds rows of its +own. The only shipped pruning is Solid Queue's finished-job cleanup in the queue +database, which holds no business data. ## Development @@ -1653,12 +1877,13 @@ AI-assisted development guidance for this repository lives in - [`docs/DESIGN_BRIEF.md`](docs/DESIGN_BRIEF.md) — the two-page architecture summary; start here. -- [`docs/adr/`](docs/adr/) — twelve architecture decision records. +- [`docs/adr/`](docs/adr/) — thirteen architecture decision records. - [`docs/evidence/`](docs/evidence/) — dated first-party verifications of contested claims. - [`docs/SUBMISSION_CHECKLIST.md`](docs/SUBMISSION_CHECKLIST.md) — the §16 quality gates as a pre-flight checklist. - [`IMPLEMENTATION_PLAN.md`](IMPLEMENTATION_PLAN.md) — the execution plan; Appendix E - records build-time divergence and Appendix F the durable-backlog correction. + records build-time divergence, Appendix F the durable-backlog correction, and + Appendix G the staged-batch enrichment design. ## License diff --git a/app/services/github/enrichment_runner.rb b/app/services/github/enrichment_runner.rb deleted file mode 100644 index 4814508..0000000 --- a/app/services/github/enrichment_runner.rb +++ /dev/null @@ -1,223 +0,0 @@ -module Github - # One enrichment cycle (IMPLEMENTATION_PLAN.md §13's PR 7): - # - # 1. ask §10's fairness policy which class works next, from which pool, and whether it - # may borrow - # 2. lease that entity row, so a second worker cannot take it - # 3. fetch it through Github.executor — the one and only network call - # 4. record the global rate-limit consequence, then the entity outcome - # - # **At most one entity per call.** §5 names EnrichActorJob and EnrichRepositoryJob, and one - # entity is what each of them performs; batching is the caller's loop, which - # Github::Enrichment::OneShot is for the operator and the 60-second reconciler tick is for - # the worker. - # - # **No source lock, ever.** §8 step 1: "Enrichment jobs skip this step — they take only - # the request gate." This class is never handed an EventSource and never reaches for - # Github::SourceLock, which is the structural half of Appendix D item 1; - # Github::LockOrder is the enforced half. - # - # **No transaction spans the fetch.** This class opens none, every collaborator's write - # is a single statement, and the three that write are constructed with no executor and - # no transport — Github::Ingestion::PageWriter's technique. Github::BudgetLedger's - # assert_committable! is the enforced backstop: every attempt goes through the executor, - # and it raises if a transaction is open. - class EnrichmentRunner - MONOTONIC = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) } - - # One return type for every outcome, so a caller branches once instead of rescuing - # four classes and eventually missing one — IngestionRunner::Result's rule. - # - # enriched the document was stored - # failed the entity reached permanent_failure or retryable_failure - # deferred the request never happened, or produced nothing chargeable to this - # entity: a ledger denial, a busy gate, a rate limit - # idle nothing was eligible — the ordinary steady state once a corpus is - # fully enriched, and a different fact from "we asked and were refused" - # lease_lost the outcome arrived after another worker had claimed the row - class Result < Data.define(:status, :entity_type, :github_id, :pool, :borrow, - :classification, :enrichment_status, :last_error, - :error_code, :deferral_reason, :next_retry_at, - :duration_ms, :enrichment_attempt) - STATUSES = %w[ enriched failed deferred idle lease_lost ].freeze - - EVENTS = { - "enriched" => "enrichment.completed", "failed" => "enrichment.failed", - "deferred" => "enrichment.deferred", "idle" => "enrichment.idle", - "lease_lost" => "enrichment.lease_lost" - }.freeze - - def initialize(status:, entity_type: nil, github_id: nil, pool: nil, borrow: false, - classification: nil, enrichment_status: nil, last_error: nil, - error_code: nil, deferral_reason: nil, next_retry_at: nil, - duration_ms: nil, enrichment_attempt: nil) - raise ArgumentError, "unknown status #{status.inspect}" unless STATUSES.include?(status) - - super - end - - def enriched? = status == "enriched" - def failed? = status == "failed" - def deferred? = status == "deferred" - def idle? = status == "idle" - def lease_lost? = status == "lease_lost" - - # Whether a live GitHub request was made. A deferral spends nothing. - def attempted? = enriched? || failed? || lease_lost? - - def to_log - { enrichment_outcome: status, entity_type: entity_type, github_id: github_id, - pool: pool, borrow: (true if borrow), classification: classification, - entity_status: enrichment_status, enrichment_attempt: enrichment_attempt, - error_code: error_code, - error_message: last_error, deferral_reason: deferral_reason, - next_retry_at: next_retry_at&.utc&.iso8601, - duration_ms: duration_ms }.compact - end - end - - def initialize(executor: Github.executor, - configuration: Github.configuration, - clock: -> { Time.current }, - monotonic: MONOTONIC, - rate_limit_policy: RateLimitPolicy.new, - selector: nil, fairness: nil, claim: nil, entity_state: nil) - @executor = executor - @configuration = configuration - @clock = clock - @monotonic = monotonic - @rate_limit_policy = rate_limit_policy - @selector = selector || Enrichment::CandidateSelector.new(configuration: configuration) - @fairness = fairness || Enrichment::Fairness.new(configuration: configuration, selector: @selector) - @claim = claim || Enrichment::Claim.new(configuration: configuration, selector: @selector) - @entity_state = entity_state || Enrichment::EntityState.new - end - - # @param entity_class [Class, Symbol, nil] restrict this cycle to one class. It - # narrows selection and bypasses nothing: the allowance, the share, the reserve, the - # gate and every global block still bind. - # @return [Result] - def call(entity_class: nil) - now = @clock.call - started = @monotonic.call - - choice = @fairness.choose(entity_class: entity_class, now: now) - return idle(choice, started: started) unless choice.chosen? - - lease = @claim.acquire(choice.entity_type, pool: choice.pool, now: now) - # A lost race, or a row that moved between the query and the claim. Nothing is - # wrong, and there is nothing to report about an entity we never held. - return idle(Enrichment::Fairness::Choice.none(reason: "no_candidate"), started: started) if lease.nil? - - enrich(choice, lease, started: started) - end - - private - - def enrich(choice, lease, started:) - fetched = @executor.call(request_for(choice, lease)) - # Before the entity write: a rate limit is a fact about the IP, the global block has - # to be recorded first, and the secondary-limit branch of the write matrix reads the - # instant this returns. Github::Ingestion::PageLoop sequences a poll the same way. - decision = @rate_limit_policy.apply!(fetched, now: @clock.call) - document = fetched.ok? ? choice.entity_type.document.parse(fetched.body, github_id: lease.github_id) : nil - - written = @entity_state.record!(lease: lease, fetched: fetched, document: document, - decision: decision, now: @clock.call) - @claim.release!(lease) if written.lease_held - - complete(choice, lease, fetched, written, started: started) - rescue Errors::FixtureMiss - # §6 requires a corpus gap to be raised rather than laundered into a failed fetch. - # The lease goes back untouched: an authoring bug is not an entity outcome and must - # not cost the entity an attempt. - @claim.release!(lease) - raise - rescue StandardError => error - # PageWriter's reasoning: an unexpected error "is a defect to fix, not a payload to - # classify". Fabricating an entity status from one would make the defect durable. - @claim.release!(lease) - # enrichment.cycle_failed, not enrichment.failed: Result::EVENTS owns that name for - # the *ordinary* outcome §11 lists — INFO, with an entity status and a scheduled - # retry. This is an escaped exception with a released lease, an error pair and no - # entity outcome at all, and one event name carrying two field sets means an alert - # filtered on it matches two structurally different records. - # PollEventSourceJob's ingestion.cycle_failed is the same fact one level up, and - # shares its name deliberately. - Rails.logger.error(event: "enrichment.cycle_failed", **lease.to_log, - error_class: error.class.name, error_message: error.message) - raise - end - - # origin: :payload, because §10's SSRF boundary treats an actor or repository URL that - # arrived inside an event payload as attacker-influenced: it clears the full *live* - # policy first whatever the mode, and only then is projected onto the fixture scheme. - # - # A blank or NULL api_url needs no special case. It becomes "", which - # Github::RequestExecutor's *pre-gate* validation refuses — outside the gate hold and - # therefore before any reservation — so it arrives back as :permanent_error and costs - # no budget. That is §10's "violations mark the entity permanent_failure", satisfied - # by the chain that already exists. - # - # The context keys are §11's common fields, and they reach the DEBUG github.request - # line through Request#to_log with no change to the executor or the formatter. The - # attempt key is spelled enrichment_attempt on purpose: FetchResult#to_log merges its - # own `attempt` — the HTTP one — after the request's, and would silently overwrite it. - def request_for(choice, lease) - Request.new( - url: lease.api_url, - request_class: choice.entity_type.request_class, - origin: :payload, - borrow: choice.borrow, - context: lease.to_log - ) - end - - def complete(choice, lease, fetched, written, started:) - result = Result.new( - status: written.outcome, entity_type: choice.entity_type.key, - github_id: lease.github_id, pool: choice.pool, borrow: choice.borrow, - classification: fetched.classification, enrichment_status: written.enrichment_status, - # §11's "attempt number", spelled enrichment_attempt for the reason #request_for's - # comment gives: `attempt` on a github.* line is the HTTP one. lease.to_log already - # carries this onto the DEBUG request line; without it here it never reaches the - # INFO outcome line, which is the line §11 actually asks reviewers to read. - enrichment_attempt: lease.enrichment_attempts + 1, - last_error: written.last_error, error_code: written.error_code, - deferral_reason: (fetched.classification.to_s if written.deferred?), - next_retry_at: written.next_retry_at, - duration_ms: elapsed_ms(started) - ) - - log(result) - result - end - - def idle(choice, started:) - # A deferral the ledger would have refused is reported as such; genuinely having - # nothing to do is idle. IngestionRunner draws the same line between "not due" and - # "deferred", and for the same reason: they are different facts and an operator acts - # on them differently. - deferred = choice.reason != "no_candidate" - - result = Result.new(status: deferred ? "deferred" : "idle", - deferral_reason: choice.reason, - duration_ms: elapsed_ms(started)) - log(result) - result - end - - # A completed attempt is INFO. A deferral or an idle cycle is DEBUG: under the recurring task - # an exhausted window would otherwise emit a line a minute for the rest of the hour, - # which is the volume argument Github::BudgetLedger#log_class_exhausted already makes. - def log(result) - payload = result.to_log.merge(event: Result::EVENTS.fetch(result.status)) - - result.attempted? ? Rails.logger.info(payload) : Rails.logger.debug(payload) - end - - def elapsed_ms(started) - ((@monotonic.call - started) * 1000).round(1) - end - end -end diff --git a/docs/DESIGN_BRIEF.md b/docs/DESIGN_BRIEF.md index 9902a86..fcdfe35 100644 --- a/docs/DESIGN_BRIEF.md +++ b/docs/DESIGN_BRIEF.md @@ -22,14 +22,14 @@ when the backlog cannot drain within a bounded time. flowchart LR subgraph runtime["Docker Compose runtime"] P["Poll job / bin/ingest"] --> I["IngestionRunner
source lock"] - E["Enrichment jobs / bin/enrich"] --> N["EnrichmentRunner
fair selection + lease"] + E["EnrichmentCycleJob / bin/enrich"] --> N["CycleRunner
batch + detail lanes, leased claims"] W["Web
health · status · event API"] Q["Solid Queue"] --> P Q --> E end - I --> X["RequestExecutor
gate → ledger → URL policy → transport"] + I --> X["RequestExecutor
gate → core or search ledger → URL policy → transport"] N --> X - X --> G["api.github.com
60 requests/hour/IP"] + X --> G["api.github.com
60 core/hour + 10 search/minute per IP"] I --> D[("PostgreSQL
business data")] N --> D W --> D @@ -42,11 +42,12 @@ itself; polling also holds a per-source advisory lock for its whole cycle (the o order is source lock, then gate), and enrichment takes only the gate. Web routes have no path to the executor, so `/health/*` and `/status` cannot spend budget. -PostgreSQL is the system of record. Seven business tables hold source/run state, -`push_events`, shared actor and repository state, quarantined payloads, and the global -budget ledger; Solid Queue uses a second database in the same server. Raw payloads are -`jsonb` — JSON meaning, not byte layout — and quarantine identity is a canonical-payload -SHA-256, since malformed data may lack a usable event ID. +PostgreSQL is the system of record. Ten business tables hold source/run state, +`push_events`, shared actor and repository projections, append-only enrichment +observations, per-request batch envelopes, quarantined payloads, and the two budget +ledgers (hourly core, per-minute search); Solid Queue uses a second database in the same +server. Raw payloads are `jsonb` — JSON meaning, not byte layout — and quarantine +identity is a canonical-payload SHA-256, since malformed data may lack a usable event ID. Acceptance occurs when a `push_events` row commits. Inserts use `ON CONFLICT (github_event_id) DO NOTHING RETURNING id`. A duplicate observation cannot @@ -65,21 +66,25 @@ exactly-once execution ([ADR 0005](adr/0005-at-least-once-with-idempotent-writes ## Request budget and the `304` finding -The hourly allocation is derived rather than guessed: +Two rate-limit resources, two persisted ledgers. The **core** allocation is derived +rather than guessed: ```text poll_allowance = ceil(3600 / POLL_INTERVAL_SECONDS) × MAX_PAGES_PER_POLL × enabled_live_source_count -enrichment_allowance = rate_limit - RATE_LIMIT_RESERVE - poll_allowance -actor_guarantee = floor(enrichment_allowance × ACTOR_ENRICHMENT_SHARE) -repository_guarantee = enrichment_allowance - actor_guarantee +feasible ⇔ poll_allowance + RATE_LIMIT_RESERVE + + CORE_DETAIL_FALLBACK_ALLOWANCE ≤ rate_limit ``` -With defaults: 12 poll attempts, a reserve of 8, and 40 enrichment attempts split 20/20; -startup rejects configurations that leave no enrichment allowance, and the live source -count is read from in-service rows at window initialization -([ADR 0004](adr/0004-class-aware-budget-ledger.md), -[ADR 0009](adr/0009-runtime-source-allocation-and-shared-ip-observability.md)). +With defaults: 12 poll attempts, 4 detail-fallback attempts split 2/2 by +`ACTOR_ENRICHMENT_SHARE`, a reserve of 8, and 36 core requests deliberately unspent. +Normal-path enrichment spends the **search** resource instead — a separate singleton +ledger over GitHub's per-minute Search window (ceiling 10, reserve 2, 6-second pacing), +reconciled against its own `x-ratelimit-resource: search` headers. Startup rejects +infeasible core configurations, and the live source count is read from in-service rows at +window initialization ([ADR 0004](adr/0004-class-aware-budget-ledger.md), +[ADR 0009](adr/0009-runtime-source-allocation-and-shared-ip-observability.md), +[ADR 0013](adr/0013-derivation-first-staged-batch-enrichment.md)). GitHub's endpoint documentation broadly describes `304` responses as free, but its REST best-practices guidance scopes that to correctly authorized requests — and a dated @@ -88,26 +93,36 @@ unauthenticated [probe](evidence/2026-07-30-unauthenticated-304-quota-probe.md) request: ETags save bandwidth, not budget. The asymmetry decides it — a wrongly-free `304` can exhaust the shared window, while a wrongly-charged one costs only local opportunity. -## Durable, fair, and safe enrichment +## Durable, staged, and safe enrichment One observed page held roughly 180 distinct entities — a cold-demand pressure scenario, not a measured unique arrival rate, because identities deduplicate into shared rows. -Defaults reserve 12 attempts for polls, 40 for the enrichment class, and 8 for safety; -durable FIFO backlog has priority over refresh within the 40. Quota exhaustion only defers -rows. If unique arrivals exceed service, backlog size and age can grow without a bounded -completion time. - -`/status` exposes per-class count, oldest pending timestamp/age, and allowance usage; it -omits an ETA because there is no durable outcome history for an honest rate. The 40 attempts -carry 20/20 actor/repository guarantees with borrowing when the other class has no -claimable work. A selection that observes a never-enriched row suppresses refresh; one -concurrent insert can cross a one-request decision/debit window before the next selection -self-corrects. Selection and leases prevent duplicate work -([ADR 0007](adr/0007-enrichment-fairness-shares-and-borrowing.md)). - -Payload, pagination, and redirect URLs are attacker-influenceable, so the SSRF boundary -allows only HTTPS URLs whose host is exactly `api.github.com` — no userinfo, non-default -port, or IP literal; redirects are bounded, revalidated, and separately debited, and +Enrichment is derivation-first and staged: ingestion persists event-native identity, +derives locally computable fields, and coalesces demand by stable GitHub ID; the normal +path then batches up to ten repeated exact `user:`/`repo:` Search qualifiers per request +(never `OR`-joined) on the search budget, validating every returned item against its +immutable ID before applying it. Only missing, renamed, mismatched, or contract-invalid +items fall back to their stored payload-provided detail URLs inside the bounded core +allowance. Completion is an explicit useful-data contract per entity — queryable fields +plus the retained raw item, nullable values valid as nulls — not every field GitHub can +return. Quota, pacing, and reserve denials only defer; no entity is ever terminal because +budget ran out. Refreshes ride the same batch path only after both backlogs are exhausted. + +Evidence and convenience are split: every raw item lands in an append-only observation +table with fingerprint, provenance, and validation outcome, and every request attempt in +a batch envelope with counts and observed headers, while entity rows stay the latest +projection pointing at their latest observation — a refresh repoints, never overwrites. +`/status` publishes per-stage backlog, batch fill ratios, measured arrival and completion +rates, and a tri-state catch-up verdict; when completions do not exceed arrivals it says +`not_keeping_up` rather than promising eventual catch-up, and it still publishes no ETA +([ADR 0007](adr/0007-enrichment-fairness-shares-and-borrowing.md), +[ADR 0013](adr/0013-derivation-first-staged-batch-enrichment.md)). + +Enrichment URLs have two origins: Search URLs are application-built constants over stored +identifiers, while detail URLs are payload-supplied and attacker-influenceable — so the +SSRF boundary allows only HTTPS URLs whose host is exactly `api.github.com` — no +userinfo, non-default port, or IP literal; redirects are bounded, revalidated, and +separately debited; a Search miss never constructs a detail URL from an identifier; and fixture mode fails closed rather than falling back to the network ([ADR 0003](adr/0003-event-source-and-transport-seams.md)). diff --git a/docs/SUBMISSION_CHECKLIST.md b/docs/SUBMISSION_CHECKLIST.md index 1e950d1..7f342b6 100644 --- a/docs/SUBMISSION_CHECKLIST.md +++ b/docs/SUBMISSION_CHECKLIST.md @@ -95,8 +95,11 @@ globally named `github-push-ingestor_pgdata` volume. grep -E 'Non-push events ignored:[[:space:]]+1' "$fixture_ingest_output" ``` -- [ ] `GITHUB_MODE=fixture docker compose run --rm enrich --limit 6` exits 0 and leaves - `complete 2 / permanent_failure 1` in each entity class. +- [ ] `GITHUB_MODE=fixture docker compose run --rm -e SEARCH_PACING_SECONDS=0 enrich + --limit 6` exits 0 and leaves `complete 2 / permanent_failure 1` in each entity + class — two Search batches, then two detail-fallback `404` terminals (the pacing + override lets the second batch run immediately instead of reporting a pacing + deferral). - [ ] SQL state is exactly 4 events, 3 actors, 3 repositories, 3 quarantine rows, and 3 total quarantine occurrences: @@ -208,6 +211,18 @@ particular, read the erratum atop - [ ] Malformed data quarantined durably per the taxonomy (canonical fingerprints, occurrence-counted) and does not terminate the batch — 3 rows, occurrences 3 → 6 on replay, and 4 events persisted beside them +- [ ] **Both staged batch paths demonstrably run** (plan Appendix G): the fixture + `default` walkthrough completes one actor and one repository Search batch, then two + payload-URL detail fallbacks that meet `404`s and go terminal — two `complete` plus + one `permanent_failure` per class from exactly 2 search + 2 detail requests + (`fixtures/github/README.md`, the enrichment end-to-end specs) +- [ ] Batch results apply only on a **stable-ID match** — the `search_renamed_repository` + and `search_unrequested_result` scenarios show a renamed and an unrequested item + observed but never applied, routed to fallback or recorded as `unrequested_result` +- [ ] **No quota-based terminal state exists** — `git grep -n skipped_budget -- app lib + db/schema.rb` returns nothing, and every search/core denial reason + (`ceiling`, `reserve`, `pacing`, `blocked`, class/share exhaustion) defers rather + than terminates --- @@ -251,6 +266,14 @@ particular, read the erratum atop oldest pending timestamp/age, reserved allowance usage, and coverage percentages by the defined formulas — without initiating GitHub requests or fabricating a drain ETA — `spec/requests/status_spec.rb`, `Github::Enrichment::Coverage` +- [ ] `/status` carries the Appendix G blocks with their exact keys: `ledger` uses + `detail_fallback` (renamed from `enrichment`); `search_ledger` publishes + ceiling/reserve/spendable/used/per-lane usage/`blocked_until`/ + `next_request_earliest_at`; `scheduler` publishes every staged tunable; each entity + class publishes `contract_backlog_count` and all seven `stages` with counts and + oldest ages; `batches` publishes all four request-kind × entity-kind groups with + fill ratios; and `throughput.catch_up.state` is exactly one of + `keeping_up | not_keeping_up | insufficient_sample` - [ ] During §1's empty-volume fixture phase, while the worker has never been started, hash the complete budget row and record all request counters. Call `/health/live`, `/health/ready`, and `/status` repeatedly, then require the hash and counters to be @@ -349,13 +372,16 @@ particular, read the erratum atop ## 7. Forbidden-claim scan ```bash -claim_pattern='exactly[ -]?once|effectively[ -]?once|once-only[[:space:]]+(execution|processing|delivery)|(complete|full|exhaustive)[[:space:]]+(upstream[[:space:]]+)?(event[[:space:]]+)?capture|captur(e|es|ed|ing)[[:space:]]+(all|every)[[:space:]]+(upstream[[:space:]]+|public[[:space:]]+|GitHub[[:space:]]+)?events?|(complete|full|exhaustive)[[:space:]]+enrichment([[:space:]]+coverage)?|enrich(es|ed|ing)?[[:space:]]+(all|every)[[:space:]]+(actors?|repositories|entities)|sampling[[:space:]]+becomes[[:space:]]+coverage|100%[[:space:]]+(capture|enrichment|coverage)' +claim_pattern='exactly[ -]?once|effectively[ -]?once|once-only[[:space:]]+(execution|processing|delivery)|(complete|full|exhaustive)[[:space:]]+(upstream[[:space:]]+)?(event[[:space:]]+)?capture|captur(e|es|ed|ing)[[:space:]]+(all|every)[[:space:]]+(upstream[[:space:]]+|public[[:space:]]+|GitHub[[:space:]]+)?events?|(complete|full|exhaustive)[[:space:]]+enrichment([[:space:]]+coverage)?|enrich(es|ed|ing)?[[:space:]]+(all|every)[[:space:]]+(actors?|repositories|entities)|sampling[[:space:]]+becomes[[:space:]]+coverage|100%[[:space:]]+(capture|enrichment|coverage)|guaranteed?[[:space:]]+catch[ -]?up|catch[ -]?up[[:space:]]+(is[[:space:]]+)?guaranteed|(will|shall)[[:space:]]+(always[[:space:]]+)?catch[[:space:]]+up|eventual(ly)?[[:space:]]+catch(es)?[ -]?(up|ing[[:space:]]+up)|backlog[[:space:]]+(always|will)[[:space:]]+drains?|bounded[[:space:]]+drain[[:space:]]+time' git grep -nI -i -E "$claim_pattern" -- . ``` **The rule: every prose hit must explicitly reject the guarantee.** The regex assignment itself is scan vocabulary, not a claim. Any affirmative system-level promise of singular -execution, exhaustive upstream capture, or exhaustive enrichment fails the gate; do not -approve it merely because it avoids one exact phrase. +execution, exhaustive upstream capture, exhaustive enrichment, or guaranteed catch-up +fails the gate; do not approve it merely because it avoids one exact phrase. Catch-up may +be described only as a dated, measured comparison of completion and arrival rates — +`/status` says `not_keeping_up` when the comparison fails, and no document promises the +backlog drains. - [ ] Every hit is a negation diff --git a/docs/adr/0004-class-aware-budget-ledger.md b/docs/adr/0004-class-aware-budget-ledger.md index ea4d1d4..ff6cd5f 100644 --- a/docs/adr/0004-class-aware-budget-ledger.md +++ b/docs/adr/0004-class-aware-budget-ledger.md @@ -2,7 +2,7 @@ Date: 2026-07-30 -Status: Accepted +Status: Accepted; staged-batch enrichment amended 2026-08-02 ## Context @@ -108,3 +108,17 @@ the startup-validation rejection path is exercisable by lowering `POLL_INTERVAL_ instead. Deriving `ENABLED_LIVE_SOURCE_COUNT` from `event_sources` at runtime is PR 9's "dynamic multi-source allocation validation"; doing it at boot would reintroduce the database dependency that keeps validation safe to run before migrations. + +## Amendment (2026-08-02): the core ledger is now one of two resource ledgers + +Plan Appendix G ([ADR 0013](0013-derivation-first-staged-batch-enrichment.md)) moves +normal-path enrichment onto GitHub's per-minute **search** resource, accounted by its own +singleton ledger (`Github::SearchBudgetLedger` over `github_search_budget`). This ledger's +mechanics — transactional reservation, failures-stay-spent, monotonic reconciliation, +resource verification, per-window bootstrap — are unchanged, but its +`enrichment_allowance`/`enrichment_used` pair is **redefined**: it now budgets the bounded +payload-URL detail-fallback lane (`CORE_DETAIL_FALLBACK_ALLOWANCE`, default 4/hour) rather +than the remainder formula, and feasibility becomes +`poll + reserve + detail_fallback ≤ limit`, with the remainder deliberately unspent. +The resource-mismatch skip above now cuts both ways: this ledger ignores `search` headers, +and the search ledger ignores `core` headers. diff --git a/docs/adr/0007-enrichment-fairness-shares-and-borrowing.md b/docs/adr/0007-enrichment-fairness-shares-and-borrowing.md index 5755616..d29b093 100644 --- a/docs/adr/0007-enrichment-fairness-shares-and-borrowing.md +++ b/docs/adr/0007-enrichment-fairness-shares-and-borrowing.md @@ -2,7 +2,7 @@ Date: 2026-07-30 -Status: Accepted; durable-backlog policy amended 2026-08-02 +Status: Accepted; durable-backlog policy amended 2026-08-02; staged-batch enrichment amended 2026-08-02 ## Context @@ -181,3 +181,18 @@ This policy deliberately does not promise a bounded completion time. If unique e arrive faster than 40 attempts per hour can serve them, backlog size and oldest pending age will grow. That pressure is reported directly; work is never converted into a terminal budget outcome merely because the quota window ended. + +## Amendment (2026-08-02): share fairness is now detail-lane only; search lanes use weights + +Plan Appendix G ([ADR 0013](0013-derivation-first-staged-batch-enrichment.md)) makes +Search batches the normal enrichment path, so this ADR's share arithmetic — +`floor(enrichment_allowance × ACTOR_ENRICHMENT_SHARE)`, borrowing on the caller's word, +`:share_exhausted` as a denial — now governs only the bounded core **detail-fallback** +lane, whose allowance is `CORE_DETAIL_FALLBACK_ALLOWANCE` (default 4, so the guarantees +default to 2/2). The batch lanes are balanced differently: a weighted rotation +(`ACTOR_ENRICHMENT_WEIGHT` / `REPOSITORY_ENRICHMENT_WEIGHT`, defaults 1/1) over whole +Search requests, with a lane that has nothing claimable yielding its slot — batch +capacity is per-request rather than per-entity, so a per-entity share would misdescribe +it. The deleted `Enrichment::Fairness` class's decisions survive in `BatchClaim` +(claimability), `CycleRunner`'s lane schedule (rotation and borrowed slots), and the +ledger's unchanged share enforcement for detail requests. diff --git a/docs/adr/0008-post-commit-enqueue-and-entity-scoped-reconciliation.md b/docs/adr/0008-post-commit-enqueue-and-entity-scoped-reconciliation.md index 2a6d696..b99c25b 100644 --- a/docs/adr/0008-post-commit-enqueue-and-entity-scoped-reconciliation.md +++ b/docs/adr/0008-post-commit-enqueue-and-entity-scoped-reconciliation.md @@ -2,7 +2,7 @@ Date: 2026-07-31 -Status: Accepted; durable-backlog ordering amended 2026-08-02 +Status: Accepted; durable-backlog ordering amended 2026-08-02; staged-batch enrichment amended 2026-08-02 ## Context @@ -137,6 +137,19 @@ iterates them. The global request gate makes outbound concurrency exactly one application-wide, so the fanned-out jobs would serialize on the same advisory lock while each held a database connection. Deferred to PR 9, which owns multi-source allocation. +## Amendment (2026-08-02): dispatch enqueues staged cycles; observations commit with the event + +Plan Appendix G ([ADR 0013](0013-derivation-first-staged-batch-enrichment.md)) replaces +the per-class `EnrichActorJob`/`EnrichRepositoryJob` with a single argument-less +`EnrichmentCycleJob`: `Github::Enrichment::Dispatch` enqueues at most one cycle when the +dual-ledger admission and claimability checks say work could proceed, and a cycle runs +batch lanes then detail lanes inside its time budget. The decisions here carry over +unweakened — the enqueue is still a hint, the entity rows (now stage-carrying) are still +the durable record, and `ReconcilePendingEnrichmentsJob` still sweeps them every 60 +seconds. One addition strengthens the durability boundary: event-source +`enrichment_observations` rows are written **inside** the ingest transaction, so the raw +evidence an entity's derivation rests on commits atomically with the push event itself. + ## Related - ADR 0002 — advisory locks and the request gate (the crash-safety property this decision @@ -145,3 +158,4 @@ each held a database connection. Deferred to PR 9, which owns multi-source alloc - ADR 0005 — repeated execution with duplicate-safe event writes (the narrow event-row and entity-activity guarantees under redelivery) - ADR 0007 — enrichment fairness shares and borrowing (why a job cannot carry an entity id) +- ADR 0013 — derivation-first staged batch enrichment (the staged cycle this dispatch feeds) diff --git a/docs/adr/0010-secondary-limit-escalation-and-refresh-pool-fairness.md b/docs/adr/0010-secondary-limit-escalation-and-refresh-pool-fairness.md index afcecec..b36e5f7 100644 --- a/docs/adr/0010-secondary-limit-escalation-and-refresh-pool-fairness.md +++ b/docs/adr/0010-secondary-limit-escalation-and-refresh-pool-fairness.md @@ -2,7 +2,7 @@ Date: 2026-07-31 -Status: Accepted; durable-backlog refresh priority amended 2026-08-02 +Status: Accepted; durable-backlog refresh priority amended 2026-08-02; staged-batch enrichment amended 2026-08-02 ## Context @@ -134,3 +134,18 @@ non-positive delta, which `#fallback_instant` already treats exactly as an absen leave capacity idle whenever the other class has no stale rows — while §10:812's borrowing rule is stated generally rather than scoped to the pending pool. Borrowing remains valid after the durable never-enriched backlog is empty. + +## Amendment (2026-08-02): refresh-pool fairness superseded by staged refresh composition + +Plan Appendix G ([ADR 0013](0013-derivation-first-staged-batch-enrichment.md)) deletes the +per-entity refresh pool this ADR's second half repaired: there is no separate refresh +request shape any more, so `#refresh_choice` and its `borrowed_refresh` reason are gone +with `Enrichment::Fairness`. Refresh now rides the same Search batch path under the +composition rule — a batch fills from its own class's never-enriched backlog first, tops +up spare slots with TTL-stale, recently active (`REFRESH_ACTIVE_WITHIN_SECONDS`) complete +rows only when neither class has claimable backlog, and refresh-only batches run only +when neither class has backlog at all. The property this ADR fought for survives in +stronger form: backlog outranks freshness in every lane, and refresh capacity cannot +starve a class because lanes rotate by weight over whole batches. The secondary-limit +escalation in the first half is untouched, and the search ledger gains its own +`blocked_until` handling for search-resource limits. diff --git a/docs/adr/0013-derivation-first-staged-batch-enrichment.md b/docs/adr/0013-derivation-first-staged-batch-enrichment.md new file mode 100644 index 0000000..938d6b2 --- /dev/null +++ b/docs/adr/0013-derivation-first-staged-batch-enrichment.md @@ -0,0 +1,120 @@ +# 13. Derivation-first staged batch enrichment over per-entity detail fetching + +Date: 2026-08-02 + +Status: Accepted + +## Context + +PR #43's durable backlog (plan Appendix F) fixed the wrong-outcome problem — quota +scarcity no longer terminates work — but not the capacity problem. The service model +underneath was still one core request per entity, at most 40 attempts per hour, against +Appendix A's cold-demand pressure sample of roughly 2,172–2,280 entity references per +hour. Durable-but-never-draining is honest and still unsatisfying: a no-token capacity +wall, with authentication explicitly out of scope. + +A dated, live, unauthenticated probe of GitHub Search established the facts that make a +different shape possible: + +- Repeated exact `user:` qualifiers in one `/search/users` request returned **5 of 5** + requested users, `incomplete_results: false`. +- Repeated exact `repo:` qualifiers in one `/search/repositories` request returned + **9 of 10** requested repositories, `incomplete_results: false`. The miss was + `facebook/react`, which redirects to `react/react` — a rename, which is precisely why + results must be validated against stable IDs and why a fallback lane must exist. +- Both responses reported `x-ratelimit-resource: search` with a **limit of 10** — a + separate per-minute budget that the core ledger does not account for. +- Joining exact qualifiers with `OR` produced **HTTP 422**: the space-joined repeated + qualifier is the supported batching form, not an optimization over it. + +So up to ten entities can be resolved per request on a budget the ingestion pipeline was +not spending at all. + +## Decision + +Adopt **derivation-first, lossless staged batch enrichment** (plan Appendix G; issue #45): + +1. **Derive before fetching.** Ingestion persists event-native identity, appends an + event-source observation in the same transaction, derives every locally computable + field, and coalesces demand by stable GitHub ID. Network requests are spent only on + facts the stored payload does not determine. +2. **Search batches are the normal path.** Up to `SEARCH_BATCH_SIZE` (≤ 10) repeated + exact qualifiers per request, never `OR`; results mapped and validated by stable + integer ID, never by result order or mutable login/name alone. +3. **Payload-URL detail fallback is the amendment, not the rule.** Only missing, + renamed, identity-mismatched, or contract-invalid batch items fetch their stored + payload-provided `api_url`, through the core ledger's + `CORE_DETAIL_FALLBACK_ALLOWANCE` (4/hour). No identifier is ever turned into a + constructed detail URL; the polling allocation is never touched. +4. **Dual ledgers.** `github_api_budget` (core, hourly: 12 poll + 4 detail + 8 reserve + ≤ 60, remainder deliberately unspent) and `github_search_budget` (search, per-minute: + ceiling 10, reserve 2, 6-second pacing, header-less window roll) — each reconciled + only against headers naming its own resource, both behind the one global request gate. +5. **Observations and projections split.** Every raw item is an append-only + `enrichment_observations` row (source `event | search | detail`, fingerprint, + provenance, validation outcome); every request attempt is an `enrichment_batches` + envelope (counts, `total_count`/`incomplete_results`, observed rate-limit headers). + Entity tables remain the latest projection and point at their latest successful + observation; a refresh repoints the projection and never overwrites retained + evidence. +6. **A stage machine with seven resting stages** — `batch_pending`, `batch_in_flight`, + `detail_pending`, `detail_in_flight`, `retry_scheduled`, `contract_complete`, + `terminal` — under durable leases, with instant timestamps for the conditions no row + rests in. Completion is the explicit useful-data contract per entity kind, nullable + fields valid as nulls. Terminal outcomes exist only for entity-specific facts (a + 404/410 immediately; other detail failures after `DETAIL_FALLBACK_MAX_ATTEMPTS`); + **no quota-based terminal outcome exists anywhere**. +7. **Refresh rides the same batch path** under the composition rule: own-class backlog + fills first, TTL-stale refresh tops up spare slots only when neither class has + claimable backlog, refresh-only batches only when neither has backlog at all. + +## Consequences + +What this buys: + +- The theoretical service ceiling moves from 40 entities/hour to 4,800 items/hour + (8 spendable search requests/minute × batches of 10) — stated as a **capacity + hypothesis**, since misses, fallback, retries, and pacing all subtract from it. +- Core polling is better protected than before: normal-path enrichment no longer + competes on core at all, and the detail lane is capped at 4 rather than 40. +- Every enrichment claim is auditable from durable state: what was asked, what came + back, what validated, what was applied, and which raw evidence supports the current + projection. +- A rename or identity mismatch is detected rather than silently applied, because + application requires a stable-ID match. + +What it costs, stated plainly: + +- Roughly double the write volume per enriched entity (observation + projection + batch + envelope), and three new tables to operate. +- Search documents are shallower than detail documents; the completion contract is + deliberately narrower than a full profile, and fields outside it (actor name, company, + location, bio, follower counts) are explicitly not promised. +- Two ledgers mean two exhaustion vocabularies; `/status` publishes both so a denial is + attributable. + +**The measured-catch-up honesty rule.** Capacity arithmetic is a hypothesis and is +labeled as one. The acceptance gate is measured: `/status` publishes arrivals, +completions, backlog delta, and a tri-state `catch_up.state` +(`keeping_up | not_keeping_up | insufficient_sample`, gated by +`CATCH_UP_MIN_SAMPLE_SECONDS`). When completions do not exceed arrivals the service +reports `not_keeping_up`; it never claims eventual catch-up, and it publishes no drain +ETA. Any documentation stating otherwise fails plan §16's forbidden-claims gate. + +## Rejected alternatives + +- **Keep per-entity core enrichment and tune it.** No tuning escapes the arithmetic: 40 + core attempts per hour cannot meet a demand sample fifty times larger, and raising the + core enrichment slice can only cannibalize polling or the reserve. +- **Authenticate.** Out of scope by the issue and plan §2, and it would change the + problem rather than solve it — an authenticated budget is larger, not unbounded, and + the durability and honesty requirements are identical. +- **Drop or sample work under pressure.** Rejected by Appendix F already; quota is a + scheduling constraint, not an entity outcome. This ADR extends that: even the new + search denials (`ceiling`, `reserve`, `pacing`, `blocked`) only ever defer. +- **`OR`-joined qualifiers or free-text search.** The probe answered this: `OR`-joined + exact qualifiers return 422, and free-text matching would reintroduce the + wrong-entity risk that stable-ID validation exists to eliminate. +- **Constructing detail URLs from identifiers after a search miss.** Rejected on SSRF + grounds: the boundary distinguishes application-origin constants from payload-origin + URLs, and synthesizing URLs from data would blur the one line that keeps it auditable. From 453fea6e15054afc6534012aee9797bbea849cc9 Mon Sep 17 00:00:00 2001 From: Umang Date: Sun, 2 Aug 2026 16:05:55 -0500 Subject: [PATCH 11/12] Close review findings on the staged enrichment path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Seven findings from an independent review of the branch, plus the default the live measurement corrected. Correctness: - A secondary rate limit is IP-scoped, so it binds both ledgers (§10). Search reservations now refuse while the core ledger holds a global block, and a secondary limit provoked by a Search response writes that block so polling stops too. A *primary* Search exhaustion stays local — blocking polling on it would let the search lane starve event capture. - MAX_HTTP_RETRIES was inert on Search: the ~1s backoff landed inside the 6s pacing window, so every retry was refused as :search_pacing and the transport failure it existed to repeat was replaced by a budget denial. A Search retry now waits at least the pacing interval. - The borrow flag stated which slot was taken; the ledger reads it as "the other class has no eligible candidate". They differ exactly when a backlog is one-sided, which stranded it at half the allowance with the rest idle. It is now computed from the other lane's real claimability — and in bin/enrich from the unfiltered pool, so --class cannot claim a lane is empty when the operator merely excluded it. - Only one exact English 422 message reached the fallback. Every deterministic client error now does: a rejected query is a fact about this batch's identifiers and URL, and the ladder would resend both unchanged, hourly, forever. Matching on GitHub's wording would also break when they reword it. - A deferred batch restored every row to batch_pending, including rows claimed from retry_scheduled — a status/stage pair Enrichable does not list as legal. release! now restores the stage it claimed. Operability: - Compose forwards the staged-enrichment configuration. Without it the variables .env.example documents and /status publishes set nothing inside a container. - CORE_DETAIL_FALLBACK_ALLOWANCE defaults to 40 rather than 4. The live run measured a 1.6-1.9% Search miss rate against ~2,000 arrivals an hour, so the fallback lane needs on the order of 40 requests an hour and had four; the core budget leaves exactly 40 after polling and the reserve. The evidence file is explicit that this is arithmetic from a measured miss rate, not itself a measured outcome. - The cycle budget bounds when a request may start, not how long one already in flight may run. Nothing can preempt a spent reservation, so the overrun is documented and bounded rather than papered over. Co-Authored-By: Claude Fable 5 --- app/services/github/configuration.rb | 9 +- app/services/github/enrichment/batch_claim.rb | 15 +- .../github/enrichment/batch_runner.rb | 29 +- .../github/enrichment/cycle_runner.rb | 25 +- app/services/github/enrichment/one_shot.rb | 8 +- app/services/github/request_executor.rb | 16 +- app/services/github/search_budget_ledger.rb | 33 +- docker-compose.yml | 21 ++ ...2026-08-02-live-staged-batch-enrichment.md | 298 ++++++++++++++++++ spec/config/github_initializer_spec.rb | 6 +- spec/requests/status_spec.rb | 2 +- spec/services/github/allowances_spec.rb | 34 +- .../github/budget_ledger_bootstrap_spec.rb | 2 +- spec/services/github/budget_ledger_spec.rb | 19 +- spec/services/github/configuration_spec.rb | 13 +- .../github/enrichment/batch_runner_spec.rb | 37 ++- .../github/enrichment/cycle_runner_spec.rb | 19 ++ spec/services/github/request_executor_spec.rb | 42 +++ .../github/search_budget_ledger_spec.rb | 66 ++++ .../github/status/scheduler_settings_spec.rb | 2 +- spec/services/github/status/snapshot_spec.rb | 2 +- 21 files changed, 637 insertions(+), 61 deletions(-) create mode 100644 docs/evidence/2026-08-02-live-staged-batch-enrichment.md diff --git a/app/services/github/configuration.rb b/app/services/github/configuration.rb index 732aea3..55f6b62 100644 --- a/app/services/github/configuration.rb +++ b/app/services/github/configuration.rb @@ -30,7 +30,14 @@ class Configuration "MAX_PAGES_PER_POLL" => "1", "ENABLED_LIVE_SOURCE_COUNT" => "1", "RATE_LIMIT_RESERVE" => "8", - "CORE_DETAIL_FALLBACK_ALLOWANCE" => "4", + # The whole core remainder after polling and the reserve (60 - 12 - 8), which is + # what the pre-staged design spent on every entity and this one spends only on the + # Search-miss residue. A live sample measured that residue at ~1.9% of arrivals — + # roughly 40 fallbacks an hour at the observed rate — so a smaller allowance + # leaves a subset that never completes. It still cannot cover a peak-rate residue + # unauthenticated, which is why /status publishes the measured verdict rather than + # a promise. + "CORE_DETAIL_FALLBACK_ALLOWANCE" => "40", "SEARCH_REQUEST_CEILING" => "10", "SEARCH_SAFETY_RESERVE" => "2", "SEARCH_BATCH_SIZE" => "10", diff --git a/app/services/github/enrichment/batch_claim.rb b/app/services/github/enrichment/batch_claim.rb index bfcb0a1..6ff14a8 100644 --- a/app/services/github/enrichment/batch_claim.rb +++ b/app/services/github/enrichment/batch_claim.rb @@ -103,9 +103,15 @@ def acquire(entity_type, now: Time.current) lease end + # A deferral must put the row back where it was claimed from, not where a new row + # starts. Restoring a retry_scheduled/retryable_failure row to batch_pending would + # leave a status/stage pair Enrichable does not list as legal and would read as + # work that had never been attempted. previous_stage is the claimed value; + # batch_in_flight is excluded because that is what an expired lease was reclaimed + # from, and returning a row to it would re-orphan it. def release!(lease, stage: nil, now: Time.current) lease.items.each do |item| - restored = stage || (item.enrichment_status == "complete" ? "contract_complete" : "batch_pending") + restored = stage || restored_stage(item) lease.entity_type.model.where(id: item.id, lease_token: lease.token, current_enrichment_batch_id: lease.batch.id).update_all( enrichment_stage: restored, lease_token: nil, leased_until: nil, @@ -116,6 +122,13 @@ def release!(lease, stage: nil, now: Time.current) private + def restored_stage(item) + return item.previous_stage if item.previous_stage.present? && + item.previous_stage != "batch_in_flight" + + item.enrichment_status == "complete" ? "contract_complete" : "batch_pending" + end + # The single due predicate: a live lease or a scheduled retry excludes a row # from every claim; expiry re-admits it. def due(model, now:) diff --git a/app/services/github/enrichment/batch_runner.rb b/app/services/github/enrichment/batch_runner.rb index 2cc8260..00097b4 100644 --- a/app/services/github/enrichment/batch_runner.rb +++ b/app/services/github/enrichment/batch_runner.rb @@ -251,28 +251,37 @@ def malformed_batch(lease, response) # batch of ten mixes them in and simply omits them, so this only occurs once the # remaining members are all in that state. # - # It is an authoritative per-item answer, so the members are admitted to the - # detail lane exactly as an omitted item would be. Retrying the search instead - # would reproduce the same 422 forever, and the stored payload URL is what - # actually resolves a rename. - UNSEARCHABLE_STATUS = 422 + # Every deterministic client error is treated the same way, not just that one + # message. A rejected query is a fact about *this batch's* identifiers and URL, + # and the retry ladder would resend both unchanged: the same 4xx, hourly, with no + # terminal condition and no way to self-heal. Routing the members to the detail + # lane is bounded (its own allowance, its own attempt ladder, its own terminal + # outcome) and is the one path that can actually resolve them. Matching on the + # English phrase alone would also break the moment GitHub rewords it. + # + # Rate-limit responses are excluded by the caller: 403/429 classify as + # rate_limited or secondary_limited and defer rather than reaching here. UNSEARCHABLE_SIGNAL = "cannot be searched".freeze def unsearchable?(fetched) - fetched.status == UNSEARCHABLE_STATUS && - fetched.body.to_s.include?(UNSEARCHABLE_SIGNAL) + fetched.classification == :client_error + end + + def unsearchable_reason(fetched) + fetched.body.to_s.include?(UNSEARCHABLE_SIGNAL) ? "unsearchable_identifier" : "search_query_rejected" end def unsearchable_batch(lease, fetched) now = @clock.call + reason = unsearchable_reason(fetched) lease.items.each do |item| - admit_fallback(lease, item, "unsearchable_identifier", now: now) + admit_fallback(lease, item, reason, now: now) end lease.batch.update!(status: "failed", completed_at: now, - last_error: "unsearchable_identifiers", + last_error: reason, returned_count: 0, missing_count: lease.items.length) Rails.logger.warn(event: "enrichment.batch_unsearchable", **lease.to_log, - response_status: fetched.status) + reason: reason, response_status: fetched.status) Result.new(status: "completed", entity_type: lease.entity_type.key, batch_id: lease.batch.id, requested_count: lease.items.length, returned_count: 0, valid_count: 0, diff --git a/app/services/github/enrichment/cycle_runner.rb b/app/services/github/enrichment/cycle_runner.rb index a4a41f8..77fef69 100644 --- a/app/services/github/enrichment/cycle_runner.rb +++ b/app/services/github/enrichment/cycle_runner.rb @@ -10,6 +10,15 @@ module Enrichment # (~4,800/hour theoretical) where a job-per-request design at the tick cadence # could never exceed ~1,200/hour. The pacing sleep happens here, on the dedicated # single-thread enrichment worker, never inside a gate hold or a ledger lock. + # + # The cycle budget bounds when a *new* request may start, not how long one already + # in flight may take: a single fetch can legitimately run to the gate wait plus its + # timeouts across every retry and redirect hop, which exceeds the budget. Nothing + # preempts it, and nothing should — the reservation is already spent. The overrun is + # bounded by Configuration#worst_case_fetch_seconds and is safe to overlap the next + # tick: the enrichment queue has one thread, so a cycle enqueued meanwhile waits + # rather than running beside this one, and it finds the pacing and ceiling state + # this cycle left behind. class CycleRunner # Two consecutive idle claims (a claimable? race that found nothing to lock) # end the phase rather than spinning on it. @@ -36,16 +45,22 @@ def initialize(actor_weight:, repository_weight:) end # @param claimable [Proc] lane key -> Boolean - # @return [Array(Symbol, Boolean), nil] lane and whether the slot was borrowed + # @return [Array(Symbol, Boolean), nil] the lane to run, and whether the *other* + # class has no claimable candidate — which is what a borrow asserts to + # Github::BudgetLedger. It is deliberately not "this slot was borrowed": a + # one-sided backlog would then stop at its own guarantee on every scheduled + # turn, even though the capacity it needs is provably idle. def next_claimable(claimable) scheduled = @rotation[@cursor % @rotation.length] @cursor += 1 - return [ scheduled, false ] if claimable.call(scheduled) - other = scheduled == :actor ? :repository : :actor - return [ other, true ] if claimable.call(other) - nil + lane = if claimable.call(scheduled) then scheduled + elsif claimable.call(other) then other + end + return nil if lane.nil? + + [ lane, !claimable.call(lane == :actor ? :repository : :actor) ] end end diff --git a/app/services/github/enrichment/one_shot.rb b/app/services/github/enrichment/one_shot.rb index 06de0b9..82b8462 100644 --- a/app/services/github/enrichment/one_shot.rb +++ b/app/services/github/enrichment/one_shot.rb @@ -166,7 +166,13 @@ def detail_lane(lanes) }) return if choice.nil? - lane, borrowed = choice + lane = choice.first + # Recomputed against the *unfiltered* pool, never taken from the schedule: + # --class narrows what this command will work on, and a borrow is a claim + # about what the other class has to do. Letting the CLI filter answer it would + # tell the ledger a lane was idle when the operator had merely excluded it. + other = lane == :actor ? :repository : :actor + borrowed = !@detail_claim.claimable?(EntityType.fetch(other)) result = @detail_runner.call(entity_class: lane, borrow: borrowed) @tally = @tally.record_detail(result) case result.status diff --git a/app/services/github/request_executor.rb b/app/services/github/request_executor.rb index f0d5c82..6358c11 100644 --- a/app/services/github/request_executor.rb +++ b/app/services/github/request_executor.rb @@ -26,6 +26,7 @@ def initialize(transport: Github.transport, retry_policy: RetryPolicy.new, mode: Github.configuration.mode, max_redirects: Github.configuration.max_redirects, + search_pacing_seconds: Github.configuration.search_pacing_seconds, request_gate_wait: RequestGate::WAIT_SECONDS, sleeper: ->(seconds) { Kernel.sleep(seconds) }, clock: -> { Time.current }) @@ -35,6 +36,7 @@ def initialize(transport: Github.transport, @retry_policy = retry_policy @mode = mode.to_sym @max_redirects = max_redirects + @search_pacing_seconds = search_pacing_seconds @request_gate_wait = request_gate_wait @sleeper = sleeper @clock = clock @@ -53,7 +55,7 @@ def call(request) # Computed once and both slept and logged, never recomputed: RetryPolicy jitters, so # asking twice would report a delay this process never took. - backoff_seconds = @retry_policy.backoff_seconds(attempt) + backoff_seconds = retry_delay_for(request, attempt) log_retry_scheduled(result, backoff_seconds: backoff_seconds) # The backoff happens with no lock held and no reservation outstanding: the @@ -65,6 +67,18 @@ def call(request) private + # A Search retry must outwait that ledger's pacing, or MAX_HTTP_RETRIES is inert on + # this resource: the default backoff is around a second, pacing is six, so every + # retry would be refused as :search_pacing and the transport failure it was meant to + # retry would be replaced by a budget denial. Taking the larger of the two keeps one + # meaning for "retry" across both resources. + def retry_delay_for(request, attempt) + backoff = @retry_policy.backoff_seconds(attempt) + return backoff unless request.search? + + [ backoff, @search_pacing_seconds ].max + end + # A retryable failure at hop n restarts from the original request rather than from # the last hop, so MAX_HTTP_RETRIES keeps its plain meaning: this logical fetch was # attempted N times. diff --git a/app/services/github/search_budget_ledger.rb b/app/services/github/search_budget_ledger.rb index f7f81ed..2633e46 100644 --- a/app/services/github/search_budget_ledger.rb +++ b/app/services/github/search_budget_ledger.rb @@ -32,6 +32,16 @@ def reserve!(request_class, now: Time.current, borrow: false) end bootstrap!(now: now) + + # §10: a secondary rate limit is IP-scoped, so it stops *all* live requests, not + # only the resource that provoked it. The two ledgers meter separate resources but + # share one outbound address, so each honours the other's global block. + if (blocked_until = global_block(now: now)) + Rails.logger.info(event: "search_budget.globally_blocked", + blocked_until: blocked_until.utc.iso8601) + raise Errors::BudgetExhausted.new(request_class, :globally_blocked) + end + reason = nil GithubSearchBudget.transaction do @@ -98,7 +108,13 @@ def reconcile!(snapshot, request_class: nil, now: Time.current) end end - def block_from!(fetched, now: Time.current) + # @param core_ledger [Github::BudgetLedger] the writer of global_blocked_until. A + # secondary limit provoked by a Search request is IP-scoped like any other, so it + # has to stop polling too — §10 is explicit that one timestamp covers every live + # request. A primary Search exhaustion is *not* global: it bounds only this + # resource, and blocking polling on it would hand the search lane the power to + # starve event capture. + def block_from!(fetched, now: Time.current, core_ledger: BudgetLedger.new) return unless %i[rate_limited secondary_limited].include?(fetched.classification) snapshot = fetched.rate_limit(observed_at: now) @@ -109,6 +125,11 @@ def block_from!(fetched, now: Time.current) snapshot.reset_at || now + SEARCH_WINDOW_SECONDS end + if fetched.classification == :secondary_limited + core_ledger.block_globally!(until_at: until_at, reason: "search_secondary_limit", + now: now) + end + # GREATEST ignores NULL, so a block only ever moves later — the core ledger's # BLOCK_SQL rule, restated for the search row. GithubSearchBudget.where(id: SINGLETON_ID).update_all( @@ -138,6 +159,16 @@ def bootstrap!(now: Time.current) private + # The core ledger owns global_blocked_until because a secondary limit can arise on + # any live request and Github::RateLimitPolicy already writes it there. Read with + # find_by, never through BudgetLedger: this path must not create that row. + def global_block(now:) + blocked_until = GithubApiBudget.find_by(id: GithubApiBudget::SINGLETON_ID) + &.global_blocked_until + + blocked_until if blocked_until&.>(now) + end + # The window has moved on when GitHub's own reset instant passed, or — when no # response ever told us one — when a full Search window elapsed since the last # outbound attempt. diff --git a/docker-compose.yml b/docker-compose.yml index cade1f3..8f90d26 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -53,6 +53,27 @@ x-app-env: &app_env # member here it changes only what /status *reports*, never what any process *does*, so # a disagreement between two processes would be cosmetic rather than a policy split. ENRICHMENT_COVERAGE_WINDOW_SECONDS: ${ENRICHMENT_COVERAGE_WINDOW_SECONDS:-86400} + # Appendix G's staged-enrichment scheduler, forwarded for the reason the allowance + # inputs are: without these lines the variables .env.example documents and /status + # publishes would set nothing inside any container. They belong in the shared anchor + # because both ledgers are single rows serving every process — a worker pacing at six + # seconds beside a one-shot pacing at zero would be two policies against one row. + CORE_DETAIL_FALLBACK_ALLOWANCE: ${CORE_DETAIL_FALLBACK_ALLOWANCE:-40} + SEARCH_REQUEST_CEILING: ${SEARCH_REQUEST_CEILING:-10} + SEARCH_SAFETY_RESERVE: ${SEARCH_SAFETY_RESERVE:-2} + SEARCH_BATCH_SIZE: ${SEARCH_BATCH_SIZE:-10} + SEARCH_PACING_SECONDS: ${SEARCH_PACING_SECONDS:-6} + SEARCH_WORKER_CONCURRENCY: ${SEARCH_WORKER_CONCURRENCY:-1} + ACTOR_ENRICHMENT_WEIGHT: ${ACTOR_ENRICHMENT_WEIGHT:-1} + REPOSITORY_ENRICHMENT_WEIGHT: ${REPOSITORY_ENRICHMENT_WEIGHT:-1} + DETAIL_FALLBACK_MAX_ATTEMPTS: ${DETAIL_FALLBACK_MAX_ATTEMPTS:-3} + ENRICHMENT_LEASE_SECONDS: ${ENRICHMENT_LEASE_SECONDS:-600} + ENRICHMENT_CYCLE_BUDGET_SECONDS: ${ENRICHMENT_CYCLE_BUDGET_SECONDS:-55} + ENRICHMENT_RETRY_BASE_SECONDS: ${ENRICHMENT_RETRY_BASE_SECONDS:-60} + ENRICHMENT_RETRY_MAX_SECONDS: ${ENRICHMENT_RETRY_MAX_SECONDS:-3600} + ENRICHMENT_METRICS_WINDOW_SECONDS: ${ENRICHMENT_METRICS_WINDOW_SECONDS:-3600} + CATCH_UP_MIN_SAMPLE_SECONDS: ${CATCH_UP_MIN_SAMPLE_SECONDS:-900} + REFRESH_ACTIVE_WITHIN_SECONDS: ${REFRESH_ACTIVE_WITHIN_SECONDS:-604800} services: db: diff --git a/docs/evidence/2026-08-02-live-staged-batch-enrichment.md b/docs/evidence/2026-08-02-live-staged-batch-enrichment.md new file mode 100644 index 0000000..fc3dce4 --- /dev/null +++ b/docs/evidence/2026-08-02-live-staged-batch-enrichment.md @@ -0,0 +1,298 @@ +# Live staged batch enrichment verification + +```text +Probe date: 2026-08-02 (UTC) +Runtime: working tree of agent/durable-enrichment-backlog (issue #45) + parent commit a18382e525a9642d376fa05b9c081b552f6a2ad5 +Run window: 2026-08-02T20:20:35Z → 2026-08-02T20:27:00Z (phase A) + 2026-08-02T20:35:29Z → 2026-08-02T21:01:30Z (phase B) +Docker: 28.3.0 +Docker Compose: 2.38.1-desktop.1 +API version: 2022-11-28 +Authorization: none sent +GitHub mode: live +Isolation: two fresh Compose projects, each with a fresh PostgreSQL volume; + the development stack was stopped for the duration so the probes + owned the outbound IP's quota +``` + +## Questions + +1. Does one unauthenticated Search request actually settle a whole batch of entities, + at the fill ratio Appendix G's capacity hypothesis assumes? +2. Are the Search rate-limit headers a separate resource from `core`, and does the + application account for them separately? +3. Does a renamed or unsearchable identifier reach the payload-URL fallback and resolve + there, rather than looping on the Search lane? +4. Does queued work survive quota exhaustion and a database restart unchanged? +5. Under a sustained run at the shipped defaults, does the measured completion rate + exceed the measured arrival rate, with a negative backlog slope while draining? + +Questions 1–4 are answered **yes** below. Question 5's answer is recorded in +[Phase B](#phase-b--sustained-catch-up-measurement) from the measurement itself, not +from the capacity arithmetic. + +## Method and safety boundary + +Both phases used the current working-tree image and explicitly named Compose projects +(`gpi-live-a`, `gpi-live-b`), each created and destroyed with its own volume. Phase A ran +`db` plus one-shot commands only — no `web`, no `worker` — so every outbound request was +one this transcript names. Phase B ran the full topology at the shipped defaults. + +Sanitization: aggregate counts, stable GitHub ids, classifications, stage names, and +rate-limit fields are reproduced verbatim. Response bodies and third-party logins are not, +except where a login is itself the finding (`github-actions[bot]`, `facebook/react`) and is +already public. + +Budget accounting for the whole session: 2 core polls, 3 core detail requests, and 14 +Search requests in phase A; phase B stayed inside the shipped per-window allowances by +construction (12 poll and the detail-fallback allowance per hour on `core`, 8 spendable +per minute on `search`). +Neither phase approached the 60/hour core limit. + +## Phase A — staged path, boundaries, and durability + +### One poll, 176 cold entities, zero enrichment requests + +```text +event=ingestion.run_completed events_received=99 push_events_seen=92 events_created=92 +event=budget.window_initialized rate_limit_resource=core rate_limit_limit=60 + rate_limit_remaining=59 rate_limit_used=1 poll_used=1 + +cold_actors=84 cold_repos=92 total=176 +stages_actor={"batch_pending" => 84} +stages_repo={"batch_pending" => 92} +observations={"event" => 184} +sample_repo={"github_id":1320389400,"full_name":"reddupney66/gfkapf", + "owner_login":"reddupney66","enrichment_stage":"batch_pending"} +``` + +`owner_login` is derived locally from the event's `repo.name`; 184 event-source +observations committed with the events. No enrichment request had been issued at this +point — the derivation-first stage costs no quota. + +### One Search request settles nine actors; one settles ten repositories + +```text +event=enrichment.fallback_admitted entity_type=actor github_actor_id=41898282 + reason=missing_search_result enrichment_batch_id=1 +event=enrichment.batch_completed entity_type=actor batch_id=1 + requested_count=10 returned_count=9 valid_count=9 fallback_count=1 + incomplete_results=false +event=enrichment.batch_completed entity_type=repository batch_id=2 + requested_count=10 returned_count=10 valid_count=10 fallback_count=0 + incomplete_results=false +``` + +The persisted batch envelopes carry the answer to question 2: + +```text +batch=1 kind=search/actor status=succeeded req=10 ret=9 valid=9 miss=1 + total_count=9 incomplete=false rl_resource=search rl_limit=10 rl_remaining=9 +batch=2 kind=search/repository status=succeeded req=10 ret=10 valid=10 miss=0 + total_count=10 incomplete=false rl_resource=search rl_limit=10 rl_remaining=9 +core_ledger=poll_used=1 enrichment_used=0 +``` + +`x-ratelimit-resource: search` with a limit of **10**, reconciled onto the search ledger +while the core ledger's enrichment counter stayed at zero. Nineteen entities reached the +useful-data contract for two requests, and the contract fields are populated from the +search items themselves: + +```text +completed_actor={"github_id":311997514,"account_type":"User", + "enrichment_status":"complete","latest_observation_source":"search"} +completed_repo={"github_id":1320389400,"language":null,"fork":false,"archived":false, + "default_branch":"main","owner_github_id":311997514} +``` + +`language: null` is a valid contract value, not a gap: the contract is "a valid response +was durably observed", never "every nullable field is populated". + +### A full cycle: 74 entities in 42.5 seconds, stopping at the reserve + +The production loop (`Github::Enrichment::CycleRunner`, what `EnrichmentCycleJob` runs) +against the remaining backlog: + +```text +before={"actor":74,"repo":0} +cycle={"batches_attempted":8,"batches_completed":8,"batches_deferred":0,"batches_failed":0, + "items_requested":74,"items_valid":74,"fallbacks_admitted":0, + "details_attempted":1,"details_completed":0,"details_terminal":1, + "batch_stop_reason":"search_blocked","detail_stop_reason":"no_detail_work", + "duration_ms":42518} +after={"actor":0,"repo":0} +search_ledger={"limit":10,"remaining":2,"request_ceiling":10,"reserve":2,"used":8, + "available":0,"blocked_until":"2026-08-02T20:26:21Z"} +``` + +**74 requested, 74 valid — a fill ratio of 1.00** across eight paced requests, stopping +cleanly when the observed `remaining` reached the configured reserve rather than by +running the limit to zero. This is the capacity hypothesis measured rather than assumed: +8 spendable requests per minute at a batch size of 10 is an **80/minute ceiling**, and the +sample reached it. + +### Two defects this phase found + +Both were invisible to design review and to the offline corpus, and both are fixed and +covered by regression tests in the same change. + +**1. An unparsable payload URL was treated as retryable.** The actor `github-actions[bot]` +carries a login with brackets, so the URL its own event supplies is not a valid URI and +`Github::UrlPolicy` refuses it before the request gate: + +```text +event=enrichment.detail_retry_scheduled github_actor_id=41898282 detail_attempts=1 + reason=refused "https://api.github.com/users/github-actions[bot]": unparsable +``` + +The refusal is correct; the disposition was not. §10 classifies a policy violation as +permanent, and the ladder would have spent three of the four hourly core detail requests +re-refusing the same stored string. `DetailRunner` now terminates on +`not_found`, `client_error`, and `permanent_error` alike. + +**2. A batch of entirely unsearchable identifiers answered 422, not an empty result.** +Seeding `facebook/react` — which redirects to `react/react` — as the only pending +repository produced: + +```text +url=https://api.github.com/search/repositories?q=repo%3Afacebook%2Freact&per_page=1 +status=422 +body={"message":"Validation Failed","errors":[{"message":"The listed users and + repositories cannot be searched either because the resources do not exist or you + do not have permission to view them.", ... }]} +``` + +The exploratory probe never saw this because a batch of ten simply omits its unsearchable +members. Treating it as a generic client error left the rename retrying on the Search lane +forever — the one path that can never resolve it. A 422 carrying that signature is now +read as "every requested identifier is missing", and the members are admitted to the +fallback exactly as an omitted item is: + +```text +event=enrichment.fallback_admitted github_repository_id=10270250 + reason=unsearchable_identifier enrichment_batch_id=5 +event=enrichment.batch_unsearchable entity_kind=repository response_status=422 +event=enrichment.detail_completed github_repository_id=10270250 detail_attempts=1 + +react={"github_id":10270250,"full_name":"facebook/react","enrichment_status":"complete", + "enrichment_stage":"contract_complete","language":"JavaScript", + "default_branch":"main","owner_github_id":102812, + "latest_observation_source":"detail"} +``` + +The fallback followed the stored payload URL through GitHub's redirect and validated the +result against the stable id — the rename resolved without ever constructing a URL from a +mutable name. + +### Quota boundary and restart durability + +With the search ledger blocked at its reserve, five pending rows were fingerprinted, the +database container was restarted, and the fingerprint recomputed: + +```text +BEFORE pending=5 fingerprint=ac3f8d66c6f93e225ec507ccc4d0fafb +BEFORE search_ledger used=8 blocked_until=2026-08-02T20:26:21Z +--- docker compose restart db --- +AFTER pending=5 fingerprint=ac3f8d66c6f93e225ec507ccc4d0fafb +AFTER search_ledger used=8 blocked_until=2026-08-02T20:26:21Z +AFTER batches={["search","succeeded"]=>10, ["search","failed"]=>2, + ["detail","succeeded"]=>1, ["detail","failed"]=>2} observations=278 +``` + +Identical fingerprint, identical ledger state, and every batch envelope and observation +retained. Quota exhaustion deferred work; it did not terminate any of it. + +A forced poll during the same window was refused by the poll class's own allowance +(`deferral_reason=poll_class_blocked_until`) — enrichment pressure never borrows from +polling, and neither does an operator's `--force`. + +### A third defect, found by the topology rather than the API + +Phase B initially made no progress at all: recurring jobs accumulated in +`solid_queue_ready_executions` while both workers registered, heartbeated, and claimed +nothing. `config/queue.yml` declared `queues: polling,control` as a bare string, and +`SolidQueue::QueueSelector` wraps its input in `Array()` — so that worker was polling for +one queue literally named `"polling,control"`, which no job is ever enqueued into. + +This predates issue #45 (the file is untouched by this change), and it means the always-on +`worker` container never polled or reconciled: only the single-name `enrichment` worker +functioned. The suite did not catch it because the spec split the configured string itself +and so asserted the author's intent rather than the runtime's reading of it. The queue is +now declared as a YAML list, and both queue specs assert through +`SolidQueue::QueueSelector` and `Array()` instead. + +## Phase B — sustained catch-up measurement + +Full topology (`db`, `web`, `worker`) at the shipped defaults, sampling `GET /status` +every 60 seconds. `/status` performs no writes and issues no GitHub request, so sampling +cannot perturb the run. + +Configuration: the shipped defaults **at the time of the run**, which included +`CORE_DETAIL_FALLBACK_ALLOWANCE=4`. That value is what this measurement put under +pressure, and the result is why the default is now 40 — see "What the measurement +changed" below. + +```text +Run window: 2026-08-02T20:35:29Z → 2026-08-02T21:01:30Z (26 minutes) +Samples: 53 (60s cadence), 24 of them past CATCH_UP_MIN_SAMPLE_SECONDS +Trailing window: ENRICHMENT_METRICS_WINDOW_SECONDS = 3600 + +Arrivals: 872 entities +Completions: 865 +Terminal outcomes: 5 +Exits: 870 (99.8% of arrivals) +Arrival rate: 1,975.58 / hour (measured, trailing window) +Completion rate: 1,959.72 / hour (measured, trailing window) +Backlog delta: +2 at the final sample + +Search batches: 45 actor + 47 repository = 92 requests +Items requested: 429 actor + 443 repository = 872 +Fill ratio: 0.981 actor, 0.986 repository +Missing items: 8 actor + 6 repository = 14 (1.6% of requested) +Detail fallbacks: 6 actor + 7 repository = 13 requests +Observations: 1,797 rows +Core ledger: poll_used 2 of 12, detail_fallback used 4 of 4 +``` + +**Verdict across the 24 mature samples: 8 `keeping_up`, 16 `not_keeping_up`.** The +verdict oscillates, and the reason is visible in the numbers rather than a mystery: the +Search lane settled 872 entities in 92 requests and drained each five-minute arrival +burst within roughly two minutes, while a residue that Search does not return — 1.6% of +requested items — waits on the detail lane. With that lane capped at 4 requests an hour, +the residue outlives the window it arrived in, and any sample taken while it is +outstanding reports a positive backlog delta. + +At the final sample the outstanding work was exactly two repositories, both +`detail_pending` with `missing_search_result`, and the core detail lane reading 4 of 4 +used. Nothing was stuck: everything was deferred, durably, by a budget doing what it was +configured to do. + +**Question 5 answered honestly: the Search lane keeps up; the service as configured +during this run did not, and it said so.** `/status` reported `not_keeping_up` in exactly +the samples where the backlog had not returned to zero — no eventual-catch-up claim was +made, and none is made here. + +### What the measurement changed + +At a 1.6–1.9% miss rate and roughly 2,000 arrivals an hour, the fallback lane needs on the +order of 40 requests an hour. It had 4. The core budget leaves exactly 40 after polling +(12) and the reserve (8) — the same allocation the pre-staged design spent on *every* +entity, which the staged path needs only for the residue — so +`CORE_DETAIL_FALLBACK_ALLOWANCE` now defaults to 40. + +This is arithmetic from a measured miss rate, not a measured outcome: the sustained run +above was performed at 4, and no equivalent run at 40 was performed within this session's +unauthenticated quota. Even 40 does not guarantee the residue clears at peak arrival +rates, which is the honest reason `/status` publishes a measured verdict rather than a +promise. + +## Caveats + +- One sample, one IP, one feed. Arrival rates on GitHub's public feed vary by hour; a + measurement is evidence about the window it covers, not a guarantee about future ones. +- The catch-up verdict published by `/status` is a comparison of measured rates over a + trailing window. It is deliberately not a forecast, and no drain estimate is published + anywhere in this service. +- Search returns items the query matched; an entity absent from GitHub's search index is + handled by the fallback rather than by a larger batch. diff --git a/spec/config/github_initializer_spec.rb b/spec/config/github_initializer_spec.rb index 51b6022..e6cf217 100644 --- a/spec/config/github_initializer_spec.rb +++ b/spec/config/github_initializer_spec.rb @@ -18,8 +18,8 @@ def boot = Rails.application.reloader.prepare! it "logs the resolved allowances at boot" do expect(Rails.logger).to receive(:info).with(hash_including( event: "config.budget_resolved", mode: "live", - poll_allowance: 12, enrichment_allowance: 4, reserve: 8, - actor_guarantee: 2, repository_guarantee: 2 + poll_allowance: 12, enrichment_allowance: 40, reserve: 8, + actor_guarantee: 20, repository_guarantee: 20 )) boot @@ -28,7 +28,7 @@ def boot = Rails.application.reloader.prepare! it "leaves Github.configuration validated and memoized for the process" do boot - expect(Github.configuration.allowances).to have_attributes(poll_allowance: 12, enrichment_allowance: 4) + expect(Github.configuration.allowances).to have_attributes(poll_allowance: 12, enrichment_allowance: 40) end # The over-commitment the allowance formula cannot see: it counts one attempt per page, diff --git a/spec/requests/status_spec.rb b/spec/requests/status_spec.rb index 1c79226..9584414 100644 --- a/spec/requests/status_spec.rb +++ b/spec/requests/status_spec.rb @@ -85,7 +85,7 @@ .to eq("request_ceiling" => 10, "safety_reserve" => 2, "batch_size" => 10, "pacing_seconds" => 6, "worker_concurrency" => 1) expect(scheduler["core"]) - .to eq("detail_fallback_allowance" => 4, "rate_limit_reserve" => 8) + .to eq("detail_fallback_allowance" => 40, "rate_limit_reserve" => 8) expect(scheduler["metrics"]) .to eq("window_seconds" => 3600, "catch_up_min_sample_seconds" => 900) end diff --git a/spec/services/github/allowances_spec.rb b/spec/services/github/allowances_spec.rb index 3bfbd89..a7f10b0 100644 --- a/spec/services/github/allowances_spec.rb +++ b/spec/services/github/allowances_spec.rb @@ -10,15 +10,16 @@ def configuration(**overrides) expect(described_class.derive(configuration: configuration, limit: 60).poll_allowance).to eq(12) end - # Appendix F: the detail-fallback allowance is a configured cap, not the remainder + # Appendix G: the detail-fallback allowance is a configured cap, not the remainder # of the limit. Search batches carry the enrichment volume on their own per-minute - # ledger, so the core budget only funds the few per-entity fallback fetches. + # ledger, so the core budget funds only the per-entity fallback fetches — the whole + # remainder after polling and the reserve, spent on the Search-miss residue. it "takes the configured detail-fallback allowance rather than deriving it from the limit" do - expect(described_class.derive(configuration: configuration, limit: 60).enrichment_allowance).to eq(4) + expect(described_class.derive(configuration: configuration, limit: 60).enrichment_allowance).to eq(40) end it "keeps that allowance fixed whatever limit GitHub reports" do - expect(described_class.derive(configuration: configuration, limit: 5000).enrichment_allowance).to eq(4) + expect(described_class.derive(configuration: configuration, limit: 5000).enrichment_allowance).to eq(40) end it "reads CORE_DETAIL_FALLBACK_ALLOWANCE, so the cap is an operator's number" do @@ -55,13 +56,13 @@ def configuration(**overrides) it "keeps the configured allowance under an over-committed cadence, leaving #feasible? the verdict" do derived = described_class.derive(configuration: configuration(POLL_INTERVAL_SECONDS: "60"), limit: 60) - expect(derived.enrichment_allowance).to eq(4) + expect(derived.enrichment_allowance).to eq(40) expect(derived).not_to be_feasible end end describe "#feasible?" do - it "accepts the pinned defaults, which commit twenty-four of sixty attempts" do + it "accepts the pinned defaults, which commit the whole limit" do expect(described_class.derive(configuration: configuration, limit: 60)).to be_feasible end @@ -69,15 +70,20 @@ def configuration(**overrides) # number is a real commitment now, so a sum that lands exactly on the limit is a # fully-funded plan rather than a starved one. it "accepts the exact boundary, where the three commitments fill the limit precisely" do - derived = described_class.derive(configuration: configuration(RATE_LIMIT_RESERVE: "44"), limit: 60) + derived = described_class.derive( + configuration: configuration(RATE_LIMIT_RESERVE: "44", CORE_DETAIL_FALLBACK_ALLOWANCE: "4"), + limit: 60 + ) expect(derived.poll_allowance + derived.reserve + derived.enrichment_allowance).to eq(60) expect(derived).to be_feasible end it "rejects the first sum past the limit" do - expect(described_class.derive(configuration: configuration(RATE_LIMIT_RESERVE: "45"), limit: 60)) - .not_to be_feasible + expect(described_class.derive( + configuration: configuration(RATE_LIMIT_RESERVE: "45", CORE_DETAIL_FALLBACK_ALLOWANCE: "4"), + limit: 60 + )).not_to be_feasible end # Detail fallback is the exception path behind search batches, so an operator may @@ -127,7 +133,7 @@ def configuration(**overrides) it "never raises the allowance above its configured cap, however much headroom the limit leaves" do clamped = described_class.derive(configuration: configuration, limit: 5000).clamped - expect(clamped.enrichment_allowance).to eq(4) + expect(clamped.enrichment_allowance).to eq(40) end it "never derives a negative allowance, which the schema's CHECK would reject" do @@ -160,7 +166,7 @@ def configuration(**overrides) it "recomputes the guarantees from the clamped allowance rather than from the derived one" do derived = described_class.derive(configuration: configuration, limit: 15) - expect(derived).to have_attributes(actor_guarantee: 2, repository_guarantee: 2) + expect(derived).to have_attributes(actor_guarantee: 20, repository_guarantee: 20) expect(derived.clamped).to have_attributes(actor_guarantee: 0, repository_guarantee: 0) end end @@ -168,10 +174,10 @@ def configuration(**overrides) describe "the fairness split (plan §10)" do def split(allowance, share) = described_class.split(allowance, Rational(share)) - it "splits the pinned defaults into two actor and two repository fallback attempts" do + it "splits the pinned defaults into twenty actor and twenty repository fallback attempts" do derived = described_class.derive(configuration: configuration, limit: 60) - expect(derived).to have_attributes(actor_guarantee: 2, repository_guarantee: 2) + expect(derived).to have_attributes(actor_guarantee: 20, repository_guarantee: 20) end # §10 writes the formula as a floor and a subtraction, so the odd attempt goes to @@ -231,7 +237,7 @@ def split(allowance, share) = described_class.split(allowance, Rational(share)) it "reports both guarantees in the line the ledger logs at window initialization" do derived = described_class.derive(configuration: configuration, limit: 60) - expect(derived.to_log).to include(actor_guarantee: 2, repository_guarantee: 2) + expect(derived.to_log).to include(actor_guarantee: 20, repository_guarantee: 20) end end end diff --git a/spec/services/github/budget_ledger_bootstrap_spec.rb b/spec/services/github/budget_ledger_bootstrap_spec.rb index 60e2c4d..4f45255 100644 --- a/spec/services/github/budget_ledger_bootstrap_spec.rb +++ b/spec/services/github/budget_ledger_bootstrap_spec.rb @@ -180,7 +180,7 @@ def bootstrap_poll! ledger.reconcile!(snapshot, request_class: :poll, now: frozen_time) - expect(budget).to have_attributes(poll_allowance: 12, enrichment_allowance: 4) + expect(budget).to have_attributes(poll_allowance: 12, enrichment_allowance: 40) end # A block set before the window was ever initialized. denial_reason derives blocking diff --git a/spec/services/github/budget_ledger_spec.rb b/spec/services/github/budget_ledger_spec.rb index 4ac2a89..5e3ffa2 100644 --- a/spec/services/github/budget_ledger_spec.rb +++ b/spec/services/github/budget_ledger_spec.rb @@ -29,7 +29,7 @@ def snapshot(**overrides) ledger.bootstrap!(now: frozen_time) expect(budget).to have_attributes( - window_status: "uninitialized", poll_allowance: 12, enrichment_allowance: 4, + window_status: "uninitialized", poll_allowance: 12, enrichment_allowance: 40, reserve: 8, poll_used: 0, enrichment_used: 0, limit: nil, remaining: nil, reset_at: nil ) end @@ -682,7 +682,7 @@ def block(reason, until_at: frozen_time + 60) ledger.reserve!(:poll, now: frozen_time) ledger.reconcile!(snapshot, now: frozen_time) - expect(budget).to have_attributes(poll_allowance: 12, enrichment_allowance: 4, reserve: 8) + expect(budget).to have_attributes(poll_allowance: 12, enrichment_allowance: 40, reserve: 8) end # A limit lower than the configured default is GitHub's business, not an operator @@ -709,7 +709,7 @@ def block(reason, until_at: frozen_time + 60) expect(Rails.logger).to have_received(:warn).with( hash_including(event: "budget.allowances_clamped", - requested_poll_allowance: 12, requested_enrichment_allowance: 4, + requested_poll_allowance: 12, requested_enrichment_allowance: 40, poll_allowance: 7, enrichment_allowance: 0) ) end @@ -742,7 +742,7 @@ def block(reason, until_at: frozen_time + 60) observed_at: later ), now: later) - expect(budget).to have_attributes(poll_allowance: 12, enrichment_allowance: 4, limit: 60) + expect(budget).to have_attributes(poll_allowance: 12, enrichment_allowance: 40, limit: 60) end end @@ -754,6 +754,9 @@ def live_sources(count) count.times { create_event_source(source_type: "github_public_events") } end + # Two sources double the poll commitment, and polling has priority: the configured + # detail-fallback allowance no longer fits beside it, so #clamped funds what is left + # (60 - 8 reserve - 24 poll) rather than over-committing the limit. it "derives the poll allowance from the rows that exist when a window opens" do live_sources(2) ledger.bootstrap!(now: frozen_time) @@ -761,7 +764,7 @@ def live_sources(count) ledger.reconcile!(snapshot, now: frozen_time) - expect(budget).to have_attributes(poll_allowance: 24, enrichment_allowance: 4) + expect(budget).to have_attributes(poll_allowance: 24, enrichment_allowance: 28) end it "re-derives it at rollover, so an added source takes effect within the hour" do @@ -770,7 +773,7 @@ def live_sources(count) ledger.reserve!(:poll, now: window_reset + 1) - expect(budget).to have_attributes(poll_allowance: 24, enrichment_allowance: 4) + expect(budget).to have_attributes(poll_allowance: 24, enrichment_allowance: 28) end # #bootstrap! runs ahead of every reservation, so asking event_sources there would put a @@ -780,7 +783,7 @@ def live_sources(count) ledger.bootstrap!(now: frozen_time) - expect(budget).to have_attributes(poll_allowance: 12, enrichment_allowance: 4) + expect(budget).to have_attributes(poll_allowance: 12, enrichment_allowance: 40) end it "ignores a disabled or failed source, which will never spend a poll attempt" do @@ -792,7 +795,7 @@ def live_sources(count) ledger.reserve!(:poll, now: frozen_time) ledger.reconcile!(snapshot, now: frozen_time) - expect(budget).to have_attributes(poll_allowance: 12, enrichment_allowance: 4) + expect(budget).to have_attributes(poll_allowance: 12, enrichment_allowance: 40) end end diff --git a/spec/services/github/configuration_spec.rb b/spec/services/github/configuration_spec.rb index f7a4a64..395ae52 100644 --- a/spec/services/github/configuration_spec.rb +++ b/spec/services/github/configuration_spec.rb @@ -33,7 +33,7 @@ def configuration(**overrides) enrichment_coverage_window_seconds: 86_400, # Appendix F's staged-enrichment block: the search lane's per-minute budget, the # core detail-fallback cap, and the cycle/retry/lease timings around them. - core_detail_fallback_allowance: 4, + core_detail_fallback_allowance: 40, search_request_ceiling: 10, search_safety_reserve: 2, search_batch_size: 10, @@ -258,9 +258,9 @@ def configuration(**overrides) end describe "startup validation of the allowance split (plan §10, Appendix F)" do - it "accepts the pinned defaults, which commit twelve poll and four fallback attempts" do + it "accepts the pinned defaults, which commit twelve poll and forty fallback attempts" do expect(configuration.validate!.allowances) - .to have_attributes(poll_allowance: 12, enrichment_allowance: 4) + .to have_attributes(poll_allowance: 12, enrichment_allowance: 40) end # Polling every 60 seconds is 60 attempts an hour — the entire unauthenticated @@ -274,14 +274,15 @@ def configuration(**overrides) # is a configured commitment now, so lowering it is a legitimate way out. it "names the offending numbers so an operator can fix it without reading the code" do expect { configuration(POLL_INTERVAL_SECONDS: "60").validate! } - .to raise_error(/poll_allowance \(60\).*CORE_DETAIL_FALLBACK_ALLOWANCE \(4\).*RATE_LIMIT_RESERVE \(8\)/m) + .to raise_error(/poll_allowance \(60\).*CORE_DETAIL_FALLBACK_ALLOWANCE \(40\).*RATE_LIMIT_RESERVE \(8\)/m) end # Appendix F's predicate is <= : the three commitments may fill the limit exactly, # because each is a real, funded plan — and the first request past it is rejected. it "accepts the sum landing exactly on the limit, and rejects one attempt more" do - expect { configuration(RATE_LIMIT_RESERVE: "44").validate! }.not_to raise_error - expect { configuration(RATE_LIMIT_RESERVE: "45").validate! } + # The pinned defaults already land exactly on it: 12 + 40 + 8 = 60. + expect { configuration.validate! }.not_to raise_error + expect { configuration(RATE_LIMIT_RESERVE: "9").validate! } .to raise_error(Github::Errors::ConfigurationError, /CORE_DETAIL_FALLBACK_ALLOWANCE/) end diff --git a/spec/services/github/enrichment/batch_runner_spec.rb b/spec/services/github/enrichment/batch_runner_spec.rb index 2116fda..5c5fb3e 100644 --- a/spec/services/github/enrichment/batch_runner_spec.rb +++ b/spec/services/github/enrichment/batch_runner_spec.rb @@ -354,11 +354,11 @@ def create_pending_actor(github_id:, login:, created_at: now - 60, **overrides) ) end - it "fails the batch on a 422 and leaves the debited request spent" do + it "fails the batch on a server error and leaves the debited request spent" do create_pending_actor(github_id: 721, login: "alpha") allow(Rails.logger).to receive(:warn) executor = recording_executor do |request, _call| - failure_response(request, status: 422, body: "Validation Failed") + failure_response(request, status: 500, body: "Internal Server Error") end result = runner(executor).call(entity_class: GithubActor) @@ -368,11 +368,36 @@ def create_pending_actor(github_id:, login:, created_at: now - 60, **overrides) # One attempt was made and there is no refund path: the executor saw exactly # one request, and the batch retains the failure evidence. expect(executor.requests.length).to eq(1) - expect(batch).to have_attributes(status: "failed", response_status: 422, - response_body: "Validation Failed") + expect(batch).to have_attributes(status: "failed", response_status: 500, + response_body: "Internal Server Error") + # A server error says nothing about the query, so the identical batch is worth + # sending again — unlike a 4xx, which is deterministic and takes the fallback. expect(GithubActor.find_by(github_id: 721).enrichment_stage).to eq("retry_scheduled") expect(Rails.logger).to have_received(:warn).with( - hash_including(event: "enrichment.batch_failed", response_status: 422) + hash_including(event: "enrichment.batch_failed", response_status: 500) + ) + end + + # A rejected query is a fact about these identifiers and this URL, and the ladder + # would resend both unchanged — the same 4xx, hourly, forever. Every deterministic + # client error therefore takes the bounded fallback, whatever GitHub's wording. + it "routes any rejected query to the detail lane rather than resending it" do + create_pending_actor(github_id: 751, login: "alpha") + allow(Rails.logger).to receive(:warn) + allow(Rails.logger).to receive(:info) + executor = recording_executor do |request, _call| + failure_response(request, status: 422, body: '{"message":"Validation Failed"}') + end + + result = runner(executor).call(entity_class: GithubActor) + + expect(result).to have_attributes(status: "completed", fallback_count: 1) + expect(GithubActor.find_by(github_id: 751)) + .to have_attributes(enrichment_stage: "detail_pending", + last_error: "search_query_rejected") + expect(Rails.logger).to have_received(:warn).with( + hash_including(event: "enrichment.batch_unsearchable", + reason: "search_query_rejected", response_status: 422) ) end @@ -407,7 +432,7 @@ def create_pending_actor(github_id:, login:, created_at: now - 60, **overrides) # No member is left on the search lane, and the batch records why. expect(batch).to have_attributes(status: "failed", missing_count: 2, returned_count: 0, - last_error: "unsearchable_identifiers") + last_error: "unsearchable_identifier") expect(Rails.logger).to have_received(:warn).with( hash_including(event: "enrichment.batch_unsearchable", response_status: 422) ) diff --git a/spec/services/github/enrichment/cycle_runner_spec.rb b/spec/services/github/enrichment/cycle_runner_spec.rb index 2c31a67..1e9b6d9 100644 --- a/spec/services/github/enrichment/cycle_runner_spec.rb +++ b/spec/services/github/enrichment/cycle_runner_spec.rb @@ -226,6 +226,25 @@ def quiet_batch_phase expect(detail_runner).to have_received(:call).with(entity_class: :actor, borrow: false) end + # The borrow states a fact about the *other* class, not about which slot was taken. + # A one-sided backlog is the case that separates the two readings: the actor lane is + # scheduled and claims its own slot, and repository work is provably absent, so the + # ledger is told it may spend past the actor guarantee. Reporting borrow: false here + # would strand a one-sided backlog at half the allowance with the rest idle. + it "borrows on its own scheduled turn when the other class has nothing claimable" do + quiet_batch_phase + allow(admission).to receive(:detail) + .and_return(granted, verdict(:class_exhausted, retry_in: 120.0)) + allow(detail_claim).to receive(:claimable?) do |entity_type, now:| + entity_type.key == :actor + end + allow(detail_runner).to receive(:call).and_return(detail_result(status: "completed")) + + cycle_runner.call + + expect(detail_runner).to have_received(:call).with(entity_class: :actor, borrow: true) + end + it "stops the phase when a detail request comes back deferred" do quiet_batch_phase allow(admission).to receive(:detail).and_return(granted) diff --git a/spec/services/github/request_executor_spec.rb b/spec/services/github/request_executor_spec.rb index d83c1ce..b9db4b2 100644 --- a/spec/services/github/request_executor_spec.rb +++ b/spec/services/github/request_executor_spec.rb @@ -305,6 +305,48 @@ def always(status) .with(hash_including(event: "github.retry_scheduled", next_attempt: 2)).once end + # SEARCH_PACING_SECONDS is six and the retry backoff is around a second, so a + # Search retry that slept only the backoff would arrive before its own ledger would + # admit it: the reservation is refused as :search_pacing and the transport failure + # the retry existed to repeat is replaced by a budget denial. MAX_HTTP_RETRIES would + # be inert on this resource. + it "waits out Search pacing before retrying, so the retry is a retry" do + active_search_window(last_request_at: nil) + slept = [] + # The clock advances with the sleep, as it does in production: pacing is measured + # against wall time, so a frozen clock would refuse the retry however long the + # process actually waited. + current = frozen_time + request = Github::Request.new( + url: "https://api.github.com/search/users?q=user%3Aoctocat&per_page=10", + request_class: :actor_search + ) + headers = { "x-ratelimit-resource" => "search", "x-ratelimit-limit" => "10", + "x-ratelimit-remaining" => "8", + "x-ratelimit-reset" => (frozen_time + 3600).to_i.to_s } + + result = executor(recording_transport { response(status: 500, headers: headers) }, + sleeper: ->(seconds) { slept << seconds; current += seconds }, + clock: -> { current }, + search_pacing_seconds: 6).call(request) + + expect(slept).to all(be >= 6) + # Every attempt reached the transport, so the failure that survives is the + # server's rather than a pacing denial. + expect(result.classification).to eq(:server_error) + expect(current_search_budget.used).to eq(3) + end + + it "leaves a core retry on its own backoff, which no pacing constrains" do + active_budget_window + slept = [] + + executor(always(500), sleeper: ->(seconds) { slept << seconds }, + search_pacing_seconds: 6).call(poll_request) + + expect(slept).to all(be < 6) + end + # RetryPolicy jitters, so a line that recomputed the delay would report a number this # process never actually slept. it "reports the delay it actually slept, not a freshly jittered one" do diff --git a/spec/services/github/search_budget_ledger_spec.rb b/spec/services/github/search_budget_ledger_spec.rb index bbf53e8..eecc32e 100644 --- a/spec/services/github/search_budget_ledger_spec.rb +++ b/spec/services/github/search_budget_ledger_spec.rb @@ -480,6 +480,72 @@ def limited(status: 403, **headers) # separate PostgreSQL sessions reserving at once must serialise on the row, with no # lost debit and no deadlock. Transactional tests are off for the reason # spec/support/concurrency_helpers.rb documents, paid for with explicit cleanup. + # §10: a secondary rate limit is IP-scoped, so it stops every live request rather + # than only the resource that provoked it. Two ledgers meter two resources, but they + # share one outbound address — a block written by either has to bind both. + describe "the global block both ledgers share" do + def search_request + Github::Request.new(url: "https://api.github.com/search/users?q=user%3Aoctocat&per_page=1", + request_class: :actor_search) + end + + it "refuses a Search reservation while the core ledger holds a global block" do + active_window + Github::BudgetLedger.new.bootstrap!(now: frozen_time) + Github::BudgetLedger.new.block_globally!(until_at: frozen_time + 300, + reason: "secondary_limit", now: frozen_time) + + expect { ledger.reserve!(:actor_search, now: frozen_time) } + .to raise_error(Github::Errors::BudgetExhausted) { |error| + expect(error.reason).to eq(:globally_blocked) + } + expect(budget.used).to eq(0) + end + + it "grants the reservation once that block has expired" do + active_window + Github::BudgetLedger.new.bootstrap!(now: frozen_time) + Github::BudgetLedger.new.block_globally!(until_at: frozen_time + 300, + reason: "secondary_limit", now: frozen_time) + + expect { ledger.reserve!(:actor_search, now: frozen_time + 301) }.not_to raise_error + end + + # The mirror image: a secondary limit provoked by Search stops polling too. + it "writes the core global block when a Search response is secondary-limited" do + active_window + Github::BudgetLedger.new.bootstrap!(now: frozen_time) + secondary = Github::FetchResult.from_response( + request: search_request, status: 403, + headers: { "x-ratelimit-remaining" => "5", "retry-after" => "60" }, + body: "", duration_ms: 1.0 + ) + + ledger.block_from!(secondary, now: frozen_time) + + expect(current_budget.global_blocked_until).to eq(frozen_time + 60) + expect(budget.blocked_until).to eq(frozen_time + 60) + end + + # A primary Search exhaustion bounds only this resource. Blocking polling on it + # would let the search lane starve event capture, which no quota rule permits. + it "leaves polling alone when Search merely exhausted its own limit" do + active_window + Github::BudgetLedger.new.bootstrap!(now: frozen_time) + exhausted = Github::FetchResult.from_response( + request: search_request, status: 403, + headers: { "x-ratelimit-remaining" => "0", + "x-ratelimit-reset" => window_reset.to_i.to_s }, + body: "", duration_ms: 1.0 + ) + + ledger.block_from!(exhausted, now: frozen_time) + + expect(current_budget.global_blocked_until).to be_nil + expect(budget.blocked_until).to eq(window_reset) + end + end + describe "under concurrency" do self.use_transactional_tests = false diff --git a/spec/services/github/status/scheduler_settings_spec.rb b/spec/services/github/status/scheduler_settings_spec.rb index 2707c75..5a9ae04 100644 --- a/spec/services/github/status/scheduler_settings_spec.rb +++ b/spec/services/github/status/scheduler_settings_spec.rb @@ -44,7 +44,7 @@ request_ceiling: 10, safety_reserve: 2, batch_size: 10, pacing_seconds: 6, worker_concurrency: 1 }, - core: { detail_fallback_allowance: 4, rate_limit_reserve: 8 }, + core: { detail_fallback_allowance: 40, rate_limit_reserve: 8 }, metrics: { window_seconds: 3600, catch_up_min_sample_seconds: 900 } ) end diff --git a/spec/services/github/status/snapshot_spec.rb b/spec/services/github/status/snapshot_spec.rb index 4f9a859..2581156 100644 --- a/spec/services/github/status/snapshot_spec.rb +++ b/spec/services/github/status/snapshot_spec.rb @@ -33,7 +33,7 @@ def payload expect(scheduler[:search]).to eq(request_ceiling: 10, safety_reserve: 2, batch_size: 10, pacing_seconds: 6, worker_concurrency: 1) - expect(scheduler[:core]).to eq(detail_fallback_allowance: 4, rate_limit_reserve: 8) + expect(scheduler[:core]).to eq(detail_fallback_allowance: 40, rate_limit_reserve: 8) end end From ae2f1f63f55c6974241cc66b8b79a614215f7fcc Mon Sep 17 00:00:00 2001 From: Umang Date: Sun, 2 Aug 2026 16:24:34 -0500 Subject: [PATCH 12/12] Correct the two-ledger blocking policy and the documented defaults MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second review round on the branch. The global block was shared too broadly. github_api_budget.global_blocked_until carries core primary exhaustion and reserve breaches as well as secondary limits, so reading it from the Search ledger made a spent core hour stop the Search resource for the rest of that hour — the opposite of why the two ledgers exist. Only a secondary limit is IP-scoped. Github::RateLimitPolicy is now the single decider for both resources, which is what it already claimed to be: it takes the resource that answered, returns no global verdict for a primary exhaustion of Search, and on a secondary limit writes both ledgers. Search limits therefore reach the same escalation ladder they always should have — the streak counter, the one-minute floor, the one-hour cap, and the clearing on the next good response — instead of the bespoke block_globally! call that bypassed all four. SearchBudgetLedger#block_from! now records only this resource's own exhaustion. Also: - Dispatch enqueues nothing while a cycle is unfinished. A cycle can outlive the 60-second tick, and the work it would find is still claimable, so the reconciler would otherwise add a wake-up every minute for as long as the overrun lasted. Serializing them was never the same as bounding them. - The 40-request detail-fallback default reached the runtime and Compose but not the prose: .env.example, README, CLAUDE.md, the plan, the design brief and three ADRs still said 4, and Compose now forwards the variable — so uncommenting the documented line would have restored the old cap. The derived arithmetic is corrected with it: the three core lanes fill the limit exactly, and a deeper poll clamps the fallback lane rather than leaving headroom. - The evidence document overstated two things. A 26-minute run cannot hold 53 samples at a 60-second cadence — two sampling loops were running — and the Search lane requested 872 items rather than settling them: 858 were applied directly, 14 went to the fallback, and the pipeline produced 870 exits. One test-only fix: advisory_lock_holders queried pg_locks without a database filter. pg_locks is instance-wide and this project runs the development stack against the same container as the test databases, so a worker mid-poll holding the request gate in the development database failed the multi-poller example here. It failed exactly that way during this session. Co-Authored-By: Claude Fable 5 --- .env.example | 17 +- CLAUDE.md | 2 +- IMPLEMENTATION_PLAN.md | 21 +-- README.md | 48 +++--- .../github/enrichment/batch_runner.rb | 6 + app/services/github/enrichment/dispatch.rb | 33 +++- app/services/github/rate_limit_policy.rb | 25 ++- app/services/github/search_budget_ledger.rb | 59 ++----- docs/DESIGN_BRIEF.md | 2 +- docs/adr/0004-class-aware-budget-ledger.md | 2 +- ...nrichment-fairness-shares-and-borrowing.md | 2 +- ...erivation-first-staged-batch-enrichment.md | 2 +- ...2026-08-02-live-staged-batch-enrichment.md | 14 +- .../github/enrichment/batch_runner_spec.rb | 11 +- .../github/search_budget_ledger_spec.rb | 146 +++++++++++------- spec/support/advisory_lock_helpers.rb | 8 + 16 files changed, 238 insertions(+), 160 deletions(-) diff --git a/.env.example b/.env.example index 0bcdacf..c9747b5 100644 --- a/.env.example +++ b/.env.example @@ -73,17 +73,18 @@ GITHUB_MODE=live # + CORE_DETAIL_FALLBACK_ALLOWANCE <= rate_limit # # With these defaults: ceil(3600/300) x 1 x 1 = 12 poll attempts/hour, and -# 12 poll + 4 detail fallback + 8 reserve = 24 of 60 — the remaining 36 core -# requests are deliberately unspent headroom. Normal-path enrichment runs on the -# separate per-minute *search* budget below, not on core. Startup fails when the -# three core lanes together exceed the limit. +# 12 poll + 40 detail fallback + 8 reserve = exactly 60. Normal-path enrichment runs +# on the separate per-minute *search* budget below, not on core; the 40 covers the +# ~2% of entities Search does not return. Startup fails when the three core lanes +# together exceed the limit, and a deeper poll clamps the fallback lane rather than +# over-committing it. # # Both are formula inputs and both are obeyed at runtime. POLL_INTERVAL_SECONDS is # the cadence: a source polled at T is due again at T + this, and an unforced run # before then is deferred rather than made. MAX_PAGES_PER_POLL caps how many -# `Link`-followed pages one poll may fetch — raising it spends the unspent core -# headroom on capture depth (at 5 pages the formula is infeasible and boot -# refuses); it no longer cuts enrichment, which lives on the search budget. +# `Link`-followed pages one poll may fetch — raising it clamps the detail-fallback +# lane down to what is left (at 5 pages the formula is infeasible and boot refuses); +# it no longer cuts batch enrichment, which lives on the search budget. # # The poll allowance is enforced on top of the cadence: every # `docker compose run --rm ingest` spends one poll attempt per page, so at these @@ -146,7 +147,7 @@ GITHUB_MODE=live # stored payload-provided URL. It is the third term of the core feasibility rule # above and can never take the polling allocation. Zero is legal and turns the # fallback off. -# CORE_DETAIL_FALLBACK_ALLOWANCE=4 +# CORE_DETAIL_FALLBACK_ALLOWANCE=40 # The per-minute Search budget — a separate GitHub rate-limit resource from core. # The ceiling is the observed unauthenticated header limit (10/minute); the diff --git a/CLAUDE.md b/CLAUDE.md index cffc7b2..203f39b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -86,7 +86,7 @@ Entity rows are durable work, coalesced by stable GitHub ID, selected FIFO by `SEARCH_BATCH_SIZE` repeated exact `user:`/`repo:` qualifiers — never joined with `OR` — on the minute-scoped search ledger (ceiling 10, reserve 2, 6-second pacing). The payload-URL detail fallback serves only missing/renamed/mismatched/contract-invalid batch -items and is bounded by `CORE_DETAIL_FALLBACK_ALLOWANCE` (4/hour); it never takes the +items and is bounded by `CORE_DETAIL_FALLBACK_ALLOWANCE` (40/hour); it never takes the polling allocation. Quota, pacing, reserve, and fairness denials defer work — they never terminate an entity. Batch results apply only on a stable-ID match. Observations are append-only; a refresh repoints the projection and never overwrites retained evidence. diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md index 812094e..3577a38 100644 --- a/IMPLEMENTATION_PLAN.md +++ b/IMPLEMENTATION_PLAN.md @@ -431,7 +431,7 @@ Single-row global ledger (constrained singleton), through which **every** outbou - `window_initialized_at` - `poll_allowance`, `poll_used` - `enrichment_allowance`, `enrichment_used` — since Appendix G these budget the - **detail-fallback** lane only (`CORE_DETAIL_FALLBACK_ALLOWANCE`, default 4); the + **detail-fallback** lane only (`CORE_DETAIL_FALLBACK_ALLOWANCE`, default 40); the batch normal path spends the separate search ledger - `actor_share_used`, `repository_share_used` (fairness accounting — Section 10) - `reserve` @@ -877,14 +877,15 @@ ceil(3600 / 300) × 1 × 1 = 12 poll attempts/hour | Allocation | Default | |---|---:| | Scheduled polling | 12 request-attempts/hour | -| Detail fallback (bounded core lane — Appendix G) | up to 4 request-attempts/hour | +| Detail fallback (bounded core lane — Appendix G) | up to 40 request-attempts/hour | | Intentionally unspent reserve | 8 requests/hour | -| **Deliberately unspent remainder** | 36 requests/hour | | **Total** | **60 requests/hour** | -The remainder is deliberately unspent: since Appendix G, normal-path enrichment runs on -the **search** rate-limit resource, not on core, so leaving core headroom costs enrichment -nothing and protects polling from co-tenant pressure. The stored +The three lanes fill the limit exactly. Normal-path enrichment does not appear because +since Appendix G it runs on the **search** rate-limit resource, not on core; the core +detail lane funds only the residue Search does not return, measured at roughly 2% of +arrivals. Polling keeps priority: a configuration that raises the poll allowance clamps +the fallback lane rather than over-committing the limit. The stored `enrichment_allowance`/`enrichment_used` pair now budgets the detail-fallback lane. Startup validation **rejects** any configuration where @@ -963,7 +964,7 @@ fairness shares with explicit rounding: actor_guarantee = floor(enrichment_allowance × ACTOR_ENRICHMENT_SHARE) repository_guarantee = enrichment_allowance − actor_guarantee -Defaults: 4 × 0.50 → 2 actor / 2 repository detail-fallback attempts/hour +Defaults: 40 × 0.50 → 20 actor / 20 repository detail-fallback attempts/hour Borrowing: a class may borrow the other’s unused detail capacity only when the other class has no CURRENTLY CLAIMABLE detail candidate (not merely no rows). @@ -1074,9 +1075,9 @@ Never disable the event source because one enrichment target disappeared. On the **core** resource: 1. Polling for new events (from `poll_attempt_allowance`) -2. Detail fallback, only within its explicit `CORE_DETAIL_FALLBACK_ALLOWANCE` (4/hour), +2. Detail fallback, only within its explicit `CORE_DETAIL_FALLBACK_ALLOWANCE` (40/hour), under the fairness guarantees — it can never take the polling allocation -3. Nothing else spends core; the remainder is deliberately unspent headroom +3. Nothing else spends core; polling and the reserve are never borrowed against The **search** resource is independent: batch enrichment — backlog first, then refresh under Appendix G's composition rule — spends the per-minute search ledger and competes @@ -1649,7 +1650,7 @@ with a limit of 10; and joining exact qualifiers with `OR` produced HTTP 422. - **Payload-URL detail fallback, bounded.** Only items a batch could not settle — missing, renamed, identity-mismatched, or contract-invalid — fetch their stored payload-provided `api_url` through the core ledger's - `CORE_DETAIL_FALLBACK_ALLOWANCE` (4/hour). The fallback never constructs a URL from an + `CORE_DETAIL_FALLBACK_ALLOWANCE` (40/hour). The fallback never constructs a URL from an identifier and never touches the polling allocation. - **Dual ledgers.** `github_api_budget` (core: 12 poll + 4 detail fallback + 8 reserve ≤ 60, remainder deliberately unspent) and `github_search_budget` (per-minute search) diff --git a/README.md b/README.md index 7ce5876..29f683c 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ What it does, running: ten entities per GitHub Search request on Search's own per-minute budget (10 ceiling, 2 reserved, 6-second pacing); only items a batch could not settle fall back to individual payload-URL fetches inside a bounded core allowance of 4 per hour. The core ledger keeps -12 requests for polling and 8 in reserve, leaving the rest deliberately unspent. Entity +12 requests for polling, 8 in reserve, and the rest for the detail-fallback lane. Entity rows remain actionable until enrichment satisfies the useful-data contract or an entity-specific terminal outcome is established; quota exhaustion, pacing, and reserve denials only defer them. Selection remains FIFO, oldest-first within each class, and delay @@ -570,11 +570,12 @@ plan §9's rule that every fetched page is processed in full and `github_event_i uniqueness absorbs the overlap — there is no stop-on-known-event, because documented event latency is 30 seconds to 6 hours and a delayed event can surface beside one already seen. And **raising the cap is not free**: at -`MAX_PAGES_PER_POLL=3` the poll allowance becomes 12 × 3 = 36 attempts an hour, and the -core headroom left deliberately unspent shrinks from 36 to 60 − 36 − 8 − 4 = 12; one more -page and it is zero, and at 5 the formula is infeasible and boot refuses. Since plan -Appendix G, capture depth spends headroom rather than enrichment capacity — batch -enrichment lives on the separate search budget. +`MAX_PAGES_PER_POLL=2` the poll allowance becomes 12 × 2 = 24 attempts an hour, and the +three core lanes ask for 24 + 40 + 8 = 72 of 60 — over-committed, so the detail-fallback +allowance is clamped down to what is left (60 − 8 − 24 = 28). At 5 pages polling alone +would take the whole limit and boot refuses. Since plan Appendix G, capture depth trades +against the *fallback* lane rather than against enrichment as a whole — batch enrichment +lives on the separate search budget. At `MAX_PAGES_PER_POLL=2` the counts are identical except `Pages fetched: 2` — page 3 is empty — and the stop reason on the `ingestion.pagination_stopped` debug line @@ -1185,11 +1186,13 @@ feasible ⇔ poll_attempt_allowance + RATE_LIMIT_RESERVE + CORE_DETAIL_FALLBACK_ALLOWANCE <= rate_limit ``` -With the defaults: 12 poll attempts, 4 detail-fallback attempts, and 8 in reserve — -24 of GitHub's unauthenticated 60, with **36 core requests an hour deliberately -unspent**. Enrichment's normal path does not appear in this formula because it does not -spend core at all. **The process refuses to boot** if the three core lanes together -exceed the limit. +With the defaults: 12 poll attempts, 40 detail-fallback attempts, and 8 in reserve — +**exactly GitHub's unauthenticated 60**. Enrichment's normal path does not appear in this +formula because it does not spend core at all; the 40 is what the ~2% of entities Search +does not return need, measured in +[the live run](docs/evidence/2026-08-02-live-staged-batch-enrichment.md). **The process +refuses to boot** if the three core lanes together exceed the limit, and clamps the +fallback lane when polling is raised. The detail-fallback allowance is then split by `ACTOR_ENRICHMENT_SHARE` (plan §10): @@ -1198,25 +1201,26 @@ actor_guarantee = floor(enrichment_allowance x ACTOR_ENRICHMENT_SHARE) repository_guarantee = enrichment_allowance - actor_guarantee ``` -With the defaults: 2 actor and 2 repository detail attempts an hour. The remainder +With the defaults: 20 actor and 20 repository detail attempts an hour. The remainder always goes to repositories, because the formula floors one side and subtracts for the other — so the two always add up to the whole allowance. A class may borrow the other's unused detail capacity when the other has no claimable detail candidate. ### The core budget table -At limit 60, reserve 8, detail allowance 4, cadence 300s, one source — the only variable -is page depth: +At limit 60, reserve 8, configured detail allowance 40, cadence 300s, one source — the +only variable is page depth. Polling has priority, so a deeper poll clamps the fallback +lane rather than over-committing the limit: -| `MAX_PAGES_PER_POLL` | Poll allowance | Detail fallback | Reserve | Deliberately unspent | +| `MAX_PAGES_PER_POLL` | Poll allowance | Detail fallback | Reserve | Unspent | |---|---|---|---|---| -| 1 (default) | 12 | 4 | 8 | 36 | -| 2 | 24 | 4 | 8 | 24 | -| 3 | 36 | 4 | 8 | 12 | -| 4 | 48 | 4 | 8 | 0 | +| 1 (default) | 12 | 40 | 8 | 0 | +| 2 | 24 | 28 (clamped) | 8 | 0 | +| 3 | 36 | 16 (clamped) | 8 | 0 | +| 4 | 48 | 4 (clamped) | 8 | 0 | | 5 | 60 | — | — | **refuses to boot** | -Capture depth now spends the unspent headroom rather than enrichment capacity: batch +Capture depth now clamps the detail-fallback lane rather than enrichment as a whole: batch enrichment lives on the search budget, so deeper polls no longer starve it — the formula simply becomes infeasible at 5 pages. @@ -1323,10 +1327,10 @@ is [`.env.example`](.env.example). | `MAX_REDIRECTS` | `2` | Redirect hops followed per request, each re-validated and separately reserved | | `SOURCE_LOCK_WAIT_SECONDS` | `30` | How long the one-shot waits for a busy source lock; the poller attempts once (plan §9) | | `POLL_INTERVAL_SECONDS` | `300` | The poll cadence, and an allowance-formula input. A source polled at T is due again at T + this; an unforced run before then is deferred rather than made. The worker's 60-second tick checks the schedule; it does not replace it (plan §9, §10) | -| `MAX_PAGES_PER_POLL` | `1` | How many `Link`-followed pages one poll may fetch, and an allowance-formula input. Raising it spends the deliberately unspent core headroom on capture depth — see [the core budget table](#the-core-budget-table) (plan §9, §10) | +| `MAX_PAGES_PER_POLL` | `1` | How many `Link`-followed pages one poll may fetch, and an allowance-formula input. Raising it takes core capacity from the detail-fallback lane, which is clamped down to fit — see [the core budget table](#the-core-budget-table) (plan §9, §10) | | `ENABLED_LIVE_SOURCE_COUNT` | `1` | Allowance-formula input: live sources sharing one per-IP budget. **A fallback rather than the authority** — at window initialization and rollover the formula counts the enabled, in-service `event_sources` rows of the running mode and uses that instead, falling back to this value only when there are none yet. A disagreement is logged as `budget.source_allocation_drift` (plan §10, [ADR 0009](docs/adr/0009-runtime-source-allocation-and-shared-ip-observability.md)) | | `RATE_LIMIT_RESERVE` | `8` | Core requests per hour left deliberately unspent (plan §10) | -| `CORE_DETAIL_FALLBACK_ALLOWANCE` | `4` | The bounded core lane for individual detail fetches — only batch items that came back missing, renamed, mismatched, or contract-invalid reach it. Third term of the core feasibility rule; can never take the polling allocation (plan §10, Appendix G) | +| `CORE_DETAIL_FALLBACK_ALLOWANCE` | `40` | The bounded core lane for individual detail fetches — only batch items that came back missing, renamed, mismatched, or contract-invalid reach it. Third term of the core feasibility rule; can never take the polling allocation (plan §10, Appendix G) | | `SEARCH_REQUEST_CEILING` | `10` | Per-minute Search request ceiling — the observed unauthenticated header limit (Appendix G) | | `SEARCH_SAFETY_RESERVE` | `2` | Search requests per minute never spent, leaving 8 spendable | | `SEARCH_BATCH_SIZE` | `10` | Exact `user:`/`repo:` qualifiers per Search request, capped at 10 and never `OR`-joined | diff --git a/app/services/github/enrichment/batch_runner.rb b/app/services/github/enrichment/batch_runner.rb index 00097b4..b9f60a5 100644 --- a/app/services/github/enrichment/batch_runner.rb +++ b/app/services/github/enrichment/batch_runner.rb @@ -22,12 +22,14 @@ def attempted? = %w[completed failed].include?(status) def initialize(executor: Github.executor, configuration: Github.configuration, claim: BatchClaim.new(configuration: configuration), search_ledger: SearchBudgetLedger.new(configuration: configuration), + rate_limit_policy: RateLimitPolicy.new(search_ledger: search_ledger), backoff: Backoff.new(configuration: configuration), clock: -> { Time.current }) @executor = executor @configuration = configuration @claim = claim @search_ledger = search_ledger + @rate_limit_policy = rate_limit_policy @backoff = backoff @clock = clock end @@ -38,6 +40,10 @@ def call(entity_class:) return idle(entity_type) if lease.nil? fetched = @executor.call(request_for(lease)) + # The policy owns the IP-scoped verdict — a secondary limit here stops polling + # too, escalates the shared streak, and is cleared by the next good response. + # The search ledger records only this resource's own primary exhaustion. + @rate_limit_policy.apply!(fetched, now: @clock.call, resource: :search) @search_ledger.block_from!(fetched, now: @clock.call) finish(lease, fetched) rescue StandardError => error diff --git a/app/services/github/enrichment/dispatch.rb b/app/services/github/enrichment/dispatch.rb index 85f1fba..1a9438c 100644 --- a/app/services/github/enrichment/dispatch.rb +++ b/app/services/github/enrichment/dispatch.rb @@ -52,12 +52,23 @@ def call(reason:) detail_work = detail_verdict.granted? && EntityType.all.any? { |type| @detail_claim.claimable?(type, now: now) } - enqueue = batch_work || detail_work + # One unfinished cycle is enough. A cycle can outlive the 60-second tick when a + # single fetch runs long, and the work it would find is still claimable, so an + # unguarded reconciler would enqueue another every minute for as long as the + # overrun lasted. The single-thread queue serializes them but does not bound + # them: each surplus job is a wake-up that will find the state the running cycle + # left behind. Counted rather than assumed — the queue database answers it. + pending = cycle_pending? + enqueue = (batch_work || detail_work) && !pending JOB.constantize.perform_later if enqueue blocked_by = unless enqueue - [ (search_verdict.reason || :no_batch_work), - (detail_verdict.reason || :no_detail_work) ] + if pending + [ :cycle_in_flight ] + else + [ (search_verdict.reason || :no_batch_work), + (detail_verdict.reason || :no_detail_work) ] + end end log({ cycle_enqueued: enqueue ? 1 : 0, reason: reason, @@ -66,6 +77,22 @@ def call(reason:) private + # Queued or running, in Solid Queue's own terms: a job row exists until it + # finishes, so "not finished" covers both. Failures are excluded — a failed + # execution is not going to run again on its own, and treating it as in flight + # would stop dispatch permanently. + def cycle_pending? + SolidQueue::Job.where(class_name: JOB, finished_at: nil) + .where.missing(:failed_execution) + .exists? + rescue StandardError => error + # The queue database is not the source of truth for enrichment work; if it + # cannot answer, fall back to enqueueing rather than stalling the pipeline. + Rails.logger.warn(event: "enrichment.dispatch_probe_failed", + error_class: error.class.name) + false + end + # INFO only when it scheduled something: a tick that enqueued nothing is the # ordinary steady state of an exhausted window, and at 60-second cadence it would # emit a line a minute for the rest of the hour. The rich per-stage summary lives diff --git a/app/services/github/rate_limit_policy.rb b/app/services/github/rate_limit_policy.rb index 9f86de5..950f561 100644 --- a/app/services/github/rate_limit_policy.rb +++ b/app/services/github/rate_limit_policy.rb @@ -51,9 +51,11 @@ def to_log end end - def initialize(ledger: BudgetLedger.new, backoff: PollBackoff.new) + def initialize(ledger: BudgetLedger.new, backoff: PollBackoff.new, + search_ledger: SearchBudgetLedger.new) @ledger = ledger @backoff = backoff + @search_ledger = search_ledger end # Called once per FetchResult a poll observes, including pages after the first: a rate @@ -63,30 +65,41 @@ def initialize(ledger: BudgetLedger.new, backoff: PollBackoff.new) # open — which is what keeps the ledger's row lock innermost. # # @param fetched [Github::FetchResult] + # @param resource [Symbol] :core or :search — which rate-limit resource answered. + # It changes only the *primary* verdict: a primary exhaustion bounds the resource + # that reported it, so a spent Search minute must not stop polling for the hour, + # and Github::SearchBudgetLedger records that one locally. Secondary limits are + # IP-scoped whichever resource surfaced them, so they are decided here once and + # written to both ledgers. # @return [Decision] - def apply!(fetched, now: Time.current) - decision = decide(fetched, now: now) + def apply!(fetched, now: Time.current, resource: :core) + decision = decide(fetched, now: now, resource: resource) unless decision.blocking? # A live request that completed without a secondary limit is the evidence that the # IP is no longer being throttled, and §10's backoff is exponential in # *consecutive* limits. Asked of successful? rather than of the decision alone: a # 5xx or a timeout produced no verdict from GitHub about throttling, so it must - # neither escalate the streak nor end it. + # neither escalate the streak nor end it. A successful Search response is that + # same evidence, which is why the streak is cleared for either resource. @ledger.clear_secondary_limit_streak!(now: now) if fetched.successful? return decision end @ledger.block_globally!(until_at: decision.blocked_until, reason: decision.kind, window_status: decision.window_status, now: now) + # The one condition that stops every live request stops the other resource too. + if decision.kind == :secondary_rate_limit + @search_ledger.block_until!(until_at: decision.blocked_until, now: now) + end decision end private - def decide(fetched, now:) + def decide(fetched, now:, resource: :core) case fetched.classification - when :rate_limited then primary_limit(fetched, now: now) + when :rate_limited then resource == :search ? Decision.none : primary_limit(fetched, now: now) when :secondary_limited then secondary_limit(fetched, now: now) when :budget_denied then reserve_breach(fetched, now: now) else Decision.none diff --git a/app/services/github/search_budget_ledger.rb b/app/services/github/search_budget_ledger.rb index 2633e46..cb1c4d1 100644 --- a/app/services/github/search_budget_ledger.rb +++ b/app/services/github/search_budget_ledger.rb @@ -32,16 +32,6 @@ def reserve!(request_class, now: Time.current, borrow: false) end bootstrap!(now: now) - - # §10: a secondary rate limit is IP-scoped, so it stops *all* live requests, not - # only the resource that provoked it. The two ledgers meter separate resources but - # share one outbound address, so each honours the other's global block. - if (blocked_until = global_block(now: now)) - Rails.logger.info(event: "search_budget.globally_blocked", - blocked_until: blocked_until.utc.iso8601) - raise Errors::BudgetExhausted.new(request_class, :globally_blocked) - end - reason = nil GithubSearchBudget.transaction do @@ -108,30 +98,26 @@ def reconcile!(snapshot, request_class: nil, now: Time.current) end end - # @param core_ledger [Github::BudgetLedger] the writer of global_blocked_until. A - # secondary limit provoked by a Search request is IP-scoped like any other, so it - # has to stop polling too — §10 is explicit that one timestamp covers every live - # request. A primary Search exhaustion is *not* global: it bounds only this - # resource, and blocking polling on it would hand the search lane the power to - # starve event capture. - def block_from!(fetched, now: Time.current, core_ledger: BudgetLedger.new) - return unless %i[rate_limited secondary_limited].include?(fetched.classification) + # Primary exhaustion of *this* resource only: Search says it is out of requests for + # the minute, which bounds this lane and nothing else. A secondary limit is not + # handled here — it is IP-scoped, so Github::RateLimitPolicy decides it once for both + # resources and calls #block_until! on this row, which is what keeps the escalation + # ladder, the one-minute floor, the one-hour cap, and the streak counter in a single + # place rather than reimplemented per ledger. + def block_from!(fetched, now: Time.current) + return unless fetched.classification == :rate_limited snapshot = fetched.rate_limit(observed_at: now) - retry_seconds = snapshot.retry_after_seconds - until_at = if retry_seconds.is_a?(Integer) && retry_seconds.positive? - now + retry_seconds - else - snapshot.reset_at || now + SEARCH_WINDOW_SECONDS - end + block_until!(until_at: snapshot.reset_at || now + SEARCH_WINDOW_SECONDS, + classification: fetched.classification, now: now) + end - if fetched.classification == :secondary_limited - core_ledger.block_globally!(until_at: until_at, reason: "search_secondary_limit", - now: now) - end + # @param until_at [Time] an instant another component decided. GREATEST ignores NULL, + # so a block only ever moves later — the core ledger's BLOCK_SQL rule, restated for + # the search row. + def block_until!(until_at:, classification: :secondary_limited, now: Time.current) + return if until_at.nil? - # GREATEST ignores NULL, so a block only ever moves later — the core ledger's - # BLOCK_SQL rule, restated for the search row. GithubSearchBudget.where(id: SINGLETON_ID).update_all( blocked_until: Arel::Nodes::NamedFunction.new( "GREATEST", @@ -139,8 +125,7 @@ def block_from!(fetched, now: Time.current, core_ledger: BudgetLedger.new) ), updated_at: now ) - Rails.logger.info(event: "search_budget.blocked", - classification: fetched.classification, + Rails.logger.info(event: "search_budget.blocked", classification: classification, blocked_until: until_at.utc.iso8601) end @@ -159,16 +144,6 @@ def bootstrap!(now: Time.current) private - # The core ledger owns global_blocked_until because a secondary limit can arise on - # any live request and Github::RateLimitPolicy already writes it there. Read with - # find_by, never through BudgetLedger: this path must not create that row. - def global_block(now:) - blocked_until = GithubApiBudget.find_by(id: GithubApiBudget::SINGLETON_ID) - &.global_blocked_until - - blocked_until if blocked_until&.>(now) - end - # The window has moved on when GitHub's own reset instant passed, or — when no # response ever told us one — when a full Search window elapsed since the last # outbound attempt. diff --git a/docs/DESIGN_BRIEF.md b/docs/DESIGN_BRIEF.md index fcdfe35..1c06f28 100644 --- a/docs/DESIGN_BRIEF.md +++ b/docs/DESIGN_BRIEF.md @@ -77,7 +77,7 @@ feasible ⇔ poll_allowance + RATE_LIMIT_RESERVE ``` With defaults: 12 poll attempts, 4 detail-fallback attempts split 2/2 by -`ACTOR_ENRICHMENT_SHARE`, a reserve of 8, and 36 core requests deliberately unspent. +`ACTOR_ENRICHMENT_SHARE`, and a reserve of 8 — the three lanes fill the limit exactly. Normal-path enrichment spends the **search** resource instead — a separate singleton ledger over GitHub's per-minute Search window (ceiling 10, reserve 2, 6-second pacing), reconciled against its own `x-ratelimit-resource: search` headers. Startup rejects diff --git a/docs/adr/0004-class-aware-budget-ledger.md b/docs/adr/0004-class-aware-budget-ledger.md index ff6cd5f..b1927a5 100644 --- a/docs/adr/0004-class-aware-budget-ledger.md +++ b/docs/adr/0004-class-aware-budget-ledger.md @@ -117,7 +117,7 @@ singleton ledger (`Github::SearchBudgetLedger` over `github_search_budget`). Thi mechanics — transactional reservation, failures-stay-spent, monotonic reconciliation, resource verification, per-window bootstrap — are unchanged, but its `enrichment_allowance`/`enrichment_used` pair is **redefined**: it now budgets the bounded -payload-URL detail-fallback lane (`CORE_DETAIL_FALLBACK_ALLOWANCE`, default 4/hour) rather +payload-URL detail-fallback lane (`CORE_DETAIL_FALLBACK_ALLOWANCE`, default 40/hour) rather than the remainder formula, and feasibility becomes `poll + reserve + detail_fallback ≤ limit`, with the remainder deliberately unspent. The resource-mismatch skip above now cuts both ways: this ledger ignores `search` headers, diff --git a/docs/adr/0007-enrichment-fairness-shares-and-borrowing.md b/docs/adr/0007-enrichment-fairness-shares-and-borrowing.md index d29b093..30cfcb8 100644 --- a/docs/adr/0007-enrichment-fairness-shares-and-borrowing.md +++ b/docs/adr/0007-enrichment-fairness-shares-and-borrowing.md @@ -188,7 +188,7 @@ Plan Appendix G ([ADR 0013](0013-derivation-first-staged-batch-enrichment.md)) m Search batches the normal enrichment path, so this ADR's share arithmetic — `floor(enrichment_allowance × ACTOR_ENRICHMENT_SHARE)`, borrowing on the caller's word, `:share_exhausted` as a denial — now governs only the bounded core **detail-fallback** -lane, whose allowance is `CORE_DETAIL_FALLBACK_ALLOWANCE` (default 4, so the guarantees +lane, whose allowance is `CORE_DETAIL_FALLBACK_ALLOWANCE` (default 40, so the guarantees default to 2/2). The batch lanes are balanced differently: a weighted rotation (`ACTOR_ENRICHMENT_WEIGHT` / `REPOSITORY_ENRICHMENT_WEIGHT`, defaults 1/1) over whole Search requests, with a lane that has nothing claimable yielding its slot — batch diff --git a/docs/adr/0013-derivation-first-staged-batch-enrichment.md b/docs/adr/0013-derivation-first-staged-batch-enrichment.md index 938d6b2..979478e 100644 --- a/docs/adr/0013-derivation-first-staged-batch-enrichment.md +++ b/docs/adr/0013-derivation-first-staged-batch-enrichment.md @@ -44,7 +44,7 @@ Adopt **derivation-first, lossless staged batch enrichment** (plan Appendix G; i 3. **Payload-URL detail fallback is the amendment, not the rule.** Only missing, renamed, identity-mismatched, or contract-invalid batch items fetch their stored payload-provided `api_url`, through the core ledger's - `CORE_DETAIL_FALLBACK_ALLOWANCE` (4/hour). No identifier is ever turned into a + `CORE_DETAIL_FALLBACK_ALLOWANCE` (40/hour). No identifier is ever turned into a constructed detail URL; the polling allocation is never touched. 4. **Dual ledgers.** `github_api_budget` (core, hourly: 12 poll + 4 detail + 8 reserve ≤ 60, remainder deliberately unspent) and `github_search_budget` (search, per-minute: diff --git a/docs/evidence/2026-08-02-live-staged-batch-enrichment.md b/docs/evidence/2026-08-02-live-staged-batch-enrichment.md index fc3dce4..b96ffe3 100644 --- a/docs/evidence/2026-08-02-live-staged-batch-enrichment.md +++ b/docs/evidence/2026-08-02-live-staged-batch-enrichment.md @@ -235,7 +235,11 @@ changed" below. ```text Run window: 2026-08-02T20:35:29Z → 2026-08-02T21:01:30Z (26 minutes) -Samples: 53 (60s cadence), 24 of them past CATCH_UP_MIN_SAMPLE_SECONDS +Samples: 53, 24 of them past CATCH_UP_MIN_SAMPLE_SECONDS. Two sampling + loops were running against the same endpoint, so the effective + cadence was ~30s rather than the 60s each loop used; the + duplicates are harmless to the counts below, which are read from + the last sample rather than summed across samples. Trailing window: ENRICHMENT_METRICS_WINDOW_SECONDS = 3600 Arrivals: 872 entities @@ -257,9 +261,11 @@ Core ledger: poll_used 2 of 12, detail_fallback used 4 of 4 **Verdict across the 24 mature samples: 8 `keeping_up`, 16 `not_keeping_up`.** The verdict oscillates, and the reason is visible in the numbers rather than a mystery: the -Search lane settled 872 entities in 92 requests and drained each five-minute arrival -burst within roughly two minutes, while a residue that Search does not return — 1.6% of -requested items — waits on the detail lane. With that lane capped at 4 requests an hour, +Search lane requested 872 items across 92 requests and had 858 of them returned and +applied directly, sending the other 14 to the fallback, and it drained each five-minute +arrival burst within roughly two minutes. The pipeline as a whole produced 870 exits, +leaving two entities outstanding at the final sample — a residue that Search does not +return, waiting on the detail lane. With that lane capped at 4 requests an hour, the residue outlives the window it arrived in, and any sample taken while it is outstanding reports a positive backlog delta. diff --git a/spec/services/github/enrichment/batch_runner_spec.rb b/spec/services/github/enrichment/batch_runner_spec.rb index 5c5fb3e..1588ced 100644 --- a/spec/services/github/enrichment/batch_runner_spec.rb +++ b/spec/services/github/enrichment/batch_runner_spec.rb @@ -475,8 +475,15 @@ def create_pending_actor(github_id:, login:, created_at: now - 60, **overrides) expect(EnrichmentBatch.find(result.batch_id)).to have_attributes( status: "deferred", last_error: expected[:reason] ) - # block_from! propagated the response's reset instant to the search ledger. - expect(current_search_budget.blocked_until).to eq(Time.zone.at((frozen_time + 60).to_i)) + # Both block the Search lane, by different routes: a primary exhaustion is this + # resource's own fact and takes the reset instant it reported, while a secondary + # limit is IP-scoped and takes Github::RateLimitPolicy's escalating floor. + if expected[:reason] == "rate_limited" + expect(current_search_budget.blocked_until).to eq(Time.zone.at((frozen_time + 60).to_i)) + else + expect(current_search_budget.blocked_until) + .to be >= frozen_time + Github::RateLimitPolicy::MIN_BLOCK_SECONDS + end expect(Rails.logger).to have_received(:info).with( hash_including(event: "enrichment.batch_deferred", deferral_reason: expected[:reason]) ) diff --git a/spec/services/github/search_budget_ledger_spec.rb b/spec/services/github/search_budget_ledger_spec.rb index eecc32e..44aa063 100644 --- a/spec/services/github/search_budget_ledger_spec.rb +++ b/spec/services/github/search_budget_ledger_spec.rb @@ -407,6 +407,9 @@ def superseding(remaining: "9") end end + # Primary exhaustion of this resource only. A secondary limit is IP-scoped and is + # decided once by Github::RateLimitPolicy for both resources, so it does not arrive + # through here — see "the two resources and one IP" below. describe "#block_from!" do def limited(status: 403, **headers) request = Github::Request.new(url: "https://api.github.com/search/users?q=user%3Aoctocat&per_page=1", @@ -416,22 +419,17 @@ def limited(status: 403, **headers) body: "", duration_ms: 1.0) end - # 403 with a non-zero remaining classifies as :secondary_limited; with "0" it is - # :rate_limited — the same discriminator Github::ResponseClassifier applies to core. - it "prefers Retry-After, the server's explicit instruction" do - active_window - - ledger.block_from!(limited("retry-after" => "120", - "x-ratelimit-reset" => window_reset.to_i.to_s), - now: frozen_time) - - expect(budget.blocked_until).to eq(frozen_time + 120) + # 403 with x-ratelimit-remaining "0" classifies as :rate_limited — the same + # discriminator Github::ResponseClassifier applies to core. + def exhausted(**headers) + limited(**{ "x-ratelimit-remaining" => "0" }.merge(headers.transform_keys(&:to_s))) end - it "falls back to the reset header when no Retry-After was sent" do + it "blocks until the reset the exhausted response reported" do active_window - ledger.block_from!(limited("x-ratelimit-reset" => window_reset.to_i.to_s), now: frozen_time) + ledger.block_from!(exhausted("x-ratelimit-reset" => window_reset.to_i.to_s), + now: frozen_time) expect(budget.blocked_until).to eq(window_reset) end @@ -439,38 +437,31 @@ def limited(status: 403, **headers) it "falls back to one search window when the response named no instant at all" do active_window - ledger.block_from!(limited, now: frozen_time) + ledger.block_from!(exhausted, now: frozen_time) expect(budget.blocked_until).to eq(frozen_time + described_class::SEARCH_WINDOW_SECONDS) end - it "blocks on a primary exhaustion as well as a secondary limit" do - active_window - - ledger.block_from!(limited("x-ratelimit-remaining" => "0", "retry-after" => "90"), - now: frozen_time) - - expect(budget.blocked_until).to eq(frozen_time + 90) - end - # GREATEST ignores NULL, so a block only ever moves later — a short block landing # after a long one must not resume searching into an exhausted quota. it "only ever moves a block later" do active_window - ledger.block_from!(limited("retry-after" => "300"), now: frozen_time) - ledger.block_from!(limited("retry-after" => "60"), now: frozen_time) + ledger.block_until!(until_at: frozen_time + 300, now: frozen_time) + ledger.block_until!(until_at: frozen_time + 60, now: frozen_time) expect(budget.blocked_until).to eq(frozen_time + 300) - ledger.block_from!(limited("retry-after" => "600"), now: frozen_time) + ledger.block_until!(until_at: frozen_time + 600, now: frozen_time) expect(budget.blocked_until).to eq(frozen_time + 600) end - it "ignores every classification that is not a rate limit" do + it "ignores every classification that is not this resource's own exhaustion" do active_window ledger.block_from!(limited(status: 500), now: frozen_time) ledger.block_from!(limited(status: 200), now: frozen_time) + # A secondary limit reaches the row through the policy, not through here. + ledger.block_from!(limited("retry-after" => "60"), now: frozen_time) expect(budget.blocked_until).to be_nil end @@ -480,66 +471,105 @@ def limited(status: 403, **headers) # separate PostgreSQL sessions reserving at once must serialise on the row, with no # lost debit and no deadlock. Transactional tests are off for the reason # spec/support/concurrency_helpers.rb documents, paid for with explicit cleanup. - # §10: a secondary rate limit is IP-scoped, so it stops every live request rather - # than only the resource that provoked it. Two ledgers meter two resources, but they - # share one outbound address — a block written by either has to bind both. - describe "the global block both ledgers share" do + # §10: a secondary rate limit is IP-scoped, so it stops every live request rather than + # only the resource that provoked it. A *primary* exhaustion is the opposite — it bounds + # the resource that reported it and nothing else. Github::RateLimitPolicy is the single + # decider for both, so the escalation ladder, the one-minute floor, the one-hour cap and + # the streak counter live in one place rather than being reimplemented per ledger. + describe "the two resources and one IP" do def search_request Github::Request.new(url: "https://api.github.com/search/users?q=user%3Aoctocat&per_page=1", request_class: :actor_search) end - it "refuses a Search reservation while the core ledger holds a global block" do + def policy = Github::RateLimitPolicy.new(search_ledger: ledger) + + def search_response(status:, **headers) + Github::FetchResult.from_response(request: search_request, status: status, + headers: headers.transform_keys(&:to_s), + body: "", duration_ms: 1.0) + end + + # The regression this replaced: reading github_api_budget.global_blocked_until made + # *core* primary exhaustion stop Search for the rest of the hour. That column carries + # primary exhaustion and reserve breaches as well as secondary limits, and neither of + # the first two says anything about the Search resource. + it "keeps searching while core is merely out of its own hourly requests" do active_window Github::BudgetLedger.new.bootstrap!(now: frozen_time) - Github::BudgetLedger.new.block_globally!(until_at: frozen_time + 300, - reason: "secondary_limit", now: frozen_time) + Github::RateLimitPolicy.new(search_ledger: ledger).apply!( + Github::FetchResult.from_response( + request: Github::Request.new(url: "https://api.github.com/events?per_page=100", + request_class: :poll), + status: 403, + headers: { "x-ratelimit-remaining" => "0", + "x-ratelimit-reset" => (frozen_time + 3600).to_i.to_s }, + body: "", duration_ms: 1.0 + ), now: frozen_time + ) + expect(current_budget.global_blocked_until).to eq(frozen_time + 3600) + expect(budget.blocked_until).to be_nil + expect { ledger.reserve!(:actor_search, now: frozen_time) }.not_to raise_error + end + + # The condition that genuinely is IP-scoped stops both, whichever resource saw it. + it "stops Search when a poll reports a secondary limit" do + active_window + Github::BudgetLedger.new.bootstrap!(now: frozen_time) + poll_secondary = Github::FetchResult.from_response( + request: Github::Request.new(url: "https://api.github.com/events?per_page=100", + request_class: :poll), + status: 403, headers: { "x-ratelimit-remaining" => "5", "retry-after" => "90" }, + body: "", duration_ms: 1.0 + ) + + Github::RateLimitPolicy.new(search_ledger: ledger).apply!(poll_secondary, now: frozen_time) + + expect(budget.blocked_until).to eq(frozen_time + 90) expect { ledger.reserve!(:actor_search, now: frozen_time) } .to raise_error(Github::Errors::BudgetExhausted) { |error| - expect(error.reason).to eq(:globally_blocked) + expect(error.reason).to eq(:search_blocked) } - expect(budget.used).to eq(0) end - it "grants the reservation once that block has expired" do + it "stops polling when a Search response reports a secondary limit" do active_window Github::BudgetLedger.new.bootstrap!(now: frozen_time) - Github::BudgetLedger.new.block_globally!(until_at: frozen_time + 300, - reason: "secondary_limit", now: frozen_time) - expect { ledger.reserve!(:actor_search, now: frozen_time + 301) }.not_to raise_error + policy.apply!(search_response(status: 403, "x-ratelimit-remaining" => "5", + "retry-after" => "90"), + now: frozen_time, resource: :search) + + expect(current_budget.global_blocked_until).to eq(frozen_time + 90) + expect(budget.blocked_until).to eq(frozen_time + 90) end - # The mirror image: a secondary limit provoked by Search stops polling too. - it "writes the core global block when a Search response is secondary-limited" do + # The escalation ladder is the policy's, so a Search limit advances the same streak + # a poll's would — and the next good Search response ends it. + it "advances and clears the shared secondary streak" do active_window Github::BudgetLedger.new.bootstrap!(now: frozen_time) - secondary = Github::FetchResult.from_response( - request: search_request, status: 403, - headers: { "x-ratelimit-remaining" => "5", "retry-after" => "60" }, - body: "", duration_ms: 1.0 - ) - ledger.block_from!(secondary, now: frozen_time) + policy.apply!(search_response(status: 403, "x-ratelimit-remaining" => "5"), + now: frozen_time, resource: :search) + expect(current_budget.consecutive_secondary_limits).to eq(1) + # No Retry-After, so the block is the policy's floor rather than nothing at all. + expect(budget.blocked_until).to be >= frozen_time + Github::RateLimitPolicy::MIN_BLOCK_SECONDS - expect(current_budget.global_blocked_until).to eq(frozen_time + 60) - expect(budget.blocked_until).to eq(frozen_time + 60) + policy.apply!(search_response(status: 200), now: frozen_time, resource: :search) + expect(current_budget.consecutive_secondary_limits).to eq(0) end - # A primary Search exhaustion bounds only this resource. Blocking polling on it - # would let the search lane starve event capture, which no quota rule permits. + # A spent Search minute must not cost polling its hour. it "leaves polling alone when Search merely exhausted its own limit" do active_window Github::BudgetLedger.new.bootstrap!(now: frozen_time) - exhausted = Github::FetchResult.from_response( - request: search_request, status: 403, - headers: { "x-ratelimit-remaining" => "0", - "x-ratelimit-reset" => window_reset.to_i.to_s }, - body: "", duration_ms: 1.0 - ) + response = search_response(status: 403, "x-ratelimit-remaining" => "0", + "x-ratelimit-reset" => window_reset.to_i.to_s) - ledger.block_from!(exhausted, now: frozen_time) + policy.apply!(response, now: frozen_time, resource: :search) + ledger.block_from!(response, now: frozen_time) expect(current_budget.global_blocked_until).to be_nil expect(budget.blocked_until).to eq(window_reset) diff --git a/spec/support/advisory_lock_helpers.rb b/spec/support/advisory_lock_helpers.rb index 0bdfd16..abb2269 100644 --- a/spec/support/advisory_lock_helpers.rb +++ b/spec/support/advisory_lock_helpers.rb @@ -85,12 +85,20 @@ def terminate_second_session! # Which backends hold this lock right now, straight from the server. Uncached, because the # query cache would happily answer a second question with the first answer. + # + # Scoped to the current database, and that is load-bearing rather than tidy: pg_locks is + # instance-wide, and this project's Compose file runs the development stack against the + # same PostgreSQL container as the test databases. Without the filter, a `worker` + # container mid-poll holds the request gate in the *development* database and this helper + # reports it as a holder here — the suite then fails, intermittently, on whether someone + # happened to be polling. Observed exactly that way. def advisory_lock_holders(namespace, key) ActiveRecord::Base.uncached do ActiveRecord::Base.connection.select_values( ActiveRecord::Base.sanitize_sql_array([ <<~SQL.squish, namespace, key ]) SELECT pid FROM pg_locks WHERE locktype = 'advisory' AND classid::bigint = ? AND objid::bigint = ? + AND database = (SELECT oid FROM pg_database WHERE datname = current_database()) SQL ) end