From 3fc1ac968c920ac40371958b1ab11e2c4d893d26 Mon Sep 17 00:00:00 2001 From: Dominik Ledzion Date: Thu, 9 Jul 2026 10:51:00 +0200 Subject: [PATCH 01/34] Add Supabase agent skills; ignore node_modules, .next, root package files Install supabase and supabase-postgres-best-practices skills via `npx skills add supabase/agent-skills` (.agents/ + skills-lock.json). Ignore the root node_modules/, package.json, package-lock.json pulled in by the installer, plus Next.js .next/ build output. Co-Authored-By: Claude Opus 4.8 --- .../CHANGELOG.md | 29 +++ .../supabase-postgres-best-practices/SKILL.md | 64 +++++++ .../references/_contributing.md | 170 ++++++++++++++++++ .../references/_sections.md | 39 ++++ .../references/_template.md | 34 ++++ .../references/advanced-full-text-search.md | 55 ++++++ .../references/advanced-jsonb-indexing.md | 49 +++++ .../references/conn-idle-timeout.md | 46 +++++ .../references/conn-limits.md | 44 +++++ .../references/conn-pooling.md | 41 +++++ .../references/conn-prepared-statements.md | 46 +++++ .../references/data-batch-inserts.md | 54 ++++++ .../references/data-n-plus-one.md | 53 ++++++ .../references/data-pagination.md | 50 ++++++ .../references/data-upsert.md | 50 ++++++ .../references/lock-advisory.md | 56 ++++++ .../references/lock-deadlock-prevention.md | 68 +++++++ .../references/lock-short-transactions.md | 50 ++++++ .../references/lock-skip-locked.md | 54 ++++++ .../references/monitor-explain-analyze.md | 45 +++++ .../references/monitor-pg-stat-statements.md | 55 ++++++ .../references/monitor-vacuum-analyze.md | 55 ++++++ .../references/query-composite-indexes.md | 44 +++++ .../references/query-covering-indexes.md | 40 +++++ .../references/query-index-types.md | 48 +++++ .../references/query-missing-indexes.md | 43 +++++ .../references/query-partial-indexes.md | 45 +++++ .../references/schema-constraints.md | 80 +++++++++ .../references/schema-data-types.md | 46 +++++ .../references/schema-foreign-key-indexes.md | 59 ++++++ .../schema-lowercase-identifiers.md | 55 ++++++ .../references/schema-partitioning.md | 55 ++++++ .../references/schema-primary-keys.md | 61 +++++++ .../references/security-privileges.md | 54 ++++++ .../references/security-rls-basics.md | 50 ++++++ .../references/security-rls-performance.md | 63 +++++++ .agents/skills/supabase/CHANGELOG.md | 35 ++++ .agents/skills/supabase/SKILL.md | 135 ++++++++++++++ .../assets/feedback-issue-template.md | 17 ++ .../supabase/references/skill-feedback.md | 17 ++ .gitignore | 8 + skills-lock.json | 17 ++ 42 files changed, 2179 insertions(+) create mode 100644 .agents/skills/supabase-postgres-best-practices/CHANGELOG.md create mode 100644 .agents/skills/supabase-postgres-best-practices/SKILL.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/_contributing.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/_sections.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/_template.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/advanced-full-text-search.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/advanced-jsonb-indexing.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/conn-idle-timeout.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/conn-limits.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/conn-pooling.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/conn-prepared-statements.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/data-batch-inserts.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/data-n-plus-one.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/data-pagination.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/data-upsert.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/lock-advisory.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/lock-deadlock-prevention.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/lock-short-transactions.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/lock-skip-locked.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/monitor-explain-analyze.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/monitor-pg-stat-statements.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/monitor-vacuum-analyze.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/query-composite-indexes.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/query-covering-indexes.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/query-index-types.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/query-missing-indexes.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/query-partial-indexes.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/schema-constraints.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/schema-data-types.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/schema-foreign-key-indexes.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/schema-lowercase-identifiers.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/schema-partitioning.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/schema-primary-keys.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/security-privileges.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/security-rls-basics.md create mode 100644 .agents/skills/supabase-postgres-best-practices/references/security-rls-performance.md create mode 100644 .agents/skills/supabase/CHANGELOG.md create mode 100644 .agents/skills/supabase/SKILL.md create mode 100644 .agents/skills/supabase/assets/feedback-issue-template.md create mode 100644 .agents/skills/supabase/references/skill-feedback.md create mode 100644 skills-lock.json diff --git a/.agents/skills/supabase-postgres-best-practices/CHANGELOG.md b/.agents/skills/supabase-postgres-best-practices/CHANGELOG.md new file mode 100644 index 0000000..94aee48 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/CHANGELOG.md @@ -0,0 +1,29 @@ +# Changelog + +## [1.3.0](https://github.com/supabase/agent-skills/compare/v1.2.0...v1.3.0) (2026-06-05) + + +### Features + +* add schema-constraints reference for safe migration patterns ([#30](https://github.com/supabase/agent-skills/issues/30)) ([9b236f3](https://github.com/supabase/agent-skills/commit/9b236f3ebd65d76a2c570f19931353da9c858d5a)) +* using Supabase agent skills ([#12](https://github.com/supabase/agent-skills/issues/12)) ([7c2e389](https://github.com/supabase/agent-skills/commit/7c2e3894fddfde8eb6c77d2a8921904543b9be7a)) + + +### Bug Fixes + +* correct broken reference link in postgres best practices skill ([#58](https://github.com/supabase/agent-skills/issues/58)) ([f4e2277](https://github.com/supabase/agent-skills/commit/f4e22777fd8573537297b568c16e5a45a25927da)) +* cover SECURITY DEFINER, auth.role() deprecation, and BOLA in security checklist ([#85](https://github.com/supabase/agent-skills/issues/85)) ([133f43e](https://github.com/supabase/agent-skills/commit/133f43e8c2ffc48823ff0630c692cabecea3e3a3)) + +## [1.2.0](https://github.com/supabase/agent-skills/compare/v1.1.1...v1.2.0) (2026-06-02) + + +### Features + +* add schema-constraints reference for safe migration patterns ([#30](https://github.com/supabase/agent-skills/issues/30)) ([9b236f3](https://github.com/supabase/agent-skills/commit/9b236f3ebd65d76a2c570f19931353da9c858d5a)) +* using Supabase agent skills ([#12](https://github.com/supabase/agent-skills/issues/12)) ([7c2e389](https://github.com/supabase/agent-skills/commit/7c2e3894fddfde8eb6c77d2a8921904543b9be7a)) + + +### Bug Fixes + +* correct broken reference link in postgres best practices skill ([#58](https://github.com/supabase/agent-skills/issues/58)) ([f4e2277](https://github.com/supabase/agent-skills/commit/f4e22777fd8573537297b568c16e5a45a25927da)) +* cover SECURITY DEFINER, auth.role() deprecation, and BOLA in security checklist ([#85](https://github.com/supabase/agent-skills/issues/85)) ([133f43e](https://github.com/supabase/agent-skills/commit/133f43e8c2ffc48823ff0630c692cabecea3e3a3)) diff --git a/.agents/skills/supabase-postgres-best-practices/SKILL.md b/.agents/skills/supabase-postgres-best-practices/SKILL.md new file mode 100644 index 0000000..d9ef194 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/SKILL.md @@ -0,0 +1,64 @@ +--- +name: supabase-postgres-best-practices +description: Postgres performance optimization and best practices from Supabase. Use this skill when writing, reviewing, or optimizing Postgres queries, schema designs, or database configurations. +license: MIT +metadata: + author: supabase + version: "1.1.1" + organization: Supabase + date: January 2026 + abstract: Comprehensive Postgres performance optimization guide for developers using Supabase and Postgres. Contains performance rules across 8 categories, prioritized by impact from critical (query performance, connection management) to incremental (advanced features). Each rule includes detailed explanations, incorrect vs. correct SQL examples, query plan analysis, and specific performance metrics to guide automated optimization and code generation. +--- + +# Supabase Postgres Best Practices + +Comprehensive performance optimization guide for Postgres, maintained by Supabase. Contains rules across 8 categories, prioritized by impact to guide automated query optimization and schema design. + +## When to Apply + +Reference these guidelines when: +- Writing SQL queries or designing schemas +- Implementing indexes or query optimization +- Reviewing database performance issues +- Configuring connection pooling or scaling +- Optimizing for Postgres-specific features +- Working with Row-Level Security (RLS) + +## Rule Categories by Priority + +| Priority | Category | Impact | Prefix | +|----------|----------|--------|--------| +| 1 | Query Performance | CRITICAL | `query-` | +| 2 | Connection Management | CRITICAL | `conn-` | +| 3 | Security & RLS | CRITICAL | `security-` | +| 4 | Schema Design | HIGH | `schema-` | +| 5 | Concurrency & Locking | MEDIUM-HIGH | `lock-` | +| 6 | Data Access Patterns | MEDIUM | `data-` | +| 7 | Monitoring & Diagnostics | LOW-MEDIUM | `monitor-` | +| 8 | Advanced Features | LOW | `advanced-` | + +## How to Use + +Read individual rule files for detailed explanations and SQL examples: + +``` +references/query-missing-indexes.md +references/query-partial-indexes.md +references/_sections.md +``` + +Each rule file contains: +- Brief explanation of why it matters +- Incorrect SQL example with explanation +- Correct SQL example with explanation +- Optional EXPLAIN output or metrics +- Additional context and references +- Supabase-specific notes (when applicable) + +## References + +- https://www.postgresql.org/docs/current/ +- https://supabase.com/docs +- https://wiki.postgresql.org/wiki/Performance_Optimization +- https://supabase.com/docs/guides/database/overview +- https://supabase.com/docs/guides/auth/row-level-security diff --git a/.agents/skills/supabase-postgres-best-practices/references/_contributing.md b/.agents/skills/supabase-postgres-best-practices/references/_contributing.md new file mode 100644 index 0000000..1c055af --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/_contributing.md @@ -0,0 +1,170 @@ +# Writing Guidelines for Postgres References + +This document provides guidelines for creating effective Postgres best +practice references that work well with AI agents and LLMs. + +## Key Principles + +### 1. Concrete Transformation Patterns + +Show exact SQL rewrites. Avoid philosophical advice. + +**Good:** "Use `WHERE id = ANY(ARRAY[...])` instead of +`WHERE id IN (SELECT ...)`" **Bad:** "Design good schemas" + +### 2. Error-First Structure + +Always show the problematic pattern first, then the solution. This trains agents +to recognize anti-patterns. + +```markdown +**Incorrect (sequential queries):** [bad example] + +**Correct (batched query):** [good example] +``` + +### 3. Quantified Impact + +Include specific metrics. Helps agents prioritize fixes. + +**Good:** "10x faster queries", "50% smaller index", "Eliminates N+1" +**Bad:** "Faster", "Better", "More efficient" + +### 4. Self-Contained Examples + +Examples should be complete and runnable (or close to it). Include `CREATE TABLE` +if context is needed. + +```sql +-- Include table definition when needed for clarity +CREATE TABLE users ( + id bigint PRIMARY KEY, + email text NOT NULL, + deleted_at timestamptz +); + +-- Now show the index +CREATE INDEX users_active_email_idx ON users(email) WHERE deleted_at IS NULL; +``` + +### 5. Semantic Naming + +Use meaningful table/column names. Names carry intent for LLMs. + +**Good:** `users`, `email`, `created_at`, `is_active` +**Bad:** `table1`, `col1`, `field`, `flag` + +--- + +## Code Example Standards + +### SQL Formatting + +```sql +-- Use lowercase keywords, clear formatting +CREATE INDEX CONCURRENTLY users_email_idx + ON users(email) + WHERE deleted_at IS NULL; + +-- Not cramped or ALL CAPS +CREATE INDEX CONCURRENTLY USERS_EMAIL_IDX ON USERS(EMAIL) WHERE DELETED_AT IS NULL; +``` + +### Comments + +- Explain _why_, not _what_ +- Highlight performance implications +- Point out common pitfalls + +### Language Tags + +- `sql` - Standard SQL queries +- `plpgsql` - Stored procedures/functions +- `typescript` - Application code (when needed) +- `python` - Application code (when needed) + +--- + +## When to Include Application Code + +**Default: SQL Only** + +Most references should focus on pure SQL patterns. This keeps examples portable. + +**Include Application Code When:** + +- Connection pooling configuration +- Transaction management in application context +- ORM anti-patterns (N+1 in Prisma/TypeORM) +- Prepared statement usage + +**Format for Mixed Examples:** + +````markdown +**Incorrect (N+1 in application):** + +```typescript +for (const user of users) { + const posts = await db.query("SELECT * FROM posts WHERE user_id = $1", [ + user.id, + ]); +} +``` +```` + +**Correct (batch query):** + +```typescript +const posts = await db.query("SELECT * FROM posts WHERE user_id = ANY($1)", [ + userIds, +]); +``` + +--- + +## Impact Level Guidelines + +| Level | Improvement | Use When | +|-------|-------------|----------| +| **CRITICAL** | 10-100x | Missing indexes, connection exhaustion, sequential scans on large tables | +| **HIGH** | 5-20x | Wrong index types, poor partitioning, missing covering indexes | +| **MEDIUM-HIGH** | 2-5x | N+1 queries, inefficient pagination, RLS optimization | +| **MEDIUM** | 1.5-3x | Redundant indexes, query plan instability | +| **LOW-MEDIUM** | 1.2-2x | VACUUM tuning, configuration tweaks | +| **LOW** | Incremental | Advanced patterns, edge cases | + +--- + +## Reference Standards + +**Primary Sources:** + +- Official Postgres documentation +- Supabase documentation +- Postgres wiki +- Established blogs (2ndQuadrant, Crunchy Data) + +**Format:** + +```markdown +Reference: +[Postgres Indexes](https://www.postgresql.org/docs/current/indexes.html) +``` + +--- + +## Review Checklist + +Before submitting a reference: + +- [ ] Title is clear and action-oriented +- [ ] Impact level matches the performance gain +- [ ] impactDescription includes quantification +- [ ] Explanation is concise (1-2 sentences) +- [ ] Has at least 1 **Incorrect** SQL example +- [ ] Has at least 1 **Correct** SQL example +- [ ] SQL uses semantic naming +- [ ] Comments explain _why_, not _what_ +- [ ] Trade-offs mentioned if applicable +- [ ] Reference links included +- [ ] `pnpm test` passes diff --git a/.agents/skills/supabase-postgres-best-practices/references/_sections.md b/.agents/skills/supabase-postgres-best-practices/references/_sections.md new file mode 100644 index 0000000..8ba57c2 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/_sections.md @@ -0,0 +1,39 @@ +# Section Definitions + +This file defines the rule categories for Postgres best practices. Rules are automatically assigned to sections based on their filename prefix. + +Take the examples below as pure demonstrative. Replace each section with the actual rule categories for Postgres best practices. + +--- + +## 1. Query Performance (query) +**Impact:** CRITICAL +**Description:** Slow queries, missing indexes, inefficient query plans. The most common source of Postgres performance issues. + +## 2. Connection Management (conn) +**Impact:** CRITICAL +**Description:** Connection pooling, limits, and serverless strategies. Critical for applications with high concurrency or serverless deployments. + +## 3. Security & RLS (security) +**Impact:** CRITICAL +**Description:** Row-Level Security policies, privilege management, and authentication patterns. + +## 4. Schema Design (schema) +**Impact:** HIGH +**Description:** Table design, index strategies, partitioning, and data type selection. Foundation for long-term performance. + +## 5. Concurrency & Locking (lock) +**Impact:** MEDIUM-HIGH +**Description:** Transaction management, isolation levels, deadlock prevention, and lock contention patterns. + +## 6. Data Access Patterns (data) +**Impact:** MEDIUM +**Description:** N+1 query elimination, batch operations, cursor-based pagination, and efficient data fetching. + +## 7. Monitoring & Diagnostics (monitor) +**Impact:** LOW-MEDIUM +**Description:** Using pg_stat_statements, EXPLAIN ANALYZE, metrics collection, and performance diagnostics. + +## 8. Advanced Features (advanced) +**Impact:** LOW +**Description:** Full-text search, JSONB optimization, PostGIS, extensions, and advanced Postgres features. diff --git a/.agents/skills/supabase-postgres-best-practices/references/_template.md b/.agents/skills/supabase-postgres-best-practices/references/_template.md new file mode 100644 index 0000000..91ace90 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/_template.md @@ -0,0 +1,34 @@ +--- +title: Clear, Action-Oriented Title (e.g., "Use Partial Indexes for Filtered Queries") +impact: MEDIUM +impactDescription: 5-20x query speedup for filtered queries +tags: indexes, query-optimization, performance +--- + +## [Rule Title] + +[1-2 sentence explanation of the problem and why it matters. Focus on performance impact.] + +**Incorrect (describe the problem):** + +```sql +-- Comment explaining what makes this slow/problematic +CREATE INDEX users_email_idx ON users(email); + +SELECT * FROM users WHERE email = 'user@example.com' AND deleted_at IS NULL; +-- This scans deleted records unnecessarily +``` + +**Correct (describe the solution):** + +```sql +-- Comment explaining why this is better +CREATE INDEX users_active_email_idx ON users(email) WHERE deleted_at IS NULL; + +SELECT * FROM users WHERE email = 'user@example.com' AND deleted_at IS NULL; +-- Only indexes active users, 10x smaller index, faster queries +``` + +[Optional: Additional context, edge cases, or trade-offs] + +Reference: [Postgres Docs](https://www.postgresql.org/docs/current/) diff --git a/.agents/skills/supabase-postgres-best-practices/references/advanced-full-text-search.md b/.agents/skills/supabase-postgres-best-practices/references/advanced-full-text-search.md new file mode 100644 index 0000000..582cbea --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/advanced-full-text-search.md @@ -0,0 +1,55 @@ +--- +title: Use tsvector for Full-Text Search +impact: MEDIUM +impactDescription: 100x faster than LIKE, with ranking support +tags: full-text-search, tsvector, gin, search +--- + +## Use tsvector for Full-Text Search + +LIKE with wildcards can't use indexes. Full-text search with tsvector is orders of magnitude faster. + +**Incorrect (LIKE pattern matching):** + +```sql +-- Cannot use index, scans all rows +select * from articles where content like '%postgresql%'; + +-- Case-insensitive makes it worse +select * from articles where lower(content) like '%postgresql%'; +``` + +**Correct (full-text search with tsvector):** + +```sql +-- Add tsvector column and index +alter table articles add column search_vector tsvector + generated always as (to_tsvector('english', coalesce(title,'') || ' ' || coalesce(content,''))) stored; + +create index articles_search_idx on articles using gin (search_vector); + +-- Fast full-text search +select * from articles +where search_vector @@ to_tsquery('english', 'postgresql & performance'); + +-- With ranking +select *, ts_rank(search_vector, query) as rank +from articles, to_tsquery('english', 'postgresql') query +where search_vector @@ query +order by rank desc; +``` + +Search multiple terms: + +```sql +-- AND: both terms required +to_tsquery('postgresql & performance') + +-- OR: either term +to_tsquery('postgresql | mysql') + +-- Prefix matching +to_tsquery('post:*') +``` + +Reference: [Full Text Search](https://supabase.com/docs/guides/database/full-text-search) diff --git a/.agents/skills/supabase-postgres-best-practices/references/advanced-jsonb-indexing.md b/.agents/skills/supabase-postgres-best-practices/references/advanced-jsonb-indexing.md new file mode 100644 index 0000000..e3d261e --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/advanced-jsonb-indexing.md @@ -0,0 +1,49 @@ +--- +title: Index JSONB Columns for Efficient Querying +impact: MEDIUM +impactDescription: 10-100x faster JSONB queries with proper indexing +tags: jsonb, gin, indexes, json +--- + +## Index JSONB Columns for Efficient Querying + +JSONB queries without indexes scan the entire table. Use GIN indexes for containment queries. + +**Incorrect (no index on JSONB):** + +```sql +create table products ( + id bigint primary key, + attributes jsonb +); + +-- Full table scan for every query +select * from products where attributes @> '{"color": "red"}'; +select * from products where attributes->>'brand' = 'Nike'; +``` + +**Correct (GIN index for JSONB):** + +```sql +-- GIN index for containment operators (@>, ?, ?&, ?|) +create index products_attrs_gin on products using gin (attributes); + +-- Now containment queries use the index +select * from products where attributes @> '{"color": "red"}'; + +-- For specific key lookups, use expression index +create index products_brand_idx on products ((attributes->>'brand')); +select * from products where attributes->>'brand' = 'Nike'; +``` + +Choose the right operator class: + +```sql +-- jsonb_ops (default): supports all operators, larger index +create index idx1 on products using gin (attributes); + +-- jsonb_path_ops: only @> operator, but 2-3x smaller index +create index idx2 on products using gin (attributes jsonb_path_ops); +``` + +Reference: [JSONB Indexes](https://www.postgresql.org/docs/current/datatype-json.html#JSON-INDEXING) diff --git a/.agents/skills/supabase-postgres-best-practices/references/conn-idle-timeout.md b/.agents/skills/supabase-postgres-best-practices/references/conn-idle-timeout.md new file mode 100644 index 0000000..40b9cc5 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/conn-idle-timeout.md @@ -0,0 +1,46 @@ +--- +title: Configure Idle Connection Timeouts +impact: HIGH +impactDescription: Reclaim 30-50% of connection slots from idle clients +tags: connections, timeout, idle, resource-management +--- + +## Configure Idle Connection Timeouts + +Idle connections waste resources. Configure timeouts to automatically reclaim them. + +**Incorrect (connections held indefinitely):** + +```sql +-- No timeout configured +show idle_in_transaction_session_timeout; -- 0 (disabled) + +-- Connections stay open forever, even when idle +select pid, state, state_change, query +from pg_stat_activity +where state = 'idle in transaction'; +-- Shows transactions idle for hours, holding locks +``` + +**Correct (automatic cleanup of idle connections):** + +```sql +-- Terminate connections idle in transaction after 30 seconds +alter system set idle_in_transaction_session_timeout = '30s'; + +-- Terminate completely idle connections after 10 minutes +alter system set idle_session_timeout = '10min'; + +-- Reload configuration +select pg_reload_conf(); +``` + +For pooled connections, configure at the pooler level: + +```ini +# pgbouncer.ini +server_idle_timeout = 60 +client_idle_timeout = 300 +``` + +Reference: [Connection Timeouts](https://www.postgresql.org/docs/current/runtime-config-client.html#GUC-IDLE-IN-TRANSACTION-SESSION-TIMEOUT) diff --git a/.agents/skills/supabase-postgres-best-practices/references/conn-limits.md b/.agents/skills/supabase-postgres-best-practices/references/conn-limits.md new file mode 100644 index 0000000..cb3e400 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/conn-limits.md @@ -0,0 +1,44 @@ +--- +title: Set Appropriate Connection Limits +impact: CRITICAL +impactDescription: Prevent database crashes and memory exhaustion +tags: connections, max-connections, limits, stability +--- + +## Set Appropriate Connection Limits + +Too many connections exhaust memory and degrade performance. Set limits based on available resources. + +**Incorrect (unlimited or excessive connections):** + +```sql +-- Default max_connections = 100, but often increased blindly +show max_connections; -- 500 (way too high for 4GB RAM) + +-- Each connection uses 1-3MB RAM +-- 500 connections * 2MB = 1GB just for connections! +-- Out of memory errors under load +``` + +**Correct (calculate based on resources):** + +```sql +-- Formula: max_connections = (RAM in MB / 5MB per connection) - reserved +-- For 4GB RAM: (4096 / 5) - 10 = ~800 theoretical max +-- But practically, 100-200 is better for query performance + +-- Recommended settings for 4GB RAM +alter system set max_connections = 100; + +-- Also set work_mem appropriately +-- work_mem * max_connections should not exceed 25% of RAM +alter system set work_mem = '8MB'; -- 8MB * 100 = 800MB max +``` + +Monitor connection usage: + +```sql +select count(*), state from pg_stat_activity group by state; +``` + +Reference: [Database Connections](https://supabase.com/docs/guides/platform/performance#connection-management) diff --git a/.agents/skills/supabase-postgres-best-practices/references/conn-pooling.md b/.agents/skills/supabase-postgres-best-practices/references/conn-pooling.md new file mode 100644 index 0000000..e2ebd58 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/conn-pooling.md @@ -0,0 +1,41 @@ +--- +title: Use Connection Pooling for All Applications +impact: CRITICAL +impactDescription: Handle 10-100x more concurrent users +tags: connection-pooling, pgbouncer, performance, scalability +--- + +## Use Connection Pooling for All Applications + +Postgres connections are expensive (1-3MB RAM each). Without pooling, applications exhaust connections under load. + +**Incorrect (new connection per request):** + +```sql +-- Each request creates a new connection +-- Application code: db.connect() per request +-- Result: 500 concurrent users = 500 connections = crashed database + +-- Check current connections +select count(*) from pg_stat_activity; -- 487 connections! +``` + +**Correct (connection pooling):** + +```sql +-- Use a pooler like PgBouncer between app and database +-- Application connects to pooler, pooler reuses a small pool to Postgres + +-- Configure pool_size based on: (CPU cores * 2) + spindle_count +-- Example for 4 cores: pool_size = 10 + +-- Result: 500 concurrent users share 10 actual connections +select count(*) from pg_stat_activity; -- 10 connections +``` + +Pool modes: + +- **Transaction mode**: connection returned after each transaction (best for most apps) +- **Session mode**: connection held for entire session (needed for prepared statements, temp tables) + +Reference: [Connection Pooling](https://supabase.com/docs/guides/database/connecting-to-postgres#connection-pooler) diff --git a/.agents/skills/supabase-postgres-best-practices/references/conn-prepared-statements.md b/.agents/skills/supabase-postgres-best-practices/references/conn-prepared-statements.md new file mode 100644 index 0000000..555547d --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/conn-prepared-statements.md @@ -0,0 +1,46 @@ +--- +title: Use Prepared Statements Correctly with Pooling +impact: HIGH +impactDescription: Avoid prepared statement conflicts in pooled environments +tags: prepared-statements, connection-pooling, transaction-mode +--- + +## Use Prepared Statements Correctly with Pooling + +Prepared statements are tied to individual database connections. In transaction-mode pooling, connections are shared, causing conflicts. + +**Incorrect (named prepared statements with transaction pooling):** + +```sql +-- Named prepared statement +prepare get_user as select * from users where id = $1; + +-- In transaction mode pooling, next request may get different connection +execute get_user(123); +-- ERROR: prepared statement "get_user" does not exist +``` + +**Correct (use unnamed statements or session mode):** + +```sql +-- Option 1: Use unnamed prepared statements (most ORMs do this automatically) +-- The query is prepared and executed in a single protocol message + +-- Option 2: Deallocate after use in transaction mode +prepare get_user as select * from users where id = $1; +execute get_user(123); +deallocate get_user; + +-- Option 3: Use session mode pooling (port 5432 vs 6543) +-- Connection is held for entire session, prepared statements persist +``` + +Check your driver settings: + +```sql +-- Many drivers use prepared statements by default +-- Node.js pg: { prepare: false } to disable +-- JDBC: prepareThreshold=0 to disable +``` + +Reference: [Prepared Statements with Pooling](https://supabase.com/docs/guides/database/connecting-to-postgres#connection-pool-modes) diff --git a/.agents/skills/supabase-postgres-best-practices/references/data-batch-inserts.md b/.agents/skills/supabase-postgres-best-practices/references/data-batch-inserts.md new file mode 100644 index 0000000..997947c --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/data-batch-inserts.md @@ -0,0 +1,54 @@ +--- +title: Batch INSERT Statements for Bulk Data +impact: MEDIUM +impactDescription: 10-50x faster bulk inserts +tags: batch, insert, bulk, performance, copy +--- + +## Batch INSERT Statements for Bulk Data + +Individual INSERT statements have high overhead. Batch multiple rows in single statements or use COPY. + +**Incorrect (individual inserts):** + +```sql +-- Each insert is a separate transaction and round trip +insert into events (user_id, action) values (1, 'click'); +insert into events (user_id, action) values (1, 'view'); +insert into events (user_id, action) values (2, 'click'); +-- ... 1000 more individual inserts + +-- 1000 inserts = 1000 round trips = slow +``` + +**Correct (batch insert):** + +```sql +-- Multiple rows in single statement +insert into events (user_id, action) values + (1, 'click'), + (1, 'view'), + (2, 'click'), + -- ... up to ~1000 rows per batch + (999, 'view'); + +-- One round trip for 1000 rows +``` + +For large imports, use COPY: + +```sql +-- COPY is fastest for bulk loading +copy events (user_id, action, created_at) +from '/path/to/data.csv' +with (format csv, header true); + +-- Or from stdin in application +copy events (user_id, action) from stdin with (format csv); +1,click +1,view +2,click +\. +``` + +Reference: [COPY](https://www.postgresql.org/docs/current/sql-copy.html) diff --git a/.agents/skills/supabase-postgres-best-practices/references/data-n-plus-one.md b/.agents/skills/supabase-postgres-best-practices/references/data-n-plus-one.md new file mode 100644 index 0000000..2109186 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/data-n-plus-one.md @@ -0,0 +1,53 @@ +--- +title: Eliminate N+1 Queries with Batch Loading +impact: MEDIUM-HIGH +impactDescription: 10-100x fewer database round trips +tags: n-plus-one, batch, performance, queries +--- + +## Eliminate N+1 Queries with Batch Loading + +N+1 queries execute one query per item in a loop. Batch them into a single query using arrays or JOINs. + +**Incorrect (N+1 queries):** + +```sql +-- First query: get all users +select id from users where active = true; -- Returns 100 IDs + +-- Then N queries, one per user +select * from orders where user_id = 1; +select * from orders where user_id = 2; +select * from orders where user_id = 3; +-- ... 97 more queries! + +-- Total: 101 round trips to database +``` + +**Correct (single batch query):** + +```sql +-- Collect IDs and query once with ANY +select * from orders where user_id = any(array[1, 2, 3, ...]); + +-- Or use JOIN instead of loop +select u.id, u.name, o.* +from users u +left join orders o on o.user_id = u.id +where u.active = true; + +-- Total: 1 round trip +``` + +Application pattern: + +```sql +-- Instead of looping in application code: +-- for user in users: db.query("SELECT * FROM orders WHERE user_id = $1", user.id) + +-- Pass array parameter: +select * from orders where user_id = any($1::bigint[]); +-- Application passes: [1, 2, 3, 4, 5, ...] +``` + +Reference: [N+1 Query Problem](https://supabase.com/docs/guides/database/query-optimization) diff --git a/.agents/skills/supabase-postgres-best-practices/references/data-pagination.md b/.agents/skills/supabase-postgres-best-practices/references/data-pagination.md new file mode 100644 index 0000000..633d839 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/data-pagination.md @@ -0,0 +1,50 @@ +--- +title: Use Cursor-Based Pagination Instead of OFFSET +impact: MEDIUM-HIGH +impactDescription: Consistent O(1) performance regardless of page depth +tags: pagination, cursor, keyset, offset, performance +--- + +## Use Cursor-Based Pagination Instead of OFFSET + +OFFSET-based pagination scans all skipped rows, getting slower on deeper pages. Cursor pagination is O(1). + +**Incorrect (OFFSET pagination):** + +```sql +-- Page 1: scans 20 rows +select * from products order by id limit 20 offset 0; + +-- Page 100: scans 2000 rows to skip 1980 +select * from products order by id limit 20 offset 1980; + +-- Page 10000: scans 200,000 rows! +select * from products order by id limit 20 offset 199980; +``` + +**Correct (cursor/keyset pagination):** + +```sql +-- Page 1: get first 20 +select * from products order by id limit 20; +-- Application stores last_id = 20 + +-- Page 2: start after last ID +select * from products where id > 20 order by id limit 20; +-- Uses index, always fast regardless of page depth + +-- Page 10000: same speed as page 1 +select * from products where id > 199980 order by id limit 20; +``` + +For multi-column sorting: + +```sql +-- Cursor must include all sort columns +select * from products +where (created_at, id) > ('2024-01-15 10:00:00', 12345) +order by created_at, id +limit 20; +``` + +Reference: [Pagination](https://supabase.com/docs/guides/database/pagination) diff --git a/.agents/skills/supabase-postgres-best-practices/references/data-upsert.md b/.agents/skills/supabase-postgres-best-practices/references/data-upsert.md new file mode 100644 index 0000000..bc95e23 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/data-upsert.md @@ -0,0 +1,50 @@ +--- +title: Use UPSERT for Insert-or-Update Operations +impact: MEDIUM +impactDescription: Atomic operation, eliminates race conditions +tags: upsert, on-conflict, insert, update +--- + +## Use UPSERT for Insert-or-Update Operations + +Using separate SELECT-then-INSERT/UPDATE creates race conditions. Use INSERT ... ON CONFLICT for atomic upserts. + +**Incorrect (check-then-insert race condition):** + +```sql +-- Race condition: two requests check simultaneously +select * from settings where user_id = 123 and key = 'theme'; +-- Both find nothing + +-- Both try to insert +insert into settings (user_id, key, value) values (123, 'theme', 'dark'); +-- One succeeds, one fails with duplicate key error! +``` + +**Correct (atomic UPSERT):** + +```sql +-- Single atomic operation +insert into settings (user_id, key, value) +values (123, 'theme', 'dark') +on conflict (user_id, key) +do update set value = excluded.value, updated_at = now(); + +-- Returns the inserted/updated row +insert into settings (user_id, key, value) +values (123, 'theme', 'dark') +on conflict (user_id, key) +do update set value = excluded.value +returning *; +``` + +Insert-or-ignore pattern: + +```sql +-- Insert only if not exists (no update) +insert into page_views (page_id, user_id) +values (1, 123) +on conflict (page_id, user_id) do nothing; +``` + +Reference: [INSERT ON CONFLICT](https://www.postgresql.org/docs/current/sql-insert.html#SQL-ON-CONFLICT) diff --git a/.agents/skills/supabase-postgres-best-practices/references/lock-advisory.md b/.agents/skills/supabase-postgres-best-practices/references/lock-advisory.md new file mode 100644 index 0000000..572eaf0 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/lock-advisory.md @@ -0,0 +1,56 @@ +--- +title: Use Advisory Locks for Application-Level Locking +impact: MEDIUM +impactDescription: Efficient coordination without row-level lock overhead +tags: advisory-locks, coordination, application-locks +--- + +## Use Advisory Locks for Application-Level Locking + +Advisory locks provide application-level coordination without requiring database rows to lock. + +**Incorrect (creating rows just for locking):** + +```sql +-- Creating dummy rows to lock on +create table resource_locks ( + resource_name text primary key +); + +insert into resource_locks values ('report_generator'); + +-- Lock by selecting the row +select * from resource_locks where resource_name = 'report_generator' for update; +``` + +**Correct (advisory locks):** + +```sql +-- Session-level advisory lock (released on disconnect or unlock) +select pg_advisory_lock(hashtext('report_generator')); +-- ... do exclusive work ... +select pg_advisory_unlock(hashtext('report_generator')); + +-- Transaction-level lock (released on commit/rollback) +begin; +select pg_advisory_xact_lock(hashtext('daily_report')); +-- ... do work ... +commit; -- Lock automatically released +``` + +Try-lock for non-blocking operations: + +```sql +-- Returns immediately with true/false instead of waiting +select pg_try_advisory_lock(hashtext('resource_name')); + +-- Use in application +if (acquired) { + -- Do work + select pg_advisory_unlock(hashtext('resource_name')); +} else { + -- Skip or retry later +} +``` + +Reference: [Advisory Locks](https://www.postgresql.org/docs/current/explicit-locking.html#ADVISORY-LOCKS) diff --git a/.agents/skills/supabase-postgres-best-practices/references/lock-deadlock-prevention.md b/.agents/skills/supabase-postgres-best-practices/references/lock-deadlock-prevention.md new file mode 100644 index 0000000..974da5e --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/lock-deadlock-prevention.md @@ -0,0 +1,68 @@ +--- +title: Prevent Deadlocks with Consistent Lock Ordering +impact: MEDIUM-HIGH +impactDescription: Eliminate deadlock errors, improve reliability +tags: deadlocks, locking, transactions, ordering +--- + +## Prevent Deadlocks with Consistent Lock Ordering + +Deadlocks occur when transactions lock resources in different orders. Always +acquire locks in a consistent order. + +**Incorrect (inconsistent lock ordering):** + +```sql +-- Transaction A -- Transaction B +begin; begin; +update accounts update accounts +set balance = balance - 100 set balance = balance - 50 +where id = 1; where id = 2; -- B locks row 2 + +update accounts update accounts +set balance = balance + 100 set balance = balance + 50 +where id = 2; -- A waits for B where id = 1; -- B waits for A + +-- DEADLOCK! Both waiting for each other +``` + +**Correct (lock rows in consistent order first):** + +```sql +-- Explicitly acquire locks in ID order before updating +begin; +select * from accounts where id in (1, 2) order by id for update; + +-- Now perform updates in any order - locks already held +update accounts set balance = balance - 100 where id = 1; +update accounts set balance = balance + 100 where id = 2; +commit; +``` + +Alternative: use a single statement to update atomically: + +```sql +-- Single statement acquires all locks atomically +begin; +update accounts +set balance = balance + case id + when 1 then -100 + when 2 then 100 +end +where id in (1, 2); +commit; +``` + +Detect deadlocks in logs: + +```sql +-- Check for recent deadlocks +select * from pg_stat_database where deadlocks > 0; + +-- Enable deadlock logging +set log_lock_waits = on; +set deadlock_timeout = '1s'; +``` + +Reference: +[Deadlocks](https://www.postgresql.org/docs/current/explicit-locking.html#LOCKING-DEADLOCKS) diff --git a/.agents/skills/supabase-postgres-best-practices/references/lock-short-transactions.md b/.agents/skills/supabase-postgres-best-practices/references/lock-short-transactions.md new file mode 100644 index 0000000..e6b8ef2 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/lock-short-transactions.md @@ -0,0 +1,50 @@ +--- +title: Keep Transactions Short to Reduce Lock Contention +impact: MEDIUM-HIGH +impactDescription: 3-5x throughput improvement, fewer deadlocks +tags: transactions, locking, contention, performance +--- + +## Keep Transactions Short to Reduce Lock Contention + +Long-running transactions hold locks that block other queries. Keep transactions as short as possible. + +**Incorrect (long transaction with external calls):** + +```sql +begin; +select * from orders where id = 1 for update; -- Lock acquired + +-- Application makes HTTP call to payment API (2-5 seconds) +-- Other queries on this row are blocked! + +update orders set status = 'paid' where id = 1; +commit; -- Lock held for entire duration +``` + +**Correct (minimal transaction scope):** + +```sql +-- Validate data and call APIs outside transaction +-- Application: response = await paymentAPI.charge(...) + +-- Only hold lock for the actual update +begin; +update orders +set status = 'paid', payment_id = $1 +where id = $2 and status = 'pending' +returning *; +commit; -- Lock held for milliseconds +``` + +Use `statement_timeout` to prevent runaway transactions: + +```sql +-- Abort queries running longer than 30 seconds +set statement_timeout = '30s'; + +-- Or per-session +set local statement_timeout = '5s'; +``` + +Reference: [Transaction Management](https://www.postgresql.org/docs/current/tutorial-transactions.html) diff --git a/.agents/skills/supabase-postgres-best-practices/references/lock-skip-locked.md b/.agents/skills/supabase-postgres-best-practices/references/lock-skip-locked.md new file mode 100644 index 0000000..77bdbb9 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/lock-skip-locked.md @@ -0,0 +1,54 @@ +--- +title: Use SKIP LOCKED for Non-Blocking Queue Processing +impact: MEDIUM-HIGH +impactDescription: 10x throughput for worker queues +tags: skip-locked, queue, workers, concurrency +--- + +## Use SKIP LOCKED for Non-Blocking Queue Processing + +When multiple workers process a queue, SKIP LOCKED allows workers to process different rows without waiting. + +**Incorrect (workers block each other):** + +```sql +-- Worker 1 and Worker 2 both try to get next job +begin; +select * from jobs where status = 'pending' order by created_at limit 1 for update; +-- Worker 2 waits for Worker 1's lock to release! +``` + +**Correct (SKIP LOCKED for parallel processing):** + +```sql +-- Each worker skips locked rows and gets the next available +begin; +select * from jobs +where status = 'pending' +order by created_at +limit 1 +for update skip locked; + +-- Worker 1 gets job 1, Worker 2 gets job 2 (no waiting) + +update jobs set status = 'processing' where id = $1; +commit; +``` + +Complete queue pattern: + +```sql +-- Atomic claim-and-update in one statement +update jobs +set status = 'processing', worker_id = $1, started_at = now() +where id = ( + select id from jobs + where status = 'pending' + order by created_at + limit 1 + for update skip locked +) +returning *; +``` + +Reference: [SELECT FOR UPDATE SKIP LOCKED](https://www.postgresql.org/docs/current/sql-select.html#SQL-FOR-UPDATE-SHARE) diff --git a/.agents/skills/supabase-postgres-best-practices/references/monitor-explain-analyze.md b/.agents/skills/supabase-postgres-best-practices/references/monitor-explain-analyze.md new file mode 100644 index 0000000..542978c --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/monitor-explain-analyze.md @@ -0,0 +1,45 @@ +--- +title: Use EXPLAIN ANALYZE to Diagnose Slow Queries +impact: LOW-MEDIUM +impactDescription: Identify exact bottlenecks in query execution +tags: explain, analyze, diagnostics, query-plan +--- + +## Use EXPLAIN ANALYZE to Diagnose Slow Queries + +EXPLAIN ANALYZE executes the query and shows actual timings, revealing the true performance bottlenecks. + +**Incorrect (guessing at performance issues):** + +```sql +-- Query is slow, but why? +select * from orders where customer_id = 123 and status = 'pending'; +-- "It must be missing an index" - but which one? +``` + +**Correct (use EXPLAIN ANALYZE):** + +```sql +explain (analyze, buffers, format text) +select * from orders where customer_id = 123 and status = 'pending'; + +-- Output reveals the issue: +-- Seq Scan on orders (cost=0.00..25000.00 rows=50 width=100) (actual time=0.015..450.123 rows=50 loops=1) +-- Filter: ((customer_id = 123) AND (status = 'pending'::text)) +-- Rows Removed by Filter: 999950 +-- Buffers: shared hit=5000 read=15000 +-- Planning Time: 0.150 ms +-- Execution Time: 450.500 ms +``` + +Key things to look for: + +```sql +-- Seq Scan on large tables = missing index +-- Rows Removed by Filter = poor selectivity or missing index +-- Buffers: read >> hit = data not cached, needs more memory +-- Nested Loop with high loops = consider different join strategy +-- Sort Method: external merge = work_mem too low +``` + +Reference: [EXPLAIN](https://supabase.com/docs/guides/database/inspect) diff --git a/.agents/skills/supabase-postgres-best-practices/references/monitor-pg-stat-statements.md b/.agents/skills/supabase-postgres-best-practices/references/monitor-pg-stat-statements.md new file mode 100644 index 0000000..d7e82f1 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/monitor-pg-stat-statements.md @@ -0,0 +1,55 @@ +--- +title: Enable pg_stat_statements for Query Analysis +impact: LOW-MEDIUM +impactDescription: Identify top resource-consuming queries +tags: pg-stat-statements, monitoring, statistics, performance +--- + +## Enable pg_stat_statements for Query Analysis + +pg_stat_statements tracks execution statistics for all queries, helping identify slow and frequent queries. + +**Incorrect (no visibility into query patterns):** + +```sql +-- Database is slow, but which queries are the problem? +-- No way to know without pg_stat_statements +``` + +**Correct (enable and query pg_stat_statements):** + +```sql +-- Enable the extension +create extension if not exists pg_stat_statements; + +-- Find slowest queries by total time +select + calls, + round(total_exec_time::numeric, 2) as total_time_ms, + round(mean_exec_time::numeric, 2) as mean_time_ms, + query +from pg_stat_statements +order by total_exec_time desc +limit 10; + +-- Find most frequent queries +select calls, query +from pg_stat_statements +order by calls desc +limit 10; + +-- Reset statistics after optimization +select pg_stat_statements_reset(); +``` + +Key metrics to monitor: + +```sql +-- Queries with high mean time (candidates for optimization) +select query, mean_exec_time, calls +from pg_stat_statements +where mean_exec_time > 100 -- > 100ms average +order by mean_exec_time desc; +``` + +Reference: [pg_stat_statements](https://supabase.com/docs/guides/database/extensions/pg_stat_statements) diff --git a/.agents/skills/supabase-postgres-best-practices/references/monitor-vacuum-analyze.md b/.agents/skills/supabase-postgres-best-practices/references/monitor-vacuum-analyze.md new file mode 100644 index 0000000..e0e8ea0 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/monitor-vacuum-analyze.md @@ -0,0 +1,55 @@ +--- +title: Maintain Table Statistics with VACUUM and ANALYZE +impact: MEDIUM +impactDescription: 2-10x better query plans with accurate statistics +tags: vacuum, analyze, statistics, maintenance, autovacuum +--- + +## Maintain Table Statistics with VACUUM and ANALYZE + +Outdated statistics cause the query planner to make poor decisions. VACUUM reclaims space, ANALYZE updates statistics. + +**Incorrect (stale statistics):** + +```sql +-- Table has 1M rows but stats say 1000 +-- Query planner chooses wrong strategy +explain select * from orders where status = 'pending'; +-- Shows: Seq Scan (because stats show small table) +-- Actually: Index Scan would be much faster +``` + +**Correct (maintain fresh statistics):** + +```sql +-- Manually analyze after large data changes +analyze orders; + +-- Analyze specific columns used in WHERE clauses +analyze orders (status, created_at); + +-- Check when tables were last analyzed +select + relname, + last_vacuum, + last_autovacuum, + last_analyze, + last_autoanalyze +from pg_stat_user_tables +order by last_analyze nulls first; +``` + +Autovacuum tuning for busy tables: + +```sql +-- Increase frequency for high-churn tables +alter table orders set ( + autovacuum_vacuum_scale_factor = 0.05, -- Vacuum at 5% dead tuples (default 20%) + autovacuum_analyze_scale_factor = 0.02 -- Analyze at 2% changes (default 10%) +); + +-- Check autovacuum status +select * from pg_stat_progress_vacuum; +``` + +Reference: [VACUUM](https://supabase.com/docs/guides/database/database-size#vacuum-operations) diff --git a/.agents/skills/supabase-postgres-best-practices/references/query-composite-indexes.md b/.agents/skills/supabase-postgres-best-practices/references/query-composite-indexes.md new file mode 100644 index 0000000..fea6452 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/query-composite-indexes.md @@ -0,0 +1,44 @@ +--- +title: Create Composite Indexes for Multi-Column Queries +impact: HIGH +impactDescription: 5-10x faster multi-column queries +tags: indexes, composite-index, multi-column, query-optimization +--- + +## Create Composite Indexes for Multi-Column Queries + +When queries filter on multiple columns, a composite index is more efficient than separate single-column indexes. + +**Incorrect (separate indexes require bitmap scan):** + +```sql +-- Two separate indexes +create index orders_status_idx on orders (status); +create index orders_created_idx on orders (created_at); + +-- Query must combine both indexes (slower) +select * from orders where status = 'pending' and created_at > '2024-01-01'; +``` + +**Correct (composite index):** + +```sql +-- Single composite index (leftmost column first for equality checks) +create index orders_status_created_idx on orders (status, created_at); + +-- Query uses one efficient index scan +select * from orders where status = 'pending' and created_at > '2024-01-01'; +``` + +**Column order matters** - place equality columns first, range columns last: + +```sql +-- Good: status (=) before created_at (>) +create index idx on orders (status, created_at); + +-- Works for: WHERE status = 'pending' +-- Works for: WHERE status = 'pending' AND created_at > '2024-01-01' +-- Does NOT work for: WHERE created_at > '2024-01-01' (leftmost prefix rule) +``` + +Reference: [Multicolumn Indexes](https://www.postgresql.org/docs/current/indexes-multicolumn.html) diff --git a/.agents/skills/supabase-postgres-best-practices/references/query-covering-indexes.md b/.agents/skills/supabase-postgres-best-practices/references/query-covering-indexes.md new file mode 100644 index 0000000..9d2a494 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/query-covering-indexes.md @@ -0,0 +1,40 @@ +--- +title: Use Covering Indexes to Avoid Table Lookups +impact: MEDIUM-HIGH +impactDescription: 2-5x faster queries by eliminating heap fetches +tags: indexes, covering-index, include, index-only-scan +--- + +## Use Covering Indexes to Avoid Table Lookups + +Covering indexes include all columns needed by a query, enabling index-only scans that skip the table entirely. + +**Incorrect (index scan + heap fetch):** + +```sql +create index users_email_idx on users (email); + +-- Must fetch name and created_at from table heap +select email, name, created_at from users where email = 'user@example.com'; +``` + +**Correct (index-only scan with INCLUDE):** + +```sql +-- Include non-searchable columns in the index +create index users_email_idx on users (email) include (name, created_at); + +-- All columns served from index, no table access needed +select email, name, created_at from users where email = 'user@example.com'; +``` + +Use INCLUDE for columns you SELECT but don't filter on: + +```sql +-- Searching by status, but also need customer_id and total +create index orders_status_idx on orders (status) include (customer_id, total); + +select status, customer_id, total from orders where status = 'shipped'; +``` + +Reference: [Index-Only Scans](https://www.postgresql.org/docs/current/indexes-index-only-scans.html) diff --git a/.agents/skills/supabase-postgres-best-practices/references/query-index-types.md b/.agents/skills/supabase-postgres-best-practices/references/query-index-types.md new file mode 100644 index 0000000..93b3259 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/query-index-types.md @@ -0,0 +1,48 @@ +--- +title: Choose the Right Index Type for Your Data +impact: HIGH +impactDescription: 10-100x improvement with correct index type +tags: indexes, btree, gin, gist, brin, hash, index-types +--- + +## Choose the Right Index Type for Your Data + +Different index types excel at different query patterns. The default B-tree isn't always optimal. + +**Incorrect (B-tree for JSONB containment):** + +```sql +-- B-tree cannot optimize containment operators +create index products_attrs_idx on products (attributes); +select * from products where attributes @> '{"color": "red"}'; +-- Full table scan - B-tree doesn't support @> operator +``` + +**Correct (GIN for JSONB):** + +```sql +-- GIN supports @>, ?, ?&, ?| operators +create index products_attrs_idx on products using gin (attributes); +select * from products where attributes @> '{"color": "red"}'; +``` + +Index type guide: + +```sql +-- B-tree (default): =, <, >, BETWEEN, IN, IS NULL +create index users_created_idx on users (created_at); + +-- GIN: arrays, JSONB, full-text search +create index posts_tags_idx on posts using gin (tags); + +-- GiST: geometric data, range types, nearest-neighbor (KNN) queries +create index locations_idx on places using gist (location); + +-- BRIN: large time-series tables (10-100x smaller) +create index events_time_idx on events using brin (created_at); + +-- Hash: equality-only (slightly faster than B-tree for =) +create index sessions_token_idx on sessions using hash (token); +``` + +Reference: [Index Types](https://www.postgresql.org/docs/current/indexes-types.html) diff --git a/.agents/skills/supabase-postgres-best-practices/references/query-missing-indexes.md b/.agents/skills/supabase-postgres-best-practices/references/query-missing-indexes.md new file mode 100644 index 0000000..e6daace --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/query-missing-indexes.md @@ -0,0 +1,43 @@ +--- +title: Add Indexes on WHERE and JOIN Columns +impact: CRITICAL +impactDescription: 100-1000x faster queries on large tables +tags: indexes, performance, sequential-scan, query-optimization +--- + +## Add Indexes on WHERE and JOIN Columns + +Queries filtering or joining on unindexed columns cause full table scans, which become exponentially slower as tables grow. + +**Incorrect (sequential scan on large table):** + +```sql +-- No index on customer_id causes full table scan +select * from orders where customer_id = 123; + +-- EXPLAIN shows: Seq Scan on orders (cost=0.00..25000.00 rows=100 width=85) +``` + +**Correct (index scan):** + +```sql +-- Create index on frequently filtered column +create index orders_customer_id_idx on orders (customer_id); + +select * from orders where customer_id = 123; + +-- EXPLAIN shows: Index Scan using orders_customer_id_idx (cost=0.42..8.44 rows=100 width=85) +``` + +For JOIN columns, always index the foreign key side: + +```sql +-- Index the referencing column +create index orders_customer_id_idx on orders (customer_id); + +select c.name, o.total +from customers c +join orders o on o.customer_id = c.id; +``` + +Reference: [Query Optimization](https://supabase.com/docs/guides/database/query-optimization) diff --git a/.agents/skills/supabase-postgres-best-practices/references/query-partial-indexes.md b/.agents/skills/supabase-postgres-best-practices/references/query-partial-indexes.md new file mode 100644 index 0000000..3e61a34 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/query-partial-indexes.md @@ -0,0 +1,45 @@ +--- +title: Use Partial Indexes for Filtered Queries +impact: HIGH +impactDescription: 5-20x smaller indexes, faster writes and queries +tags: indexes, partial-index, query-optimization, storage +--- + +## Use Partial Indexes for Filtered Queries + +Partial indexes only include rows matching a WHERE condition, making them smaller and faster when queries consistently filter on the same condition. + +**Incorrect (full index includes irrelevant rows):** + +```sql +-- Index includes all rows, even soft-deleted ones +create index users_email_idx on users (email); + +-- Query always filters active users +select * from users where email = 'user@example.com' and deleted_at is null; +``` + +**Correct (partial index matches query filter):** + +```sql +-- Index only includes active users +create index users_active_email_idx on users (email) +where deleted_at is null; + +-- Query uses the smaller, faster index +select * from users where email = 'user@example.com' and deleted_at is null; +``` + +Common use cases for partial indexes: + +```sql +-- Only pending orders (status rarely changes once completed) +create index orders_pending_idx on orders (created_at) +where status = 'pending'; + +-- Only non-null values +create index products_sku_idx on products (sku) +where sku is not null; +``` + +Reference: [Partial Indexes](https://www.postgresql.org/docs/current/indexes-partial.html) diff --git a/.agents/skills/supabase-postgres-best-practices/references/schema-constraints.md b/.agents/skills/supabase-postgres-best-practices/references/schema-constraints.md new file mode 100644 index 0000000..1d2ef8f --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/schema-constraints.md @@ -0,0 +1,80 @@ +--- +title: Add Constraints Safely in Migrations +impact: HIGH +impactDescription: Prevents migration failures and enables idempotent schema changes +tags: constraints, migrations, schema, alter-table +--- + +## Add Constraints Safely in Migrations + +PostgreSQL does not support `ADD CONSTRAINT IF NOT EXISTS`. Migrations using this syntax will fail. + +**Incorrect (causes syntax error):** + +```sql +-- ERROR: syntax error at or near "not" (SQLSTATE 42601) +alter table public.profiles +add constraint if not exists profiles_birthchart_id_unique unique (birthchart_id); +``` + +**Correct (idempotent constraint creation):** + +```sql +-- Use DO block to check before adding +do $$ +begin + if not exists ( + select 1 from pg_constraint + where conname = 'profiles_birthchart_id_unique' + and conrelid = 'public.profiles'::regclass + ) then + alter table public.profiles + add constraint profiles_birthchart_id_unique unique (birthchart_id); + end if; +end $$; +``` + +For all constraint types: + +```sql +-- Check constraints +do $$ +begin + if not exists ( + select 1 from pg_constraint + where conname = 'check_age_positive' + ) then + alter table users add constraint check_age_positive check (age > 0); + end if; +end $$; + +-- Foreign keys +do $$ +begin + if not exists ( + select 1 from pg_constraint + where conname = 'profiles_birthchart_id_fkey' + ) then + alter table profiles + add constraint profiles_birthchart_id_fkey + foreign key (birthchart_id) references birthcharts(id); + end if; +end $$; +``` + +Check if constraint exists: + +```sql +-- Query to check constraint existence +select conname, contype, pg_get_constraintdef(oid) +from pg_constraint +where conrelid = 'public.profiles'::regclass; + +-- contype values: +-- 'p' = PRIMARY KEY +-- 'f' = FOREIGN KEY +-- 'u' = UNIQUE +-- 'c' = CHECK +``` + +Reference: [Constraints](https://www.postgresql.org/docs/current/ddl-constraints.html) diff --git a/.agents/skills/supabase-postgres-best-practices/references/schema-data-types.md b/.agents/skills/supabase-postgres-best-practices/references/schema-data-types.md new file mode 100644 index 0000000..f253a58 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/schema-data-types.md @@ -0,0 +1,46 @@ +--- +title: Choose Appropriate Data Types +impact: HIGH +impactDescription: 50% storage reduction, faster comparisons +tags: data-types, schema, storage, performance +--- + +## Choose Appropriate Data Types + +Using the right data types reduces storage, improves query performance, and prevents bugs. + +**Incorrect (wrong data types):** + +```sql +create table users ( + id int, -- Will overflow at 2.1 billion + email varchar(255), -- Unnecessary length limit + created_at timestamp, -- Missing timezone info + is_active varchar(5), -- String for boolean + price varchar(20) -- String for numeric +); +``` + +**Correct (appropriate data types):** + +```sql +create table users ( + id bigint generated always as identity primary key, -- 9 quintillion max + email text, -- No artificial limit, same performance as varchar + created_at timestamptz, -- Always store timezone-aware timestamps + is_active boolean default true, -- 1 byte vs variable string length + price numeric(10,2) -- Exact decimal arithmetic +); +``` + +Key guidelines: + +```sql +-- IDs: use bigint, not int (future-proofing) +-- Strings: use text, not varchar(n) unless constraint needed +-- Time: use timestamptz, not timestamp +-- Money: use numeric, not float (precision matters) +-- Enums: use text with check constraint or create enum type +``` + +Reference: [Data Types](https://www.postgresql.org/docs/current/datatype.html) diff --git a/.agents/skills/supabase-postgres-best-practices/references/schema-foreign-key-indexes.md b/.agents/skills/supabase-postgres-best-practices/references/schema-foreign-key-indexes.md new file mode 100644 index 0000000..6c3d6ff --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/schema-foreign-key-indexes.md @@ -0,0 +1,59 @@ +--- +title: Index Foreign Key Columns +impact: HIGH +impactDescription: 10-100x faster JOINs and CASCADE operations +tags: foreign-key, indexes, joins, schema +--- + +## Index Foreign Key Columns + +Postgres does not automatically index foreign key columns. Missing indexes cause slow JOINs and CASCADE operations. + +**Incorrect (unindexed foreign key):** + +```sql +create table orders ( + id bigint generated always as identity primary key, + customer_id bigint references customers(id) on delete cascade, + total numeric(10,2) +); + +-- No index on customer_id! +-- JOINs and ON DELETE CASCADE both require full table scan +select * from orders where customer_id = 123; -- Seq Scan +delete from customers where id = 123; -- Locks table, scans all orders +``` + +**Correct (indexed foreign key):** + +```sql +create table orders ( + id bigint generated always as identity primary key, + customer_id bigint references customers(id) on delete cascade, + total numeric(10,2) +); + +-- Always index the FK column +create index orders_customer_id_idx on orders (customer_id); + +-- Now JOINs and cascades are fast +select * from orders where customer_id = 123; -- Index Scan +delete from customers where id = 123; -- Uses index, fast cascade +``` + +Find missing FK indexes: + +```sql +select + conrelid::regclass as table_name, + a.attname as fk_column +from pg_constraint c +join pg_attribute a on a.attrelid = c.conrelid and a.attnum = any(c.conkey) +where c.contype = 'f' + and not exists ( + select 1 from pg_index i + where i.indrelid = c.conrelid and a.attnum = any(i.indkey) + ); +``` + +Reference: [Foreign Keys](https://www.postgresql.org/docs/current/ddl-constraints.html#DDL-CONSTRAINTS-FK) diff --git a/.agents/skills/supabase-postgres-best-practices/references/schema-lowercase-identifiers.md b/.agents/skills/supabase-postgres-best-practices/references/schema-lowercase-identifiers.md new file mode 100644 index 0000000..f007294 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/schema-lowercase-identifiers.md @@ -0,0 +1,55 @@ +--- +title: Use Lowercase Identifiers for Compatibility +impact: MEDIUM +impactDescription: Avoid case-sensitivity bugs with tools, ORMs, and AI assistants +tags: naming, identifiers, case-sensitivity, schema, conventions +--- + +## Use Lowercase Identifiers for Compatibility + +PostgreSQL folds unquoted identifiers to lowercase. Quoted mixed-case identifiers require quotes forever and cause issues with tools, ORMs, and AI assistants that may not recognize them. + +**Incorrect (mixed-case identifiers):** + +```sql +-- Quoted identifiers preserve case but require quotes everywhere +CREATE TABLE "Users" ( + "userId" bigint PRIMARY KEY, + "firstName" text, + "lastName" text +); + +-- Must always quote or queries fail +SELECT "firstName" FROM "Users" WHERE "userId" = 1; + +-- This fails - Users becomes users without quotes +SELECT firstName FROM Users; +-- ERROR: relation "users" does not exist +``` + +**Correct (lowercase snake_case):** + +```sql +-- Unquoted lowercase identifiers are portable and tool-friendly +CREATE TABLE users ( + user_id bigint PRIMARY KEY, + first_name text, + last_name text +); + +-- Works without quotes, recognized by all tools +SELECT first_name FROM users WHERE user_id = 1; +``` + +Common sources of mixed-case identifiers: + +```sql +-- ORMs often generate quoted camelCase - configure them to use snake_case +-- Migrations from other databases may preserve original casing +-- Some GUI tools quote identifiers by default - disable this + +-- If stuck with mixed-case, create views as a compatibility layer +CREATE VIEW users AS SELECT "userId" AS user_id, "firstName" AS first_name FROM "Users"; +``` + +Reference: [Identifiers and Key Words](https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS) diff --git a/.agents/skills/supabase-postgres-best-practices/references/schema-partitioning.md b/.agents/skills/supabase-postgres-best-practices/references/schema-partitioning.md new file mode 100644 index 0000000..13137a0 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/schema-partitioning.md @@ -0,0 +1,55 @@ +--- +title: Partition Large Tables for Better Performance +impact: MEDIUM-HIGH +impactDescription: 5-20x faster queries and maintenance on large tables +tags: partitioning, large-tables, time-series, performance +--- + +## Partition Large Tables for Better Performance + +Partitioning splits a large table into smaller pieces, improving query performance and maintenance operations. + +**Incorrect (single large table):** + +```sql +create table events ( + id bigint generated always as identity, + created_at timestamptz, + data jsonb +); + +-- 500M rows, queries scan everything +select * from events where created_at > '2024-01-01'; -- Slow +vacuum events; -- Takes hours, locks table +``` + +**Correct (partitioned by time range):** + +```sql +create table events ( + id bigint generated always as identity, + created_at timestamptz not null, + data jsonb +) partition by range (created_at); + +-- Create partitions for each month +create table events_2024_01 partition of events + for values from ('2024-01-01') to ('2024-02-01'); + +create table events_2024_02 partition of events + for values from ('2024-02-01') to ('2024-03-01'); + +-- Queries only scan relevant partitions +select * from events where created_at > '2024-01-15'; -- Only scans events_2024_01+ + +-- Drop old data instantly +drop table events_2023_01; -- Instant vs DELETE taking hours +``` + +When to partition: + +- Tables > 100M rows +- Time-series data with date-based queries +- Need to efficiently drop old data + +Reference: [Table Partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html) diff --git a/.agents/skills/supabase-postgres-best-practices/references/schema-primary-keys.md b/.agents/skills/supabase-postgres-best-practices/references/schema-primary-keys.md new file mode 100644 index 0000000..fb0fbb1 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/schema-primary-keys.md @@ -0,0 +1,61 @@ +--- +title: Select Optimal Primary Key Strategy +impact: HIGH +impactDescription: Better index locality, reduced fragmentation +tags: primary-key, identity, uuid, serial, schema +--- + +## Select Optimal Primary Key Strategy + +Primary key choice affects insert performance, index size, and replication +efficiency. + +**Incorrect (problematic PK choices):** + +```sql +-- identity is the SQL-standard approach +create table users ( + id serial primary key -- Works, but IDENTITY is recommended +); + +-- Random UUIDs (v4) cause index fragmentation +create table orders ( + id uuid default gen_random_uuid() primary key -- UUIDv4 = random = scattered inserts +); +``` + +**Correct (optimal PK strategies):** + +```sql +-- Use IDENTITY for sequential IDs (SQL-standard, best for most cases) +create table users ( + id bigint generated always as identity primary key +); + +-- For distributed systems needing UUIDs, use UUIDv7 (time-ordered) +-- Requires pg_uuidv7 extension: create extension pg_uuidv7; +create table orders ( + id uuid default uuid_generate_v7() primary key -- Time-ordered, no fragmentation +); + +-- Alternative: time-prefixed IDs for sortable, distributed IDs (no extension needed) +create table events ( + id text default concat( + to_char(now() at time zone 'utc', 'YYYYMMDDHH24MISSMS'), + gen_random_uuid()::text + ) primary key +); +``` + +Guidelines: + +- Single database: `bigint identity` (sequential, 8 bytes, SQL-standard) +- Distributed/exposed IDs: UUIDv7 (requires pg_uuidv7) or ULID (time-ordered, no + fragmentation) +- `serial` works but `identity` is SQL-standard and preferred for new + applications +- Avoid random UUIDs (v4) as primary keys on large tables (causes index + fragmentation) + +Reference: +[Identity Columns](https://www.postgresql.org/docs/current/sql-createtable.html#SQL-CREATETABLE-PARMS-GENERATED-IDENTITY) diff --git a/.agents/skills/supabase-postgres-best-practices/references/security-privileges.md b/.agents/skills/supabase-postgres-best-practices/references/security-privileges.md new file mode 100644 index 0000000..448ec34 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/security-privileges.md @@ -0,0 +1,54 @@ +--- +title: Apply Principle of Least Privilege +impact: MEDIUM +impactDescription: Reduced attack surface, better audit trail +tags: privileges, security, roles, permissions +--- + +## Apply Principle of Least Privilege + +Grant only the minimum permissions required. Never use superuser for application queries. + +**Incorrect (overly broad permissions):** + +```sql +-- Application uses superuser connection +-- Or grants ALL to application role +grant all privileges on all tables in schema public to app_user; +grant all privileges on all sequences in schema public to app_user; + +-- Any SQL injection becomes catastrophic +-- drop table users; cascades to everything +``` + +**Correct (minimal, specific grants):** + +```sql +-- Create role with no default privileges +create role app_readonly nologin; + +-- Grant only SELECT on specific tables +grant usage on schema public to app_readonly; +grant select on public.products, public.categories to app_readonly; + +-- Create role for writes with limited scope +create role app_writer nologin; +grant usage on schema public to app_writer; +grant select, insert, update on public.orders to app_writer; +grant usage on sequence orders_id_seq to app_writer; +-- No DELETE permission + +-- Login role inherits from these +create role app_user login password 'xxx'; +grant app_writer to app_user; +``` + +Revoke public defaults: + +```sql +-- Revoke default public access +revoke all on schema public from public; +revoke all on all tables in schema public from public; +``` + +Reference: [Roles and Privileges](https://supabase.com/blog/postgres-roles-and-privileges) diff --git a/.agents/skills/supabase-postgres-best-practices/references/security-rls-basics.md b/.agents/skills/supabase-postgres-best-practices/references/security-rls-basics.md new file mode 100644 index 0000000..c61e1a8 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/security-rls-basics.md @@ -0,0 +1,50 @@ +--- +title: Enable Row Level Security for Multi-Tenant Data +impact: CRITICAL +impactDescription: Database-enforced tenant isolation, prevent data leaks +tags: rls, row-level-security, multi-tenant, security +--- + +## Enable Row Level Security for Multi-Tenant Data + +Row Level Security (RLS) enforces data access at the database level, ensuring users only see their own data. + +**Incorrect (application-level filtering only):** + +```sql +-- Relying only on application to filter +select * from orders where user_id = $current_user_id; + +-- Bug or bypass means all data is exposed! +select * from orders; -- Returns ALL orders +``` + +**Correct (database-enforced RLS):** + +```sql +-- Enable RLS on the table +alter table orders enable row level security; + +-- Create policy for users to see only their orders +create policy orders_user_policy on orders + for all + using (user_id = current_setting('app.current_user_id')::bigint); + +-- Force RLS even for table owners +alter table orders force row level security; + +-- Set user context and query +set app.current_user_id = '123'; +select * from orders; -- Only returns orders for user 123 +``` + +Policy for authenticated role: + +```sql +create policy orders_user_policy on orders + for all + to authenticated + using (user_id = auth.uid()); +``` + +Reference: [Row Level Security](https://supabase.com/docs/guides/database/postgres/row-level-security) diff --git a/.agents/skills/supabase-postgres-best-practices/references/security-rls-performance.md b/.agents/skills/supabase-postgres-best-practices/references/security-rls-performance.md new file mode 100644 index 0000000..c3c7c41 --- /dev/null +++ b/.agents/skills/supabase-postgres-best-practices/references/security-rls-performance.md @@ -0,0 +1,63 @@ +--- +title: Optimize RLS Policies for Performance +impact: HIGH +impactDescription: 5-10x faster RLS queries with proper patterns +tags: rls, performance, security, optimization +--- + +## Optimize RLS Policies for Performance + +Poorly written RLS policies can cause severe performance issues. Use subqueries and indexes strategically. + +**Incorrect (function called for every row):** + +```sql +create policy orders_policy on orders + using (auth.uid() = user_id); -- auth.uid() called per row! + +-- With 1M rows, auth.uid() is called 1M times +``` + +**Correct (wrap functions in SELECT):** + +```sql +create policy orders_policy on orders + using ((select auth.uid()) = user_id); -- Called once, cached + +-- 100x+ faster on large tables +``` + +Use security definer functions for complex checks: + +`SECURITY DEFINER` functions run with the creator's privileges and bypass RLS on any tables they touch — which is what makes them useful for internal lookups, but also what makes them dangerous if misused. Always include an explicit `auth.uid()` check inside the function body, keep them in a non-exposed schema, and revoke `EXECUTE` from any role that shouldn't call them directly. + +```sql +-- Create helper function in a private schema +create or replace function private.is_team_member(team_id bigint) +returns boolean +language sql +security definer +set search_path = '' +as $$ + select exists ( + select 1 from public.team_members + -- always check the calling user's identity inside the function + where team_id = $1 and user_id = (select auth.uid()) + ); +$$; + +-- Revoke direct execution from public roles +revoke execute on function private.is_team_member(bigint) from PUBLIC, anon, authenticated, service_role; + +-- Use in policy (indexed lookup, not per-row check) +create policy team_orders_policy on orders + using ((select private.is_team_member(team_id))); +``` + +Always add indexes on columns used in RLS policies: + +```sql +create index orders_user_id_idx on orders (user_id); +``` + +Reference: [RLS Performance](https://supabase.com/docs/guides/database/postgres/row-level-security#rls-performance-recommendations) diff --git a/.agents/skills/supabase/CHANGELOG.md b/.agents/skills/supabase/CHANGELOG.md new file mode 100644 index 0000000..a62015d --- /dev/null +++ b/.agents/skills/supabase/CHANGELOG.md @@ -0,0 +1,35 @@ +# Changelog + +## [0.1.4](https://github.com/supabase/agent-skills/compare/v0.1.3...v0.1.4) (2026-06-05) + + +### Features + +* add instructions to check changelog ([#74](https://github.com/supabase/agent-skills/issues/74)) ([4bb13d8](https://github.com/supabase/agent-skills/commit/4bb13d858d19f1f848505a66f46fc9603fdcde95)) +* add npm supply-chain security guidance to supabase skill ([#94](https://github.com/supabase/agent-skills/issues/94)) ([82df90a](https://github.com/supabase/agent-skills/commit/82df90a5de1cd84386d8bc192746e50343b86dc0)) +* instructions on exposing tables to the data api ([#71](https://github.com/supabase/agent-skills/issues/71)) ([f15a5a4](https://github.com/supabase/agent-skills/commit/f15a5a40779072a530c9e53c3f14ec4131118ea6)) +* using Supabase agent skills ([#12](https://github.com/supabase/agent-skills/issues/12)) ([7c2e389](https://github.com/supabase/agent-skills/commit/7c2e3894fddfde8eb6c77d2a8921904543b9be7a)) + + +### Bug Fixes + +* bump supabase skill to v0.1.1 and fix Data API broken link ([#72](https://github.com/supabase/agent-skills/issues/72)) ([5a6542e](https://github.com/supabase/agent-skills/commit/5a6542e08fc026d90c9a6a0f5a67749e9ceb9946)) +* cover SECURITY DEFINER, auth.role() deprecation, and BOLA in security checklist ([#85](https://github.com/supabase/agent-skills/issues/85)) ([133f43e](https://github.com/supabase/agent-skills/commit/133f43e8c2ffc48823ff0630c692cabecea3e3a3)) +* update Data API doc link and bump supabase skill to v0.1.1 ([#73](https://github.com/supabase/agent-skills/issues/73)) ([e5f7a7c](https://github.com/supabase/agent-skills/commit/e5f7a7cfd697765848ffd6a4505f3c02e1ee17ee)) + +## [0.1.3](https://github.com/supabase/agent-skills/compare/v0.1.2...v0.1.3) (2026-06-02) + + +### Features + +* add instructions to check changelog ([#74](https://github.com/supabase/agent-skills/issues/74)) ([4bb13d8](https://github.com/supabase/agent-skills/commit/4bb13d858d19f1f848505a66f46fc9603fdcde95)) +* add npm supply-chain security guidance to supabase skill ([#94](https://github.com/supabase/agent-skills/issues/94)) ([82df90a](https://github.com/supabase/agent-skills/commit/82df90a5de1cd84386d8bc192746e50343b86dc0)) +* instructions on exposing tables to the data api ([#71](https://github.com/supabase/agent-skills/issues/71)) ([f15a5a4](https://github.com/supabase/agent-skills/commit/f15a5a40779072a530c9e53c3f14ec4131118ea6)) +* using Supabase agent skills ([#12](https://github.com/supabase/agent-skills/issues/12)) ([7c2e389](https://github.com/supabase/agent-skills/commit/7c2e3894fddfde8eb6c77d2a8921904543b9be7a)) + + +### Bug Fixes + +* bump supabase skill to v0.1.1 and fix Data API broken link ([#72](https://github.com/supabase/agent-skills/issues/72)) ([5a6542e](https://github.com/supabase/agent-skills/commit/5a6542e08fc026d90c9a6a0f5a67749e9ceb9946)) +* cover SECURITY DEFINER, auth.role() deprecation, and BOLA in security checklist ([#85](https://github.com/supabase/agent-skills/issues/85)) ([133f43e](https://github.com/supabase/agent-skills/commit/133f43e8c2ffc48823ff0630c692cabecea3e3a3)) +* update Data API doc link and bump supabase skill to v0.1.1 ([#73](https://github.com/supabase/agent-skills/issues/73)) ([e5f7a7c](https://github.com/supabase/agent-skills/commit/e5f7a7cfd697765848ffd6a4505f3c02e1ee17ee)) diff --git a/.agents/skills/supabase/SKILL.md b/.agents/skills/supabase/SKILL.md new file mode 100644 index 0000000..198c985 --- /dev/null +++ b/.agents/skills/supabase/SKILL.md @@ -0,0 +1,135 @@ +--- +name: supabase +description: "Use when doing ANY task involving Supabase. Triggers: Supabase products (Database, Auth, Edge Functions, Realtime, Storage, Vectors, Cron, Queues); client libraries and SSR integrations (supabase-js, @supabase/ssr) in Next.js, React, SvelteKit, Astro, Remix; auth issues (login, logout, sessions, JWT, cookies, getSession, getUser, getClaims, RLS); Supabase CLI or MCP server; schema changes, migrations, security audits, Postgres extensions (pg_graphql, pg_cron, pg_vector)." +metadata: + author: supabase + version: "0.1.2" +--- + +# Supabase + +## Core Principles + +**1. Supabase changes frequently — verify against changelog and current docs before implementing.** +Do not rely on training data for Supabase features. Function signatures, config.toml settings, and API conventions change between versions. + +First, fetch `https://supabase.com/changelog.md` (a lightweight summary index — not a heavy pull), scan for `breaking-change` tags relevant to your task, and follow the linked page for any that apply. Then look up the relevant topic using the documentation access methods below. + +**2. Verify your work.** +After implementing any fix, run a test query to confirm the change works. A fix without verification is incomplete. + +**3. Recover from errors, don't loop.** +If an approach fails after 2-3 attempts, stop and reconsider. Try a different method, check documentation, inspect the error more carefully, and review relevant logs when available. Supabase issues are not always solved by retrying the same command, and the answer is not always in the logs, but logs are often worth checking before proceeding. + +**4. Exposing tables to the Data API:** Depending on the user's [Data API settings](https://supabase.com/dashboard/project//integrations/data_api/settings), newly created tables may not be automatically exposed via the Data (REST) API. If this is the case, `anon` and `authenticated` roles will need to be explicitly granted access. + +> Note that this is separate from RLS, which controls which _rows_ are visible once a table is accessible, not whether the table is accessible at all. + +When a user reports a SQL-created table is unexpectedly inaccessible, check their Data API settings and whether the roles have been granted access via explicit `GRANT` SQL. When granting public (`anon`/`authenticated`) access, always enable RLS too. See [Exposing a Table to the Data API](https://supabase.com/docs/guides/api/securing-your-api.md) for the full setup workflow. + +**5. RLS in exposed schemas.** +Enable RLS on every table in any exposed schema, which includes `public` by default. This is critical in Supabase because tables in exposed schemas can be reachable through the Data API when the `anon`/`authenticated` roles have access (see [Exposing a Table to the Data API](https://supabase.com/docs/guides/api/securing-your-api.md)). For private schemas, prefer RLS as defense in depth. After enabling RLS, create policies that match the actual access model rather than defaulting every table to the same `auth.uid()` pattern. + +**6. Security checklist.** +When working on any Supabase task that touches auth, RLS, views, storage, or user data, run through this checklist. These are Supabase-specific security traps that silently create vulnerabilities: + +- **Auth and session security** + - **Never use `user_metadata` claims in JWT-based authorization decisions.** In Supabase, `raw_user_meta_data` is user-editable and can appear in `auth.jwt()`, so it is unsafe for RLS policies or any other authorization logic. Store authorization data in `raw_app_meta_data` / `app_metadata` instead. + - **Deleting a user does not invalidate existing access tokens.** Sign out or revoke sessions first, keep JWT expiry short for sensitive apps, and for strict guarantees validate `session_id` against `auth.sessions` on sensitive operations. + - **If you use `app_metadata` or `auth.jwt()` for authorization, remember JWT claims are not always fresh until the user's token is refreshed.** + +- **API key and client exposure** + - **Never expose the `service_role` or secret key in public clients.** Prefer publishable keys for frontend code. Legacy `anon` keys are only for compatibility. In Next.js, any `NEXT_PUBLIC_` env var is sent to the browser. + +- **RLS, views, and privileged database code** + - **Views bypass RLS by default.** In Postgres 15 and above, use `CREATE VIEW ... WITH (security_invoker = true)`. In older versions of Postgres, protect your views by revoking access from the `anon` and `authenticated` roles, or by putting them in an unexposed schema. + - **UPDATE requires a SELECT policy.** In Postgres RLS, an UPDATE needs to first SELECT the row. Without a SELECT policy, updates silently return 0 rows — no error, just no change. + - **`auth.role()` is deprecated — use the `TO` clause instead.** Supabase has deprecated `auth.role()` in favour of specifying the target role directly on the policy with `TO authenticated` or `TO anon`. Beyond deprecation, `auth.role() = 'authenticated'` breaks silently when anonymous sign-ins are enabled, because anonymous users carry the `authenticated` Postgres role and pass the check regardless of whether the user is genuinely signed in. + ```sql + -- Deprecated (do not use) + create policy "example" on table_name for select + using ( auth.role() = 'authenticated' ); + ``` + - **`TO authenticated` alone is authentication without authorization (BOLA / IDOR).** Using `TO authenticated` only checks the role — it does not restrict which rows a user can access. The correct pattern combines `TO authenticated` with an ownership predicate in `USING`: + ```sql + create policy "example" on table_name for select + to authenticated + using ( (select auth.uid()) = user_id ); + ``` + - **UPDATE policies require both `USING` and `WITH CHECK`.** Without `WITH CHECK`, a user can reassign a row's `user_id` to another user: + ```sql + create policy "example" on table_name for update + to authenticated + using ( (select auth.uid()) = user_id ) + with check ( (select auth.uid()) = user_id ); + ``` + - **`SECURITY DEFINER` functions bypass RLS.** A `SECURITY DEFINER` function runs with its creator's privileges — typically a role with `bypassrls` (e.g., `postgres`). Never add `SECURITY DEFINER` to resolve a permission error; it silently removes access control without fixing the underlying cause. Prefer `SECURITY INVOKER`. + - **`SECURITY DEFINER` functions in `public` are callable by all roles.** Postgres grants `EXECUTE` to `PUBLIC` by default for every new function, so any `SECURITY DEFINER` function in `public` is a public API endpoint callable by `anon` and `authenticated` (which inherit from `PUBLIC`) without any additional grant. When `SECURITY DEFINER` is genuinely needed (e.g., bypassing RLS on an internal lookup table), keep the function in a non-exposed schema, always include an `auth.uid()` check in the function body, and run `supabase db advisors` after making changes. + +- **Storage access control** + - **Storage upsert requires INSERT + SELECT + UPDATE.** Granting only INSERT allows new uploads but file replacement (upsert) silently fails. You need all three. + +- **Dependency and supply-chain security** + - **Always pin package versions and commit lockfiles** when installing Supabase packages (`supabase-js`, `@supabase/ssr`, `supabase-py`, etc.). See the [npm security guide](https://supabase.com/docs/guides/security/npm-security.md) for the full checklist. + +For any security concern not covered above, fetch the Supabase product security index: `https://supabase.com/docs/guides/security/product-security.md` + +## Supabase CLI + +Always discover commands via `--help` — never guess. The CLI structure changes between versions. + +```bash +supabase --help # All top-level commands +supabase --help # Subcommands (e.g., supabase db --help) +supabase --help # Flags for a specific command +``` + +**Supabase CLI Known gotchas:** + +- `supabase db query` requires **CLI v2.79.0+** → use MCP `execute_sql` or `psql` as fallback +- `supabase db advisors` requires **CLI v2.81.3+** → use MCP `get_advisors` as fallback +- When you need a new migration SQL file, **always** create it with `supabase migration new ` first. Never invent a migration filename or rely on memory for the expected format. + +**Version check and upgrade:** Run `supabase --version` to check. For CLI changelogs and version-specific features, consult the [CLI documentation](https://supabase.com/docs/reference/cli/introduction) or [GitHub releases](https://github.com/supabase/cli/releases). + +## Supabase MCP Server + +For setup instructions, server URL, and configuration, see the [MCP setup guide](https://supabase.com/docs/guides/getting-started/mcp). + +**Troubleshooting connection issues** — follow these steps in order: + +1. **Check if the server is reachable:** + `curl -so /dev/null -w "%{http_code}" https://mcp.supabase.com/mcp` + A `401` is expected (no token) and means the server is up. Timeout or "connection refused" means it may be down. + +2. **Check `.mcp.json` configuration:** + Verify the project root has a valid `.mcp.json` with the correct server URL. If missing, create one pointing to `https://mcp.supabase.com/mcp`. + +3. **Authenticate the MCP server:** + If the server is reachable and `.mcp.json` is correct but tools aren't visible, the user needs to authenticate. The Supabase MCP server uses OAuth 2.1 — tell the user to trigger the auth flow in their agent, complete it in the browser, and reload the session. + +## Supabase Documentation + +Before implementing any Supabase feature, find the relevant documentation. Use these methods in priority order: + +1. **MCP `search_docs` tool** (preferred — returns relevant snippets directly) +2. **Fetch docs pages as markdown** — any docs page can be fetched by appending `.md` to the URL path. +3. **Web search** for Supabase-specific topics when you don't know which page to look at. + +## Making and Committing Schema Changes + +**To make schema changes, use `execute_sql` (MCP) or `supabase db query` (CLI).** These run SQL directly on the database without creating migration history entries, so you can iterate freely and generate a clean migration when ready. + +Do NOT use `apply_migration` to change a local database schema — it writes a migration history entry on every call, which means you can't iterate, and `supabase db diff` / `supabase db pull` will produce empty or conflicting diffs. If you use it, you'll be stuck with whatever SQL you passed on the first try. + +**When ready to commit** your changes to a migration file: + +1. **Run advisors** → `supabase db advisors` (CLI v2.81.3+) or MCP `get_advisors`. Fix any issues. +2. **Review the Security Checklist above** if your changes involve views, functions, triggers, or storage. +3. **Generate the migration** → `supabase db pull --local --yes` +4. **Verify** → `supabase migration list --local` + +## Reference Guides + +- **Skill Feedback** → [references/skill-feedback.md](references/skill-feedback.md) + **MUST read when** the user reports that this skill gave incorrect guidance or is missing information. diff --git a/.agents/skills/supabase/assets/feedback-issue-template.md b/.agents/skills/supabase/assets/feedback-issue-template.md new file mode 100644 index 0000000..e810825 --- /dev/null +++ b/.agents/skills/supabase/assets/feedback-issue-template.md @@ -0,0 +1,17 @@ +## What happened + +**Task:** + +**Skill said:** + +**Expected:** + +## Source + +**File:** + +**Section:** + +## Fix suggestion + + diff --git a/.agents/skills/supabase/references/skill-feedback.md b/.agents/skills/supabase/references/skill-feedback.md new file mode 100644 index 0000000..9c047d5 --- /dev/null +++ b/.agents/skills/supabase/references/skill-feedback.md @@ -0,0 +1,17 @@ +# Skill Feedback + +Use this when the user reports that the skill gave incorrect guidance, is missing information, or could be improved. This is about the skill (agent instructions), not about Supabase the product. + +## Steps + +1. **Ask permission** — Ask the user if they'd like to submit feedback to the skill maintainers. If they decline, move on. + +2. **Draft the issue** — Use the template at [assets/feedback-issue-template.md](../assets/feedback-issue-template.md) to structure the feedback. Fill in the fields based on the conversation. Always identify which specific reference file and section caused the problem. + +3. **Submit** — Create a GitHub Issue on the `supabase/agent-skills` repository using the draft as the issue body. The title must follow this format: `user-feedback: `. + +4. **Share the result** — Share the issue URL with the user after submission. If submission fails, give the user this link to create the issue manually: + +``` +https://github.com/supabase/agent-skills/issues/new +``` diff --git a/.gitignore b/.gitignore index 9e01f38..8231ac6 100644 --- a/.gitignore +++ b/.gitignore @@ -47,6 +47,14 @@ coverage.xml .vscode/ *.swp +# Node (root-level, e.g. from `npx skills add`) +/node_modules/ +/package.json +/package-lock.json + +# Next.js build output +.next/ + # OS .DS_Store Thumbs.db diff --git a/skills-lock.json b/skills-lock.json new file mode 100644 index 0000000..25262a7 --- /dev/null +++ b/skills-lock.json @@ -0,0 +1,17 @@ +{ + "version": 1, + "skills": { + "supabase": { + "source": "supabase/agent-skills", + "sourceType": "github", + "skillPath": "skills/supabase/SKILL.md", + "computedHash": "61638e85394d2e39d1109cbf607593afb9733e0de19cdf0b52ec9bc32d95ea74" + }, + "supabase-postgres-best-practices": { + "source": "supabase/agent-skills", + "sourceType": "github", + "skillPath": "skills/supabase-postgres-best-practices/SKILL.md", + "computedHash": "3639bed1f40b3fbadae79fee631c42c89b2d1f5c30b05d5aa3cca06422a6bbbc" + } + } +} From d6968fe3a2196bf6e62c26bc78f732731de78905 Mon Sep 17 00:00:00 2001 From: Dominik Ledzion Date: Thu, 9 Jul 2026 13:02:39 +0200 Subject: [PATCH 02/34] Add design spec: browser/LinkedIn safety, queue & rate limiting Co-Authored-By: Claude Opus 4.8 --- ...26-07-09-browser-linkedin-safety-design.md | 148 ++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-09-browser-linkedin-safety-design.md diff --git a/docs/superpowers/specs/2026-07-09-browser-linkedin-safety-design.md b/docs/superpowers/specs/2026-07-09-browser-linkedin-safety-design.md new file mode 100644 index 0000000..4b9d9b3 --- /dev/null +++ b/docs/superpowers/specs/2026-07-09-browser-linkedin-safety-design.md @@ -0,0 +1,148 @@ +# Browser / LinkedIn Web Bridge — Safety, Queue & Rate Limiting + +**Date:** 2026-07-09 +**Status:** Approved (design) +**Scope:** Harden the `browser` module (LinkedIn Sales Navigator automation via +Kimi WebBridge) against excessive activity and account-ban risk, without +removing existing models or breaking the current UI/API contract. + +## Problem + +The path `User → Django → BrowserJob → Celery → BrowserService → KimiBrowserProvider +→ LinkedIn Sales Navigator` currently has no application-level throttling: + +| Risk | Location | Current state | +|------|----------|---------------| +| A. No single-session guard | `run_browser_job.delay` + worker | The `browser` Celery queue exists but nothing enforces `concurrency=1`. A worker with `-c N` runs N browsers on one LinkedIn session → ban risk. `config/settings/base.py:252` says "needs to be serialized" but it is not implemented. | +| B. No daily search cap | `TriggerBrowserJobView` | A user can trigger "Find Leads" unlimited times. | +| C. No daily profile cap | provider | Only a per-job cap (`KIMI_WEBBRIDGE_MAX_RESULTS=100`, `MAX_PAGES=20`). | +| D. No cooldown between actions | view/task | Jobs start back-to-back; only `next_page_delay=1.0s` between pages exists. | +| E. No task timeout | Celery | No `time_limit`/`soft_time_limit`; a hung daemon blocks the worker forever. | +| F. No cancellation | everywhere | A running/pending job cannot be stopped. | + +Missing status: `CANCELLED` (absent from `core.JobStatus`). + +## What already exists (reused, not duplicated) + +- Dedicated Celery queue `browser` (`CELERY_TASK_ROUTES`) — foundation for serialization. +- `IsReady` permission — blocks non-READY users (PENDING/DISABLED). +- `core.JobBase` state machine: `status/celery_task_id/error/created_at/started_at/finished_at`. +- `core.ActivityLog` + `log_activity()` — already writes `browser_job.succeeded/failed` + with `actor/verb/target_type/target_id/metadata/created_at`. Covers ~80% of the + requested "action log" via generic columns. +- Per-job provider caps: `max_results`, `max_pages`, `next_page_delay`, `wait_timeout`. +- Failure-on-empty + exception handling in `run_browser_job`. +- Ownership scoping in `BrowserJobStatusView` (application-layer RLS re-application). + +## Design decisions (approved) + +1. **Single session:** Django-cache distributed lock in the task + belt-and-suspenders + `--concurrency=1` documented. (Not worker-only; not DB-only.) +2. **Limits storage:** fields on `UserProfile` (alongside existing `daily_sending_limit`). +3. **Action log:** reuse `core.ActivityLog` with richer verbs + metadata (no new table). +4. **Statuses:** add `CANCELLED` to shared `JobStatus`; keep `SUCCEEDED` as the + "completed" state (label-only in UI/admin). No data migration, no rename. + +## Design + +### A. Queue + statuses +- Add `CANCELLED = "cancelled", "Cancelled"` to `core.JobStatus`. Shared enum, so the + existing `browser_browserjob_status_valid` CheckConstraint (built from + `[choice.value for choice in JobStatus]`) automatically accepts it — but a new + migration is required because the constraint's rendered SQL changes. +- Every "Find Leads" remains a separate `BrowserJob` (unchanged). + +### B. Single LinkedIn session (risks A, E) +- **Lock via Django cache** `cache.add(LOCK_KEY, job.pk, timeout=LOCK_TTL)` — works with + Redis in prod and locmem in tests, no hard redis-py dependency. In `run_browser_job`: + - acquire; if taken → set job back to `PENDING`, `apply_async(countdown=REQUEUE_DELAY)`, + log `browser_job.requeued`, return. + - `try/finally` releases the lock even on exception (compare-and-delete: only the + holder releases, so a slow job that lost its lock to TTL can't delete a successor's). +- **Task timeout:** `@shared_task(bind=True, soft_time_limit=SOFT, time_limit=HARD)`. + `SoftTimeLimitExceeded` is caught → job `FAILED`, `error` recorded, lock released, + `browser_job.failed` logged. +- **Eager mode note:** in dev/tests (`CELERY_TASK_ALWAYS_EAGER=True`) tasks run inline and + sequentially, so the lock is always free when a task runs; the requeue branch is + exercised by tests that pre-seed the lock, not by normal eager execution. + +### C. Cancellation (risk F) +- Endpoint `POST /api/browser-jobs//cancel/` → `CancelBrowserJobView` + (`SupabaseJWTAuthentication`, `IsAuthenticated`, scoped by `requested_by`). + - `PENDING` → set `CANCELLED` immediately + `AsyncResult(celery_task_id).revoke()`. + - `RUNNING` → request cancellation (revoke + a cache/DB cancel flag keyed by job pk); + `run_browser_job` checks the flag between pagination pages and, if set, stops and + finishes the job as `CANCELLED`. + - `SUCCEEDED/FAILED/CANCELLED` → 409 (nothing to cancel). +- Cancellation state helpers live in `services/browser/cancellation.py` (business logic + out of the view, per CLAUDE.md). + +### D. Rate limiting (risks B, C, D) +- New `UserProfile` fields (all `PositiveIntegerField`, `0` = unlimited, matching + `daily_sending_limit`): `daily_search_limit` (default 20), `daily_profile_limit` + (default 500), `action_cooldown_seconds` (default 60). +- Logic in a new pure module `services/browser/rate_limit.py`: + `check_rate_limit(profile) -> RateLimitDecision` computing, for "today" + (`timezone.localdate`, Europe/Warsaw): + - count of this user's `BrowserJob`s created today vs `daily_search_limit`; + - seconds since the user's most recent job `started_at`/`created_at` vs `action_cooldown_seconds`; + - sum of `result_count` for today's jobs vs `daily_profile_limit`. +- **Two-layer enforcement:** + - `TriggerBrowserJobView` calls it before creating a job → on breach, **HTTP 429** with + a clear message and `Retry-After` where relevant; logs `browser_job.rate_limited`. + No `BrowserJob` row is created on a hard breach (search/profile cap); cooldown returns 429 too. + - `run_browser_job` re-checks the profile cap before the upsert loop (state may have + changed between enqueue and start); on breach it finishes the job `CANCELLED` with a + reason and logs `browser_job.rate_limited`. + +### E. Activity logging (reuse `core.ActivityLog`) +- Verbs: `browser_job.queued`, `browser_job.requeued`, `browser_job.started`, + `browser_job.succeeded`, `browser_job.failed`, `browser_job.cancelled`, + `browser_job.rate_limited`. +- `metadata` carries the LinkedIn context the requested log columns imply: + `criteria`, `result_count`, `error`, `reason`, `cooldown_remaining`. + Mapping to the requested table: user=`actor`, action=`verb`, timestamp=`created_at`, + linkedin object=`target_type`/`target_id`+`metadata`, result/error=`metadata`. + +### F. Admin dashboard +- Enhanced `BrowserJobAdmin`: add duration and queue-position display, `list_filter` by + status + created date, admin action "Cancel selected jobs", search by `requested_by`. +- New read-only staff view "LinkedIn Queue" (custom `ModelAdmin` changelist template or a + registered `AdminSite` view, `is_staff` only): active jobs (RUNNING), pending queue + (PENDING by `created_at`), recent actions (latest `ActivityLog`), and per-user limit + usage (each `UserProfile` with today's search/profile counts vs its limits). + +### G. Tests (CLAUDE.md: every new function needs a test) +- Lock acquired/blocked/requeued; lock released on success and on exception. +- Task soft-timeout → `FAILED` + lock released. +- Rate limit: search cap, profile cap, cooldown (each 429 at view; profile re-check → + `CANCELLED` at task). +- Cancel: PENDING → CANCELLED immediately; RUNNING → CANCELLED via mid-pagination flag; + already-terminal → 409; ownership (another user's job → 404). +- Migrations apply cleanly (core, users, browser). +- Activity verbs emitted for each transition. +- Admin queue view returns 200 for staff, 302/403 for non-staff. + +## Files + +**Changed:** `core/models.py`, `browser/tasks.py`, `browser/views.py`, `browser/admin.py`, +`users/models.py`, `config/settings/base.py`, `api/urls.py`. + +**New:** `services/browser/rate_limit.py`, `services/browser/cancellation.py`, +migrations (`core/`, `users/`, `browser/`), tests (`browser/`, `services/browser/`, +`users/`), docs update. + +## Out of scope (YAGNI) + +- Separate `RateLimitPolicy` model. +- Renaming `SUCCEEDED → COMPLETED` (label-only mapping instead). +- Sales Navigator URN filter mapping. +- Per-browser-click granular logging (job-level transitions are the audit unit). + +## Non-goals / constraints honored + +- Existing models are not removed; changes are additive. +- Business logic stays in `services/` (rate_limit, cancellation), not views. +- Supabase remains the sole lead store; nothing writes leads to SQLite. +- Existing UI/API contract preserved: trigger + status endpoints unchanged in shape + (429 is a new failure response; cancel is a new additive endpoint). From 76d6d045dfe8f2d2ac87eead6892d5d953f0b8d6 Mon Sep 17 00:00:00 2001 From: Dominik Ledzion Date: Thu, 9 Jul 2026 13:17:15 +0200 Subject: [PATCH 03/34] Extend browser safety spec: emergency stop, health check, action budget, retry policy, manual control Co-Authored-By: Claude Opus 4.8 --- ...26-07-09-browser-linkedin-safety-design.md | 172 ++++++++++++++---- 1 file changed, 140 insertions(+), 32 deletions(-) diff --git a/docs/superpowers/specs/2026-07-09-browser-linkedin-safety-design.md b/docs/superpowers/specs/2026-07-09-browser-linkedin-safety-design.md index 4b9d9b3..278a2fd 100644 --- a/docs/superpowers/specs/2026-07-09-browser-linkedin-safety-design.md +++ b/docs/superpowers/specs/2026-07-09-browser-linkedin-safety-design.md @@ -42,6 +42,17 @@ Missing status: `CANCELLED` (absent from `core.JobStatus`). 3. **Action log:** reuse `core.ActivityLog` with richer verbs + metadata (no new table). 4. **Statuses:** add `CANCELLED` to shared `JobStatus`; keep `SUCCEEDED` as the "completed" state (label-only in UI/admin). No data migration, no rename. +5. **Emergency stop:** DB-backed singleton toggle (`BrowserControlState`), flipped from + admin, read via cache; the worker checks it before starting and between pages. +6. **Session health:** provider detects logout / CAPTCHA / checkpoint / unexpected + redirect and raises a typed, non-retryable `BrowserSessionHealthError`. +7. **Action budget:** the rate-limit mechanism generalizes to per-action-type budgets with + per-action cost weights, superseding the standalone `daily_search_limit` / + `daily_profile_limit` fields (those become the `search` / `profile_view` action caps). +8. **Retry policy:** retry only transient failures (timeout, network, temporary); never + retry health failures (CAPTCHA / checkpoint / login) or emergency stop. +9. **Manual control:** admin sees, per user, who triggered a job, actions performed, + remaining budget, and live browser-worker status. ## Design @@ -77,60 +88,157 @@ Missing status: `CANCELLED` (absent from `core.JobStatus`). - Cancellation state helpers live in `services/browser/cancellation.py` (business logic out of the view, per CLAUDE.md). -### D. Rate limiting (risks B, C, D) -- New `UserProfile` fields (all `PositiveIntegerField`, `0` = unlimited, matching - `daily_sending_limit`): `daily_search_limit` (default 20), `daily_profile_limit` - (default 500), `action_cooldown_seconds` (default 60). -- Logic in a new pure module `services/browser/rate_limit.py`: - `check_rate_limit(profile) -> RateLimitDecision` computing, for "today" - (`timezone.localdate`, Europe/Warsaw): - - count of this user's `BrowserJob`s created today vs `daily_search_limit`; - - seconds since the user's most recent job `started_at`/`created_at` vs `action_cooldown_seconds`; - - sum of `result_count` for today's jobs vs `daily_profile_limit`. +### D. Rate limiting & Action Budget (risks B, C, D) + +Rate limiting is expressed as a **per-action-type daily budget with cost weights**, not +just a job count. This supersedes standalone `daily_search_limit` / `daily_profile_limit` +fields: `search` and `profile_view` are simply two of the tracked action types. + +**Action-type registry** (`services/browser/actions.py`): + +| action type | wired in v1 | current meaning | +|----------------|-------------|-----------------| +| `search` | yes | one `BrowserJob` run (one Sales Nav query) | +| `profile_view` | yes | one profile card extracted (`result_count` units) | +| `export` | reserved | bulk export/download of results (not performed in v1) | +| `enrichment` | yes | one `EnrichmentJob` run (Apollo/RichAPI), counted cross-app | +| `message` | reserved | outbound LinkedIn message — no such action exists yet; reserved so the budget model needn't change when/if added | + +- **Cost weights** in `settings.BROWSER_ACTION_COSTS` (overridable), e.g. + `{"search": 1, "profile_view": 1, "export": 5, "enrichment": 1, "message": 3}`. +- **`UserProfile` fields** (all `0` = unlimited, matching `daily_sending_limit`): + - `daily_action_budget` (`PositiveIntegerField`, default 500) — cap on the **weighted sum** + of today's actions (`Σ count(type) × cost(type)`); + - `action_budgets` (`JSONField`, default `{"search": 20, "profile_view": 500}`) — + optional **per-type hard caps** (absent/0 = unlimited for that type); + - `action_cooldown_seconds` (`PositiveIntegerField`, default 60) — min gap between jobs. +- **Accounting source:** today's actions are counted from `core.ActivityLog` (the same + rows written in §E), so the budget has one source of truth and no separate counters to + keep in sync. "Today" = `timezone.localdate()` (Europe/Warsaw). +- Logic in `services/browser/action_budget.py`: + `check_budget(profile, action_type, units=1) -> BudgetDecision` returning + allowed/blocked + reason + `remaining` (weighted) + `cooldown_remaining`. - **Two-layer enforcement:** - - `TriggerBrowserJobView` calls it before creating a job → on breach, **HTTP 429** with - a clear message and `Retry-After` where relevant; logs `browser_job.rate_limited`. - No `BrowserJob` row is created on a hard breach (search/profile cap); cooldown returns 429 too. - - `run_browser_job` re-checks the profile cap before the upsert loop (state may have - changed between enqueue and start); on breach it finishes the job `CANCELLED` with a - reason and logs `browser_job.rate_limited`. + - `TriggerBrowserJobView` checks `search` (+ cooldown + projected `profile_view` budget) + before creating a job → on breach, **HTTP 429** with a clear message and `Retry-After` + where relevant; logs `browser_job.rate_limited`. No `BrowserJob` row is created on a + hard breach. + - `run_browser_job` re-checks the `profile_view` budget before the upsert loop (state may + have changed between enqueue and start) and, if the remaining budget is smaller than the + result set, **truncates to the remaining budget** rather than discarding the run; if the + budget is already exhausted it finishes the job `CANCELLED` with a reason. Either way it + logs the outcome. ### E. Activity logging (reuse `core.ActivityLog`) - Verbs: `browser_job.queued`, `browser_job.requeued`, `browser_job.started`, `browser_job.succeeded`, `browser_job.failed`, `browser_job.cancelled`, - `browser_job.rate_limited`. + `browser_job.rate_limited`, `browser_job.retried`, `browser_job.health_stopped`, + `browser_job.emergency_stopped`. Per-action-type rows for budget accounting use verbs + `linkedin.search` / `linkedin.profile_view` / `linkedin.export` / `linkedin.enrichment`. - `metadata` carries the LinkedIn context the requested log columns imply: - `criteria`, `result_count`, `error`, `reason`, `cooldown_remaining`. + `criteria`, `result_count`, `error`, `reason`, `cooldown_remaining`, `action_type`, + `units`, `health_kind`, `retry_count`. Mapping to the requested table: user=`actor`, action=`verb`, timestamp=`created_at`, linkedin object=`target_type`/`target_id`+`metadata`, result/error=`metadata`. -### F. Admin dashboard +### F. Admin dashboard & manual control (point 4) - Enhanced `BrowserJobAdmin`: add duration and queue-position display, `list_filter` by status + created date, admin action "Cancel selected jobs", search by `requested_by`. -- New read-only staff view "LinkedIn Queue" (custom `ModelAdmin` changelist template or a - registered `AdminSite` view, `is_staff` only): active jobs (RUNNING), pending queue - (PENDING by `created_at`), recent actions (latest `ActivityLog`), and per-user limit - usage (each `UserProfile` with today's search/profile counts vs its limits). +- Enhanced `BrowserControlStateAdmin`: the emergency-stop toggle (see §H) with + who/when it was last changed. +- New read-only staff view "LinkedIn Queue" (custom `AdminSite` view, `is_staff` only) + showing everything point 4 asks for: + - **active jobs** (RUNNING) — with **who triggered** (`requested_by` → user), start time, + live duration; + - **pending queue** (PENDING by `created_at`); + - **per user**: **actions performed today** (by type), **remaining budget** + (weighted + per-type), cooldown state — computed via `services/browser/action_budget.py`; + - **browser-worker status**: emergency-stop on/off, whether the session lock is held and + by which job, last health-check result/heartbeat; + - **recent actions**: latest `ActivityLog` rows. + +### G. Emergency Stop (point 1) +- Singleton model `BrowserControlState` (single row, id=1) in `browser/models.py`: + `emergency_stop` (bool), `updated_by`, `updated_at`, optional `reason`. +- Toggled from Django admin (a clear on/off with confirmation). A helper + `services/browser/control.py::emergency_stop_active()` reads it through the cache + (short TTL) so the worker doesn't hit the DB on every check. +- **Worker honors it before every action:** at task start and between pagination pages, + `run_browser_job` calls `emergency_stop_active()`; if set, it stops immediately, finishes + the job `CANCELLED` with reason `emergency_stop`, releases the session lock, and logs + `browser_job.emergency_stopped`. Pending jobs that reach the worker while stop is active + are likewise short-circuited to `CANCELLED` without touching the browser. +- The trigger view also refuses new jobs (HTTP 409) while emergency stop is active. + +### H. Browser Session Health Check (point 2) +- Detection lives with the provider (it is the only layer that sees the live page) but + raises **provider-agnostic** typed exceptions from `services/browser/exceptions.py` so + the task never imports Kimi-specific types. +- After navigation and after each `_wait_for_results` / page transition, the provider + inspects the current URL + page markers and classifies: + - LinkedIn **logout / auth wall** → `SessionHealthError(kind="logged_out")` + - **CAPTCHA** challenge → `SessionHealthError(kind="captcha")` + - **checkpoint / security verification** (`/checkpoint/`, `/uas/`) → `kind="checkpoint"` + - **unexpected redirect** (URL host/path not the expected Sales Nav results) → `kind="unexpected_redirect"` +- `SessionHealthError` is **non-retryable** (see §I). On detection `run_browser_job` + stops the job → `FAILED` (or `CANCELLED` for a clean stop), records `error`/`health_kind`, + releases the lock, logs `browser_job.health_stopped`. +- **Safety escalation:** `checkpoint` and `captcha` mean LinkedIn has flagged the account, + so continuing *any* queued job is dangerous. On those two kinds the worker also engages + the global emergency stop (§G), configurable via + `settings.BROWSER_HEALTH_AUTOSTOP` (default `True`). `logged_out` / + `unexpected_redirect` stop only the current job. + +### I. Retry policy (point 5) +- Typed exception hierarchy in `services/browser/exceptions.py`: + - `RetryableBrowserError` — base for transient faults: timeouts, network/transport + errors (`httpx.TransportError`), daemon-temporarily-unavailable. `run_browser_job` + retries these with `self.retry(countdown=backoff, max_retries=BROWSER_MAX_RETRIES)` + (capped, exponential backoff), logging `browser_job.retried`. + - `SessionHealthError` and the emergency-stop path — **never retried**; the job goes + straight to `FAILED`/`CANCELLED`. + - Any other/unknown exception → `FAILED`, not retried (fail visible, don't hammer LinkedIn). +- The provider maps its low-level failures onto these types (e.g. an `httpx` timeout → + `RetryableBrowserError`; an auth wall → `SessionHealthError(kind="logged_out")`), + replacing the current bare `raise`. Retries respect the session lock and cooldown so a + retry storm cannot bypass the single-session guarantee. -### G. Tests (CLAUDE.md: every new function needs a test) +### J. Tests (CLAUDE.md: every new function needs a test) - Lock acquired/blocked/requeued; lock released on success and on exception. - Task soft-timeout → `FAILED` + lock released. -- Rate limit: search cap, profile cap, cooldown (each 429 at view; profile re-check → - `CANCELLED` at task). +- Action budget: `search` cap, `profile_view` cap, weighted `daily_action_budget`, cooldown + (429 at view); task-side `profile_view` re-check → truncation and → `CANCELLED` when exhausted. - Cancel: PENDING → CANCELLED immediately; RUNNING → CANCELLED via mid-pagination flag; already-terminal → 409; ownership (another user's job → 404). +- Emergency stop: active stop → new trigger 409; worker short-circuits pending → CANCELLED; + running job stops between pages → CANCELLED + lock released. +- Health check: each `kind` classified from a stubbed page/URL; job stops non-retryably; + `checkpoint`/`captcha` engage emergency stop when `BROWSER_HEALTH_AUTOSTOP=True`, don't when False. +- Retry policy: `RetryableBrowserError` retries up to the cap then FAILED; + `SessionHealthError` never retries; unknown exception → FAILED, no retry. - Migrations apply cleanly (core, users, browser). - Activity verbs emitted for each transition. -- Admin queue view returns 200 for staff, 302/403 for non-staff. +- Admin queue view returns 200 for staff, 302/403 for non-staff, and shows triggerer / + actions / remaining budget / worker status. ## Files -**Changed:** `core/models.py`, `browser/tasks.py`, `browser/views.py`, `browser/admin.py`, -`users/models.py`, `config/settings/base.py`, `api/urls.py`. +**Changed:** `core/models.py` (CANCELLED), `browser/models.py` (`BrowserControlState`), +`browser/tasks.py` (lock, timeout, cancel-check, emergency-stop check, budget re-check, +health handling, retry policy), `browser/views.py` (budget gate, cancel view, emergency-stop +refusal), `browser/admin.py` (dashboard + control toggle + queue view), +`users/models.py` (`daily_action_budget`, `action_budgets`, `action_cooldown_seconds`), +`config/settings/base.py` (`BROWSER_ACTION_COSTS`, lock TTL, task time limits, requeue delay, +`BROWSER_MAX_RETRIES`, `BROWSER_HEALTH_AUTOSTOP`), `api/urls.py` (cancel route), +`services/browser/providers/kimi.py` (raise typed exceptions, health detection), +`services/browser/providers/base.py` (health-check contract if needed). -**New:** `services/browser/rate_limit.py`, `services/browser/cancellation.py`, -migrations (`core/`, `users/`, `browser/`), tests (`browser/`, `services/browser/`, -`users/`), docs update. +**New:** `services/browser/actions.py` (action-type registry), `services/browser/action_budget.py`, +`services/browser/cancellation.py`, `services/browser/control.py` (emergency stop), +`services/browser/exceptions.py` (typed `RetryableBrowserError`, `SessionHealthError`), +`services/browser/health.py` (page/URL classification helpers), +migrations (`core/`, `users/`, `browser/`), tests (`browser/`, `services/browser/`, `users/`), +docs update. ## Out of scope (YAGNI) From ffba789dd01106326809f68e3464b6b25d375d7c Mon Sep 17 00:00:00 2001 From: Dominik Ledzion Date: Thu, 9 Jul 2026 13:24:45 +0200 Subject: [PATCH 04/34] Add implementation plan: browser/LinkedIn safety, queue & rate limiting Co-Authored-By: Claude Opus 4.8 --- .../2026-07-09-browser-linkedin-safety.md | 2351 +++++++++++++++++ 1 file changed, 2351 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-09-browser-linkedin-safety.md diff --git a/docs/superpowers/plans/2026-07-09-browser-linkedin-safety.md b/docs/superpowers/plans/2026-07-09-browser-linkedin-safety.md new file mode 100644 index 0000000..00f80a9 --- /dev/null +++ b/docs/superpowers/plans/2026-07-09-browser-linkedin-safety.md @@ -0,0 +1,2351 @@ +# Browser / LinkedIn Safety, Queue & Rate Limiting — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Harden the `browser` module (LinkedIn Sales Navigator automation via Kimi WebBridge) with a serialized single-session queue, per-action budgets, cancellation, emergency stop, session-health detection, and a typed retry policy — without removing existing models or breaking the current UI/API contract. + +**Architecture:** Business logic lands in new pure modules under `services/browser/` (exceptions, action registry, budgets, session lock, emergency-stop control, cancellation flags, health classification). `browser/tasks.py::run_browser_job` orchestrates them; `browser/views.py` gates triggering; `browser/admin.py` surfaces control + a queue dashboard. Statuses reuse the shared `core.JobStatus` (only `CANCELLED` is added). Cross-request state (session lock, emergency stop, cancel flags) uses Django's cache; durable state (budgets, emergency-stop record) uses the DB. + +**Tech Stack:** Django 6, Django REST Framework, Celery (eager in dev/tests via `CELERY_TASK_ALWAYS_EAGER=True`), Django cache (LocMemCache in tests, Redis in prod), pytest + Django `TestCase`. + +## Global Constraints + +- **Test settings:** `DJANGO_SETTINGS_MODULE=config.settings.dev` (see `pytest.ini`); Celery is eager, so `run_browser_job.delay(pk)` runs inline and `.get()` re-raises. +- **Test provider:** always decorate task/view tests with `@override_settings(BROWSER_AUTOMATION_PROVIDER="services.browser.providers.fixture.FixtureBrowserProvider")`. The fixture returns **2** profiles (`fixture-lead-1`, `fixture-lead-2`, company `fixturecorp.example`). +- **Cache isolation:** any test touching the lock / emergency stop / cancel flags must call `from django.core.cache import cache; cache.clear()` in `setUp` (LocMemCache persists across tests in a process). +- **Leads store:** Supabase Postgres only. Never write leads to SQLite (CLAUDE.md). +- **Business logic in `services/`**, not views (CLAUDE.md). Views call service functions. +- **Every new function needs a test** (CLAUDE.md). +- **Ownership key:** `BrowserJob.requested_by` is the Supabase Auth **UUID**, matched to `UserProfile.supabase_user_id`. `ActivityLog.actor` is the local `auth.User` (`profile.user`). +- **`log_activity` signature (existing):** `log_activity(actor, verb, target_type, target_id, metadata=None)` from `core.activity`. +- **`0` means "unlimited"** for every limit field, matching existing `UserProfile.daily_sending_limit`. +- **Commit** after every task with the shown message. + +### Canonical interfaces (defined once, referenced by all tasks) + +``` +# core/models.py +JobStatus.CANCELLED = "cancelled" + +# services/browser/exceptions.py +class BrowserAutomationError(Exception) +class RetryableBrowserError(BrowserAutomationError) # timeout / network / temporary +class SessionHealthError(BrowserAutomationError) # .kind: str +HEALTH_AUTOSTOP_KINDS: set[str] = {"captcha", "checkpoint"} + +# services/browser/actions.py +SEARCH="search"; PROFILE_VIEW="profile_view"; EXPORT="export"; ENRICHMENT="enrichment"; MESSAGE="message" +ACTION_TYPES: set[str] +DEFAULT_ACTION_COSTS: dict[str,int] +action_cost(action_type: str) -> int + +# services/browser/locks.py +acquire_session_lock(job_id: int, ttl: int | None = None) -> bool +release_session_lock(job_id: int) -> bool +session_lock_holder() -> int | None + +# services/browser/control.py +emergency_stop_active() -> bool +engage_emergency_stop(reason: str = "", updated_by=None) -> None +release_emergency_stop(updated_by=None) -> None + +# services/browser/cancellation.py +request_cancel(job_id: int) -> None +is_cancel_requested(job_id: int) -> bool +clear_cancel(job_id: int) -> None + +# services/browser/health.py +classify_page(url: str, page_markers: dict | None = None) -> str | None # returns a health kind or None + +# services/browser/action_budget.py +@dataclass BudgetDecision: allowed: bool; reason: str; remaining: int; cooldown_remaining: int +actions_today(user, action_type: str) -> int +weighted_spend_today(user) -> int +seconds_since_last_action(user) -> int | None +check_search_allowed(profile) -> BudgetDecision +remaining_profile_views(profile) -> int # -1 == unlimited +record_action(user, action_type: str, units: int = 1, target_id: int | None = None) -> None +``` + +--- + +## File Structure + +**New files** +- `services/browser/exceptions.py` — typed error hierarchy (health vs retryable). +- `services/browser/actions.py` — action-type registry + cost lookup. +- `services/browser/locks.py` — single-session cache lock. +- `services/browser/control.py` — emergency-stop read/engage/release. +- `services/browser/cancellation.py` — per-job cancel flags. +- `services/browser/health.py` — URL/page → health-kind classifier. +- `services/browser/action_budget.py` — budget accounting + decisions + `record_action`. +- Migrations: `core/migrations/000X_jobstatus_cancelled.py`, `users/migrations/000X_action_budget_fields.py`, `browser/migrations/000X_browsercontrolstate.py`. +- Tests: `services/browser/test_locks.py`, `test_control.py`, `test_cancellation.py`, `test_health.py`, `test_action_budget.py`; `browser/test_safety.py` (task hardening), `browser/test_views_safety.py` (view gates + cancel), `browser/test_admin.py` (queue view). + +**Modified files** +- `core/models.py` — add `CANCELLED`. +- `config/settings/base.py` — new constants. +- `users/models.py` — budget fields. +- `browser/models.py` — `BrowserControlState`. +- `browser/tasks.py` — orchestrate lock/stop/budget/cancel/health/retry/timeout. +- `browser/views.py` — trigger gate + `CancelBrowserJobView`. +- `browser/admin.py` — enhanced job admin, control admin, queue view. +- `api/urls.py` — cancel route. +- `services/browser/providers/kimi.py` — raise typed exceptions + health detection. +- `docs/workflows/browser.md` — ops note (`--concurrency=1`, emergency stop). + +--- + +## Task 1: Add CANCELLED to the shared job status enum + +**Files:** +- Modify: `core/models.py:5-9` +- Create: `core/migrations/000X_jobstatus_cancelled.py` (generated) +- Test: `browser/test_safety.py` + +**Interfaces:** +- Produces: `JobStatus.CANCELLED == "cancelled"`, accepted by `browser_browserjob_status_valid`. + +- [ ] **Step 1: Write the failing test** + +Create `browser/test_safety.py`: +```python +import uuid + +from django.test import TestCase + +from core.models import JobStatus +from browser.models import BrowserJob + + +class JobStatusCancelledTests(TestCase): + def test_cancelled_is_a_valid_status_value(self): + self.assertEqual(JobStatus.CANCELLED, "cancelled") + + def test_browserjob_accepts_cancelled_under_check_constraint(self): + job = BrowserJob.objects.create( + requested_by=uuid.uuid4(), status=JobStatus.CANCELLED + ) + job.refresh_from_db() + self.assertEqual(job.status, JobStatus.CANCELLED) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python -m pytest browser/test_safety.py::JobStatusCancelledTests -v` +Expected: FAIL — `AttributeError: CANCELLED` / IntegrityError on the check constraint. + +- [ ] **Step 3: Add the enum member** + +In `core/models.py`, extend `JobStatus`: +```python +class JobStatus(models.TextChoices): + PENDING = "pending", "Pending" + RUNNING = "running", "Running" + SUCCEEDED = "succeeded", "Succeeded" + FAILED = "failed", "Failed" + # Additive: a job stopped before completion — by the user (cancel), + # the administrator (emergency stop), or an exhausted budget. Shared + # by every JobBase subclass; browser is the only producer today. + CANCELLED = "cancelled", "Cancelled" +``` + +- [ ] **Step 4: Generate migrations** + +The `browser_browserjob_status_valid` CheckConstraint renders `status IN (...)` from the enum, so its SQL changes and Django will emit a migration for `browser` (and none for `core`, which has no concrete model change). Run: +```bash +python manage.py makemigrations core browser +``` +Expected: a new `browser/migrations/000X_*.py` altering the constraint. (If `core` reports "No changes", that is correct.) + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `python -m pytest browser/test_safety.py::JobStatusCancelledTests -v` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add core/models.py browser/migrations/ browser/test_safety.py +git commit -m "feat(core): add CANCELLED job status + browser constraint migration" +``` + +--- + +## Task 2: Add safety settings constants + +**Files:** +- Modify: `config/settings/base.py` (after the existing CELERY block, ~line 273) +- Test: `browser/test_safety.py` + +**Interfaces:** +- Produces settings: `BROWSER_ACTION_COSTS`, `BROWSER_SESSION_LOCK_TTL`, `BROWSER_REQUEUE_DELAY`, `BROWSER_TASK_SOFT_TIME_LIMIT`, `BROWSER_TASK_TIME_LIMIT`, `BROWSER_MAX_RETRIES`, `BROWSER_RETRY_BACKOFF`, `BROWSER_HEALTH_AUTOSTOP`, `BROWSER_CANCEL_FLAG_TTL`. + +- [ ] **Step 1: Write the failing test** + +Append to `browser/test_safety.py`: +```python +from django.conf import settings + + +class SafetySettingsTests(TestCase): + def test_action_costs_cover_every_action_type(self): + from services.browser.actions import ACTION_TYPES + self.assertTrue(ACTION_TYPES.issubset(set(settings.BROWSER_ACTION_COSTS))) + + def test_time_limits_are_ordered(self): + self.assertLess( + settings.BROWSER_TASK_SOFT_TIME_LIMIT, settings.BROWSER_TASK_TIME_LIMIT + ) + + def test_health_autostop_defaults_on(self): + self.assertTrue(settings.BROWSER_HEALTH_AUTOSTOP) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python -m pytest browser/test_safety.py::SafetySettingsTests -v` +Expected: FAIL — `AttributeError` on missing settings (and `actions` module, added in Task 6 — that import failure also counts as a fail here; it passes once both exist). + +- [ ] **Step 3: Add the constants** + +In `config/settings/base.py`, after the `CELERY_TASK_ROUTES` block: +```python +# --- Browser / LinkedIn safety -------------------------------------------- +# See docs/superpowers/specs/2026-07-09-browser-linkedin-safety-design.md. +# Every limit uses 0 = "unlimited", matching UserProfile.daily_sending_limit. + +# Weighted cost per LinkedIn action type (services/browser/actions.py). +BROWSER_ACTION_COSTS = { + 'search': int(os.environ.get('BROWSER_COST_SEARCH', '1')), + 'profile_view': int(os.environ.get('BROWSER_COST_PROFILE_VIEW', '1')), + 'export': int(os.environ.get('BROWSER_COST_EXPORT', '5')), + 'enrichment': int(os.environ.get('BROWSER_COST_ENRICHMENT', '1')), + 'message': int(os.environ.get('BROWSER_COST_MESSAGE', '3')), +} + +# Only one LinkedIn browser session may run at a time. The lock auto-expires +# after this many seconds so a crashed worker cannot wedge the queue forever; +# keep it comfortably above BROWSER_TASK_TIME_LIMIT. +BROWSER_SESSION_LOCK_TTL = int(os.environ.get('BROWSER_SESSION_LOCK_TTL', '900')) + +# When the session lock is busy, a queued job re-enqueues itself after this +# many seconds instead of running a second browser. +BROWSER_REQUEUE_DELAY = int(os.environ.get('BROWSER_REQUEUE_DELAY', '30')) + +# Celery soft/hard time limits for run_browser_job. Soft raises +# SoftTimeLimitExceeded (caught → FAILED, lock released); hard kills the worker. +BROWSER_TASK_SOFT_TIME_LIMIT = int(os.environ.get('BROWSER_TASK_SOFT_TIME_LIMIT', '600')) +BROWSER_TASK_TIME_LIMIT = int(os.environ.get('BROWSER_TASK_TIME_LIMIT', '660')) + +# Retry policy (only transient failures; never health failures). +BROWSER_MAX_RETRIES = int(os.environ.get('BROWSER_MAX_RETRIES', '3')) +BROWSER_RETRY_BACKOFF = int(os.environ.get('BROWSER_RETRY_BACKOFF', '30')) + +# On a CAPTCHA/checkpoint health failure, also engage the global emergency +# stop so queued jobs don't keep hitting a flagged account. +BROWSER_HEALTH_AUTOSTOP = os.environ.get('BROWSER_HEALTH_AUTOSTOP', '1') == '1' + +# TTL for the per-job cancel flag (cache key lifetime). +BROWSER_CANCEL_FLAG_TTL = int(os.environ.get('BROWSER_CANCEL_FLAG_TTL', '900')) +``` + +- [ ] **Step 4: Run test to verify it passes (after Task 6)** + +Run: `python -m pytest browser/test_safety.py::SafetySettingsTests -v` +Expected: PASS once `services/browser/actions.py` exists (Task 6). If running Task 2 in isolation, the two non-actions assertions pass; re-run after Task 6. + +- [ ] **Step 5: Commit** + +```bash +git add config/settings/base.py browser/test_safety.py +git commit -m "feat(config): add browser safety settings (costs, lock TTL, time limits, retry, autostop)" +``` + +--- + +## Task 3: Add action-budget fields to UserProfile + +**Files:** +- Modify: `users/models.py:123` (after `daily_sending_limit`) +- Create: `users/migrations/000X_action_budget_fields.py` (generated) +- Test: `users/test_action_budget_fields.py` + +**Interfaces:** +- Produces: `UserProfile.daily_action_budget` (int), `.action_budgets` (dict), `.action_cooldown_seconds` (int). + +- [ ] **Step 1: Write the failing test** + +Create `users/test_action_budget_fields.py`: +```python +from django.contrib.auth import get_user_model +from django.test import TestCase + +from users.models import UserProfile + + +class ActionBudgetFieldsTests(TestCase): + def test_defaults(self): + user = get_user_model().objects.create(username="u1") + profile = UserProfile.objects.create(user=user) + self.assertEqual(profile.daily_action_budget, 500) + self.assertEqual(profile.action_budgets, {"search": 20, "profile_view": 500}) + self.assertEqual(profile.action_cooldown_seconds, 60) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python -m pytest users/test_action_budget_fields.py -v` +Expected: FAIL — attributes do not exist. + +- [ ] **Step 3: Add the fields** + +In `users/models.py`, immediately after the `daily_sending_limit` field: +```python + # --- LinkedIn action budget (docs/.../browser-linkedin-safety-design.md) --- + # Cap on the weighted sum of today's LinkedIn actions (Σ count × cost). + # 0 = unlimited, matching daily_sending_limit. + daily_action_budget = models.PositiveIntegerField(default=500) + # Optional per-action-type hard caps; absent/0 = unlimited for that type. + action_budgets = models.JSONField( + default=_default_action_budgets, + blank=True, + help_text="Per-action-type daily caps, e.g. {'search': 20, 'profile_view': 500}.", + ) + # Minimum seconds between two searches by this user. 0 = no cooldown. + action_cooldown_seconds = models.PositiveIntegerField(default=60) +``` + +And add this module-level callable **above** the `UserProfile` class (JSONField defaults must be a named callable, not a lambda, so migrations can serialize it): +```python +def _default_action_budgets(): + """Default per-action-type daily caps for a new UserProfile.""" + return {"search": 20, "profile_view": 500} +``` + +- [ ] **Step 4: Generate + run migration** + +```bash +python manage.py makemigrations users +``` +Expected: `users/migrations/000X_action_budget_fields.py` adding three fields. + +- [ ] **Step 5: Run test to verify it passes** + +Run: `python -m pytest users/test_action_budget_fields.py -v` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add users/models.py users/migrations/ users/test_action_budget_fields.py +git commit -m "feat(users): add LinkedIn action-budget fields to UserProfile" +``` + +--- + +## Task 4: BrowserControlState model (emergency-stop storage) + +**Files:** +- Modify: `browser/models.py` (append) +- Create: `browser/migrations/000X_browsercontrolstate.py` (generated) +- Test: `browser/test_safety.py` + +**Interfaces:** +- Produces: `BrowserControlState` singleton with `.load()` classmethod, fields `emergency_stop`, `reason`, `updated_by`, `updated_at`. + +- [ ] **Step 1: Write the failing test** + +Append to `browser/test_safety.py`: +```python +from browser.models import BrowserControlState + + +class BrowserControlStateTests(TestCase): + def test_load_returns_singleton_row(self): + a = BrowserControlState.load() + b = BrowserControlState.load() + self.assertEqual(a.pk, 1) + self.assertEqual(a.pk, b.pk) + self.assertFalse(a.emergency_stop) + + def test_save_always_pins_pk_1(self): + state = BrowserControlState.load() + state.pk = 5 + state.emergency_stop = True + state.save() + self.assertEqual(BrowserControlState.objects.count(), 1) + self.assertTrue(BrowserControlState.load().emergency_stop) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python -m pytest browser/test_safety.py::BrowserControlStateTests -v` +Expected: FAIL — `ImportError: cannot import name 'BrowserControlState'`. + +- [ ] **Step 3: Add the model** + +Append to `browser/models.py`: +```python +class BrowserControlState(models.Model): + """Singleton (pk is always 1) holding the global browser-automation + kill switch. The administrator flips ``emergency_stop`` from Django + admin; ``run_browser_job`` checks it (through the cache) before every + LinkedIn action and short-circuits running/pending jobs to CANCELLED. + See docs/superpowers/specs/2026-07-09-browser-linkedin-safety-design.md.""" + + emergency_stop = models.BooleanField( + default=False, + help_text="When on, no LinkedIn browser job may run; active jobs stop.", + ) + reason = models.CharField(max_length=255, blank=True) + updated_by = models.CharField(max_length=255, blank=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + verbose_name = "browser control state" + verbose_name_plural = "browser control state" + + def save(self, *args, **kwargs): + # Force a single row: there is exactly one global switch. + self.pk = 1 + super().save(*args, **kwargs) + + @classmethod + def load(cls): + obj, _ = cls.objects.get_or_create(pk=1) + return obj + + def __str__(self): + return f"BrowserControlState(emergency_stop={self.emergency_stop})" +``` + +- [ ] **Step 4: Generate migration** + +```bash +python manage.py makemigrations browser +``` +Expected: a migration creating `BrowserControlState`. + +- [ ] **Step 5: Run test to verify it passes** + +Run: `python -m pytest browser/test_safety.py::BrowserControlStateTests -v` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add browser/models.py browser/migrations/ browser/test_safety.py +git commit -m "feat(browser): add BrowserControlState singleton for emergency stop" +``` + +--- + +## Task 5: Typed exception hierarchy + +**Files:** +- Create: `services/browser/exceptions.py` +- Test: `services/browser/test_exceptions.py` + +**Interfaces:** +- Produces: `BrowserAutomationError`, `RetryableBrowserError`, `SessionHealthError(kind)`, `HEALTH_AUTOSTOP_KINDS`. + +- [ ] **Step 1: Write the failing test** + +Create `services/browser/test_exceptions.py`: +```python +from django.test import TestCase + +from services.browser.exceptions import ( + BrowserAutomationError, + RetryableBrowserError, + SessionHealthError, + HEALTH_AUTOSTOP_KINDS, +) + + +class ExceptionHierarchyTests(TestCase): + def test_retryable_is_a_browser_error(self): + self.assertTrue(issubclass(RetryableBrowserError, BrowserAutomationError)) + + def test_session_health_carries_kind(self): + exc = SessionHealthError("captcha") + self.assertEqual(exc.kind, "captcha") + self.assertIn("captcha", str(exc)) + + def test_health_is_not_retryable(self): + self.assertFalse(issubclass(SessionHealthError, RetryableBrowserError)) + + def test_autostop_kinds(self): + self.assertEqual(HEALTH_AUTOSTOP_KINDS, {"captcha", "checkpoint"}) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python -m pytest services/browser/test_exceptions.py -v` +Expected: FAIL — module does not exist. + +- [ ] **Step 3: Write the module** + +Create `services/browser/exceptions.py`: +```python +"""Typed browser-automation errors shared by the task layer and providers. + +Providers raise these (never provider-specific types), so ``browser/tasks.py`` +can decide retry-vs-fail from the *type*, not a string match. See +docs/superpowers/specs/2026-07-09-browser-linkedin-safety-design.md §H/§I. +""" + + +class BrowserAutomationError(Exception): + """Base for any browser-automation failure.""" + + +class RetryableBrowserError(BrowserAutomationError): + """A transient fault worth retrying: request timeout, network/transport + error, or a temporarily-unavailable daemon. NEVER used for health + problems (those must not be retried against a flagged account).""" + + +class SessionHealthError(BrowserAutomationError): + """The LinkedIn session is unhealthy and the job must stop immediately + without retry. ``kind`` is one of: 'logged_out', 'captcha', + 'checkpoint', 'unexpected_redirect'.""" + + def __init__(self, kind: str, message: str | None = None): + self.kind = kind + super().__init__(message or f"LinkedIn session health check failed: {kind}") + + +# Kinds severe enough that the whole queue should stop, not just this job: +# LinkedIn has actively challenged the account. +HEALTH_AUTOSTOP_KINDS = {"captcha", "checkpoint"} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `python -m pytest services/browser/test_exceptions.py -v` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add services/browser/exceptions.py services/browser/test_exceptions.py +git commit -m "feat(services/browser): typed exception hierarchy (retryable vs health)" +``` + +--- + +## Task 6: Action-type registry + cost lookup + +**Files:** +- Create: `services/browser/actions.py` +- Test: `services/browser/test_actions.py` + +**Interfaces:** +- Produces: `SEARCH/PROFILE_VIEW/EXPORT/ENRICHMENT/MESSAGE`, `ACTION_TYPES`, `DEFAULT_ACTION_COSTS`, `action_cost(action_type)`. + +- [ ] **Step 1: Write the failing test** + +Create `services/browser/test_actions.py`: +```python +from django.test import TestCase, override_settings + +from services.browser import actions + + +class ActionRegistryTests(TestCase): + def test_action_types_set(self): + self.assertEqual( + actions.ACTION_TYPES, + {"search", "profile_view", "export", "enrichment", "message"}, + ) + + def test_cost_reads_settings_first(self): + with override_settings(BROWSER_ACTION_COSTS={"search": 7}): + self.assertEqual(actions.action_cost("search"), 7) + + def test_cost_falls_back_to_default_then_one(self): + with override_settings(BROWSER_ACTION_COSTS={}): + self.assertEqual(actions.action_cost("export"), 5) # DEFAULT_ACTION_COSTS + self.assertEqual(actions.action_cost("unknown"), 1) # ultimate fallback +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python -m pytest services/browser/test_actions.py -v` +Expected: FAIL — module does not exist. + +- [ ] **Step 3: Write the module** + +Create `services/browser/actions.py`: +```python +"""Registry of LinkedIn action types and their budget cost weights. + +Cost weights are configurable via ``settings.BROWSER_ACTION_COSTS``; this +module holds the canonical type names and safe defaults. ``search`` and +``profile_view`` are wired in v1; ``export``/``message`` are reserved so the +budget model needn't change when they are added. See the safety design spec. +""" + +from django.conf import settings + +SEARCH = "search" +PROFILE_VIEW = "profile_view" +EXPORT = "export" +ENRICHMENT = "enrichment" +MESSAGE = "message" + +ACTION_TYPES = {SEARCH, PROFILE_VIEW, EXPORT, ENRICHMENT, MESSAGE} + +DEFAULT_ACTION_COSTS = { + SEARCH: 1, + PROFILE_VIEW: 1, + EXPORT: 5, + ENRICHMENT: 1, + MESSAGE: 3, +} + + +def action_cost(action_type: str) -> int: + """Cost weight for an action type: settings override → module default → 1.""" + configured = getattr(settings, "BROWSER_ACTION_COSTS", {}) + if action_type in configured: + return int(configured[action_type]) + return int(DEFAULT_ACTION_COSTS.get(action_type, 1)) +``` + +- [ ] **Step 4: Run tests to verify they pass (and re-run Task 2's)** + +Run: `python -m pytest services/browser/test_actions.py browser/test_safety.py::SafetySettingsTests -v` +Expected: PASS (Task 2's settings test now passes too). + +- [ ] **Step 5: Commit** + +```bash +git add services/browser/actions.py services/browser/test_actions.py +git commit -m "feat(services/browser): action-type registry + cost lookup" +``` + +--- + +## Task 7: Single-session cache lock + +**Files:** +- Create: `services/browser/locks.py` +- Test: `services/browser/test_locks.py` + +**Interfaces:** +- Produces: `acquire_session_lock(job_id, ttl=None)`, `release_session_lock(job_id)`, `session_lock_holder()`. + +- [ ] **Step 1: Write the failing test** + +Create `services/browser/test_locks.py`: +```python +from django.core.cache import cache +from django.test import TestCase + +from services.browser import locks + + +class SessionLockTests(TestCase): + def setUp(self): + cache.clear() + + def test_first_acquire_succeeds_second_blocks(self): + self.assertTrue(locks.acquire_session_lock(1)) + self.assertFalse(locks.acquire_session_lock(2)) + self.assertEqual(locks.session_lock_holder(), 1) + + def test_release_only_by_holder(self): + locks.acquire_session_lock(1) + self.assertFalse(locks.release_session_lock(2)) # non-holder can't release + self.assertTrue(locks.release_session_lock(1)) # holder releases + self.assertIsNone(locks.session_lock_holder()) + self.assertTrue(locks.acquire_session_lock(2)) # now free +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python -m pytest services/browser/test_locks.py -v` +Expected: FAIL — module does not exist. + +- [ ] **Step 3: Write the module** + +Create `services/browser/locks.py`: +```python +"""Global single-session lock for LinkedIn automation. + +At most one BrowserJob may drive a browser at a time (docs/.../safety spec §B). +Backed by Django's cache: ``cache.add`` is atomic (set-if-absent), so it is a +correct mutual-exclusion primitive on both LocMemCache (tests) and Redis (prod). +Release is compare-and-delete: only the holder may release, so a job that lost +the lock to TTL expiry cannot delete a successor's lock. +""" + +from django.conf import settings +from django.core.cache import cache + +SESSION_LOCK_KEY = "browser:linkedin_session_lock" + + +def acquire_session_lock(job_id: int, ttl: int | None = None) -> bool: + """Try to acquire the single-session lock for ``job_id``. Returns True if + acquired, False if another job holds it.""" + if ttl is None: + ttl = getattr(settings, "BROWSER_SESSION_LOCK_TTL", 900) + return bool(cache.add(SESSION_LOCK_KEY, job_id, ttl)) + + +def release_session_lock(job_id: int) -> bool: + """Release the lock only if ``job_id`` currently holds it. Returns True if + a release happened, False otherwise (not the holder / already free).""" + if cache.get(SESSION_LOCK_KEY) == job_id: + cache.delete(SESSION_LOCK_KEY) + return True + return False + + +def session_lock_holder() -> int | None: + """The job_id currently holding the lock, or None if free.""" + return cache.get(SESSION_LOCK_KEY) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `python -m pytest services/browser/test_locks.py -v` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add services/browser/locks.py services/browser/test_locks.py +git commit -m "feat(services/browser): single-session cache lock" +``` + +--- + +## Task 8: Emergency-stop control + +**Files:** +- Create: `services/browser/control.py` +- Test: `services/browser/test_control.py` + +**Interfaces:** +- Produces: `emergency_stop_active()`, `engage_emergency_stop(reason, updated_by)`, `release_emergency_stop(updated_by)`. + +- [ ] **Step 1: Write the failing test** + +Create `services/browser/test_control.py`: +```python +from django.core.cache import cache +from django.test import TestCase + +from browser.models import BrowserControlState +from services.browser import control + + +class EmergencyStopTests(TestCase): + def setUp(self): + cache.clear() + + def test_inactive_by_default(self): + self.assertFalse(control.emergency_stop_active()) + + def test_engage_then_active_and_persisted(self): + control.engage_emergency_stop(reason="captcha", updated_by="worker") + self.assertTrue(control.emergency_stop_active()) + state = BrowserControlState.load() + self.assertTrue(state.emergency_stop) + self.assertEqual(state.reason, "captcha") + + def test_release_clears_it(self): + control.engage_emergency_stop(reason="x") + control.release_emergency_stop(updated_by="admin") + self.assertFalse(control.emergency_stop_active()) + self.assertFalse(BrowserControlState.load().emergency_stop) + + def test_reads_db_when_cache_cold(self): + state = BrowserControlState.load() + state.emergency_stop = True + state.save() + cache.clear() # simulate a fresh worker with an empty cache + self.assertTrue(control.emergency_stop_active()) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python -m pytest services/browser/test_control.py -v` +Expected: FAIL — module does not exist. + +- [ ] **Step 3: Write the module** + +Create `services/browser/control.py`: +```python +"""Global emergency stop for LinkedIn automation (docs/.../safety spec §G). + +Durable state lives in ``browser.BrowserControlState`` (admin-editable); a +short-lived cache mirror keeps the hot per-action check off the database. The +worker reads ``emergency_stop_active()`` before every LinkedIn action; the +provider/task engages it on a CAPTCHA/checkpoint; the administrator releases it. +""" + +from django.core.cache import cache + +_CACHE_KEY = "browser:emergency_stop" +_CACHE_TTL = 10 # seconds; short so an admin release is honored quickly + + +def emergency_stop_active() -> bool: + """True if the global kill switch is on. Cache-first, DB fallback.""" + cached = cache.get(_CACHE_KEY) + if cached is not None: + return bool(cached) + from browser.models import BrowserControlState # local: app-registry safety + active = BrowserControlState.load().emergency_stop + cache.set(_CACHE_KEY, active, _CACHE_TTL) + return active + + +def engage_emergency_stop(reason: str = "", updated_by=None) -> None: + """Turn the kill switch on (DB + cache).""" + from browser.models import BrowserControlState + state = BrowserControlState.load() + state.emergency_stop = True + state.reason = reason or state.reason + if updated_by is not None: + state.updated_by = str(updated_by) + state.save() + cache.set(_CACHE_KEY, True, _CACHE_TTL) + + +def release_emergency_stop(updated_by=None) -> None: + """Turn the kill switch off (DB + cache).""" + from browser.models import BrowserControlState + state = BrowserControlState.load() + state.emergency_stop = False + state.reason = "" + if updated_by is not None: + state.updated_by = str(updated_by) + state.save() + cache.set(_CACHE_KEY, False, _CACHE_TTL) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `python -m pytest services/browser/test_control.py -v` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add services/browser/control.py services/browser/test_control.py +git commit -m "feat(services/browser): global emergency-stop control" +``` + +--- + +## Task 9: Per-job cancel flags + +**Files:** +- Create: `services/browser/cancellation.py` +- Test: `services/browser/test_cancellation.py` + +**Interfaces:** +- Produces: `request_cancel(job_id)`, `is_cancel_requested(job_id)`, `clear_cancel(job_id)`. + +- [ ] **Step 1: Write the failing test** + +Create `services/browser/test_cancellation.py`: +```python +from django.core.cache import cache +from django.test import TestCase + +from services.browser import cancellation + + +class CancellationFlagTests(TestCase): + def setUp(self): + cache.clear() + + def test_request_then_detected(self): + self.assertFalse(cancellation.is_cancel_requested(7)) + cancellation.request_cancel(7) + self.assertTrue(cancellation.is_cancel_requested(7)) + + def test_clear(self): + cancellation.request_cancel(7) + cancellation.clear_cancel(7) + self.assertFalse(cancellation.is_cancel_requested(7)) + + def test_flags_are_per_job(self): + cancellation.request_cancel(7) + self.assertFalse(cancellation.is_cancel_requested(8)) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python -m pytest services/browser/test_cancellation.py -v` +Expected: FAIL — module does not exist. + +- [ ] **Step 3: Write the module** + +Create `services/browser/cancellation.py`: +```python +"""Cooperative cancellation flags for running BrowserJobs (docs/.../spec §C). + +A cancel request from the API sets a per-job cache flag; ``run_browser_job`` +polls it between pagination pages and stops cleanly as CANCELLED. Cache-based +so the flag crosses the process boundary between the web request and the worker. +""" + +from django.conf import settings +from django.core.cache import cache + + +def _key(job_id: int) -> str: + return f"browser:cancel:{job_id}" + + +def request_cancel(job_id: int) -> None: + ttl = getattr(settings, "BROWSER_CANCEL_FLAG_TTL", 900) + cache.set(_key(job_id), True, ttl) + + +def is_cancel_requested(job_id: int) -> bool: + return bool(cache.get(_key(job_id))) + + +def clear_cancel(job_id: int) -> None: + cache.delete(_key(job_id)) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `python -m pytest services/browser/test_cancellation.py -v` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add services/browser/cancellation.py services/browser/test_cancellation.py +git commit -m "feat(services/browser): per-job cancellation flags" +``` + +--- + +## Task 10: Session-health classifier + +**Files:** +- Create: `services/browser/health.py` +- Test: `services/browser/test_health.py` + +**Interfaces:** +- Produces: `classify_page(url, page_markers=None) -> str | None` returning one of `logged_out`/`captcha`/`checkpoint`/`unexpected_redirect`/`None`. + +- [ ] **Step 1: Write the failing test** + +Create `services/browser/test_health.py`: +```python +from django.test import TestCase + +from services.browser import health + + +class ClassifyPageTests(TestCase): + def test_healthy_sales_nav_results(self): + url = "https://www.linkedin.com/sales/search/people?query=x" + self.assertIsNone(health.classify_page(url)) + + def test_login_redirect_is_logged_out(self): + self.assertEqual( + health.classify_page("https://www.linkedin.com/login"), "logged_out" + ) + self.assertEqual( + health.classify_page("https://www.linkedin.com/uas/login"), "logged_out" + ) + + def test_checkpoint_url(self): + self.assertEqual( + health.classify_page("https://www.linkedin.com/checkpoint/challenge"), + "checkpoint", + ) + + def test_captcha_marker(self): + url = "https://www.linkedin.com/sales/search/people" + self.assertEqual( + health.classify_page(url, {"has_captcha": True}), "captcha" + ) + + def test_offsite_redirect_is_unexpected(self): + self.assertEqual( + health.classify_page("https://example.com/anything"), + "unexpected_redirect", + ) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python -m pytest services/browser/test_health.py -v` +Expected: FAIL — module does not exist. + +- [ ] **Step 3: Write the module** + +Create `services/browser/health.py`: +```python +"""Classify the current browser page into a LinkedIn session-health kind. + +Pure URL/marker inspection so it is trivially unit-testable and provider- +agnostic. The provider gathers ``page_markers`` from the live DOM (e.g. a +CAPTCHA iframe present) and passes them in; the task turns a non-None result +into a non-retryable SessionHealthError. See docs/.../safety spec §H. +""" + +from urllib.parse import urlparse + +_EXPECTED_HOSTS = {"www.linkedin.com", "linkedin.com"} + + +def classify_page(url: str, page_markers: dict | None = None) -> str | None: + """Return a health kind, or None if the page looks like a healthy + LinkedIn/Sales Navigator page. + + Precedence: explicit challenge markers first (captcha), then URL-path + signals (checkpoint, login), then off-site host (unexpected_redirect). + """ + markers = page_markers or {} + parsed = urlparse(url or "") + host = (parsed.hostname or "").lower() + path = (parsed.path or "").lower() + + # CAPTCHA can appear on an otherwise-expected URL, so check it first. + if markers.get("has_captcha"): + return "captcha" + + if host in _EXPECTED_HOSTS: + if path.startswith("/checkpoint"): + return "checkpoint" + if path.startswith("/login") or path.startswith("/uas"): + return "logged_out" + return None + + # Empty/relative URL: can't judge — treat as healthy (caller decides). + if not host: + return None + + # Any other host means we were redirected off LinkedIn unexpectedly. + return "unexpected_redirect" +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `python -m pytest services/browser/test_health.py -v` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add services/browser/health.py services/browser/test_health.py +git commit -m "feat(services/browser): session-health page classifier" +``` + +--- + +## Task 11: Action-budget accounting + decisions + +**Files:** +- Create: `services/browser/action_budget.py` +- Test: `services/browser/test_action_budget.py` + +**Interfaces:** +- Consumes: `services.browser.actions.action_cost`; `core.ActivityLog`; `browser.BrowserJob`. +- Produces: `BudgetDecision`, `actions_today(user, action_type)`, `weighted_spend_today(user)`, `seconds_since_last_action(user)`, `check_search_allowed(profile)`, `remaining_profile_views(profile)`, `record_action(user, action_type, units=1, target_id=None)`. + +**Accounting model:** every LinkedIn action writes one `core.ActivityLog` row with `verb=f"linkedin.{action_type}"` and `metadata={"units": n}`. Budgets read those rows for "today" (`timezone.localdate()`, Europe/Warsaw). `record_action` is the single writer. + +- [ ] **Step 1: Write the failing test** + +Create `services/browser/test_action_budget.py`: +```python +from django.contrib.auth import get_user_model +from django.test import TestCase, override_settings +from django.utils import timezone + +from core.models import ActivityLog +from services.browser import action_budget as ab +from users.models import UserProfile + + +class ActionBudgetTests(TestCase): + def _profile(self, **kw): + user = get_user_model().objects.create(username=f"u{UserProfile.objects.count()}") + defaults = dict( + user=user, daily_action_budget=0, + action_budgets={}, action_cooldown_seconds=0, + ) + defaults.update(kw) + return UserProfile.objects.create(**defaults) + + def test_record_and_count(self): + p = self._profile() + ab.record_action(p.user, "profile_view", units=5) + self.assertEqual(ab.actions_today(p.user, "profile_view"), 5) + self.assertEqual(ab.actions_today(p.user, "search"), 0) + + def test_weighted_spend(self): + p = self._profile() + ab.record_action(p.user, "search", units=2) # cost 1 → 2 + ab.record_action(p.user, "profile_view", units=3) # cost 1 → 3 + self.assertEqual(ab.weighted_spend_today(p.user), 5) + + def test_search_blocked_by_type_cap(self): + p = self._profile(action_budgets={"search": 1}) + ab.record_action(p.user, "search", units=1) + decision = ab.check_search_allowed(p) + self.assertFalse(decision.allowed) + self.assertIn("search", decision.reason) + + def test_search_blocked_by_weighted_budget(self): + p = self._profile(daily_action_budget=2) + ab.record_action(p.user, "profile_view", units=2) + self.assertFalse(ab.check_search_allowed(p).allowed) + + def test_cooldown_blocks_then_allows(self): + p = self._profile(action_cooldown_seconds=300) + ab.record_action(p.user, "search", units=1) + d = ab.check_search_allowed(p) + self.assertFalse(d.allowed) + self.assertGreater(d.cooldown_remaining, 0) + + def test_unlimited_when_zero(self): + p = self._profile() # all zeros + for _ in range(50): + ab.record_action(p.user, "search", units=1) + self.assertTrue(ab.check_search_allowed(p).allowed) + + def test_remaining_profile_views(self): + p = self._profile(action_budgets={"profile_view": 10}, daily_action_budget=0) + ab.record_action(p.user, "profile_view", units=4) + self.assertEqual(ab.remaining_profile_views(p), 6) + + def test_remaining_profile_views_unlimited(self): + p = self._profile() # all zeros + self.assertEqual(ab.remaining_profile_views(p), -1) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python -m pytest services/browser/test_action_budget.py -v` +Expected: FAIL — module does not exist. + +- [ ] **Step 3: Write the module** + +Create `services/browser/action_budget.py`: +```python +"""Per-action-type daily budget accounting and decisions (docs/.../spec §D). + +The ActivityLog is the single ledger: ``record_action`` writes one row per +LinkedIn action (verb ``linkedin.``, ``metadata['units']`` = count), and +every query below reads today's rows for a user. All limits use 0 = unlimited. +""" + +from dataclasses import dataclass, field + +from django.utils import timezone + +from core.activity import log_activity +from core.models import ActivityLog +from services.browser.actions import action_cost, ACTION_TYPES + +_VERB_PREFIX = "linkedin." + + +@dataclass +class BudgetDecision: + allowed: bool + reason: str = "" + remaining: int = 0 # weighted units left today; -1 = unlimited + cooldown_remaining: int = 0 # seconds until cooldown clears + + +def _today_start(): + """Aware datetime for the start of 'today' in the project timezone.""" + today = timezone.localdate() + return timezone.make_aware( + timezone.datetime(today.year, today.month, today.day) + ) + + +def _today_rows(user): + return ActivityLog.objects.filter( + actor=user, verb__startswith=_VERB_PREFIX, created_at__gte=_today_start() + ) + + +def record_action(user, action_type: str, units: int = 1, target_id=None) -> None: + """Write one ledger row for a LinkedIn action (the only budget writer).""" + log_activity( + actor=user, + verb=f"{_VERB_PREFIX}{action_type}", + target_type="BrowserJob", + target_id=target_id, + metadata={"units": int(units), "action_type": action_type}, + ) + + +def actions_today(user, action_type: str) -> int: + """Total units of one action type performed by ``user`` today.""" + total = 0 + for row in _today_rows(user).filter(verb=f"{_VERB_PREFIX}{action_type}"): + total += int(row.metadata.get("units", 1)) + return total + + +def weighted_spend_today(user) -> int: + """Σ units × cost across all action types today.""" + spend = 0 + for action_type in ACTION_TYPES: + spend += actions_today(user, action_type) * action_cost(action_type) + return spend + + +def seconds_since_last_action(user): + """Seconds since the user's most recent LinkedIn action, or None.""" + last = _today_rows(user).order_by("-created_at").first() + if last is None: + return None + return int((timezone.now() - last.created_at).total_seconds()) + + +def _weighted_remaining(profile) -> int: + budget = profile.daily_action_budget + if not budget: # 0 = unlimited + return -1 + return max(budget - weighted_spend_today(profile.user), 0) + + +def check_search_allowed(profile) -> BudgetDecision: + """Decide whether ``profile`` may start one more search right now.""" + # 1. Cooldown between searches. + cooldown = profile.action_cooldown_seconds + if cooldown: + since = seconds_since_last_action(profile.user) + if since is not None and since < cooldown: + return BudgetDecision( + allowed=False, + reason="cooldown", + remaining=_weighted_remaining(profile), + cooldown_remaining=cooldown - since, + ) + # 2. Per-type hard cap for 'search'. + cap = (profile.action_budgets or {}).get("search", 0) + if cap and actions_today(profile.user, "search") >= cap: + return BudgetDecision( + allowed=False, reason="search cap reached", + remaining=_weighted_remaining(profile), + ) + # 3. Weighted daily budget (a search would add action_cost('search')). + remaining = _weighted_remaining(profile) + if remaining != -1 and remaining < action_cost("search"): + return BudgetDecision( + allowed=False, reason="daily action budget reached", remaining=remaining + ) + return BudgetDecision(allowed=True, remaining=remaining) + + +def remaining_profile_views(profile) -> int: + """How many more profiles this user may pull today. -1 = unlimited. + + The min of the per-type 'profile_view' cap and what the weighted budget + still affords (weighted budget is in cost units; profile_view cost is + per profile).""" + caps = [] + type_cap = (profile.action_budgets or {}).get("profile_view", 0) + if type_cap: + caps.append(max(type_cap - actions_today(profile.user, "profile_view"), 0)) + weighted = _weighted_remaining(profile) + if weighted != -1: + pv_cost = action_cost("profile_view") or 1 + caps.append(weighted // pv_cost) + if not caps: + return -1 + return min(caps) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `python -m pytest services/browser/test_action_budget.py -v` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add services/browser/action_budget.py services/browser/test_action_budget.py +git commit -m "feat(services/browser): action-budget accounting and decisions" +``` + +--- + +## Task 12: Harden run_browser_job (lock, emergency stop, cancel, budget, health, retry, timeout) + +**Files:** +- Modify: `browser/tasks.py` (rewrite `run_browser_job`) +- Test: `browser/test_safety.py` + +**Interfaces:** +- Consumes: everything from Tasks 5–11. +- Produces: a `run_browser_job` that (a) short-circuits on emergency stop, (b) serializes on the session lock with requeue, (c) polls cancel flags, (d) truncates/cancels on profile budget, (e) records `search`/`profile_view` actions, (f) maps health/transient errors, (g) enforces a soft time limit. + +This task keeps the existing success path (upsert people, cascade enrichment) intact and wraps it with the safety controls. Steps are incremental; each adds one behavior with its test. + +- [ ] **Step 1: Write the emergency-stop test** + +Append to `browser/test_safety.py`: +```python +import uuid +from unittest import mock + +from django.core.cache import cache +from django.test import override_settings + +from core.models import JobStatus +from services.browser import control + + +FIXTURE = "services.browser.providers.fixture.FixtureBrowserProvider" + + +@override_settings(BROWSER_AUTOMATION_PROVIDER=FIXTURE) +class RunBrowserJobSafetyTests(TestCase): + def setUp(self): + cache.clear() + + def _job(self, **kw): + return BrowserJob.objects.create(requested_by=uuid.uuid4(), **kw) + + def test_emergency_stop_short_circuits_to_cancelled(self): + control.engage_emergency_stop(reason="test") + job = self._job() + from browser.tasks import run_browser_job + run_browser_job.delay(job.pk) + job.refresh_from_db() + self.assertEqual(job.status, JobStatus.CANCELLED) + self.assertIn("emergency", job.error.lower()) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python -m pytest browser/test_safety.py::RunBrowserJobSafetyTests::test_emergency_stop_short_circuits_to_cancelled -v` +Expected: FAIL — job runs to SUCCEEDED (no stop check yet). + +- [ ] **Step 3: Rewrite the task** + +Replace the body of `browser/tasks.py` with the version below. It preserves the existing docstring intent and success/enrichment logic, and adds the safety wrapper. Read carefully — this is the whole file. + +```python +import logging + +from celery import shared_task +from celery.exceptions import SoftTimeLimitExceeded +from django.conf import settings +from django.utils import timezone + +from companies.models import Company +from core.activity import log_activity +from core.models import JobStatus +from enrichment.models import EnrichmentJob +from enrichment.tasks import run_enrichment_job +from leads.models import Person +from services.browser import action_budget, cancellation, control, health, locks +from services.browser.exceptions import RetryableBrowserError, SessionHealthError +from services.browser import BrowserService +from users.models import UserProfile + +from .models import BrowserJob + +logger = logging.getLogger("kimisystem.browser") + + +def _finish(job, status, actor, verb, *, error="", metadata=None): + """Mark a job terminal, timestamp it, and write one activity row.""" + job.status = status + if error: + job.error = error + job.finished_at = timezone.now() + job.save(update_fields=["status", "error", "finished_at"]) + log_activity(actor=actor, verb=verb, target_type="BrowserJob", + target_id=job.pk, metadata=metadata or {}) + + +@shared_task( + bind=True, + soft_time_limit=settings.BROWSER_TASK_SOFT_TIME_LIMIT, + time_limit=settings.BROWSER_TASK_TIME_LIMIT, +) +def run_browser_job(self, browser_job_id): + """Runs a BrowserJob under the safety controls (docs/.../safety spec): + emergency stop, single-session lock, cooperative cancellation, per-action + budget, session-health detection, and a transient-only retry policy. + + Ownership rule (unchanged): discovered_by is set only when a Person is + first created; rediscovery never reassigns it. + """ + job = BrowserJob.objects.get(pk=browser_job_id) + + actor = None + profile = ( + UserProfile.objects.filter(supabase_user_id=job.requested_by) + .select_related("user") + .first() + ) + if profile: + actor = profile.user + + # 1. Emergency stop: refuse before touching the browser. + if control.emergency_stop_active(): + _finish(job, JobStatus.CANCELLED, actor, "browser_job.emergency_stopped", + error="Emergency stop active — job cancelled.", + metadata={"reason": "emergency_stop"}) + logger.warning("BrowserJob %s cancelled: emergency stop active", job.pk) + return 0 + + # 2. Cancel requested while still queued. + if cancellation.is_cancel_requested(job.pk): + cancellation.clear_cancel(job.pk) + _finish(job, JobStatus.CANCELLED, actor, "browser_job.cancelled", + error="Cancelled before start.", metadata={"reason": "user_cancel"}) + return 0 + + # 3. Single-session lock: only one LinkedIn browser at a time. + if not locks.acquire_session_lock(job.pk): + job.status = JobStatus.PENDING + job.save(update_fields=["status"]) + log_activity(actor=actor, verb="browser_job.requeued", + target_type="BrowserJob", target_id=job.pk, + metadata={"holder": locks.session_lock_holder()}) + run_browser_job.apply_async( + args=[job.pk], countdown=settings.BROWSER_REQUEUE_DELAY + ) + return 0 + + try: + job.status = JobStatus.RUNNING + job.started_at = timezone.now() + job.celery_task_id = self.request.id or "" + job.save(update_fields=["status", "started_at", "celery_task_id"]) + log_activity(actor=actor, verb="browser_job.started", + target_type="BrowserJob", target_id=job.pk, metadata={}) + # Ledger: one 'search' action for this run. + if profile: + action_budget.record_action(actor, "search", units=1, target_id=job.pk) + + try: + profiles = BrowserService().run_linkedin_search(job.criteria) + except SessionHealthError as exc: + self._handle_health(job, actor, exc) + return 0 + except RetryableBrowserError as exc: + log_activity(actor=actor, verb="browser_job.retried", + target_type="BrowserJob", target_id=job.pk, + metadata={"error": str(exc), "retry_count": self.request.retries}) + # Release the lock before retrying so the retry re-queues fairly. + locks.release_session_lock(job.pk) + raise self.retry( + exc=exc, countdown=settings.BROWSER_RETRY_BACKOFF, + max_retries=settings.BROWSER_MAX_RETRIES, + ) + except SoftTimeLimitExceeded: + _finish(job, JobStatus.FAILED, actor, "browser_job.failed", + error="Browser job exceeded its time limit.", + metadata={"reason": "timeout"}) + logger.warning("BrowserJob %s hit soft time limit", job.pk) + return 0 + except Exception as exc: # unknown → fail visibly, do not retry + _finish(job, JobStatus.FAILED, actor, "browser_job.failed", + error=str(exc), metadata={"error": str(exc)}) + logger.warning("BrowserJob %s failed: %s", job.pk, exc) + raise + + # 4. Per-profile budget: truncate to what the user may still pull. + if profile: + remaining = action_budget.remaining_profile_views(profile) + if remaining == 0: + _finish(job, JobStatus.CANCELLED, actor, "browser_job.rate_limited", + error="Daily profile budget exhausted.", + metadata={"reason": "profile_budget"}) + return 0 + if remaining > 0 and len(profiles) > remaining: + profiles = profiles[:remaining] + + result_count, new_people = self._upsert(job, profiles) + + # 5. Ledger: profiles actually pulled. + if profile and result_count: + action_budget.record_action(actor, "profile_view", + units=result_count, target_id=job.pk) + + job.status = JobStatus.SUCCEEDED + job.result_count = result_count + job.finished_at = timezone.now() + job.save(update_fields=["status", "result_count", "finished_at"]) + + for person in new_people: + enrichment_job = EnrichmentJob.objects.create(person=person, browser_job=job) + run_enrichment_job.delay(enrichment_job.pk) + + log_activity(actor=actor, verb="browser_job.succeeded", + target_type="BrowserJob", target_id=job.pk, + metadata={"result_count": result_count}) + return result_count + finally: + # Always free the session for the next job, success or failure. + locks.release_session_lock(job.pk) + + # --- helpers ---------------------------------------------------------- + + # (methods below are defined on the module, see Step 5/6) +``` + +Note: `_handle_health` and `_upsert` are added as module-level helpers in later steps; for Step 3 define them as thin stsubs so the file imports (the emergency-stop test never reaches them): +```python +def _noop_health(*a, **k): + raise NotImplementedError +``` +Replace `self._handle_health(...)`/`self._upsert(...)` calls with module functions `_handle_health(self, job, actor, exc)` and `_upsert(job, profiles)` — implemented in Steps 5 and 7. To keep Step 3 runnable, add minimal versions now: + +```python +def _upsert(job, profiles): + """Upsert profile dicts into Person/Company. Returns (result_count, new_people).""" + result_count = 0 + new_people = [] + for raw_profile in profiles: + company = None + if raw_profile.get("company_domain"): + company, _ = Company.objects.get_or_create( + domain=raw_profile["company_domain"], + defaults={"name": raw_profile.get("company_name", "")}, + ) + person, created = Person.objects.get_or_create( + linkedin_url=raw_profile["linkedin_url"], + defaults={ + "discovered_by": job.requested_by, + "company": company, + "first_name": raw_profile.get("first_name", ""), + "last_name": raw_profile.get("last_name", ""), + "title": raw_profile.get("title", ""), + "headline": raw_profile.get("headline", ""), + "location": raw_profile.get("location", ""), + }, + ) + if created: + new_people.append(person) + else: + person.company = company or person.company + person.first_name = raw_profile.get("first_name", person.first_name) + person.last_name = raw_profile.get("last_name", person.last_name) + person.title = raw_profile.get("title", person.title) + person.headline = raw_profile.get("headline", person.headline) + person.location = raw_profile.get("location", person.location) + person.save() + result_count += 1 + return result_count, new_people + + +def _handle_health(task, job, actor, exc): + """Stop a job on a session-health failure; escalate the worst kinds.""" + from services.browser.exceptions import HEALTH_AUTOSTOP_KINDS + if getattr(settings, "BROWSER_HEALTH_AUTOSTOP", True) and exc.kind in HEALTH_AUTOSTOP_KINDS: + control.engage_emergency_stop(reason=f"health:{exc.kind}", updated_by="worker") + _finish(job, JobStatus.FAILED, actor, "browser_job.health_stopped", + error=str(exc), metadata={"health_kind": exc.kind}) + logger.warning("BrowserJob %s stopped by health check: %s", job.pk, exc.kind) +``` + +Change the two call sites to module functions: `_handle_health(self, job, actor, exc)` and `result_count, new_people = _upsert(job, profiles)`. + +- [ ] **Step 4: Run the emergency-stop test + the existing task suite** + +Run: `python -m pytest browser/test_safety.py::RunBrowserJobSafetyTests::test_emergency_stop_short_circuits_to_cancelled browser/test_tasks.py -v` +Expected: PASS (new test) and the **7 existing `RunBrowserJobTests` still pass** — the success path is unchanged. + +Note: `test_failure_marks_the_job_failed_and_reraises` expects a raised `RuntimeError`; the `except Exception` branch above re-raises after marking FAILED, preserving that behavior. + +- [ ] **Step 5: Add the cancellation-mid-run + requeue tests** + +Append to `RunBrowserJobSafetyTests`: +```python + def test_requeue_when_session_locked(self): + locks_holder = 999 + from services.browser import locks + locks.acquire_session_lock(locks_holder) # someone else holds it + job = self._job() + from browser.tasks import run_browser_job + run_browser_job.delay(job.pk) + job.refresh_from_db() + # Job went back to PENDING and a retry was scheduled (eager: it ran, + # but the lock is still held, so it stays PENDING after one bounce). + self.assertEqual(job.status, JobStatus.PENDING) + + def test_cancel_flag_before_start_cancels(self): + from services.browser import cancellation + job = self._job() + cancellation.request_cancel(job.pk) + from browser.tasks import run_browser_job + run_browser_job.delay(job.pk) + job.refresh_from_db() + self.assertEqual(job.status, JobStatus.CANCELLED) +``` + +Run: `python -m pytest browser/test_safety.py::RunBrowserJobSafetyTests -v` +Expected: PASS. (In eager mode `apply_async(countdown=...)` runs inline; because the foreign lock is still held, the requeued run also bounces and the job remains PENDING — asserting the serialization guarantee.) + +- [ ] **Step 6: Add the health + budget-truncation tests (with a stub provider)** + +Append to `browser/test_safety.py`: +```python +from django.contrib.auth import get_user_model +from users.models import UserProfile, OnboardingStatus + + +class HealthAndBudgetTests(TestCase): + def setUp(self): + cache.clear() + + def _profile_and_uuid(self, **budget): + uid = uuid.uuid4() + user = get_user_model().objects.create(username=f"u{uuid.uuid4().hex[:8]}") + UserProfile.objects.create( + user=user, supabase_user_id=uid, + daily_action_budget=budget.get("daily_action_budget", 0), + action_budgets=budget.get("action_budgets", {}), + action_cooldown_seconds=0, + ) + return user, uid + + @mock.patch("browser.tasks.BrowserService") + def test_health_error_stops_and_autostops(self, svc): + svc.return_value.run_linkedin_search.side_effect = SessionHealthError("captcha") + user, uid = self._profile_and_uuid() + job = BrowserJob.objects.create(requested_by=uid) + from browser.tasks import run_browser_job + run_browser_job.delay(job.pk) + job.refresh_from_db() + self.assertEqual(job.status, JobStatus.FAILED) + self.assertEqual(job.error.count("captcha") >= 1, True) + self.assertTrue(control.emergency_stop_active()) # captcha escalated + + @override_settings(BROWSER_AUTOMATION_PROVIDER=FIXTURE) + def test_profile_budget_truncates(self): + user, uid = self._profile_and_uuid(action_budgets={"profile_view": 1}) + job = BrowserJob.objects.create(requested_by=uid) + from browser.tasks import run_browser_job + run_browser_job.delay(job.pk) + job.refresh_from_db() + self.assertEqual(job.status, JobStatus.SUCCEEDED) + self.assertEqual(job.result_count, 1) # fixture yields 2, capped to 1 +``` +Add the missing import at the top of the test file: `from services.browser.exceptions import SessionHealthError`. + +Run: `python -m pytest browser/test_safety.py::HealthAndBudgetTests -v` +Expected: PASS. + +- [ ] **Step 7: Full regression + commit** + +Run: `python -m pytest browser/ services/browser/ -v` +Expected: all PASS. + +```bash +git add browser/tasks.py browser/test_safety.py +git commit -m "feat(browser): harden run_browser_job (lock, stop, cancel, budget, health, retry, timeout)" +``` + +--- + +## Task 13: Trigger view — budget gate + emergency-stop refusal + +**Files:** +- Modify: `browser/views.py` (`TriggerBrowserJobView.post`) +- Test: `browser/test_views_safety.py` + +**Interfaces:** +- Consumes: `action_budget.check_search_allowed`, `control.emergency_stop_active`. +- Produces: HTTP 429 on budget/cooldown breach; HTTP 409 while emergency stop active; unchanged 202 on success. + +- [ ] **Step 1: Write the failing test** + +Create `browser/test_views_safety.py`: +```python +import uuid + +from django.contrib.auth import get_user_model +from django.core.cache import cache +from django.test import TestCase, override_settings +from rest_framework.test import APIRequestFactory, force_authenticate + +from browser.views import TriggerBrowserJobView +from browser.models import BrowserJob +from services.browser import control +from users.models import ( + OnboardingStatus, SearchCriteria, SenderAccount, UserProfile, Workspace, +) +from campaigns.models import Campaign + + +def _ready_profile(): + user = get_user_model().objects.create(username=f"u{uuid.uuid4().hex[:8]}") + profile = UserProfile.objects.create( + user=user, supabase_user_id=uuid.uuid4(), + search_criteria=SearchCriteria.objects.create( + job_position="CFO", country="PL", industry="Logistics"), + assigned_campaign=Campaign.objects.create(name=f"c{uuid.uuid4().hex[:6]}"), + sender_account=SenderAccount.objects.create(name=f"s{uuid.uuid4().hex[:6]}"), + workspace=Workspace.objects.create(name=f"w{uuid.uuid4().hex[:6]}"), + status=OnboardingStatus.READY, + daily_action_budget=0, action_budgets={"search": 1}, + action_cooldown_seconds=0, + ) + return user, profile + + +@override_settings(BROWSER_AUTOMATION_PROVIDER="services.browser.providers.fixture.FixtureBrowserProvider") +class TriggerGateTests(TestCase): + def setUp(self): + cache.clear() + self.factory = APIRequestFactory() + + def _post(self, user): + request = self.factory.post("/api/browser-jobs/") + force_authenticate(request, user=user) + return TriggerBrowserJobView.as_view()(request) + + def test_emergency_stop_returns_409(self): + user, _ = _ready_profile() + control.engage_emergency_stop(reason="x") + resp = self._post(user) + self.assertEqual(resp.status_code, 409) + self.assertEqual(BrowserJob.objects.count(), 0) + + def test_search_cap_returns_429(self): + user, profile = _ready_profile() + from services.browser import action_budget + action_budget.record_action(user, "search", units=1) # cap is 1 + resp = self._post(user) + self.assertEqual(resp.status_code, 429) + self.assertEqual(BrowserJob.objects.count(), 0) + + def test_allowed_creates_job_202(self): + user, _ = _ready_profile() + resp = self._post(user) + self.assertEqual(resp.status_code, 202) + self.assertEqual(BrowserJob.objects.count(), 1) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python -m pytest browser/test_views_safety.py::TriggerGateTests -v` +Expected: FAIL — 409/429 not returned (view has no gate yet). + +- [ ] **Step 3: Add the gate** + +Rewrite `TriggerBrowserJobView.post` in `browser/views.py`: +```python + def post(self, request): + profile = request.user.profile + + # Global kill switch: refuse new work while emergency stop is on. + if control.emergency_stop_active(): + return Response( + {"detail": "LinkedIn automation is paused by the administrator."}, + status=status.HTTP_409_CONFLICT, + ) + + # Per-user action budget / cooldown. + decision = check_search_allowed(profile) + if not decision.allowed: + log_activity( + actor=request.user, verb="browser_job.rate_limited", + target_type="BrowserJob", target_id=None, + metadata={"reason": decision.reason, + "cooldown_remaining": decision.cooldown_remaining}, + ) + resp = Response( + {"detail": f"Rate limit reached: {decision.reason}.", + "cooldown_remaining": decision.cooldown_remaining}, + status=status.HTTP_429_TOO_MANY_REQUESTS, + ) + if decision.cooldown_remaining: + resp["Retry-After"] = str(decision.cooldown_remaining) + return resp + + criteria = {} + if profile.search_criteria: + criteria = { + "job_position": profile.search_criteria.job_position, + "country": profile.search_criteria.country, + "industry": profile.search_criteria.industry, + "job_change_window_days": profile.search_criteria.job_change_window_days, + "keywords": profile.search_criteria.keywords, + } + + job = BrowserJob.objects.create( + requested_by=profile.supabase_user_id, criteria=criteria, + ) + log_activity(actor=request.user, verb="browser_job.queued", + target_type="BrowserJob", target_id=job.pk, metadata={}) + run_browser_job.delay(job.pk) + return Response( + {"id": job.pk, "status": job.status}, status=status.HTTP_202_ACCEPTED + ) +``` + +Add imports at the top of `browser/views.py`: +```python +from core.activity import log_activity +from services.browser import control +from services.browser.action_budget import check_search_allowed +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `python -m pytest browser/test_views_safety.py::TriggerGateTests browser/test_views.py -v` +Expected: PASS (new gate tests + existing view tests). + +- [ ] **Step 5: Commit** + +```bash +git add browser/views.py browser/test_views_safety.py +git commit -m "feat(browser): gate trigger view on emergency stop + action budget" +``` + +--- + +## Task 14: Cancel endpoint + +**Files:** +- Modify: `browser/views.py` (add `CancelBrowserJobView`) +- Modify: `api/urls.py` (add route) +- Test: `browser/test_views_safety.py` + +**Interfaces:** +- Consumes: `cancellation.request_cancel`, `JobStatus`. +- Produces: `POST /api/browser-jobs//cancel/` → 200 (cancel accepted), 409 (already terminal), 404 (not owner / missing). + +- [ ] **Step 1: Write the failing test** + +Append to `browser/test_views_safety.py`: +```python +from core.models import JobStatus +from browser.views import CancelBrowserJobView + + +class CancelViewTests(TestCase): + def setUp(self): + cache.clear() + self.factory = APIRequestFactory() + + def _cancel(self, user, pk): + request = self.factory.post(f"/api/browser-jobs/{pk}/cancel/") + force_authenticate(request, user=user) + return CancelBrowserJobView.as_view()(request, pk=pk) + + def test_pending_job_cancelled_immediately(self): + user, profile = _ready_profile() + job = BrowserJob.objects.create( + requested_by=profile.supabase_user_id, status=JobStatus.PENDING) + resp = self._cancel(user, job.pk) + self.assertEqual(resp.status_code, 200) + job.refresh_from_db() + self.assertEqual(job.status, JobStatus.CANCELLED) + + def test_running_job_sets_flag_and_returns_200(self): + from services.browser import cancellation + user, profile = _ready_profile() + job = BrowserJob.objects.create( + requested_by=profile.supabase_user_id, status=JobStatus.RUNNING) + resp = self._cancel(user, job.pk) + self.assertEqual(resp.status_code, 200) + self.assertTrue(cancellation.is_cancel_requested(job.pk)) + + def test_terminal_job_returns_409(self): + user, profile = _ready_profile() + job = BrowserJob.objects.create( + requested_by=profile.supabase_user_id, status=JobStatus.SUCCEEDED) + self.assertEqual(self._cancel(user, job.pk).status_code, 409) + + def test_other_users_job_returns_404(self): + user, _ = _ready_profile() + other = BrowserJob.objects.create(requested_by=uuid.uuid4()) + self.assertEqual(self._cancel(user, other.pk).status_code, 404) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python -m pytest browser/test_views_safety.py::CancelViewTests -v` +Expected: FAIL — `CancelBrowserJobView` does not exist. + +- [ ] **Step 3: Add the view** + +Append to `browser/views.py`: +```python +class CancelBrowserJobView(APIView): + """POST cancels a BrowserJob the current user owns. PENDING → CANCELLED + immediately; RUNNING → a cancel flag the worker honors between pages; + already-terminal → 409. Ownership scoped by requested_by (as the status + view is), re-applying RLS at the application layer.""" + + authentication_classes = [SupabaseJWTAuthentication] + permission_classes = [IsAuthenticated] + + def post(self, request, pk): + try: + job = BrowserJob.objects.get( + pk=pk, requested_by=request.user.profile.supabase_user_id + ) + except BrowserJob.DoesNotExist: + return Response(status=status.HTTP_404_NOT_FOUND) + + if job.status in (JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED): + return Response( + {"detail": f"Job is already {job.status}; nothing to cancel."}, + status=status.HTTP_409_CONFLICT, + ) + + if job.status == JobStatus.PENDING: + job.status = JobStatus.CANCELLED + job.finished_at = timezone.now() + job.save(update_fields=["status", "finished_at"]) + else: # RUNNING — cooperative stop + request_cancel(job.pk) + + if job.celery_task_id: + # Best-effort: stop it from starting if still queued. + from config.celery import app as celery_app + celery_app.control.revoke(job.celery_task_id) + + log_activity(actor=request.user, verb="browser_job.cancelled", + target_type="BrowserJob", target_id=job.pk, + metadata={"from_status": job.status}) + return Response({"id": job.pk, "status": job.status}) +``` + +Add imports to `browser/views.py`: +```python +from django.utils import timezone +from core.models import JobStatus +from services.browser.cancellation import request_cancel +``` + +- [ ] **Step 4: Add the URL route** + +In `api/urls.py`, import and register (near the existing browser routes): +```python +from browser.views import ( + BrowserJobStatusView, CancelBrowserJobView, TriggerBrowserJobView, +) +``` +and add to `urlpatterns`: +```python + path( + "browser-jobs//cancel/", + CancelBrowserJobView.as_view(), + name="browser-job-cancel", + ), +``` + +- [ ] **Step 5: Run test to verify it passes** + +Run: `python -m pytest browser/test_views_safety.py::CancelViewTests -v` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add browser/views.py api/urls.py browser/test_views_safety.py +git commit -m "feat(browser): add cancel endpoint (PENDING→CANCELLED, RUNNING→flag)" +``` + +--- + +## Task 15: Provider raises typed exceptions + health detection + +**Files:** +- Modify: `services/browser/providers/kimi.py` +- Test: `services/browser/providers/tests.py` (append) or `services/browser/test_kimi_health.py` + +**Interfaces:** +- Consumes: `services.browser.exceptions`, `services.browser.health.classify_page`. +- Produces: Kimi provider maps transport/timeout failures → `RetryableBrowserError`; auth wall / detected health issues → `SessionHealthError(kind)`. + +Context: `kimi.py` already defines `KimiWebBridgeSessionError` (auth wall) and `KimiWebBridgeExtractionError`. Make the session error a `SessionHealthError` subclass and add a health check after navigation. + +- [ ] **Step 1: Write the failing test** + +Create `services/browser/test_kimi_health.py`: +```python +from unittest import mock + +from django.test import TestCase + +from services.browser.exceptions import RetryableBrowserError, SessionHealthError + + +class KimiTypedErrorsTests(TestCase): + def test_transport_error_maps_to_retryable(self): + import httpx + from services.browser.providers.kimi import KimiBrowserProvider + provider = KimiBrowserProvider() + with mock.patch.object( + provider, "_command", side_effect=httpx.ConnectError("down") + ): + with self.assertRaises(RetryableBrowserError): + provider.run_linkedin_search({}) + + def test_health_check_raises_sessionhealtherror(self): + from services.browser.providers.kimi import KimiBrowserProvider + provider = KimiBrowserProvider() + with mock.patch.object( + provider, "_current_url", + return_value="https://www.linkedin.com/checkpoint/challenge", + ), mock.patch.object(provider, "_navigate"), \ + mock.patch.object(provider, "_bring_to_front"): + with self.assertRaises(SessionHealthError) as ctx: + provider.run_linkedin_search({"keywords": []}) + self.assertEqual(ctx.exception.kind, "checkpoint") +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python -m pytest services/browser/test_kimi_health.py -v` +Expected: FAIL — provider raises its own types / has no `_current_url` health gate. + +- [ ] **Step 3: Wire typed exceptions into the provider** + +In `services/browser/providers/kimi.py`: + +(a) Make the existing session error a health error. Change its declaration: +```python +from ..exceptions import RetryableBrowserError, SessionHealthError +from ..health import classify_page + + +class KimiWebBridgeSessionError(SessionHealthError): + """LinkedIn session is not authenticated. Non-retryable health failure.""" + def __init__(self, message=None): + super().__init__("logged_out", message) +``` +(If `KimiWebBridgeExtractionError` exists, leave it as-is — an unknown failure that fails the job without retry.) + +(b) Add a health check helper and a `_current_url` accessor, then call it right after `_bring_to_front()`/`_wait_for_results()` in `run_linkedin_search`: +```python + def _current_url(self) -> str: + """Best-effort current tab URL via the daemon (CDP/target info).""" + result = self._command("cdp", method="Target.getTargets", params={}) + # The daemon returns the active target's url; fall back to "". + targets = (result or {}).get("targetInfos", []) + for t in targets: + if t.get("attached"): + return t.get("url", "") + return targets[0].get("url", "") if targets else "" + + def _check_health(self, page_markers=None): + kind = classify_page(self._current_url(), page_markers) + if kind is not None: + raise SessionHealthError(kind) +``` +In `run_linkedin_search`, after `self._bring_to_front()` and before/after `self._wait_for_results()`: +```python + self._bring_to_front() + self._check_health() # logout/checkpoint/redirect gate + self._wait_for_results() +``` + +(c) Wrap transport/timeout faults. In the low-level `_command`/HTTP path, map `httpx.TransportError`, `httpx.TimeoutException` to `RetryableBrowserError`: +```python + except (httpx.TransportError, httpx.TimeoutException) as exc: + raise RetryableBrowserError(str(exc)) from exc +``` +(Apply at the existing `httpx` call site; keep any current handling for non-transient HTTP errors that should fail hard.) + +- [ ] **Step 4: Run test to verify it passes + provider regression** + +Run: `python -m pytest services/browser/test_kimi_health.py services/browser/providers/tests.py -v` +Expected: PASS. Adjust the mock target names in Step 1 if the provider's internal method names differ (check `kimi.py` — methods `_command`, `_navigate`, `_bring_to_front`, `_wait_for_results` exist; add `_current_url`). + +- [ ] **Step 5: Commit** + +```bash +git add services/browser/providers/kimi.py services/browser/test_kimi_health.py +git commit -m "feat(kimi): raise typed retryable/health exceptions + session health gate" +``` + +--- + +## Task 16: Admin dashboard — enhanced job admin, control toggle, queue view + +**Files:** +- Modify: `browser/admin.py` +- Test: `browser/test_admin.py` + +**Interfaces:** +- Consumes: `locks.session_lock_holder`, `control.emergency_stop_active`, `action_budget`, `BrowserControlState`, `BrowserJob`, `ActivityLog`. +- Produces: registered `BrowserControlState` admin; enhanced `BrowserJobAdmin` with a "Cancel selected" action and duration column; a staff-only `browser-queue` admin view. + +- [ ] **Step 1: Write the failing test** + +Create `browser/test_admin.py`: +```python +import uuid + +from django.contrib.auth import get_user_model +from django.test import TestCase +from django.urls import reverse + +from browser.models import BrowserJob +from core.models import JobStatus + + +class AdminQueueViewTests(TestCase): + def setUp(self): + self.admin = get_user_model().objects.create_superuser( + username="admin", email="a@e.com", password="pw" + ) + + def test_queue_view_requires_staff(self): + url = reverse("admin:browser-queue") + resp = self.client.get(url) # anonymous + self.assertIn(resp.status_code, (302, 403)) + + def test_queue_view_lists_jobs_for_staff(self): + BrowserJob.objects.create(requested_by=uuid.uuid4(), status=JobStatus.PENDING) + BrowserJob.objects.create(requested_by=uuid.uuid4(), status=JobStatus.RUNNING) + self.client.force_login(self.admin) + resp = self.client.get(reverse("admin:browser-queue")) + self.assertEqual(resp.status_code, 200) + self.assertContains(resp, "LinkedIn Queue") + + def test_cancel_action_cancels_selected(self): + job = BrowserJob.objects.create( + requested_by=uuid.uuid4(), status=JobStatus.PENDING) + self.client.force_login(self.admin) + self.client.post( + reverse("admin:browser_browserjob_changelist"), + {"action": "cancel_selected_jobs", "_selected_action": [job.pk]}, + ) + job.refresh_from_db() + self.assertEqual(job.status, JobStatus.CANCELLED) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python -m pytest browser/test_admin.py -v` +Expected: FAIL — `admin:browser-queue` not reversible; no cancel action. + +- [ ] **Step 3: Rewrite the admin** + +Replace `browser/admin.py`: +```python +from django.contrib import admin +from django.template.response import TemplateResponse +from django.urls import path +from django.utils import timezone + +from core.models import ActivityLog, JobStatus +from services.browser import action_budget, control, locks +from users.models import UserProfile + +from .models import BrowserControlState, BrowserCredential, BrowserJob + + +@admin.register(BrowserCredential) +class BrowserCredentialAdmin(admin.ModelAdmin): + list_display = ("label", "is_active") + + +@admin.register(BrowserControlState) +class BrowserControlStateAdmin(admin.ModelAdmin): + """The global emergency-stop toggle. Editing the single row flips it.""" + list_display = ("emergency_stop", "reason", "updated_by", "updated_at") + + +@admin.register(BrowserJob) +class BrowserJobAdmin(admin.ModelAdmin): + list_display = ( + "id", "status", "requested_by", "credential", + "result_count", "duration", "created_at", + ) + list_filter = ("status", "created_at") + search_fields = ("requested_by",) + readonly_fields = ("created_at",) + actions = ("cancel_selected_jobs",) + + @admin.display(description="Duration") + def duration(self, obj): + if obj.started_at and obj.finished_at: + return str(obj.finished_at - obj.started_at) + return "—" + + @admin.action(description="Cancel selected jobs") + def cancel_selected_jobs(self, request, queryset): + from services.browser.cancellation import request_cancel + cancelled = 0 + for job in queryset: + if job.status in (JobStatus.PENDING, JobStatus.RUNNING): + if job.status == JobStatus.RUNNING: + request_cancel(job.pk) + job.status = JobStatus.CANCELLED + job.finished_at = timezone.now() + job.save(update_fields=["status", "finished_at"]) + cancelled += 1 + self.message_user(request, f"Cancelled {cancelled} job(s).") + + def get_urls(self): + urls = super().get_urls() + custom = [ + path( + "linkedin-queue/", + self.admin_site.admin_view(self.queue_view), + name="browser-queue", + ), + ] + return custom + urls + + def queue_view(self, request): + """Read-only operational dashboard: active jobs, pending queue, per-user + budgets, worker status, recent actions. Staff-only via admin_view().""" + running = BrowserJob.objects.filter(status=JobStatus.RUNNING).order_by("started_at") + pending = BrowserJob.objects.filter(status=JobStatus.PENDING).order_by("created_at") + users = [] + for profile in UserProfile.objects.select_related("user"): + users.append({ + "user": profile.user, + "searches_today": action_budget.actions_today(profile.user, "search"), + "profiles_today": action_budget.actions_today(profile.user, "profile_view"), + "weighted_spend": action_budget.weighted_spend_today(profile.user), + "daily_action_budget": profile.daily_action_budget, + }) + context = { + **self.admin_site.each_context(request), + "title": "LinkedIn Queue", + "running": running, + "pending": pending, + "users": users, + "emergency_stop": control.emergency_stop_active(), + "lock_holder": locks.session_lock_holder(), + "recent_actions": ActivityLog.objects.filter( + verb__startswith="browser_job." + )[:25], + } + return TemplateResponse(request, "admin/browser/queue.html", context) +``` + +- [ ] **Step 4: Add the template** + +Create `browser/templates/admin/browser/queue.html`: +```html +{% extends "admin/base_site.html" %} +{% block content %} +

LinkedIn Queue

+

Emergency stop: {{ emergency_stop|yesno:"ON,off" }}   + Session lock held by job: {{ lock_holder|default:"—" }}

+ +

Active jobs ({{ running|length }})

+ +{% for job in running %} + +{% empty %}{% endfor %} +
IDRequested byStarted
{{ job.id }}{{ job.requested_by }}{{ job.started_at }}
None
+ +

Pending queue ({{ pending|length }})

+ +{% for job in pending %} + +{% empty %}{% endfor %} +
IDRequested byCreated
{{ job.id }}{{ job.requested_by }}{{ job.created_at }}
None
+ +

Per-user usage (today)

+ +{% for row in users %} + + +{% endfor %} +
UserSearchesProfilesWeighted spendBudget
{{ row.user }}{{ row.searches_today }}{{ row.profiles_today }}{{ row.weighted_spend }}{{ row.daily_action_budget|default:"∞" }}
+ +

Recent actions

+ +{% for a in recent_actions %} + +{% endfor %} +
WhenActorVerbJob
{{ a.created_at }}{{ a.actor }}{{ a.verb }}{{ a.target_id }}
+{% endblock %} +``` + +Confirm `browser` is in `INSTALLED_APPS` and Django's app-template loader is on (default `APP_DIRS=True`); the template resolves from `browser/templates/`. + +- [ ] **Step 5: Run test to verify it passes** + +Run: `python -m pytest browser/test_admin.py -v` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add browser/admin.py browser/templates/admin/browser/queue.html browser/test_admin.py +git commit -m "feat(browser): admin control toggle, cancel action, LinkedIn queue dashboard" +``` + +--- + +## Task 17: Operations docs + +**Files:** +- Modify: `docs/workflows/browser.md` (append an "Operational safety" section) + +- [ ] **Step 1: Append the ops note** + +Add to `docs/workflows/browser.md`: +```markdown +## Operational safety (queue, limits, emergency stop) + +- **Single session:** run the browser worker with `--concurrency=1`: + `celery -A config worker -Q browser --concurrency=1`. The app also enforces + one session via a cache lock (`services/browser/locks.py`), so a mis-started + worker still can't run two LinkedIn browsers — but pin concurrency anyway. +- **Emergency stop:** Admin → *Browser control state* → toggle `emergency_stop`. + Running jobs stop as CANCELLED at the next checkpoint; new triggers get HTTP 409. +- **Per-user budgets:** Admin → *User profile* → `daily_action_budget`, + `action_budgets`, `action_cooldown_seconds` (0 = unlimited). +- **Queue dashboard:** Admin → *Browser jobs* → "LinkedIn Queue" + (`/admin/browser/browserjob/linkedin-queue/`). +- **Health auto-stop:** a CAPTCHA/checkpoint detected mid-run engages the global + emergency stop (`BROWSER_HEALTH_AUTOSTOP=1`). +``` + +- [ ] **Step 2: Full suite + commit** + +Run: `python -m pytest browser/ services/browser/ users/ core/ -v` +Expected: all PASS. + +```bash +git add docs/workflows/browser.md +git commit -m "docs(browser): operational safety — queue, budgets, emergency stop" +``` + +--- + +## Self-Review + +**Spec coverage:** +- §A queue+statuses → Task 1. §B single session/timeout → Tasks 2, 7, 12. §C cancellation → Tasks 9, 12, 14. §D budget → Tasks 2, 3, 6, 11, 12, 13. §E logging → verbs used across Tasks 11–16. §F admin+manual control → Task 16. §G emergency stop → Tasks 4, 8, 12, 13, 16. §H health → Tasks 5, 10, 15, 12. §I retry → Tasks 5, 12, 15. §J tests → each task is TDD. +- Point 1 (emergency stop) ✓ T4/8/12/13/16. Point 2 (health) ✓ T5/10/15/12. Point 3 (action budget) ✓ T3/6/11. Point 4 (manual control visibility) ✓ T16 queue view. Point 5 (retry policy) ✓ T5/12/15. + +**Placeholder scan:** No TBD/TODO; every code step shows complete code. The one stub (`_noop_health`) is explicitly replaced by real `_handle_health` in the same step. + +**Type consistency:** `BudgetDecision(allowed, reason, remaining, cooldown_remaining)` used consistently (T11 defines, T13 consumes). `SessionHealthError(kind)` / `.kind` consistent (T5/10/12/15). `acquire_session_lock`/`release_session_lock`/`session_lock_holder` consistent (T7/12/16). `record_action(user, action_type, units, target_id)` consistent (T11/12/13). `emergency_stop_active` consistent (T8/12/13/16). + +--- + +## Execution Handoff + +Plan complete. Two execution options: +1. **Subagent-Driven (recommended)** — a fresh subagent per task, review between tasks. +2. **Inline Execution** — execute tasks in this session with checkpoints. From 830524d7aa8b6fd3bbf9b5bf4d1f6be9d75323bd Mon Sep 17 00:00:00 2001 From: Dominik Ledzion Date: Thu, 9 Jul 2026 13:33:41 +0200 Subject: [PATCH 05/34] feat(core): add CANCELLED job status + browser constraint migration --- ...rowser_browserjob_status_valid_and_more.py | 44 +++++++++++++++++++ browser/test_safety.py | 18 ++++++++ core/models.py | 4 ++ 3 files changed, 66 insertions(+) create mode 100644 browser/migrations/0005_remove_browserjob_browser_browserjob_status_valid_and_more.py create mode 100644 browser/test_safety.py diff --git a/browser/migrations/0005_remove_browserjob_browser_browserjob_status_valid_and_more.py b/browser/migrations/0005_remove_browserjob_browser_browserjob_status_valid_and_more.py new file mode 100644 index 0000000..56c735f --- /dev/null +++ b/browser/migrations/0005_remove_browserjob_browser_browserjob_status_valid_and_more.py @@ -0,0 +1,44 @@ +# Generated by Django 6.0.6 on 2026-07-09 11:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("browser", "0004_browsercredential_created_at_and_more"), + ] + + operations = [ + migrations.RemoveConstraint( + model_name="browserjob", + name="browser_browserjob_status_valid", + ), + migrations.AlterField( + model_name="browserjob", + name="status", + field=models.CharField( + choices=[ + ("pending", "Pending"), + ("running", "Running"), + ("succeeded", "Succeeded"), + ("failed", "Failed"), + ("cancelled", "Cancelled"), + ], + default="pending", + max_length=16, + ), + ), + migrations.AddConstraint( + model_name="browserjob", + constraint=models.CheckConstraint( + condition=models.Q( + ( + "status__in", + ["pending", "running", "succeeded", "failed", "cancelled"], + ) + ), + name="browser_browserjob_status_valid", + ), + ), + ] diff --git a/browser/test_safety.py b/browser/test_safety.py new file mode 100644 index 0000000..b3a2fe3 --- /dev/null +++ b/browser/test_safety.py @@ -0,0 +1,18 @@ +import uuid + +from django.test import TestCase + +from core.models import JobStatus +from browser.models import BrowserJob + + +class JobStatusCancelledTests(TestCase): + def test_cancelled_is_a_valid_status_value(self): + self.assertEqual(JobStatus.CANCELLED, "cancelled") + + def test_browserjob_accepts_cancelled_under_check_constraint(self): + job = BrowserJob.objects.create( + requested_by=uuid.uuid4(), status=JobStatus.CANCELLED + ) + job.refresh_from_db() + self.assertEqual(job.status, JobStatus.CANCELLED) diff --git a/core/models.py b/core/models.py index 6d6cd87..4e84848 100644 --- a/core/models.py +++ b/core/models.py @@ -7,6 +7,10 @@ class JobStatus(models.TextChoices): RUNNING = "running", "Running" SUCCEEDED = "succeeded", "Succeeded" FAILED = "failed", "Failed" + # Additive: a job stopped before completion — by the user (cancel), + # the administrator (emergency stop), or an exhausted budget. Shared + # by every JobBase subclass; browser is the only producer today. + CANCELLED = "cancelled", "Cancelled" class JobBase(models.Model): From e9e2155df3dd5798f2a4352632ea552863fea1a7 Mon Sep 17 00:00:00 2001 From: Dominik Ledzion Date: Thu, 9 Jul 2026 13:35:50 +0200 Subject: [PATCH 06/34] feat(core): propagate CANCELLED constraint to campaigns/enrichment jobs CampaignJob and EnrichmentJob also carry JobStatus-based CheckConstraints, so the shared-enum change needs their migrations too (keeps makemigrations --check clean across all apps). Co-Authored-By: Claude Opus 4.8 --- ...aigns_campaignjob_status_valid_and_more.py | 44 ++++++++++++++++++ ...ent_enrichmentjob_status_valid_and_more.py | 46 +++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 campaigns/migrations/0003_remove_campaignjob_campaigns_campaignjob_status_valid_and_more.py create mode 100644 enrichment/migrations/0003_remove_enrichmentjob_enrichment_enrichmentjob_status_valid_and_more.py diff --git a/campaigns/migrations/0003_remove_campaignjob_campaigns_campaignjob_status_valid_and_more.py b/campaigns/migrations/0003_remove_campaignjob_campaigns_campaignjob_status_valid_and_more.py new file mode 100644 index 0000000..829de1a --- /dev/null +++ b/campaigns/migrations/0003_remove_campaignjob_campaigns_campaignjob_status_valid_and_more.py @@ -0,0 +1,44 @@ +# Generated by Django 6.0.6 on 2026-07-09 11:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("campaigns", "0002_enable_row_level_security"), + ] + + operations = [ + migrations.RemoveConstraint( + model_name="campaignjob", + name="campaigns_campaignjob_status_valid", + ), + migrations.AlterField( + model_name="campaignjob", + name="status", + field=models.CharField( + choices=[ + ("pending", "Pending"), + ("running", "Running"), + ("succeeded", "Succeeded"), + ("failed", "Failed"), + ("cancelled", "Cancelled"), + ], + default="pending", + max_length=16, + ), + ), + migrations.AddConstraint( + model_name="campaignjob", + constraint=models.CheckConstraint( + condition=models.Q( + ( + "status__in", + ["pending", "running", "succeeded", "failed", "cancelled"], + ) + ), + name="campaigns_campaignjob_status_valid", + ), + ), + ] diff --git a/enrichment/migrations/0003_remove_enrichmentjob_enrichment_enrichmentjob_status_valid_and_more.py b/enrichment/migrations/0003_remove_enrichmentjob_enrichment_enrichmentjob_status_valid_and_more.py new file mode 100644 index 0000000..225fd39 --- /dev/null +++ b/enrichment/migrations/0003_remove_enrichmentjob_enrichment_enrichmentjob_status_valid_and_more.py @@ -0,0 +1,46 @@ +# Generated by Django 6.0.6 on 2026-07-09 11:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("browser", "0005_remove_browserjob_browser_browserjob_status_valid_and_more"), + ("enrichment", "0002_enable_row_level_security"), + ("leads", "0003_person_leads_person_status_valid"), + ] + + operations = [ + migrations.RemoveConstraint( + model_name="enrichmentjob", + name="enrichment_enrichmentjob_status_valid", + ), + migrations.AlterField( + model_name="enrichmentjob", + name="status", + field=models.CharField( + choices=[ + ("pending", "Pending"), + ("running", "Running"), + ("succeeded", "Succeeded"), + ("failed", "Failed"), + ("cancelled", "Cancelled"), + ], + default="pending", + max_length=16, + ), + ), + migrations.AddConstraint( + model_name="enrichmentjob", + constraint=models.CheckConstraint( + condition=models.Q( + ( + "status__in", + ["pending", "running", "succeeded", "failed", "cancelled"], + ) + ), + name="enrichment_enrichmentjob_status_valid", + ), + ), + ] From 0e8e227bfcc6acae2c67e727bb3510a423e5a2af Mon Sep 17 00:00:00 2001 From: Dominik Ledzion Date: Thu, 9 Jul 2026 13:39:05 +0200 Subject: [PATCH 07/34] test: CampaignJob/EnrichmentJob accept CANCELLED status Parity tests for the shared-enum constraint migrations added in e9e2155, closing the Task 1 review's Important finding. Co-Authored-By: Claude Opus 4.8 --- campaigns/test_status_constraint.py | 19 +++++++++++++++++++ enrichment/test_status_constraint.py | 21 +++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 campaigns/test_status_constraint.py create mode 100644 enrichment/test_status_constraint.py diff --git a/campaigns/test_status_constraint.py b/campaigns/test_status_constraint.py new file mode 100644 index 0000000..dd09e50 --- /dev/null +++ b/campaigns/test_status_constraint.py @@ -0,0 +1,19 @@ +import uuid + +from django.test import TestCase + +from campaigns.models import Campaign, CampaignJob +from core.models import JobStatus + + +class CampaignJobCancelledStatusTests(TestCase): + """CampaignJob shares core.JobStatus, so its status CheckConstraint + (migration 0003) must accept the newly-added CANCELLED value.""" + + def test_accepts_cancelled_under_check_constraint(self): + campaign = Campaign.objects.create(name="cancel-parity") + job = CampaignJob.objects.create( + requested_by=uuid.uuid4(), campaign=campaign, status=JobStatus.CANCELLED + ) + job.refresh_from_db() + self.assertEqual(job.status, JobStatus.CANCELLED) diff --git a/enrichment/test_status_constraint.py b/enrichment/test_status_constraint.py new file mode 100644 index 0000000..a4bda85 --- /dev/null +++ b/enrichment/test_status_constraint.py @@ -0,0 +1,21 @@ +import uuid + +from django.test import TestCase + +from core.models import JobStatus +from enrichment.models import EnrichmentJob +from leads.models import Person + + +class EnrichmentJobCancelledStatusTests(TestCase): + """EnrichmentJob shares core.JobStatus, so its status CheckConstraint + (migration 0003) must accept the newly-added CANCELLED value.""" + + def test_accepts_cancelled_under_check_constraint(self): + person = Person.objects.create( + linkedin_url="https://linkedin.com/in/enrich-cancel", + discovered_by=uuid.uuid4(), + ) + job = EnrichmentJob.objects.create(person=person, status=JobStatus.CANCELLED) + job.refresh_from_db() + self.assertEqual(job.status, JobStatus.CANCELLED) From 6de4ca9a528c2b29e7f9b301a6e9e7e0f3462124 Mon Sep 17 00:00:00 2001 From: Dominik Ledzion Date: Thu, 9 Jul 2026 13:40:36 +0200 Subject: [PATCH 08/34] feat(services/browser): action-type registry + cost lookup --- services/browser/actions.py | 33 ++++++++++++++++++++++++++++++++ services/browser/test_actions.py | 20 +++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 services/browser/actions.py create mode 100644 services/browser/test_actions.py diff --git a/services/browser/actions.py b/services/browser/actions.py new file mode 100644 index 0000000..068f252 --- /dev/null +++ b/services/browser/actions.py @@ -0,0 +1,33 @@ +"""Registry of LinkedIn action types and their budget cost weights. + +Cost weights are configurable via ``settings.BROWSER_ACTION_COSTS``; this +module holds the canonical type names and safe defaults. ``search`` and +``profile_view`` are wired in v1; ``export``/``message`` are reserved so the +budget model needn't change when they are added. See the safety design spec. +""" + +from django.conf import settings + +SEARCH = "search" +PROFILE_VIEW = "profile_view" +EXPORT = "export" +ENRICHMENT = "enrichment" +MESSAGE = "message" + +ACTION_TYPES = {SEARCH, PROFILE_VIEW, EXPORT, ENRICHMENT, MESSAGE} + +DEFAULT_ACTION_COSTS = { + SEARCH: 1, + PROFILE_VIEW: 1, + EXPORT: 5, + ENRICHMENT: 1, + MESSAGE: 3, +} + + +def action_cost(action_type: str) -> int: + """Cost weight for an action type: settings override → module default → 1.""" + configured = getattr(settings, "BROWSER_ACTION_COSTS", {}) + if action_type in configured: + return int(configured[action_type]) + return int(DEFAULT_ACTION_COSTS.get(action_type, 1)) diff --git a/services/browser/test_actions.py b/services/browser/test_actions.py new file mode 100644 index 0000000..d1fe315 --- /dev/null +++ b/services/browser/test_actions.py @@ -0,0 +1,20 @@ +from django.test import TestCase, override_settings + +from services.browser import actions + + +class ActionRegistryTests(TestCase): + def test_action_types_set(self): + self.assertEqual( + actions.ACTION_TYPES, + {"search", "profile_view", "export", "enrichment", "message"}, + ) + + def test_cost_reads_settings_first(self): + with override_settings(BROWSER_ACTION_COSTS={"search": 7}): + self.assertEqual(actions.action_cost("search"), 7) + + def test_cost_falls_back_to_default_then_one(self): + with override_settings(BROWSER_ACTION_COSTS={}): + self.assertEqual(actions.action_cost("export"), 5) # DEFAULT_ACTION_COSTS + self.assertEqual(actions.action_cost("unknown"), 1) # ultimate fallback From 3c822533a9cf0dbc18a3a57014029f61aec44e81 Mon Sep 17 00:00:00 2001 From: Dominik Ledzion Date: Thu, 9 Jul 2026 13:48:37 +0200 Subject: [PATCH 09/34] feat(config): add browser safety settings (costs, lock TTL, time limits, retry, autostop) Co-Authored-By: Claude Opus 4.8 --- browser/test_safety.py | 15 +++++++++++++++ config/settings/base.py | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/browser/test_safety.py b/browser/test_safety.py index b3a2fe3..4a622a3 100644 --- a/browser/test_safety.py +++ b/browser/test_safety.py @@ -1,6 +1,7 @@ import uuid from django.test import TestCase +from django.conf import settings from core.models import JobStatus from browser.models import BrowserJob @@ -16,3 +17,17 @@ def test_browserjob_accepts_cancelled_under_check_constraint(self): ) job.refresh_from_db() self.assertEqual(job.status, JobStatus.CANCELLED) + + +class SafetySettingsTests(TestCase): + def test_action_costs_cover_every_action_type(self): + from services.browser.actions import ACTION_TYPES + self.assertTrue(ACTION_TYPES.issubset(set(settings.BROWSER_ACTION_COSTS))) + + def test_time_limits_are_ordered(self): + self.assertLess( + settings.BROWSER_TASK_SOFT_TIME_LIMIT, settings.BROWSER_TASK_TIME_LIMIT + ) + + def test_health_autostop_defaults_on(self): + self.assertTrue(settings.BROWSER_HEALTH_AUTOSTOP) diff --git a/config/settings/base.py b/config/settings/base.py index 8f13f7f..9d10e2d 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -265,6 +265,45 @@ } +# --- Browser / LinkedIn safety -------------------------------------------- +# See docs/superpowers/specs/2026-07-09-browser-linkedin-safety-design.md. +# Every limit uses 0 = "unlimited", matching UserProfile.daily_sending_limit. + +# Weighted cost per LinkedIn action type (services/browser/actions.py). +BROWSER_ACTION_COSTS = { + 'search': int(os.environ.get('BROWSER_COST_SEARCH', '1')), + 'profile_view': int(os.environ.get('BROWSER_COST_PROFILE_VIEW', '1')), + 'export': int(os.environ.get('BROWSER_COST_EXPORT', '5')), + 'enrichment': int(os.environ.get('BROWSER_COST_ENRICHMENT', '1')), + 'message': int(os.environ.get('BROWSER_COST_MESSAGE', '3')), +} + +# Only one LinkedIn browser session may run at a time. The lock auto-expires +# after this many seconds so a crashed worker cannot wedge the queue forever; +# keep it comfortably above BROWSER_TASK_TIME_LIMIT. +BROWSER_SESSION_LOCK_TTL = int(os.environ.get('BROWSER_SESSION_LOCK_TTL', '900')) + +# When the session lock is busy, a queued job re-enqueues itself after this +# many seconds instead of running a second browser. +BROWSER_REQUEUE_DELAY = int(os.environ.get('BROWSER_REQUEUE_DELAY', '30')) + +# Celery soft/hard time limits for run_browser_job. Soft raises +# SoftTimeLimitExceeded (caught → FAILED, lock released); hard kills the worker. +BROWSER_TASK_SOFT_TIME_LIMIT = int(os.environ.get('BROWSER_TASK_SOFT_TIME_LIMIT', '600')) +BROWSER_TASK_TIME_LIMIT = int(os.environ.get('BROWSER_TASK_TIME_LIMIT', '660')) + +# Retry policy (only transient failures; never health failures). +BROWSER_MAX_RETRIES = int(os.environ.get('BROWSER_MAX_RETRIES', '3')) +BROWSER_RETRY_BACKOFF = int(os.environ.get('BROWSER_RETRY_BACKOFF', '30')) + +# On a CAPTCHA/checkpoint health failure, also engage the global emergency +# stop so queued jobs don't keep hitting a flagged account. +BROWSER_HEALTH_AUTOSTOP = os.environ.get('BROWSER_HEALTH_AUTOSTOP', '1') == '1' + +# TTL for the per-job cancel flag (cache key lifetime). +BROWSER_CANCEL_FLAG_TTL = int(os.environ.get('BROWSER_CANCEL_FLAG_TTL', '900')) + + LANGUAGE_CODE = 'pl' TIME_ZONE = 'Europe/Warsaw' From e70bb2a80eb3547240ad034f93d9e03667773342 Mon Sep 17 00:00:00 2001 From: Dominik Ledzion Date: Thu, 9 Jul 2026 13:54:07 +0200 Subject: [PATCH 10/34] feat(users): onboarding state machine + auth provisioning baseline Baseline commit of the previously-uncommitted onboarding/auth work that the browser-safety feature builds on: UserProfile onboarding state (PENDING/READY/ DISABLED), Workspace/SenderAccount/SearchCriteria assignment, IsReady gate on browser + campaign triggers, JWT email-verified claim resolution, and their tests + migration 0005. Scoped to backend Python only; frontend onboarding UI, the webbridge-e2e marker, and the DB CONN_HEALTH_CHECKS tweak remain as separate uncommitted WIP. Co-Authored-By: Claude Opus 4.8 --- browser/test_views.py | 34 ++- browser/views.py | 6 +- campaigns/test_views.py | 9 +- campaigns/views.py | 5 +- docs/auth-signup-fix-report.md | 154 +++++++++++ ...profile-onboarding-state-machine-design.md | 147 ++++++++++ services/supabase/auth.py | 16 ++ services/supabase/tests.py | 31 ++- tests/test_discovery_pipeline.py | 8 + tests/test_onboarding_integration.py | 171 ++++++++++++ users/admin.py | 99 +++++-- users/authentication.py | 132 ++++++--- .../0005_userprofile_onboarding_state.py | 104 +++++++ users/models.py | 107 ++++++- users/permissions.py | 36 +++ users/test_authentication.py | 108 +++++++- users/test_onboarding.py | 261 ++++++++++++++++++ 17 files changed, 1355 insertions(+), 73 deletions(-) create mode 100644 docs/auth-signup-fix-report.md create mode 100644 docs/superpowers/specs/2026-07-09-userprofile-onboarding-state-machine-design.md create mode 100644 tests/test_onboarding_integration.py create mode 100644 users/migrations/0005_userprofile_onboarding_state.py create mode 100644 users/permissions.py create mode 100644 users/test_onboarding.py diff --git a/browser/test_views.py b/browser/test_views.py index 3fa04b1..d3a56a9 100644 --- a/browser/test_views.py +++ b/browser/test_views.py @@ -7,7 +7,7 @@ from core.models import JobStatus from leads.models import Person -from users.models import UserProfile +from users.models import OnboardingStatus, SearchCriteria, UserProfile from .models import BrowserJob @@ -18,18 +18,38 @@ def _auth_header(request_factory_client, claims): return mock.patch("users.authentication.verify_access_token", return_value=claims) +def _make_ready_profile(username, supabase_user_id): + """Build a fully-provisioned READY UserProfile (all four required + assignments present) so IsReady lets the trigger through.""" + from campaigns.models import Campaign + from users.models import SenderAccount, Workspace + + user = get_user_model().objects.create_user(username=username) + user.set_unusable_password() + user.save() + profile = UserProfile.objects.create( + user=user, + supabase_user_id=supabase_user_id, + search_criteria=SearchCriteria.objects.create( + job_position="PM", country="DE", industry="Logistics" + ), + assigned_campaign=Campaign.objects.create(name=f"{username}-campaign"), + sender_account=SenderAccount.objects.create(name=f"{username}-sender"), + workspace=Workspace.objects.create(name=f"{username}-workspace"), + daily_sending_limit=50, + status=OnboardingStatus.READY, + ) + return profile + + @override_settings( BROWSER_AUTOMATION_PROVIDER="services.browser.providers.fixture.FixtureBrowserProvider" ) class BrowserJobViewTests(TestCase): def setUp(self): self.supabase_user_id = uuid.uuid4() - self.django_user = get_user_model().objects.create_user(username="grace") - self.django_user.set_unusable_password() - self.django_user.save() - self.profile = UserProfile.objects.create( - user=self.django_user, supabase_user_id=self.supabase_user_id - ) + self.profile = _make_ready_profile("grace", self.supabase_user_id) + self.django_user = self.profile.user self.claims = {"sub": str(self.supabase_user_id)} def test_trigger_without_a_token_is_rejected(self): diff --git a/browser/views.py b/browser/views.py index 93b18c8..ac9d032 100644 --- a/browser/views.py +++ b/browser/views.py @@ -4,6 +4,7 @@ from rest_framework.views import APIView from users.authentication import SupabaseJWTAuthentication +from users.permissions import IsReady from .models import BrowserJob from .tasks import run_browser_job @@ -12,10 +13,11 @@ class TriggerBrowserJobView(APIView): """POST creates a BrowserJob for the current user, using their assigned SearchCriteria, and enqueues it. See - docs/workflows/browser.md ("Find Leads").""" + docs/workflows/browser.md ("Find Leads"). Only a READY user may trigger + a search (PENDING/DISABLED are blocked by IsReady).""" authentication_classes = [SupabaseJWTAuthentication] - permission_classes = [IsAuthenticated] + permission_classes = [IsAuthenticated, IsReady] def post(self, request): profile = request.user.profile diff --git a/campaigns/test_views.py b/campaigns/test_views.py index ae9496d..d8ab9cc 100644 --- a/campaigns/test_views.py +++ b/campaigns/test_views.py @@ -5,7 +5,7 @@ from django.test import TestCase from django.urls import reverse -from users.models import UserProfile +from users.models import OnboardingStatus, SearchCriteria, SenderAccount, UserProfile, Workspace from .models import Campaign, CampaignJob @@ -27,6 +27,13 @@ def setUp(self): user=user, supabase_user_id=self.supabase_user_id, assigned_campaign=self.campaign, + search_criteria=SearchCriteria.objects.create( + job_position="PM", country="DE", industry="Logistics" + ), + sender_account=SenderAccount.objects.create(name="grace-sender"), + workspace=Workspace.objects.create(name="grace-workspace"), + daily_sending_limit=50, + status=OnboardingStatus.READY, ) self.claims = {"sub": str(self.supabase_user_id)} diff --git a/campaigns/views.py b/campaigns/views.py index 42eed15..eac6072 100644 --- a/campaigns/views.py +++ b/campaigns/views.py @@ -4,6 +4,7 @@ from rest_framework.views import APIView from users.authentication import SupabaseJWTAuthentication +from users.permissions import IsReady from .models import CampaignJob from .tasks import run_campaign_job @@ -13,13 +14,15 @@ class TriggerCampaignJobView(APIView): """POST creates a CampaignJob for the current user's assigned campaign — the user chooses which leads, never which campaign (docs/workflows/campaign.md). Body: {"person_ids": [1, 2, ...]}. + Only a READY user may push leads to Instantly (PENDING/DISABLED are + blocked by IsReady). The verified-email hard rule is deliberately NOT checked here — it lives in services/instantly so no entry point can bypass it; this view only validates the request's shape.""" authentication_classes = [SupabaseJWTAuthentication] - permission_classes = [IsAuthenticated] + permission_classes = [IsAuthenticated, IsReady] def post(self, request): profile = request.user.profile diff --git a/docs/auth-signup-fix-report.md b/docs/auth-signup-fix-report.md new file mode 100644 index 0000000..61c88a8 --- /dev/null +++ b/docs/auth-signup-fix-report.md @@ -0,0 +1,154 @@ +# Authentication Signup Flow — Investigation & Fix Report + +Date: 2026-07-09 +Scope: signup → confirmation email → Django User → UserProfile → login. + +## TL;DR + +Signup was never broken at the Supabase layer — `auth.users` records **were** +created and confirmation emails **were** sent and clicked. The breakage was +entirely on the Django side: the auth backend (`SupabaseJWTAuthentication`) +**deliberately did not auto-provision** a Django User or `UserProfile`, so +every freshly-confirmed user hit `UNPROVISIONED` ("No local account provisioned +for this Supabase user") on first API call and could never log in. + +The fix implements **auto-provisioning on first verified login**: when a +verified Supabase JWT arrives and no local account exists, the backend now +creates a shadow Django User (unusable password) + a `UserProfile` linked to +the Supabase `sub`. No admin action, no new endpoint, no webhook, no dashboard +config required. + +## Investigation evidence + +Django's database connection **is** the Supabase Postgres instance, so the +live `auth.users` / `users_userprofile` / `auth_user` tables were queried +directly (read-only) for ground truth: + +| Table | State | Meaning | +|---|---|---| +| `auth.users` | 2 rows, **both with `email_confirmed_at` set** | Supabase signup + email confirmation **are working** | +| `users_userprofile` | **0 rows** | ← the actual breakage: no profile for anyone | +| `auth_user` | 1 row (`dledzion`, created today, no profile) | a manual admin attempt, still unlinked | + +The 39-second gap between `created_at` and `email_confirmed_at` for one user +confirms a human clicked a confirmation-email link. Confirmation is ON and +functional. + +## Answers to the 8 questions + +1. **How signup works** — Frontend `signupAction` + (`frontend/src/app/actions/auth.ts`) calls `supabase.auth.signUp({email, + password})` directly against Supabase Auth from a Next.js server action. + **No Django endpoint is hit during signup.** Supabase Auth **is** used. +2. **Supabase config** — `SUPABASE_URL` and `SUPABASE_SERVICE_ROLE_KEY` are set + in `.env`; the anon key is set in `frontend/.env.local`. The service-role + key is present but **not read by any backend code**. `signUp` is called + with no `emailRedirectTo`. Email confirmation is a Supabase dashboard + setting (not represented in-repo) — and it is **enabled**. +3. **What signup creates** — `auth.users` ✅ (proven). Django User ❌. + `UserProfile` ❌. Nothing else. +4. **Backend exceptions during signup** — none. The backend is **never + invoked** during signup, so there is nothing to throw. +5. **Silent transaction failure** — no. There is **no backend transaction at + all** during signup; the failure is architectural, not a caught exception. +6. **Why the confirmation email is "never sent"** — The DB disproves the + premise: confirmation emails **are** sent and were clicked by existing + users. If a *specific* recent signup did not receive the email, that is a + delivery issue (spam folder / Supabase auth logs), not a configuration or + code defect. The real blocker is not email — it is provisioning. +7. **Why no Django User is created** — There is no backend signup endpoint, + and `SupabaseJWTAuthentication` deliberately never created one + (`users/authentication.py`, class docstring: "administrator-controlled, no + auto-creation"). +8. **Why login requires manual provisioning** — `SupabaseJWTAuthentication` + requires a pre-existing `UserProfile` matched by `supabase_user_id` (or, on + first login, by verified email against an *existing* Django User). With no + profile and no Django User, it raised `UNPROVISIONED` → HTTP 401. + +> Note: the scary string "An Administrator must provision your workspace +> profile before the API accepts your session" is a **static frontend notice** +> (`frontend/src/app/(auth)/login/page.tsx`) shown after the +> `?notice=account-created` redirect — it was never a backend response. The +> real backend error was the different string `"No local account provisioned +> for this Supabase user"` (`users/authentication.py:42`). + +## Root cause + +The architecture split identity (Supabase Auth) from authorization (Django +`UserProfile`) but left the **bridge** between them as a manual Administrator +step. A self-service signup produced a valid Supabase identity with no Django +side, so the JWT auth backend — correctly, per its old contract — refused it. +There was no bug in any single component; the *contract itself* excluded +self-service users. + +## Fix + +Replace the "administrator-controlled, no auto-creation" contract with +**auto-provisioning on first verified login**, in `SupabaseJWTAuthentication`. + +### New flow + +1. User signs up via `signupAction` → `supabase.auth.signUp` creates + `auth.users` and Supabase sends the confirmation email (unchanged). +2. User clicks the confirmation link → Supabase marks `email_confirmed_at` + (unchanged). +3. User signs in via `loginAction` → `signInWithPassword` → Supabase issues a + JWT with `email_verified: true` (unchanged). +4. Frontend calls the Django API with `Authorization: Bearer ` + (unchanged). +5. `SupabaseJWTAuthentication.authenticate`: + - verifies the JWT via JWKS (unchanged); + - fast-path: if a `UserProfile` already exists for the JWT `sub`, return it; + - else require `email_verified` (reject unverified — prevents an attacker + registering a victim's email to claim/create a local account); + - if a Django User matches the verified email → link its blank profile + (or auto-create the missing profile); + - if **no** Django User matches → **auto-provision**: create a shadow + `auth.User` (unusable password) + `UserProfile` linked to `sub`, in one + transaction. +6. The request succeeds; the user now appears in Django Admin → Users. + +### Security invariants preserved + +- An **unverified** email never provisions anything. +- An existing `supabase_user_id` is **never overwritten** + (`filter(supabase_user_id__isnull=True)` + DB unique constraint). +- An attacker who registers a victim's email in Supabase and verifies it + gets, at most, a fresh empty profile under that email — they cannot steal + the victim's existing profile (matched first by `sub`) or overwrite its + link (rejected as `LINKED_ELSEWHERE`). +- Auto-provisioning is idempotent under concurrent first-logins: the loser of + the `supabase_user_id` unique race re-fetches the winner's profile. + +## Files modified + +| File | Change | +|---|---| +| `users/authentication.py` | Replaced the "refuse unprovisioned" branch with auto-provisioning; added `_auto_provision_user` and `_auto_provision_profile` helpers (transactional, race-idempotent); updated the class docstring. Added `IntegrityError`/`transaction` imports. | +| `users/test_authentication.py` | Rewrote the two tests encoding the old "no auto-creation" contract to assert auto-provisioning instead; added `test_first_login_auto_creates_profile_when_user_has_none`, `test_first_login_auto_provisions_new_user_for_unknown_verified_email`, `test_unverified_unknown_email_is_not_auto_provisioned`; updated the `FirstLoginLinkingTests` docstring. | +| `users/admin.py` | Updated `UserProfileAdmin` docstring: admin pre-creation is now optional, not required. | +| `frontend/src/app/(auth)/login/page.tsx` | Replaced the false "Administrator must provision…" notice with "Check your email to confirm your address, then sign in — your workspace is provisioned automatically on first login." | +| `frontend/src/app/(auth)/signup/page.tsx` | Replaced the helper text to match the new auto-provisioning flow. | +| `frontend/src/app/actions/auth.ts` | Updated the module docstring and the `signupAction` comment to reflect auto-provisioning. | + +## Verification + +- `pytest users/test_authentication.py` → **14 passed** (incl. the security + invariants: never-overwrite, verified-email-required, case-insensitive + match, fast-path second login). +- `pytest` (full suite) → **201 passed**, no regressions. +- `python manage.py check` → no issues. + +## Follow-up / optional + +- **Existing users**: the two already-confirmed Supabase users (with no + Django side) will be auto-provisioned on their next login — no migration + needed. The orphan `dledzion` Django User (no profile) will be linked to + its Supabase identity on next verified login. +- **`emailRedirectTo`**: the `signUp` call passes no redirect URL, so the + confirmation link points at the Supabase default site URL. It works, but + adding `emailRedirectTo` to a frontend callback would improve UX — requires + allowlisting the URL in the Supabase dashboard. +- **Pre-existing `.env` lines 13–15 are malformed** (stray `"`/`:` in + `ANTHROPIC_*` entries) — `python-dotenv` logs parse warnings. Unrelated to + this fix but worth cleaning up. diff --git a/docs/superpowers/specs/2026-07-09-userprofile-onboarding-state-machine-design.md b/docs/superpowers/specs/2026-07-09-userprofile-onboarding-state-machine-design.md new file mode 100644 index 0000000..d4df3d6 --- /dev/null +++ b/docs/superpowers/specs/2026-07-09-userprofile-onboarding-state-machine-design.md @@ -0,0 +1,147 @@ +# UserProfile Onboarding State Machine — Design + +Date: 2026-07-09 +Status: implementing (per user directive "continue until everything is implemented, tested and verified") + +## Goal + +Give `UserProfile` an onboarding status (`PENDING` / `READY` / `DISABLED`) so a +freshly auto-provisioned user can authenticate and see the dashboard but cannot +run billable/external work until an Administrator assigns the required +workspace configuration and marks them `READY`. + +## States & permissions + +| Status | Auth | Dashboard | Run Search | Enrich | Push to Instantly | Frontend message | +|---|---|---|---|---|---|---| +| PENDING | ✅ | ✅ | ❌ | ❌ | ❌ | "Your account has been created successfully. Your administrator is configuring your workspace. You will be able to run searches after your administrator assigns your Search Criteria, Campaign and Sender Account." | +| READY | ✅ | ✅ | ✅ | ✅ | ✅ | (none — Run Search enabled) | +| DISABLED | ✅ | ✅ | ❌ | ❌ | ❌ | "Your account has been disabled by the administrator." | + +Default for a newly auto-provisioned user: `PENDING`, `search_criteria=NULL`, +`assigned_campaign=NULL`, `sender_account=NULL`, `workspace=NULL`, +`daily_sending_limit=0`. + +## Data model + +New `OnboardingStatus(models.TextChoices)`: `PENDING`/`READY`/`DISABLED`. + +`UserProfile` new fields: +- `status` — `CharField(max_length=16, choices=OnboardingStatus.choices, default=OnboardingStatus.PENDING)` + `CheckConstraint(status__in=[...])` (matches the `Campaign`/`Person` idiom). +- `workspace` — `ForeignKey("users.Workspace", on_delete=PROTECT, null=True, blank=True)`. +- `sender_account` — `ForeignKey("users.SenderAccount", on_delete=PROTECT, null=True, blank=True)`. +- `daily_sending_limit` — `PositiveIntegerField(default=0)`. + +New models (in `users` app, to keep the assignment relations local to +`UserProfile`): +- `Workspace` — `name` (CharField 255, unique), `instantly_workspace_id` (CharField 255, blank), `created_at`/`updated_at`. +- `SenderAccount` — `name` (CharField 255, unique), `instantly_sender_id` (CharField 255, blank), `from_email` (EmailField, blank), `created_at`/`updated_at`. + +> Assumption: the spec lists Workspace & SenderAccount as admin-configurable +> *assignments* alongside SearchCriteria & Campaign (both real models with +> relations on `UserProfile`). New models are the consistent, spec-aligned +> choice. If the user intended simple free-text IDs, pivot is cheap. + +Migration: `users/0005_*` adds the two models + the four `UserProfile` fields + +the status `CheckConstraint`. Existing 0 profiles → default `PENDING`. + +### READY validation + +`UserProfile.clean()` raises `ValidationError` if `status == READY` while any of +`search_criteria`, `assigned_campaign`, `workspace`, `sender_account` is null. +`daily_sending_limit` is **not** required for READY (it is optional config). + +## Server-side enforcement + +Custom DRF permission `users.permissions.IsReady`: +- `has_permission`: if `request.user.is_authenticated` and the view starts + external work, require `request.user.profile.status == READY`; else 403. +- Applied to `TriggerBrowserJobView` (Run Search) and `TriggerCampaignJobView` + (Push to Instantly) via `permission_classes = [IsAuthenticated, IsReady]`. + +Enrichment cascades from `BrowserJob` (`browser/tasks.py`), so gating Run Search +gates enrichment transitively. No service-layer change needed. + +Unchanged: `InstantlyWebhookView` (HMAC, no user), `ActivityListView` +(harmless read), the three analytics views (already `IsAdminUser`). + +## Admin + +`UserProfileAdmin` + inline: add `workspace`, `sender_account`, +`daily_sending_limit`, `status` (read-only when not settable) to the editing +surface. Admin actions: +- `mark_ready` — sets `READY`, runs `clean()` validation; errors if required + fields missing. +- `mark_pending` — sets `PENDING`. +- `disable` — sets `DISABLED`. + +New `WorkspaceAdmin` / `SenderAccountAdmin` for CRUD. + +## API + +No new endpoint required for UX (frontend reads status from Supabase RLS via +the existing `useMyProfile()` hook — `select("*")` returns the new column +automatically). The existing action endpoints gain the `IsReady` permission. +Response bodies unchanged (hand-rolled dict convention preserved). + +> If a trusted server-side status source is later desired, add `GET /api/me/`; +> out of scope here. + +## Frontend + +- `OnboardingBanner` component (dashboard + search page) driven by + `useMyProfile().status`: + - PENDING → the spec message (info styling). + - DISABLED → "Your account has been disabled by the administrator." + (destructive/warning styling). +- Search page `Run search` button: `disabled = run.isPending || !criteria || + status !== "READY"`. +- Dashboard: render the banner above the stat grid; stat cards/jobs/activity + remain (PENDING user sees zeros — harmless). +- Proxy untouched (it is an optimistic auth guard, not an authz layer; the + docstring discourages using it for authz). + +## Auth preservation + +Auto-provision (`users/authentication.py`) is unchanged in logic — +`status` defaults to `PENDING` via the field default; no code change to the +auth backend. Existing auth tests (14) and the full suite (201) must remain +green. + +## Tests + +Backend (Django, pytest): +- Model `clean()`: READY rejected without the 4 required fields; accepted with + them. +- `IsReady` permission: allows READY, denies PENDING & DISABLED. +- `TriggerBrowserJobView`: 403 PENDING, 403 DISABLED, 202 READY. +- `TriggerCampaignJobView`: 403 PENDING, 202 READY. +- Auto-provision: a newly provisioned `UserProfile` has `status=PENDING` and + null assignments + `daily_sending_limit=0`. +- Admin action `mark_ready`: succeeds with required fields, fails without. + +Integration tests (`tests/`): new-signup → PENDING; pending-user cannot run +search; ready-user can; disabled-user cannot; administrator-activation flips +PENDING→READY when configured. + +Frontend: no test framework exists in the repo; verify via TypeScript +typecheck/build. + +## Files to touch + +- `users/models.py` — `OnboardingStatus`, `Workspace`, `SenderAccount`, new + `UserProfile` fields + `Meta`/`clean()`. +- `users/migrations/0005_*.py` — generated. +- `users/permissions.py` — `IsReady` (new file). +- `users/admin.py` — `WorkspaceAdmin`, `SenderAccountAdmin`, `UserProfileAdmin` + fields + actions. +- `browser/views.py`, `campaigns/views.py` — add `IsReady` to + `permission_classes`. +- `users/test_authentication.py` — assert new profile is PENDING (additive). +- `users/test_onboarding.py` (new) — model + permission + admin tests. +- `tests/test_onboarding_integration.py` (new) — end-to-end status gating. +- `frontend/src/components/OnboardingBanner.tsx` (new). +- `frontend/src/app/(app)/dashboard/page.tsx` — banner. +- `frontend/src/app/(app)/search/page.tsx` — banner + disabled guard. +- `frontend/src/features/search/hooks.ts` — extend `useMyProfile` return type + with `status`. diff --git a/services/supabase/auth.py b/services/supabase/auth.py index 6a579e3..37c51e8 100644 --- a/services/supabase/auth.py +++ b/services/supabase/auth.py @@ -46,3 +46,19 @@ def verify_access_token(token: str) -> dict: except jwt.PyJWTError as exc: raise InvalidSupabaseToken(str(exc)) from exc return claims + + +def claim_email_verified(claims: dict) -> bool: + """Whether Supabase's verified-email flag is set in the JWT claims. + + Supabase emits this flag at the top level of the access token in some + GoTrue releases and nested under ``user_metadata`` in others (this + project emits it only under ``user_metadata``). Either location counts + as authoritative, mirroring how the Supabase client resolves it — so a + genuinely confirmed user is never treated as unverified merely because + of where GoTrue placed the flag. + """ + if claims.get("email_verified"): + return True + metadata = claims.get("user_metadata") or {} + return bool(metadata.get("email_verified")) diff --git a/services/supabase/tests.py b/services/supabase/tests.py index cfce3bd..3c320a4 100644 --- a/services/supabase/tests.py +++ b/services/supabase/tests.py @@ -5,7 +5,11 @@ from cryptography.hazmat.primitives.asymmetric import ec from django.test import SimpleTestCase, override_settings -from services.supabase.auth import InvalidSupabaseToken, verify_access_token +from services.supabase.auth import ( + InvalidSupabaseToken, + claim_email_verified, + verify_access_token, +) ISSUER = "https://project.supabase.co/auth/v1" @@ -79,3 +83,28 @@ def test_token_signed_by_a_different_key_is_rejected(self): with self.assertRaises(InvalidSupabaseToken): verify_access_token(token) + + +class ClaimEmailVerifiedTests(SimpleTestCase): + """The verified-email flag may live at the top level or nested under + ``user_metadata`` depending on the GoTrue release — either counts.""" + + def test_true_at_top_level(self): + self.assertTrue(claim_email_verified({"email_verified": True})) + + def test_true_only_in_user_metadata(self): + # This project's Supabase emits the flag only here. + self.assertTrue( + claim_email_verified({"user_metadata": {"email_verified": True}}) + ) + + def test_false_when_absent_everywhere(self): + self.assertFalse(claim_email_verified({"sub": "x"})) + + def test_false_when_metadata_flag_is_false(self): + self.assertFalse( + claim_email_verified({"user_metadata": {"email_verified": False}}) + ) + + def test_handles_null_user_metadata(self): + self.assertFalse(claim_email_verified({"user_metadata": None})) diff --git a/tests/test_discovery_pipeline.py b/tests/test_discovery_pipeline.py index 0e6214c..4595372 100644 --- a/tests/test_discovery_pipeline.py +++ b/tests/test_discovery_pipeline.py @@ -48,6 +48,9 @@ def fake_richapi_response(self, person): ) class DiscoveryPipelineTests(TestCase): def setUp(self): + from campaigns.models import Campaign + from users.models import OnboardingStatus, SenderAccount, Workspace + self.supabase_user_id = uuid.uuid4() user = get_user_model().objects.create_user(username="grace") user.set_unusable_password() @@ -59,6 +62,11 @@ def setUp(self): user=user, supabase_user_id=self.supabase_user_id, search_criteria=criteria, + assigned_campaign=Campaign.objects.create(name="Q3"), + sender_account=SenderAccount.objects.create(name="grace-sender"), + workspace=Workspace.objects.create(name="grace-workspace"), + daily_sending_limit=50, + status=OnboardingStatus.READY, ) self.claims = {"sub": str(self.supabase_user_id)} diff --git a/tests/test_onboarding_integration.py b/tests/test_onboarding_integration.py new file mode 100644 index 0000000..a70bf4e --- /dev/null +++ b/tests/test_onboarding_integration.py @@ -0,0 +1,171 @@ +"""End-to-end onboarding integration tests. + +Walks a user through the full lifecycle against the real (eager-Celery) +pipeline, exercising the Supabase JWT auth path, the IsReady gate on both +user-triggered action endpoints, and administrator activation. + +Scenarios (per spec): + 1. new signup → auto-provisioned PENDING, no assignments, limit 0 + 2. pending user → authenticates, dashboard-readable, CANNOT run search + 3. ready user → CAN run search (and push to Instantly) + 4. disabled user → authenticates, CANNOT run search + 5. administrator activation → PENDING → READY only once fully configured +""" + +import uuid +from unittest import mock + +from django.test import RequestFactory, TestCase +from django.urls import reverse + +from campaigns.models import Campaign +from users.authentication import SupabaseJWTAuthentication +from users.models import OnboardingStatus, SearchCriteria, SenderAccount, UserProfile, Workspace + + +def auth_as(claims): + return mock.patch("users.authentication.verify_access_token", return_value=claims) + + +def _signup_via_auth(email, sub): + """Simulate the first verified login that auto-provisions a user, exactly + as SupabaseJWTAuthentication would after a confirmed email signup.""" + auth = SupabaseJWTAuthentication() + from rest_framework.test import APIRequestFactory + + request = APIRequestFactory().get("/", HTTP_AUTHORIZATION="Bearer token") + claims = {"sub": str(sub), "email": email, "email_verified": True} + with auth_as(claims): + user, _ = auth.authenticate(request) + return user + + +class OnboardingLifecycleTests(TestCase): + def setUp(self): + self.sub = uuid.uuid4() + self.email = "newuser@example.com" + self.claims = {"sub": str(self.sub), "email": self.email, "email_verified": True} + + # --- 1. new signup ------------------------------------------------- + def test_new_signup_provisions_a_pending_profile(self): + user = _signup_via_auth(self.email, self.sub) + profile = user.profile + self.assertEqual(profile.status, OnboardingStatus.PENDING) + self.assertIsNone(profile.search_criteria_id) + self.assertIsNone(profile.assigned_campaign_id) + self.assertIsNone(profile.sender_account_id) + self.assertIsNone(profile.workspace_id) + self.assertEqual(profile.daily_sending_limit, 0) + + # --- 2. pending user ---------------------------------------------- + @mock.patch("browser.views.run_browser_job.delay") + def test_pending_user_cannot_run_search(self, _delay): + _signup_via_auth(self.email, self.sub) + with auth_as(self.claims): + response = self.client.post( + reverse("api:browser-job-trigger"), HTTP_AUTHORIZATION="Bearer token" + ) + self.assertEqual(response.status_code, 403) + _delay.assert_not_called() + + @mock.patch("campaigns.views.run_campaign_job.delay") + def test_pending_user_cannot_push_to_instantly(self, _delay): + _signup_via_auth(self.email, self.sub) + with auth_as(self.claims): + response = self.client.post( + reverse("api:campaign-job-trigger"), + {"person_ids": [1]}, + content_type="application/json", + HTTP_AUTHORIZATION="Bearer token", + ) + self.assertEqual(response.status_code, 403) + _delay.assert_not_called() + + def test_pending_user_can_authenticate(self): + # The IsReady gate lives on action endpoints, not on auth. A pending + # user authenticates fine (SupabaseJWTAuthentication returns the user). + from rest_framework.test import APIRequestFactory + + _signup_via_auth(self.email, self.sub) + request = APIRequestFactory().get("/", HTTP_AUTHORIZATION="Bearer token") + auth = SupabaseJWTAuthentication() + with auth_as(self.claims): + result = auth.authenticate(request) + self.assertIsNotNone(result) + user, _ = result + self.assertEqual(user.profile.status, OnboardingStatus.PENDING) + + # --- 3. ready user ------------------------------------------------- + @mock.patch("browser.views.run_browser_job.delay") + def test_ready_user_can_run_search(self, _delay): + user = _signup_via_auth(self.email, self.sub) + profile = user.profile + profile.search_criteria = SearchCriteria.objects.create( + job_position="PM", country="DE", industry="Logistics" + ) + profile.assigned_campaign = Campaign.objects.create(name="Q3") + profile.sender_account = SenderAccount.objects.create(name="s1") + profile.workspace = Workspace.objects.create(name="w1") + profile.status = OnboardingStatus.READY + profile.full_clean(validate_unique=False) + profile.save() + + with auth_as(self.claims): + response = self.client.post( + reverse("api:browser-job-trigger"), HTTP_AUTHORIZATION="Bearer token" + ) + self.assertEqual(response.status_code, 202) + _delay.assert_called_once() + + # --- 4. disabled user --------------------------------------------- + @mock.patch("browser.views.run_browser_job.delay") + def test_disabled_user_cannot_run_search_but_authenticates(self, _delay): + user = _signup_via_auth(self.email, self.sub) + user.profile.status = OnboardingStatus.DISABLED + user.profile.save() + + with auth_as(self.claims): + response = self.client.post( + reverse("api:browser-job-trigger"), HTTP_AUTHORIZATION="Bearer token" + ) + self.assertEqual(response.status_code, 403) + _delay.assert_not_called() + + # --- 5. administrator activation --------------------------------- + def test_administrator_cannot_activate_without_full_configuration(self): + from users.admin import UserProfileAdmin + from django.contrib import admin as django_admin + + user = _signup_via_auth(self.email, self.sub) + profile = user.profile + + admin = UserProfileAdmin(UserProfile, django_admin.site) + request = RequestFactory().post("/") + request._messages = mock.Mock() + request._messages.add = mock.Mock() + admin.mark_ready(request, UserProfile.objects.filter(pk=profile.pk)) + profile.refresh_from_db() + # Missing all assignments → must remain PENDING. + self.assertEqual(profile.status, OnboardingStatus.PENDING) + + def test_administrator_activates_once_fully_configured(self): + from users.admin import UserProfileAdmin + from django.contrib import admin as django_admin + + user = _signup_via_auth(self.email, self.sub) + profile = user.profile + profile.search_criteria = SearchCriteria.objects.create( + job_position="PM", country="DE", industry="Logistics" + ) + profile.assigned_campaign = Campaign.objects.create(name="Q3") + profile.sender_account = SenderAccount.objects.create(name="s1") + profile.workspace = Workspace.objects.create(name="w1") + profile.save() + + admin = UserProfileAdmin(UserProfile, django_admin.site) + request = RequestFactory().post("/") + request._messages = mock.Mock() + request._messages.add = mock.Mock() + admin.mark_ready(request, UserProfile.objects.filter(pk=profile.pk)) + profile.refresh_from_db() + self.assertEqual(profile.status, OnboardingStatus.READY) diff --git a/users/admin.py b/users/admin.py index d083201..7d15d97 100644 --- a/users/admin.py +++ b/users/admin.py @@ -2,21 +2,27 @@ from django.contrib.auth import get_user_model from django.contrib.auth.admin import UserAdmin -from .models import SearchCriteria, UserProfile +from .models import ( + OnboardingStatus, + SearchCriteria, + SenderAccount, + UserProfile, + Workspace, +) class UserProfileInline(admin.StackedInline): """Profile fields edited from the User admin page (the classic Django "profile as inline" pattern). `extra=1, max_num=1` surfaces one blank profile form when creating a User so the Administrator can provision - supabase_user_id / criteria / campaign at the same moment, without a - separate round-trip — and never more than one (it's a OneToOne).""" + assignments / status at the same moment, without a separate round-trip + — and never more than one (it's a OneToOne).""" model = UserProfile can_delete = False extra = 1 max_num = 1 - autocomplete_fields = ("search_criteria", "assigned_campaign") + autocomplete_fields = ("search_criteria", "assigned_campaign", "workspace", "sender_account") class ExtendedUserAdmin(UserAdmin): @@ -29,25 +35,82 @@ class SearchCriteriaAdmin(admin.ModelAdmin): search_fields = ("job_position", "country", "industry") +@admin.register(Workspace) +class WorkspaceAdmin(admin.ModelAdmin): + list_display = ("name", "instantly_workspace_id", "created_at") + search_fields = ("name", "instantly_workspace_id") + + +@admin.register(SenderAccount) +class SenderAccountAdmin(admin.ModelAdmin): + list_display = ("name", "instantly_sender_id", "from_email", "created_at") + search_fields = ("name", "instantly_sender_id", "from_email") + + @admin.register(UserProfile) class UserProfileAdmin(admin.ModelAdmin): """Standalone management of the local profile that links a Django User to its Supabase Auth identity (`supabase_user_id`) and to the - administrator-assigned SearchCriteria and Campaign. - - This is the provisioning surface: the Administrator creates a User, - then creates this profile pointing at the user's Supabase Auth UUID - and assigning the criteria/campaign they may use. Authentication - (users/authentication.py) resolves a Django User by matching the JWT - `sub` against `supabase_user_id`, so a profile with no `supabase_user_id` - can never authenticate — it is not auto-created (the auth backend - deliberately does not auto-provision). The `backfill_profiles` command - creates empty shells for pre-existing users lacking a profile.""" - - list_display = ("user", "supabase_user_id", "search_criteria", "assigned_campaign") - list_filter = ("assigned_campaign",) + administrator-assigned SearchCriteria, Campaign, SenderAccount, + Workspace, daily sending limit, and onboarding status. + + Onboarding is administrator-controlled: a profile starts PENDING (an + auto-provisioned signup) and the Administrator assigns the four + required pieces (SearchCriteria, Campaign, SenderAccount, Workspace) + then flips it to READY via the `mark_ready` action. `clean()` blocks + READY until all four are present.""" + + list_display = ( + "user", + "status", + "supabase_user_id", + "search_criteria", + "assigned_campaign", + "sender_account", + "workspace", + "daily_sending_limit", + ) + list_filter = ("status", "assigned_campaign", "workspace", "sender_account") search_fields = ("user__username", "user__email", "supabase_user_id") - autocomplete_fields = ("user", "search_criteria", "assigned_campaign") + autocomplete_fields = ( + "user", + "search_criteria", + "assigned_campaign", + "workspace", + "sender_account", + ) + actions = ("mark_ready", "mark_pending", "disable") + + @admin.action(description="Mark selected profiles as READY (requires all assignments)") + def mark_ready(self, request, queryset): + updated = 0 + skipped = [] + for profile in queryset: + profile.status = OnboardingStatus.READY + try: + profile.full_clean(validate_unique=False) + except Exception as exc: # noqa: BLE001 — surface every reason + skipped.append(f"{profile}: {exc}") + continue + profile.save() + updated += 1 + self.message_user(request, f"{updated} profile(s) marked READY.") + if skipped: + self.message_user( + request, + "Skipped (missing assignments): " + "; ".join(skipped), + level=40, # WARNING + ) + + @admin.action(description="Mark selected profiles as PENDING") + def mark_pending(self, request, queryset): + count = queryset.update(status=OnboardingStatus.PENDING) + self.message_user(request, f"{count} profile(s) marked PENDING.") + + @admin.action(description="Disable selected profiles (DISABLED)") + def disable(self, request, queryset): + count = queryset.update(status=OnboardingStatus.DISABLED) + self.message_user(request, f"{count} profile(s) disabled.") admin.site.unregister(get_user_model()) diff --git a/users/authentication.py b/users/authentication.py index 1e899f4..9299b8f 100644 --- a/users/authentication.py +++ b/users/authentication.py @@ -1,10 +1,15 @@ import uuid from django.contrib.auth import get_user_model +from django.db import IntegrityError, transaction from rest_framework.authentication import BaseAuthentication from rest_framework.exceptions import AuthenticationFailed -from services.supabase.auth import InvalidSupabaseToken, verify_access_token +from services.supabase.auth import ( + InvalidSupabaseToken, + claim_email_verified, + verify_access_token, +) from .models import UserProfile @@ -17,21 +22,26 @@ class SupabaseJWTAuthentication(BaseAuthentication): Django's own ModelBackend and a real Django password; this class is never consulted for /admin/. See docs/architecture/authentication.md. - Provisioning model (administrator-controlled, no auto-creation): - * The Administrator creates a Django User and a UserProfile (admin - inline / standalone, or the `backfill_profiles` command). - * `supabase_user_id` may be left blank at provisioning time — the - Administrator no longer copies the UUID by hand. - * On the first successful JWT login, the blank profile is linked to - the Supabase identity by matching the JWT's *verified* email to - the Django User's email, and `supabase_user_id` is populated from - the JWT `sub`. Requiring a Supabase-verified email prevents an - attacker from registering a victim's email in Supabase to claim - their local profile. - * A profile is never auto-created: if no UserProfile exists for the - email, authentication fails exactly as before. + Provisioning model (auto-provisioning on first verified login): + * The Administrator *may* pre-create a Django User and a UserProfile + (admin inline / standalone, or the `backfill_profiles` command), + leaving `supabase_user_id` blank — but no longer *must*. + * On the first successful JWT login, the backend links an existing + blank profile to the Supabase identity by matching the JWT's + *verified* email to the Django User's email, and populates + `supabase_user_id` from the JWT `sub`. + * If no Django User matches the verified email at all, the backend + auto-provisions one: a shadow `auth.User` with an unusable + password (identity remains Supabase's responsibility) plus a + `UserProfile` linked to the Supabase identity. This is what lets + a self-service signup reach the API without an Administrator. + * Requiring a Supabase-VERIFIED email before any provisioning + prevents an attacker from registering a victim's email in Supabase + to claim or create a local account before confirmation. * An existing `supabase_user_id` is never overwritten — enforced at the database layer with `filter(supabase_user_id__isnull=True)`. + * SearchCriteria and Campaign assignment remain + administrator-controlled; this class never touches them. SearchCriteria and Campaign assignment remain administrator-controlled; this class never touches them. @@ -69,13 +79,12 @@ def authenticate(self, request): if profile is not None: return (profile.user, claims) - # First successful login for an Administrator-provisioned profile - # whose supabase_user_id is still blank. Link it from the JWT - # subject — but only via a Supabase-VERIFIED email, so that an - # attacker cannot register a victim's email in Supabase and claim - # their local profile. No profile is auto-created here, and an - # existing supabase_user_id is never overwritten. - if not claims.get("email_verified"): + # First verified login. A Supabase-VERIFIED email is required + # before any provisioning or linking, so that an attacker cannot + # register a victim's email in Supabase and claim or create a + # local account before confirmation. An existing supabase_user_id + # is never overwritten. + if not claim_email_verified(claims): raise AuthenticationFailed(self.UNPROVISIONED) email = claims.get("email") @@ -84,12 +93,17 @@ def authenticate(self, request): user = get_user_model().objects.filter(email__iexact=email).first() if user is None: - raise AuthenticationFailed(self.UNPROVISIONED) - - # Atomically claim the blank profile. The isnull filter makes - # "never overwrite" a database-level guarantee; the unique column - # backs it under concurrent first-logins (the loser's update - # touches zero rows and falls through to the re-fetch below). + # No local account for this verified email → auto-provision a + # shadow Django User (unusable password; identity is Supabase's + # responsibility) plus a UserProfile linked to the Supabase + # identity. This is the self-service signup path. + return self._auto_provision_user(email, supabase_user_id, claims) + + # An Administrator-provisioned User exists. Link a blank profile to + # the Supabase identity — the isnull filter makes "never overwrite" + # a database-level guarantee; the unique column backs it under + # concurrent first-logins (the loser's update touches zero rows + # and falls through to the re-fetch below). linked = ( UserProfile.objects.filter(user=user, supabase_user_id__isnull=True) .update(supabase_user_id=supabase_user_id) @@ -98,15 +112,71 @@ def authenticate(self, request): profile = UserProfile.objects.select_related("user").get(user=user) return (profile.user, claims) - # Zero rows touched: the profile is missing, or was linked between - # our lookup and our update (possibly to this same identity). + # Zero rows touched: a profile already exists (linked or not), or + # the User exists without a profile. Resolve which. try: profile = UserProfile.objects.select_related("user").get(user=user) - except UserProfile.DoesNotExist as exc: - raise AuthenticationFailed(self.UNPROVISIONED) from exc + except UserProfile.DoesNotExist: + # Administrator created the User but not the profile — + # auto-create the missing profile linked to this identity. + return self._auto_provision_profile(user, supabase_user_id, claims) if profile.supabase_user_id == supabase_user_id: return (profile.user, claims) raise AuthenticationFailed(self.LINKED_ELSEWHERE) + def _auto_provision_user(self, email, supabase_user_id, claims): + """Create a shadow Django User (unusable password) and a UserProfile + linked to the Supabase identity, for a verified email that has no + local account yet. + + Idempotent under concurrent first-logins: the loser of the + `supabase_user_id` unique race re-fetches the winner's profile + instead of creating a duplicate. A username collision (a + pre-existing User reusing the email as a username under a different + email field) falls back to an opaque, UUID-derived username. + """ + User = get_user_model() + try: + with transaction.atomic(): + # create_user(password=None) sets an unusable password — + # identity remains Supabase Auth's responsibility. + user = User.objects.create_user( + username=email[:150], email=email + ) + UserProfile.objects.create( + user=user, supabase_user_id=supabase_user_id + ) + except IntegrityError: + profile = ( + UserProfile.objects.select_related("user") + .filter(supabase_user_id=supabase_user_id) + .first() + ) + if profile is not None: + return (profile.user, claims) + with transaction.atomic(): + user = User.objects.create_user( + username=f"user-{supabase_user_id.hex[:12]}"[:150], + email=email, + ) + UserProfile.objects.create( + user=user, supabase_user_id=supabase_user_id + ) + return (user, claims) + + def _auto_provision_profile(self, user, supabase_user_id, claims): + """Create the missing UserProfile for an existing Django User, + linked to the Supabase identity. Idempotent under the same race.""" + try: + with transaction.atomic(): + profile = UserProfile.objects.create( + user=user, supabase_user_id=supabase_user_id + ) + except IntegrityError: + profile = UserProfile.objects.select_related("user").get( + supabase_user_id=supabase_user_id + ) + return (profile.user, claims) + def authenticate_header(self, request): return self.keyword diff --git a/users/migrations/0005_userprofile_onboarding_state.py b/users/migrations/0005_userprofile_onboarding_state.py new file mode 100644 index 0000000..86e7873 --- /dev/null +++ b/users/migrations/0005_userprofile_onboarding_state.py @@ -0,0 +1,104 @@ +# Generated by Django 6.0.6 on 2026-07-09 06:37 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("campaigns", "0002_enable_row_level_security"), + ("users", "0004_userprofile_assigned_campaign"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="SenderAccount", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=255, unique=True)), + ("instantly_sender_id", models.CharField(blank=True, max_length=255)), + ("from_email", models.EmailField(blank=True, max_length=254)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ], + ), + migrations.CreateModel( + name="Workspace", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=255, unique=True)), + ( + "instantly_workspace_id", + models.CharField(blank=True, max_length=255), + ), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ], + ), + migrations.AddField( + model_name="userprofile", + name="daily_sending_limit", + field=models.PositiveIntegerField(default=0), + ), + migrations.AddField( + model_name="userprofile", + name="status", + field=models.CharField( + choices=[ + ("PENDING", "Pending"), + ("READY", "Ready"), + ("DISABLED", "Disabled"), + ], + default="PENDING", + max_length=16, + ), + ), + migrations.AddField( + model_name="userprofile", + name="sender_account", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.PROTECT, + related_name="assigned_profiles", + to="users.senderaccount", + ), + ), + migrations.AddField( + model_name="userprofile", + name="workspace", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.PROTECT, + related_name="assigned_profiles", + to="users.workspace", + ), + ), + migrations.AddConstraint( + model_name="userprofile", + constraint=models.CheckConstraint( + condition=models.Q(("status__in", ["PENDING", "READY", "DISABLED"])), + name="users_userprofile_status_valid", + ), + ), + ] diff --git a/users/models.py b/users/models.py index 3abad27..883144f 100644 --- a/users/models.py +++ b/users/models.py @@ -1,4 +1,5 @@ from django.conf import settings +from django.core.exceptions import ValidationError from django.db import models @@ -21,6 +22,52 @@ def __str__(self): return f"{self.job_position} / {self.country} / {self.industry}" +class Workspace(models.Model): + """A tenant workspace an end user is assigned to. Administrator-owned. + The `instantly_workspace_id` mirrors the Campaign.instantly_campaign_id + convention — a free-text external id, not a foreign key.""" + + name = models.CharField(max_length=255, unique=True) + instantly_workspace_id = models.CharField(max_length=255, blank=True) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + def __str__(self): + return self.name + + +class SenderAccount(models.Model): + """An Instantly sender account an end user sends from. + Administrator-owned. `instantly_sender_id` is a free-text external id, + mirroring Campaign.instantly_campaign_id.""" + + name = models.CharField(max_length=255, unique=True) + instantly_sender_id = models.CharField(max_length=255, blank=True) + from_email = models.EmailField(blank=True) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + def __str__(self): + return self.name + + +class OnboardingStatus(models.TextChoices): + """Lifecycle of a UserProfile through administrator-controlled onboarding. + + PENDING — auto-provisioned; can authenticate and see the dashboard but + cannot run searches / enrich / push to Instantly until the + Administrator assigns the required workspace config and marks + the profile READY. + READY — fully provisioned; all user-triggered external actions allowed. + DISABLED — administrator-revoked; authentication still works and the + dashboard opens, but external actions are blocked. + """ + + PENDING = "PENDING", "Pending" + READY = "READY", "Ready" + DISABLED = "DISABLED", "Disabled" + + class UserProfile(models.Model): """Extends auth.User rather than replacing it, per Django convention. @@ -29,7 +76,14 @@ class UserProfile(models.Model): auth.User with an unusable password, kept only so Django's ORM, permissions, and admin have something to point at. `supabase_user_id` is the real join key, matched against the verified JWT's `sub` claim - (and against Postgres's auth.uid() for RLS policies).""" + (and against Postgres's auth.uid() for RLS policies). + + Onboarding is administrator-controlled: a freshly auto-provisioned + profile starts PENDING with no assignments and daily_sending_limit=0; + the Administrator assigns SearchCriteria, Campaign, SenderAccount and + Workspace, then marks it READY. See + docs/superpowers/specs/2026-07-09-userprofile-onboarding-state-machine-design.md. + """ user = models.OneToOneField( settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="profile" @@ -52,6 +106,57 @@ class UserProfile(models.Model): null=True, blank=True, ) + workspace = models.ForeignKey( + Workspace, + on_delete=models.PROTECT, + related_name="assigned_profiles", + null=True, + blank=True, + ) + sender_account = models.ForeignKey( + SenderAccount, + on_delete=models.PROTECT, + related_name="assigned_profiles", + null=True, + blank=True, + ) + daily_sending_limit = models.PositiveIntegerField(default=0) + status = models.CharField( + max_length=16, + choices=OnboardingStatus.choices, + default=OnboardingStatus.PENDING, + ) + + class Meta: + constraints = [ + models.CheckConstraint( + condition=models.Q( + status__in=[choice.value for choice in OnboardingStatus] + ), + name="users_userprofile_status_valid", + ), + ] + + def clean(self): + super().clean() + # READY is only valid once every required assignment is present. + # daily_sending_limit is configurable but NOT required for READY. + if self.status == OnboardingStatus.READY: + missing = [ + label + for label, value in ( + ("search_criteria", self.search_criteria_id), + ("assigned_campaign", self.assigned_campaign_id), + ("sender_account", self.sender_account_id), + ("workspace", self.workspace_id), + ) + if not value + ] + if missing: + raise ValidationError( + "Cannot mark a profile READY before assigning: " + + ", ".join(missing) + ) def __str__(self): return f"Profile({self.user})" diff --git a/users/permissions.py b/users/permissions.py new file mode 100644 index 0000000..12498f8 --- /dev/null +++ b/users/permissions.py @@ -0,0 +1,36 @@ +"""DRF permission classes for the onboarding state machine. + +`IsReady` gates user-triggered external actions (Run Search, Push to +Instantly) behind a READY UserProfile. PENDING and DISABLED users can still +authenticate and read the dashboard — only billable/external work is blocked. +""" + +from rest_framework.permissions import BasePermission + +from .models import OnboardingStatus + + +class IsReady(BasePermission): + """Require the authenticated user's UserProfile to be READY. + + The SupabaseJWTAuthentication backend guarantees a profile exists on every + authenticated request (it auto-provisions one), so `request.user.profile` + is safe here. `IsAuthenticated` must precede this class in + `permission_classes` so anonymous requests never reach `has_permission`. + """ + + message = "Your workspace is not provisioned for this action yet." + + def has_permission(self, request, view): + profile = request.user.profile + if profile.status == OnboardingStatus.DISABLED: + self.message = "Your account has been disabled by the administrator." + return False + if profile.status != OnboardingStatus.READY: + self.message = ( + "Your administrator is still configuring your workspace. " + "You will be able to run searches once your Search Criteria, " + "Campaign and Sender Account are assigned." + ) + return False + return True diff --git a/users/test_authentication.py b/users/test_authentication.py index 152d60b..791fa55 100644 --- a/users/test_authentication.py +++ b/users/test_authentication.py @@ -77,10 +77,13 @@ def test_token_missing_subject_claim_raises_authentication_failed( class FirstLoginLinkingTests(TestCase): - """First successful JWT login for an Administrator-provisioned profile - whose `supabase_user_id` is still blank: the backend links it from the - JWT subject by matching a Supabase-VERIFIED email — no UUID copying, - no auto-creation, no overwrite.""" + """First successful JWT login for a Supabase-VERIFIED email. + + The backend links an existing Administrator-provisioned profile whose + `supabase_user_id` is still blank, and — when no local account exists + at all — auto-provisions a shadow Django User plus a UserProfile linked + to the Supabase identity. A verified email is always required first; + an existing `supabase_user_id` is never overwritten.""" def setUp(self): self.factory = APIRequestFactory() @@ -117,6 +120,44 @@ def test_first_login_links_blank_profile_by_verified_email(self, mock_verify): UserProfile.objects.get(user=user).supabase_user_id, sub ) + @mock.patch("users.authentication.verify_access_token") + def test_first_login_accepts_email_verified_in_user_metadata(self, mock_verify): + # Supabase (this project's GoTrue) emits `email_verified` only nested + # under `user_metadata`, not at the top level. Provisioning must still + # succeed — otherwise every self-service signup 401s on first login. + sub = uuid.uuid4() + user = self._make_user("mona", "mona@example.com") + UserProfile.objects.create(user=user) # blank supabase_user_id + + mock_verify.return_value = { + "sub": str(sub), + "email": "mona@example.com", + "user_metadata": {"email_verified": True}, + } + + resolved, _ = self.auth.authenticate(self._request(mock_verify.return_value)) + + self.assertEqual(resolved, user) + self.assertEqual( + UserProfile.objects.get(user=user).supabase_user_id, sub + ) + + @mock.patch("users.authentication.verify_access_token") + def test_first_login_rejects_when_user_metadata_flag_is_false(self, mock_verify): + # A nested flag that is explicitly false is still unverified. + sub = uuid.uuid4() + mock_verify.return_value = { + "sub": str(sub), + "email": "nope@example.com", + "user_metadata": {"email_verified": False}, + } + + with self.assertRaises(AuthenticationFailed): + self.auth.authenticate(self._request(mock_verify.return_value)) + self.assertFalse( + get_user_model().objects.filter(email__iexact="nope@example.com").exists() + ) + @mock.patch("users.authentication.verify_access_token") def test_first_login_matches_email_case_insensitively(self, mock_verify): sub = uuid.uuid4() @@ -171,31 +212,76 @@ def test_first_login_never_overwrites_an_existing_supabase_user_id(self, mock_ve ) @mock.patch("users.authentication.verify_access_token") - def test_first_login_does_not_auto_create_a_profile(self, mock_verify): + def test_first_login_auto_creates_profile_when_user_has_none(self, mock_verify): # Django User exists by email, but no UserProfile was provisioned. + # Auto-provisioning now creates the missing profile and links it to + # the Supabase identity, instead of rejecting the login. user = self._make_user("kim", "kim@example.com") + sub = uuid.uuid4() mock_verify.return_value = { - "sub": str(uuid.uuid4()), + "sub": str(sub), "email": "kim@example.com", "email_verified": True, } - with self.assertRaises(AuthenticationFailed): - self.auth.authenticate(self._request(mock_verify.return_value)) - self.assertEqual(UserProfile.objects.filter(user=user).count(), 0) + resolved, _ = self.auth.authenticate(self._request(mock_verify.return_value)) + + self.assertEqual(resolved, user) + self.assertEqual( + UserProfile.objects.get(user=user).supabase_user_id, sub + ) @mock.patch("users.authentication.verify_access_token") - def test_first_login_fails_when_email_is_unknown(self, mock_verify): + def test_first_login_auto_provisions_new_user_for_unknown_verified_email( + self, mock_verify + ): + # No Django User at all for this verified email → the backend now + # creates a shadow User (unusable password; identity lives in + # Supabase) plus a UserProfile linked to the Supabase identity. + sub = uuid.uuid4() mock_verify.return_value = { - "sub": str(uuid.uuid4()), + "sub": str(sub), "email": "nobody@example.com", "email_verified": True, } + resolved, _ = self.auth.authenticate(self._request(mock_verify.return_value)) + + self.assertTrue( + get_user_model() + .objects.filter(email__iexact="nobody@example.com") + .exists() + ) + profile = UserProfile.objects.get(supabase_user_id=sub) + self.assertEqual(profile.user, resolved) + # Identity is Supabase's responsibility — no local password. + self.assertFalse(resolved.has_usable_password()) + + @mock.patch("users.authentication.verify_access_token") + def test_unverified_unknown_email_is_not_auto_provisioned(self, mock_verify): + # A verified email is required before any provisioning, so an + # attacker cannot register a victim's email in Supabase to create a + # local account before confirmation. + sub = uuid.uuid4() + mock_verify.return_value = { + "sub": str(sub), + "email": "unverified@example.com", + "email_verified": False, + } + with self.assertRaises(AuthenticationFailed): self.auth.authenticate(self._request(mock_verify.return_value)) + self.assertFalse( + get_user_model() + .objects.filter(email__iexact="unverified@example.com") + .exists() + ) + self.assertEqual( + UserProfile.objects.filter(supabase_user_id=sub).count(), 0 + ) + @mock.patch("users.authentication.verify_access_token") def test_second_login_uses_the_fast_path_without_email(self, mock_verify): sub = uuid.uuid4() diff --git a/users/test_onboarding.py b/users/test_onboarding.py new file mode 100644 index 0000000..379f53a --- /dev/null +++ b/users/test_onboarding.py @@ -0,0 +1,261 @@ +"""Onboarding state machine tests: PENDING / READY / DISABLED. + +Covers model validation, the IsReady permission, trigger-view gating +(Run Search + Push to Instantly), auto-provision defaults, and +administrator activation (PENDING → READY). +""" + +import uuid +from unittest import mock + +from django.contrib import admin as django_admin +from django.contrib.auth import get_user_model +from django.core.exceptions import ValidationError +from django.test import RequestFactory, TestCase +from django.urls import reverse +from rest_framework.test import APIRequestFactory + +from campaigns.models import Campaign +from users.authentication import SupabaseJWTAuthentication +from users.models import ( + OnboardingStatus, + SearchCriteria, + SenderAccount, + UserProfile, + Workspace, +) +from users.permissions import IsReady + +from .admin import UserProfileAdmin + + +def auth_as(claims): + return mock.patch("users.authentication.verify_access_token", return_value=claims) + + +def _make_user(username): + user = get_user_model().objects.create_user(username=username) + user.set_unusable_password() + user.save() + return user + + +def _make_ready_profile(username, sub): + """A fully-provisioned READY profile (all four required assignments).""" + user = _make_user(username) + return UserProfile.objects.create( + user=user, + supabase_user_id=sub, + search_criteria=SearchCriteria.objects.create( + job_position="PM", country="DE", industry="Logistics" + ), + assigned_campaign=Campaign.objects.create(name=f"{username}-campaign"), + sender_account=SenderAccount.objects.create(name=f"{username}-sender"), + workspace=Workspace.objects.create(name=f"{username}-workspace"), + daily_sending_limit=50, + status=OnboardingStatus.READY, + ) + + +class OnboardingStatusModelTests(TestCase): + def test_new_profile_defaults_to_pending_with_no_assignments(self): + profile = UserProfile.objects.create( + user=_make_user("fresh"), supabase_user_id=uuid.uuid4() + ) + self.assertEqual(profile.status, OnboardingStatus.PENDING) + self.assertIsNone(profile.search_criteria_id) + self.assertIsNone(profile.assigned_campaign_id) + self.assertIsNone(profile.sender_account_id) + self.assertIsNone(profile.workspace_id) + self.assertEqual(profile.daily_sending_limit, 0) + + def test_ready_requires_all_four_assignments(self): + profile = UserProfile( + user=_make_user("incomplete"), supabase_user_id=uuid.uuid4() + ) + profile.status = OnboardingStatus.READY + with self.assertRaises(ValidationError): + profile.full_clean(validate_unique=False) + + def test_ready_accepted_when_fully_assigned(self): + profile = _make_ready_profile("complete", uuid.uuid4()) + # Should already be READY from the factory; re-validate cleanly. + profile.full_clean(validate_unique=False) + self.assertEqual(profile.status, OnboardingStatus.READY) + + def test_pending_and_disabled_do_not_require_assignments(self): + for status_value in (OnboardingStatus.PENDING, OnboardingStatus.DISABLED): + profile = UserProfile( + user=_make_user(f"u-{status_value}"), supabase_user_id=uuid.uuid4() + ) + profile.status = status_value + profile.full_clean(validate_unique=False) # must not raise + + +class IsReadyPermissionTests(TestCase): + def setUp(self): + self.factory = APIRequestFactory() + self.permission = IsReady() + + def _request_for(self, profile): + request = self.factory.post("/") + request.user = profile.user # SupabaseJWTAuthentication sets the User + return request + + def test_ready_user_is_permitted(self): + profile = _make_ready_profile("ready", uuid.uuid4()) + self.assertTrue(self.permission.has_permission(self._request_for(profile), None)) + + def test_pending_user_is_denied(self): + profile = UserProfile.objects.create( + user=_make_user("pending"), supabase_user_id=uuid.uuid4() + ) + self.assertFalse(self.permission.has_permission(self._request_for(profile), None)) + + def test_disabled_user_is_denied_with_disabled_message(self): + profile = UserProfile.objects.create( + user=_make_user("disabled"), + supabase_user_id=uuid.uuid4(), + status=OnboardingStatus.DISABLED, + ) + self.assertFalse(self.permission.has_permission(self._request_for(profile), None)) + self.assertIn("disabled", self.permission.message.lower()) + + +class TriggerBrowserJobOnboardingTests(TestCase): + """Run Search (POST /api/browser-jobs/) is gated by IsReady.""" + + def setUp(self): + self.sub = uuid.uuid4() + self.claims = {"sub": str(self.sub)} + + def _trigger(self): + with auth_as(self.claims): + return self.client.post( + reverse("api:browser-job-trigger"), HTTP_AUTHORIZATION="Bearer token" + ) + + @mock.patch("browser.views.run_browser_job.delay") + def test_pending_user_cannot_run_search(self, _delay): + UserProfile.objects.create(user=_make_user("p"), supabase_user_id=self.sub) + self.assertEqual(self._trigger().status_code, 403) + _delay.assert_not_called() + + @mock.patch("browser.views.run_browser_job.delay") + def test_disabled_user_cannot_run_search(self, _delay): + UserProfile.objects.create( + user=_make_user("d"), supabase_user_id=self.sub, status=OnboardingStatus.DISABLED + ) + self.assertEqual(self._trigger().status_code, 403) + _delay.assert_not_called() + + @mock.patch("browser.views.run_browser_job.delay") + def test_ready_user_can_run_search(self, _delay): + _make_ready_profile("r", self.sub) + response = self._trigger() + self.assertEqual(response.status_code, 202) + _delay.assert_called_once() + + +class TriggerCampaignJobOnboardingTests(TestCase): + """Push to Instantly (POST /api/campaign-jobs/) is gated by IsReady.""" + + def setUp(self): + self.sub = uuid.uuid4() + self.claims = {"sub": str(self.sub)} + + def _trigger(self): + with auth_as(self.claims): + return self.client.post( + reverse("api:campaign-job-trigger"), + {"person_ids": [1]}, + content_type="application/json", + HTTP_AUTHORIZATION="Bearer token", + ) + + @mock.patch("campaigns.views.run_campaign_job.delay") + def test_pending_user_cannot_push_to_instantly(self, _delay): + UserProfile.objects.create(user=_make_user("p"), supabase_user_id=self.sub) + self.assertEqual(self._trigger().status_code, 403) + _delay.assert_not_called() + + @mock.patch("campaigns.views.run_campaign_job.delay") + def test_ready_user_can_push_to_instantly(self, _delay): + _make_ready_profile("r", self.sub) + response = self._trigger() + self.assertEqual(response.status_code, 202) + _delay.assert_called_once() + + +class AutoProvisionDefaultsTests(TestCase): + """A freshly auto-provisioned user starts PENDING with no assignments.""" + + def test_auto_provision_creates_a_pending_profile(self): + sub = uuid.uuid4() + auth = SupabaseJWTAuthentication() + claims = { + "sub": str(sub), + "email": "newuser@example.com", + "email_verified": True, + } + request = APIRequestFactory().get("/", HTTP_AUTHORIZATION="Bearer token") + with auth_as(claims): + user, _ = auth.authenticate(request) + + profile = UserProfile.objects.get(user=user) + self.assertEqual(profile.status, OnboardingStatus.PENDING) + self.assertIsNone(profile.search_criteria_id) + self.assertIsNone(profile.assigned_campaign_id) + self.assertIsNone(profile.sender_account_id) + self.assertIsNone(profile.workspace_id) + self.assertEqual(profile.daily_sending_limit, 0) + self.assertEqual(profile.supabase_user_id, sub) + + +class AdministratorActivationTests(TestCase): + """The Administrator flips PENDING → READY once all assignments exist.""" + + def setUp(self): + self.factory = RequestFactory() + self.site = django_admin.site + self.admin = UserProfileAdmin(UserProfile, self.site) + self.sub = uuid.uuid4() + self.profile = UserProfile.objects.create( + user=_make_user("activate-me"), supabase_user_id=self.sub + ) + self.profile.refresh_from_db() + self.assertEqual(self.profile.status, OnboardingStatus.PENDING) + + def _mark_ready(self): + request = self.factory.post("/") + request._messages = mock.Mock() # admin message_user uses messages framework + request._messages.add = mock.Mock() + self.admin.mark_ready(request, UserProfile.objects.filter(pk=self.profile.pk)) + self.profile.refresh_from_db() + + def test_mark_ready_fails_without_required_assignments(self): + self._mark_ready() + self.profile.refresh_from_db() + self.assertEqual(self.profile.status, OnboardingStatus.PENDING) + + def test_mark_ready_succeeds_when_fully_assigned(self): + self.profile.search_criteria = SearchCriteria.objects.create( + job_position="PM", country="DE", industry="Logistics" + ) + self.profile.assigned_campaign = Campaign.objects.create(name="Q3") + self.profile.sender_account = SenderAccount.objects.create(name="s1") + self.profile.workspace = Workspace.objects.create(name="w1") + self.profile.save() + self._mark_ready() + self.assertEqual(self.profile.status, OnboardingStatus.READY) + + def test_disable_and_mark_pending_actions(self): + request = self.factory.post("/") + request._messages = mock.Mock() + request._messages.add = mock.Mock() + self.admin.disable(request, UserProfile.objects.filter(pk=self.profile.pk)) + self.profile.refresh_from_db() + self.assertEqual(self.profile.status, OnboardingStatus.DISABLED) + self.admin.mark_pending(request, UserProfile.objects.filter(pk=self.profile.pk)) + self.profile.refresh_from_db() + self.assertEqual(self.profile.status, OnboardingStatus.PENDING) From 2c948ae4cb9a1b1220e6d8d8478d1d7068b3510a Mon Sep 17 00:00:00 2001 From: Dominik Ledzion Date: Thu, 9 Jul 2026 13:56:44 +0200 Subject: [PATCH 11/34] feat(users): add LinkedIn action-budget fields to UserProfile --- ...006_userprofile_action_budgets_and_more.py | 33 +++++++++++++++++++ users/models.py | 17 ++++++++++ users/test_action_budget_fields.py | 13 ++++++++ 3 files changed, 63 insertions(+) create mode 100644 users/migrations/0006_userprofile_action_budgets_and_more.py create mode 100644 users/test_action_budget_fields.py diff --git a/users/migrations/0006_userprofile_action_budgets_and_more.py b/users/migrations/0006_userprofile_action_budgets_and_more.py new file mode 100644 index 0000000..ce4316e --- /dev/null +++ b/users/migrations/0006_userprofile_action_budgets_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 6.0.6 on 2026-07-09 11:55 + +import users.models +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("users", "0005_userprofile_onboarding_state"), + ] + + operations = [ + migrations.AddField( + model_name="userprofile", + name="action_budgets", + field=models.JSONField( + blank=True, + default=users.models._default_action_budgets, + help_text="Per-action-type daily caps, e.g. {'search': 20, 'profile_view': 500}.", + ), + ), + migrations.AddField( + model_name="userprofile", + name="action_cooldown_seconds", + field=models.PositiveIntegerField(default=60), + ), + migrations.AddField( + model_name="userprofile", + name="daily_action_budget", + field=models.PositiveIntegerField(default=500), + ), + ] diff --git a/users/models.py b/users/models.py index 883144f..52a3f8a 100644 --- a/users/models.py +++ b/users/models.py @@ -51,6 +51,11 @@ def __str__(self): return self.name +def _default_action_budgets(): + """Default per-action-type daily caps for a new UserProfile.""" + return {"search": 20, "profile_view": 500} + + class OnboardingStatus(models.TextChoices): """Lifecycle of a UserProfile through administrator-controlled onboarding. @@ -121,6 +126,18 @@ class UserProfile(models.Model): blank=True, ) daily_sending_limit = models.PositiveIntegerField(default=0) + # --- LinkedIn action budget (docs/.../browser-linkedin-safety-design.md) --- + # Cap on the weighted sum of today's LinkedIn actions (Σ count × cost). + # 0 = unlimited, matching daily_sending_limit. + daily_action_budget = models.PositiveIntegerField(default=500) + # Optional per-action-type hard caps; absent/0 = unlimited for that type. + action_budgets = models.JSONField( + default=_default_action_budgets, + blank=True, + help_text="Per-action-type daily caps, e.g. {'search': 20, 'profile_view': 500}.", + ) + # Minimum seconds between two searches by this user. 0 = no cooldown. + action_cooldown_seconds = models.PositiveIntegerField(default=60) status = models.CharField( max_length=16, choices=OnboardingStatus.choices, diff --git a/users/test_action_budget_fields.py b/users/test_action_budget_fields.py new file mode 100644 index 0000000..43d317c --- /dev/null +++ b/users/test_action_budget_fields.py @@ -0,0 +1,13 @@ +from django.contrib.auth import get_user_model +from django.test import TestCase + +from users.models import UserProfile + + +class ActionBudgetFieldsTests(TestCase): + def test_defaults(self): + user = get_user_model().objects.create(username="u1") + profile = UserProfile.objects.create(user=user) + self.assertEqual(profile.daily_action_budget, 500) + self.assertEqual(profile.action_budgets, {"search": 20, "profile_view": 500}) + self.assertEqual(profile.action_cooldown_seconds, 60) From 1801222c58ebb21e7f74ce581ffea61b139653d2 Mon Sep 17 00:00:00 2001 From: Dominik Ledzion Date: Thu, 9 Jul 2026 14:00:47 +0200 Subject: [PATCH 12/34] feat(browser): add BrowserControlState singleton for emergency stop --- .../migrations/0006_browsercontrolstate.py | 41 +++++++++++++++++++ browser/models.py | 33 +++++++++++++++ browser/test_safety.py | 18 ++++++++ 3 files changed, 92 insertions(+) create mode 100644 browser/migrations/0006_browsercontrolstate.py diff --git a/browser/migrations/0006_browsercontrolstate.py b/browser/migrations/0006_browsercontrolstate.py new file mode 100644 index 0000000..ec258df --- /dev/null +++ b/browser/migrations/0006_browsercontrolstate.py @@ -0,0 +1,41 @@ +# Generated by Django 6.0.6 on 2026-07-09 11:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("browser", "0005_remove_browserjob_browser_browserjob_status_valid_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="BrowserControlState", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "emergency_stop", + models.BooleanField( + default=False, + help_text="When on, no LinkedIn browser job may run; active jobs stop.", + ), + ), + ("reason", models.CharField(blank=True, max_length=255)), + ("updated_by", models.CharField(blank=True, max_length=255)), + ("updated_at", models.DateTimeField(auto_now=True)), + ], + options={ + "verbose_name": "browser control state", + "verbose_name_plural": "browser control state", + }, + ), + ] diff --git a/browser/models.py b/browser/models.py index 7415311..fd04a58 100644 --- a/browser/models.py +++ b/browser/models.py @@ -70,3 +70,36 @@ class Meta: def __str__(self): return f"BrowserJob({self.pk}, {self.status})" + + +class BrowserControlState(models.Model): + """Singleton (pk is always 1) holding the global browser-automation + kill switch. The administrator flips ``emergency_stop`` from Django + admin; ``run_browser_job`` checks it (through the cache) before every + LinkedIn action and short-circuits running/pending jobs to CANCELLED. + See docs/superpowers/specs/2026-07-09-browser-linkedin-safety-design.md.""" + + emergency_stop = models.BooleanField( + default=False, + help_text="When on, no LinkedIn browser job may run; active jobs stop.", + ) + reason = models.CharField(max_length=255, blank=True) + updated_by = models.CharField(max_length=255, blank=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + verbose_name = "browser control state" + verbose_name_plural = "browser control state" + + def save(self, *args, **kwargs): + # Force a single row: there is exactly one global switch. + self.pk = 1 + super().save(*args, **kwargs) + + @classmethod + def load(cls): + obj, _ = cls.objects.get_or_create(pk=1) + return obj + + def __str__(self): + return f"BrowserControlState(emergency_stop={self.emergency_stop})" diff --git a/browser/test_safety.py b/browser/test_safety.py index 4a622a3..696ddfa 100644 --- a/browser/test_safety.py +++ b/browser/test_safety.py @@ -5,6 +5,7 @@ from core.models import JobStatus from browser.models import BrowserJob +from browser.models import BrowserControlState class JobStatusCancelledTests(TestCase): @@ -31,3 +32,20 @@ def test_time_limits_are_ordered(self): def test_health_autostop_defaults_on(self): self.assertTrue(settings.BROWSER_HEALTH_AUTOSTOP) + + +class BrowserControlStateTests(TestCase): + def test_load_returns_singleton_row(self): + a = BrowserControlState.load() + b = BrowserControlState.load() + self.assertEqual(a.pk, 1) + self.assertEqual(a.pk, b.pk) + self.assertFalse(a.emergency_stop) + + def test_save_always_pins_pk_1(self): + state = BrowserControlState.load() + state.pk = 5 + state.emergency_stop = True + state.save() + self.assertEqual(BrowserControlState.objects.count(), 1) + self.assertTrue(BrowserControlState.load().emergency_stop) From 376c3f6812b26cc540797e1c853211d7c5bf10a8 Mon Sep 17 00:00:00 2001 From: Dominik Ledzion Date: Thu, 9 Jul 2026 14:04:33 +0200 Subject: [PATCH 13/34] feat(services/browser): typed exception hierarchy (retryable vs health) --- services/browser/exceptions.py | 31 +++++++++++++++++++++++++++++ services/browser/test_exceptions.py | 24 ++++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 services/browser/exceptions.py create mode 100644 services/browser/test_exceptions.py diff --git a/services/browser/exceptions.py b/services/browser/exceptions.py new file mode 100644 index 0000000..906ad41 --- /dev/null +++ b/services/browser/exceptions.py @@ -0,0 +1,31 @@ +"""Typed browser-automation errors shared by the task layer and providers. + +Providers raise these (never provider-specific types), so ``browser/tasks.py`` +can decide retry-vs-fail from the *type*, not a string match. See +docs/superpowers/specs/2026-07-09-browser-linkedin-safety-design.md §H/§I. +""" + + +class BrowserAutomationError(Exception): + """Base for any browser-automation failure.""" + + +class RetryableBrowserError(BrowserAutomationError): + """A transient fault worth retrying: request timeout, network/transport + error, or a temporarily-unavailable daemon. NEVER used for health + problems (those must not be retried against a flagged account).""" + + +class SessionHealthError(BrowserAutomationError): + """The LinkedIn session is unhealthy and the job must stop immediately + without retry. ``kind`` is one of: 'logged_out', 'captcha', + 'checkpoint', 'unexpected_redirect'.""" + + def __init__(self, kind: str, message: str | None = None): + self.kind = kind + super().__init__(message or f"LinkedIn session health check failed: {kind}") + + +# Kinds severe enough that the whole queue should stop, not just this job: +# LinkedIn has actively challenged the account. +HEALTH_AUTOSTOP_KINDS = {"captcha", "checkpoint"} diff --git a/services/browser/test_exceptions.py b/services/browser/test_exceptions.py new file mode 100644 index 0000000..00979e9 --- /dev/null +++ b/services/browser/test_exceptions.py @@ -0,0 +1,24 @@ +from django.test import TestCase + +from services.browser.exceptions import ( + BrowserAutomationError, + RetryableBrowserError, + SessionHealthError, + HEALTH_AUTOSTOP_KINDS, +) + + +class ExceptionHierarchyTests(TestCase): + def test_retryable_is_a_browser_error(self): + self.assertTrue(issubclass(RetryableBrowserError, BrowserAutomationError)) + + def test_session_health_carries_kind(self): + exc = SessionHealthError("captcha") + self.assertEqual(exc.kind, "captcha") + self.assertIn("captcha", str(exc)) + + def test_health_is_not_retryable(self): + self.assertFalse(issubclass(SessionHealthError, RetryableBrowserError)) + + def test_autostop_kinds(self): + self.assertEqual(HEALTH_AUTOSTOP_KINDS, {"captcha", "checkpoint"}) From d2d9517c9f2427e53982779019ef50806850a8b3 Mon Sep 17 00:00:00 2001 From: Dominik Ledzion Date: Thu, 9 Jul 2026 14:07:25 +0200 Subject: [PATCH 14/34] feat(services/browser): single-session cache lock --- services/browser/locks.py | 35 ++++++++++++++++++++++++++++++++++ services/browser/test_locks.py | 21 ++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 services/browser/locks.py create mode 100644 services/browser/test_locks.py diff --git a/services/browser/locks.py b/services/browser/locks.py new file mode 100644 index 0000000..366438b --- /dev/null +++ b/services/browser/locks.py @@ -0,0 +1,35 @@ +"""Global single-session lock for LinkedIn automation. + +At most one BrowserJob may drive a browser at a time (docs/.../safety spec §B). +Backed by Django's cache: ``cache.add`` is atomic (set-if-absent), so it is a +correct mutual-exclusion primitive on both LocMemCache (tests) and Redis (prod). +Release is compare-and-delete: only the holder may release, so a job that lost +the lock to TTL expiry cannot delete a successor's lock. +""" + +from django.conf import settings +from django.core.cache import cache + +SESSION_LOCK_KEY = "browser:linkedin_session_lock" + + +def acquire_session_lock(job_id: int, ttl: int | None = None) -> bool: + """Try to acquire the single-session lock for ``job_id``. Returns True if + acquired, False if another job holds it.""" + if ttl is None: + ttl = getattr(settings, "BROWSER_SESSION_LOCK_TTL", 900) + return bool(cache.add(SESSION_LOCK_KEY, job_id, ttl)) + + +def release_session_lock(job_id: int) -> bool: + """Release the lock only if ``job_id`` currently holds it. Returns True if + a release happened, False otherwise (not the holder / already free).""" + if cache.get(SESSION_LOCK_KEY) == job_id: + cache.delete(SESSION_LOCK_KEY) + return True + return False + + +def session_lock_holder() -> int | None: + """The job_id currently holding the lock, or None if free.""" + return cache.get(SESSION_LOCK_KEY) diff --git a/services/browser/test_locks.py b/services/browser/test_locks.py new file mode 100644 index 0000000..24d438b --- /dev/null +++ b/services/browser/test_locks.py @@ -0,0 +1,21 @@ +from django.core.cache import cache +from django.test import TestCase + +from services.browser import locks + + +class SessionLockTests(TestCase): + def setUp(self): + cache.clear() + + def test_first_acquire_succeeds_second_blocks(self): + self.assertTrue(locks.acquire_session_lock(1)) + self.assertFalse(locks.acquire_session_lock(2)) + self.assertEqual(locks.session_lock_holder(), 1) + + def test_release_only_by_holder(self): + locks.acquire_session_lock(1) + self.assertFalse(locks.release_session_lock(2)) # non-holder can't release + self.assertTrue(locks.release_session_lock(1)) # holder releases + self.assertIsNone(locks.session_lock_holder()) + self.assertTrue(locks.acquire_session_lock(2)) # now free From 044eab722e2ed91479c385e99047df1c1c9f7ffe Mon Sep 17 00:00:00 2001 From: Dominik Ledzion Date: Thu, 9 Jul 2026 14:15:00 +0200 Subject: [PATCH 15/34] feat(services/browser): global emergency-stop control --- services/browser/control.py | 47 ++++++++++++++++++++++++++++++++ services/browser/test_control.py | 33 ++++++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 services/browser/control.py create mode 100644 services/browser/test_control.py diff --git a/services/browser/control.py b/services/browser/control.py new file mode 100644 index 0000000..0f6ce9a --- /dev/null +++ b/services/browser/control.py @@ -0,0 +1,47 @@ +"""Global emergency stop for LinkedIn automation (docs/.../safety spec §G). + +Durable state lives in ``browser.BrowserControlState`` (admin-editable); a +short-lived cache mirror keeps the hot per-action check off the database. The +worker reads ``emergency_stop_active()`` before every LinkedIn action; the +provider/task engages it on a CAPTCHA/checkpoint; the administrator releases it. +""" + +from django.core.cache import cache + +_CACHE_KEY = "browser:emergency_stop" +_CACHE_TTL = 10 # seconds; short so an admin release is honored quickly + + +def emergency_stop_active() -> bool: + """True if the global kill switch is on. Cache-first, DB fallback.""" + cached = cache.get(_CACHE_KEY) + if cached is not None: + return bool(cached) + from browser.models import BrowserControlState # local: app-registry safety + active = BrowserControlState.load().emergency_stop + cache.set(_CACHE_KEY, active, _CACHE_TTL) + return active + + +def engage_emergency_stop(reason: str = "", updated_by=None) -> None: + """Turn the kill switch on (DB + cache).""" + from browser.models import BrowserControlState + state = BrowserControlState.load() + state.emergency_stop = True + state.reason = reason or state.reason + if updated_by is not None: + state.updated_by = str(updated_by) + state.save() + cache.set(_CACHE_KEY, True, _CACHE_TTL) + + +def release_emergency_stop(updated_by=None) -> None: + """Turn the kill switch off (DB + cache).""" + from browser.models import BrowserControlState + state = BrowserControlState.load() + state.emergency_stop = False + state.reason = "" + if updated_by is not None: + state.updated_by = str(updated_by) + state.save() + cache.set(_CACHE_KEY, False, _CACHE_TTL) diff --git a/services/browser/test_control.py b/services/browser/test_control.py new file mode 100644 index 0000000..8b7f309 --- /dev/null +++ b/services/browser/test_control.py @@ -0,0 +1,33 @@ +from django.core.cache import cache +from django.test import TestCase + +from browser.models import BrowserControlState +from services.browser import control + + +class EmergencyStopTests(TestCase): + def setUp(self): + cache.clear() + + def test_inactive_by_default(self): + self.assertFalse(control.emergency_stop_active()) + + def test_engage_then_active_and_persisted(self): + control.engage_emergency_stop(reason="captcha", updated_by="worker") + self.assertTrue(control.emergency_stop_active()) + state = BrowserControlState.load() + self.assertTrue(state.emergency_stop) + self.assertEqual(state.reason, "captcha") + + def test_release_clears_it(self): + control.engage_emergency_stop(reason="x") + control.release_emergency_stop(updated_by="admin") + self.assertFalse(control.emergency_stop_active()) + self.assertFalse(BrowserControlState.load().emergency_stop) + + def test_reads_db_when_cache_cold(self): + state = BrowserControlState.load() + state.emergency_stop = True + state.save() + cache.clear() # simulate a fresh worker with an empty cache + self.assertTrue(control.emergency_stop_active()) From fcf2e1a4b2475a78385cbefe83fc64379b08b9a4 Mon Sep 17 00:00:00 2001 From: Dominik Ledzion Date: Thu, 9 Jul 2026 14:17:47 +0200 Subject: [PATCH 16/34] feat(services/browser): per-job cancellation flags --- services/browser/cancellation.py | 26 ++++++++++++++++++++++++++ services/browser/test_cancellation.py | 23 +++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 services/browser/cancellation.py create mode 100644 services/browser/test_cancellation.py diff --git a/services/browser/cancellation.py b/services/browser/cancellation.py new file mode 100644 index 0000000..22d7434 --- /dev/null +++ b/services/browser/cancellation.py @@ -0,0 +1,26 @@ +"""Cooperative cancellation flags for running BrowserJobs (docs/.../spec §C). + +A cancel request from the API sets a per-job cache flag; ``run_browser_job`` +polls it between pagination pages and stops cleanly as CANCELLED. Cache-based +so the flag crosses the process boundary between the web request and the worker. +""" + +from django.conf import settings +from django.core.cache import cache + + +def _key(job_id: int) -> str: + return f"browser:cancel:{job_id}" + + +def request_cancel(job_id: int) -> None: + ttl = getattr(settings, "BROWSER_CANCEL_FLAG_TTL", 900) + cache.set(_key(job_id), True, ttl) + + +def is_cancel_requested(job_id: int) -> bool: + return bool(cache.get(_key(job_id))) + + +def clear_cancel(job_id: int) -> None: + cache.delete(_key(job_id)) diff --git a/services/browser/test_cancellation.py b/services/browser/test_cancellation.py new file mode 100644 index 0000000..ebdd059 --- /dev/null +++ b/services/browser/test_cancellation.py @@ -0,0 +1,23 @@ +from django.core.cache import cache +from django.test import TestCase + +from services.browser import cancellation + + +class CancellationFlagTests(TestCase): + def setUp(self): + cache.clear() + + def test_request_then_detected(self): + self.assertFalse(cancellation.is_cancel_requested(7)) + cancellation.request_cancel(7) + self.assertTrue(cancellation.is_cancel_requested(7)) + + def test_clear(self): + cancellation.request_cancel(7) + cancellation.clear_cancel(7) + self.assertFalse(cancellation.is_cancel_requested(7)) + + def test_flags_are_per_job(self): + cancellation.request_cancel(7) + self.assertFalse(cancellation.is_cancel_requested(8)) From 23e24c0fa7667993ff505bc4f5369a41c2596608 Mon Sep 17 00:00:00 2001 From: Dominik Ledzion Date: Thu, 9 Jul 2026 14:20:07 +0200 Subject: [PATCH 17/34] feat(services/browser): session-health page classifier --- services/browser/health.py | 42 +++++++++++++++++++++++++++++++++ services/browser/test_health.py | 35 +++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 services/browser/health.py create mode 100644 services/browser/test_health.py diff --git a/services/browser/health.py b/services/browser/health.py new file mode 100644 index 0000000..85c1d2f --- /dev/null +++ b/services/browser/health.py @@ -0,0 +1,42 @@ +"""Classify the current browser page into a LinkedIn session-health kind. + +Pure URL/marker inspection so it is trivially unit-testable and provider- +agnostic. The provider gathers ``page_markers`` from the live DOM (e.g. a +CAPTCHA iframe present) and passes them in; the task turns a non-None result +into a non-retryable SessionHealthError. See docs/.../safety spec §H. +""" + +from urllib.parse import urlparse + +_EXPECTED_HOSTS = {"www.linkedin.com", "linkedin.com"} + + +def classify_page(url: str, page_markers: dict | None = None) -> str | None: + """Return a health kind, or None if the page looks like a healthy + LinkedIn/Sales Navigator page. + + Precedence: explicit challenge markers first (captcha), then URL-path + signals (checkpoint, login), then off-site host (unexpected_redirect). + """ + markers = page_markers or {} + parsed = urlparse(url or "") + host = (parsed.hostname or "").lower() + path = (parsed.path or "").lower() + + # CAPTCHA can appear on an otherwise-expected URL, so check it first. + if markers.get("has_captcha"): + return "captcha" + + if host in _EXPECTED_HOSTS: + if path.startswith("/checkpoint"): + return "checkpoint" + if path.startswith("/login") or path.startswith("/uas"): + return "logged_out" + return None + + # Empty/relative URL: can't judge — treat as healthy (caller decides). + if not host: + return None + + # Any other host means we were redirected off LinkedIn unexpectedly. + return "unexpected_redirect" diff --git a/services/browser/test_health.py b/services/browser/test_health.py new file mode 100644 index 0000000..e23c17e --- /dev/null +++ b/services/browser/test_health.py @@ -0,0 +1,35 @@ +from django.test import TestCase + +from services.browser import health + + +class ClassifyPageTests(TestCase): + def test_healthy_sales_nav_results(self): + url = "https://www.linkedin.com/sales/search/people?query=x" + self.assertIsNone(health.classify_page(url)) + + def test_login_redirect_is_logged_out(self): + self.assertEqual( + health.classify_page("https://www.linkedin.com/login"), "logged_out" + ) + self.assertEqual( + health.classify_page("https://www.linkedin.com/uas/login"), "logged_out" + ) + + def test_checkpoint_url(self): + self.assertEqual( + health.classify_page("https://www.linkedin.com/checkpoint/challenge"), + "checkpoint", + ) + + def test_captcha_marker(self): + url = "https://www.linkedin.com/sales/search/people" + self.assertEqual( + health.classify_page(url, {"has_captcha": True}), "captcha" + ) + + def test_offsite_redirect_is_unexpected(self): + self.assertEqual( + health.classify_page("https://example.com/anything"), + "unexpected_redirect", + ) From d3c0097c99fb236f072ad9df9ba471fc951524cd Mon Sep 17 00:00:00 2001 From: Dominik Ledzion Date: Thu, 9 Jul 2026 14:24:34 +0200 Subject: [PATCH 18/34] feat(services/browser): action-budget accounting and decisions --- services/browser/action_budget.py | 128 +++++++++++++++++++++++++ services/browser/test_action_budget.py | 64 +++++++++++++ 2 files changed, 192 insertions(+) create mode 100644 services/browser/action_budget.py create mode 100644 services/browser/test_action_budget.py diff --git a/services/browser/action_budget.py b/services/browser/action_budget.py new file mode 100644 index 0000000..83b267a --- /dev/null +++ b/services/browser/action_budget.py @@ -0,0 +1,128 @@ +"""Per-action-type daily budget accounting and decisions (docs/.../spec §D). + +The ActivityLog is the single ledger: ``record_action`` writes one row per +LinkedIn action (verb ``linkedin.``, ``metadata['units']`` = count), and +every query below reads today's rows for a user. All limits use 0 = unlimited. +""" + +from dataclasses import dataclass, field + +from django.utils import timezone + +from core.activity import log_activity +from core.models import ActivityLog +from services.browser.actions import action_cost, ACTION_TYPES + +_VERB_PREFIX = "linkedin." + + +@dataclass +class BudgetDecision: + allowed: bool + reason: str = "" + remaining: int = 0 # weighted units left today; -1 = unlimited + cooldown_remaining: int = 0 # seconds until cooldown clears + + +def _today_start(): + """Aware datetime for the start of 'today' in the project timezone.""" + today = timezone.localdate() + return timezone.make_aware( + timezone.datetime(today.year, today.month, today.day) + ) + + +def _today_rows(user): + return ActivityLog.objects.filter( + actor=user, verb__startswith=_VERB_PREFIX, created_at__gte=_today_start() + ) + + +def record_action(user, action_type: str, units: int = 1, target_id=None) -> None: + """Write one ledger row for a LinkedIn action (the only budget writer).""" + log_activity( + actor=user, + verb=f"{_VERB_PREFIX}{action_type}", + target_type="BrowserJob", + target_id=target_id, + metadata={"units": int(units), "action_type": action_type}, + ) + + +def actions_today(user, action_type: str) -> int: + """Total units of one action type performed by ``user`` today.""" + total = 0 + for row in _today_rows(user).filter(verb=f"{_VERB_PREFIX}{action_type}"): + total += int(row.metadata.get("units", 1)) + return total + + +def weighted_spend_today(user) -> int: + """Σ units × cost across all action types today.""" + spend = 0 + for action_type in ACTION_TYPES: + spend += actions_today(user, action_type) * action_cost(action_type) + return spend + + +def seconds_since_last_action(user): + """Seconds since the user's most recent LinkedIn action, or None.""" + last = _today_rows(user).order_by("-created_at").first() + if last is None: + return None + return int((timezone.now() - last.created_at).total_seconds()) + + +def _weighted_remaining(profile) -> int: + budget = profile.daily_action_budget + if not budget: # 0 = unlimited + return -1 + return max(budget - weighted_spend_today(profile.user), 0) + + +def check_search_allowed(profile) -> BudgetDecision: + """Decide whether ``profile`` may start one more search right now.""" + # 1. Cooldown between searches. + cooldown = profile.action_cooldown_seconds + if cooldown: + since = seconds_since_last_action(profile.user) + if since is not None and since < cooldown: + return BudgetDecision( + allowed=False, + reason="cooldown", + remaining=_weighted_remaining(profile), + cooldown_remaining=cooldown - since, + ) + # 2. Per-type hard cap for 'search'. + cap = (profile.action_budgets or {}).get("search", 0) + if cap and actions_today(profile.user, "search") >= cap: + return BudgetDecision( + allowed=False, reason="search cap reached", + remaining=_weighted_remaining(profile), + ) + # 3. Weighted daily budget (a search would add action_cost('search')). + remaining = _weighted_remaining(profile) + if remaining != -1 and remaining < action_cost("search"): + return BudgetDecision( + allowed=False, reason="daily action budget reached", remaining=remaining + ) + return BudgetDecision(allowed=True, remaining=remaining) + + +def remaining_profile_views(profile) -> int: + """How many more profiles this user may pull today. -1 = unlimited. + + The min of the per-type 'profile_view' cap and what the weighted budget + still affords (weighted budget is in cost units; profile_view cost is + per profile).""" + caps = [] + type_cap = (profile.action_budgets or {}).get("profile_view", 0) + if type_cap: + caps.append(max(type_cap - actions_today(profile.user, "profile_view"), 0)) + weighted = _weighted_remaining(profile) + if weighted != -1: + pv_cost = action_cost("profile_view") or 1 + caps.append(weighted // pv_cost) + if not caps: + return -1 + return min(caps) diff --git a/services/browser/test_action_budget.py b/services/browser/test_action_budget.py new file mode 100644 index 0000000..da413cc --- /dev/null +++ b/services/browser/test_action_budget.py @@ -0,0 +1,64 @@ +from django.contrib.auth import get_user_model +from django.test import TestCase, override_settings +from django.utils import timezone + +from core.models import ActivityLog +from services.browser import action_budget as ab +from users.models import UserProfile + + +class ActionBudgetTests(TestCase): + def _profile(self, **kw): + user = get_user_model().objects.create(username=f"u{UserProfile.objects.count()}") + defaults = dict( + user=user, daily_action_budget=0, + action_budgets={}, action_cooldown_seconds=0, + ) + defaults.update(kw) + return UserProfile.objects.create(**defaults) + + def test_record_and_count(self): + p = self._profile() + ab.record_action(p.user, "profile_view", units=5) + self.assertEqual(ab.actions_today(p.user, "profile_view"), 5) + self.assertEqual(ab.actions_today(p.user, "search"), 0) + + def test_weighted_spend(self): + p = self._profile() + ab.record_action(p.user, "search", units=2) # cost 1 → 2 + ab.record_action(p.user, "profile_view", units=3) # cost 1 → 3 + self.assertEqual(ab.weighted_spend_today(p.user), 5) + + def test_search_blocked_by_type_cap(self): + p = self._profile(action_budgets={"search": 1}) + ab.record_action(p.user, "search", units=1) + decision = ab.check_search_allowed(p) + self.assertFalse(decision.allowed) + self.assertIn("search", decision.reason) + + def test_search_blocked_by_weighted_budget(self): + p = self._profile(daily_action_budget=2) + ab.record_action(p.user, "profile_view", units=2) + self.assertFalse(ab.check_search_allowed(p).allowed) + + def test_cooldown_blocks_then_allows(self): + p = self._profile(action_cooldown_seconds=300) + ab.record_action(p.user, "search", units=1) + d = ab.check_search_allowed(p) + self.assertFalse(d.allowed) + self.assertGreater(d.cooldown_remaining, 0) + + def test_unlimited_when_zero(self): + p = self._profile() # all zeros + for _ in range(50): + ab.record_action(p.user, "search", units=1) + self.assertTrue(ab.check_search_allowed(p).allowed) + + def test_remaining_profile_views(self): + p = self._profile(action_budgets={"profile_view": 10}, daily_action_budget=0) + ab.record_action(p.user, "profile_view", units=4) + self.assertEqual(ab.remaining_profile_views(p), 6) + + def test_remaining_profile_views_unlimited(self): + p = self._profile() # all zeros + self.assertEqual(ab.remaining_profile_views(p), -1) From fe73e9341fa24e8dbdf4e22bccd89c69d8a47378 Mon Sep 17 00:00:00 2001 From: Dominik Ledzion Date: Thu, 9 Jul 2026 14:29:07 +0200 Subject: [PATCH 19/34] fix(services/browser): cooldown survives midnight; cover budget truncation; drop unused import --- services/browser/action_budget.py | 13 ++++++++++--- services/browser/test_action_budget.py | 11 +++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/services/browser/action_budget.py b/services/browser/action_budget.py index 83b267a..7c916b6 100644 --- a/services/browser/action_budget.py +++ b/services/browser/action_budget.py @@ -5,7 +5,7 @@ every query below reads today's rows for a user. All limits use 0 = unlimited. """ -from dataclasses import dataclass, field +from dataclasses import dataclass from django.utils import timezone @@ -66,8 +66,15 @@ def weighted_spend_today(user) -> int: def seconds_since_last_action(user): - """Seconds since the user's most recent LinkedIn action, or None.""" - last = _today_rows(user).order_by("-created_at").first() + """Seconds since the user's most recent LinkedIn action, or None. + + Not bounded to 'today' — a cooldown must still hold across the local + midnight boundary (an action at 23:59 must gate a 00:00 search).""" + last = ( + ActivityLog.objects.filter(actor=user, verb__startswith=_VERB_PREFIX) + .order_by("-created_at") + .first() + ) if last is None: return None return int((timezone.now() - last.created_at).total_seconds()) diff --git a/services/browser/test_action_budget.py b/services/browser/test_action_budget.py index da413cc..64f1e33 100644 --- a/services/browser/test_action_budget.py +++ b/services/browser/test_action_budget.py @@ -62,3 +62,14 @@ def test_remaining_profile_views(self): def test_remaining_profile_views_unlimited(self): p = self._profile() # all zeros self.assertEqual(ab.remaining_profile_views(p), -1) + + @override_settings(BROWSER_ACTION_COSTS={"profile_view": 3, "search": 1}) + def test_remaining_profile_views_truncates_and_mins(self): + # weighted budget 30, cost/profile 3 -> 10 affordable by budget; + # per-type cap 6 -> min(10, 6) == 6 + p = self._profile(daily_action_budget=30, action_budgets={"profile_view": 6}) + self.assertEqual(ab.remaining_profile_views(p), 6) + # spend 3 profiles (weighted 9): budget headroom (30-9)//3 = 7, + # type cap 6-3 = 3 -> min(7, 3) == 3 + ab.record_action(p.user, "profile_view", units=3) + self.assertEqual(ab.remaining_profile_views(p), 3) From c31400586e0596ea151b910f3b632df72168832f Mon Sep 17 00:00:00 2001 From: Dominik Ledzion Date: Thu, 9 Jul 2026 14:38:19 +0200 Subject: [PATCH 20/34] feat(browser): harden run_browser_job (lock, stop, cancel, budget, health, retry, timeout) --- browser/tasks.py | 217 ++++++++++++++++++++++++++++------------- browser/test_safety.py | 89 ++++++++++++++++- browser/test_tasks.py | 9 ++ 3 files changed, 246 insertions(+), 69 deletions(-) diff --git a/browser/tasks.py b/browser/tasks.py index 0cd7420..252f790 100644 --- a/browser/tasks.py +++ b/browser/tasks.py @@ -1,6 +1,8 @@ import logging from celery import shared_task +from celery.exceptions import SoftTimeLimitExceeded +from django.conf import settings from django.utils import timezone from companies.models import Company @@ -9,6 +11,8 @@ from enrichment.models import EnrichmentJob from enrichment.tasks import run_enrichment_job from leads.models import Person +from services.browser import action_budget, cancellation, control, health, locks +from services.browser.exceptions import RetryableBrowserError, SessionHealthError from services.browser import BrowserService from users.models import UserProfile @@ -17,50 +21,19 @@ logger = logging.getLogger("kimisystem.browser") -@shared_task(bind=True) -def run_browser_job(self, browser_job_id): - """Runs a BrowserJob: calls BrowserService, upserts the resulting - profiles into Person/Company, and updates the job's own status. - See docs/workflows/browser.md. - - Ownership rule: discovered_by is set only when a Person is first - created. If a later search (by this user or a different one) finds - the same public LinkedIn profile again, the existing Person's other - fields are refreshed but discovered_by is never reassigned — the - original discoverer keeps this lead in their own RLS-scoped view. - """ - job = BrowserJob.objects.get(pk=browser_job_id) - job.status = JobStatus.RUNNING - job.started_at = timezone.now() - job.celery_task_id = self.request.id - job.save(update_fields=["status", "started_at", "celery_task_id"]) +def _finish(job, status, actor, verb, *, error="", metadata=None): + """Mark a job terminal, timestamp it, and write one activity row.""" + job.status = status + if error: + job.error = error + job.finished_at = timezone.now() + job.save(update_fields=["status", "error", "finished_at"]) + log_activity(actor=actor, verb=verb, target_type="BrowserJob", + target_id=job.pk, metadata=metadata or {}) - actor = None - profile = ( - UserProfile.objects.filter(supabase_user_id=job.requested_by) - .select_related("user") - .first() - ) - if profile: - actor = profile.user - - try: - profiles = BrowserService().run_linkedin_search(job.criteria) - except Exception as exc: - job.status = JobStatus.FAILED - job.error = str(exc) - job.finished_at = timezone.now() - job.save(update_fields=["status", "error", "finished_at"]) - log_activity( - actor=actor, - verb="browser_job.failed", - target_type="BrowserJob", - target_id=job.pk, - metadata={"error": str(exc)}, - ) - logger.warning("BrowserJob %s failed: %s", job.pk, exc) - raise +def _upsert(job, profiles): + """Upsert profile dicts into Person/Company. Returns (result_count, new_people).""" result_count = 0 new_people = [] for raw_profile in profiles: @@ -70,7 +43,6 @@ def run_browser_job(self, browser_job_id): domain=raw_profile["company_domain"], defaults={"name": raw_profile.get("company_name", "")}, ) - person, created = Person.objects.get_or_create( linkedin_url=raw_profile["linkedin_url"], defaults={ @@ -93,31 +65,140 @@ def run_browser_job(self, browser_job_id): person.headline = raw_profile.get("headline", person.headline) person.location = raw_profile.get("location", person.location) person.save() - result_count += 1 + return result_count, new_people - job.status = JobStatus.SUCCEEDED - job.result_count = result_count - job.finished_at = timezone.now() - job.save(update_fields=["status", "result_count", "finished_at"]) - - # Enrichment cascades automatically from a successful BrowserJob, - # and only for people this run actually created — a rediscovered - # Person was already enriched (or is being enriched) by the job - # that first found them. See docs/workflows/enrichment.md. Created - # after the job is marked succeeded so a failure mid-upsert never - # enqueues enrichment for a job that ends up failed. - for person in new_people: - enrichment_job = EnrichmentJob.objects.create( - person=person, browser_job=job - ) - run_enrichment_job.delay(enrichment_job.pk) - - log_activity( - actor=actor, - verb="browser_job.succeeded", - target_type="BrowserJob", - target_id=job.pk, - metadata={"result_count": result_count}, + +def _handle_health(task, job, actor, exc): + """Stop a job on a session-health failure; escalate the worst kinds.""" + from services.browser.exceptions import HEALTH_AUTOSTOP_KINDS + if getattr(settings, "BROWSER_HEALTH_AUTOSTOP", True) and exc.kind in HEALTH_AUTOSTOP_KINDS: + control.engage_emergency_stop(reason=f"health:{exc.kind}", updated_by="worker") + _finish(job, JobStatus.FAILED, actor, "browser_job.health_stopped", + error=str(exc), metadata={"health_kind": exc.kind}) + logger.warning("BrowserJob %s stopped by health check: %s", job.pk, exc.kind) + + +@shared_task( + bind=True, + soft_time_limit=settings.BROWSER_TASK_SOFT_TIME_LIMIT, + time_limit=settings.BROWSER_TASK_TIME_LIMIT, +) +def run_browser_job(self, browser_job_id): + """Runs a BrowserJob under the safety controls (docs/.../safety spec): + emergency stop, single-session lock, cooperative cancellation, per-action + budget, session-health detection, and a transient-only retry policy. + + Ownership rule (unchanged): discovered_by is set only when a Person is + first created; rediscovery never reassigns it. + """ + job = BrowserJob.objects.get(pk=browser_job_id) + + actor = None + profile = ( + UserProfile.objects.filter(supabase_user_id=job.requested_by) + .select_related("user") + .first() ) - return result_count + if profile: + actor = profile.user + + # 1. Emergency stop: refuse before touching the browser. + if control.emergency_stop_active(): + _finish(job, JobStatus.CANCELLED, actor, "browser_job.emergency_stopped", + error="Emergency stop active — job cancelled.", + metadata={"reason": "emergency_stop"}) + logger.warning("BrowserJob %s cancelled: emergency stop active", job.pk) + return 0 + + # 2. Cancel requested while still queued. + if cancellation.is_cancel_requested(job.pk): + cancellation.clear_cancel(job.pk) + _finish(job, JobStatus.CANCELLED, actor, "browser_job.cancelled", + error="Cancelled before start.", metadata={"reason": "user_cancel"}) + return 0 + + # 3. Single-session lock: only one LinkedIn browser at a time. + if not locks.acquire_session_lock(job.pk): + job.status = JobStatus.PENDING + job.save(update_fields=["status"]) + log_activity(actor=actor, verb="browser_job.requeued", + target_type="BrowserJob", target_id=job.pk, + metadata={"holder": locks.session_lock_holder()}) + run_browser_job.apply_async( + args=[job.pk], countdown=settings.BROWSER_REQUEUE_DELAY + ) + return 0 + + try: + job.status = JobStatus.RUNNING + job.started_at = timezone.now() + job.celery_task_id = self.request.id or "" + job.save(update_fields=["status", "started_at", "celery_task_id"]) + log_activity(actor=actor, verb="browser_job.started", + target_type="BrowserJob", target_id=job.pk, metadata={}) + # Ledger: one 'search' action for this run. + if profile: + action_budget.record_action(actor, "search", units=1, target_id=job.pk) + + try: + profiles = BrowserService().run_linkedin_search(job.criteria) + except SessionHealthError as exc: + _handle_health(self, job, actor, exc) + return 0 + except RetryableBrowserError as exc: + log_activity(actor=actor, verb="browser_job.retried", + target_type="BrowserJob", target_id=job.pk, + metadata={"error": str(exc), "retry_count": self.request.retries}) + # Release the lock before retrying so the retry re-queues fairly. + locks.release_session_lock(job.pk) + raise self.retry( + exc=exc, countdown=settings.BROWSER_RETRY_BACKOFF, + max_retries=settings.BROWSER_MAX_RETRIES, + ) + except SoftTimeLimitExceeded: + _finish(job, JobStatus.FAILED, actor, "browser_job.failed", + error="Browser job exceeded its time limit.", + metadata={"reason": "timeout"}) + logger.warning("BrowserJob %s hit soft time limit", job.pk) + return 0 + except Exception as exc: # unknown → fail visibly, do not retry + _finish(job, JobStatus.FAILED, actor, "browser_job.failed", + error=str(exc), metadata={"error": str(exc)}) + logger.warning("BrowserJob %s failed: %s", job.pk, exc) + raise + + # 4. Per-profile budget: truncate to what the user may still pull. + if profile: + remaining = action_budget.remaining_profile_views(profile) + if remaining == 0: + _finish(job, JobStatus.CANCELLED, actor, "browser_job.rate_limited", + error="Daily profile budget exhausted.", + metadata={"reason": "profile_budget"}) + return 0 + if remaining > 0 and len(profiles) > remaining: + profiles = profiles[:remaining] + + result_count, new_people = _upsert(job, profiles) + + # 5. Ledger: profiles actually pulled. + if profile and result_count: + action_budget.record_action(actor, "profile_view", + units=result_count, target_id=job.pk) + + job.status = JobStatus.SUCCEEDED + job.result_count = result_count + job.finished_at = timezone.now() + job.save(update_fields=["status", "result_count", "finished_at"]) + + for person in new_people: + enrichment_job = EnrichmentJob.objects.create(person=person, browser_job=job) + run_enrichment_job.delay(enrichment_job.pk) + + log_activity(actor=actor, verb="browser_job.succeeded", + target_type="BrowserJob", target_id=job.pk, + metadata={"result_count": result_count}) + return result_count + finally: + # Always free the session for the next job, success or failure. + locks.release_session_lock(job.pk) diff --git a/browser/test_safety.py b/browser/test_safety.py index 696ddfa..a36e382 100644 --- a/browser/test_safety.py +++ b/browser/test_safety.py @@ -1,11 +1,18 @@ import uuid +from unittest import mock -from django.test import TestCase +from django.core.cache import cache +from django.test import TestCase, override_settings from django.conf import settings from core.models import JobStatus from browser.models import BrowserJob from browser.models import BrowserControlState +from services.browser import control +from services.browser.exceptions import SessionHealthError + + +FIXTURE = "services.browser.providers.fixture.FixtureBrowserProvider" class JobStatusCancelledTests(TestCase): @@ -49,3 +56,83 @@ def test_save_always_pins_pk_1(self): state.save() self.assertEqual(BrowserControlState.objects.count(), 1) self.assertTrue(BrowserControlState.load().emergency_stop) + + +@override_settings(BROWSER_AUTOMATION_PROVIDER=FIXTURE) +class RunBrowserJobSafetyTests(TestCase): + def setUp(self): + cache.clear() + + def _job(self, **kw): + return BrowserJob.objects.create(requested_by=uuid.uuid4(), **kw) + + def test_emergency_stop_short_circuits_to_cancelled(self): + control.engage_emergency_stop(reason="test") + job = self._job() + from browser.tasks import run_browser_job + run_browser_job.delay(job.pk) + job.refresh_from_db() + self.assertEqual(job.status, JobStatus.CANCELLED) + self.assertIn("emergency", job.error.lower()) + + def test_requeue_when_session_locked(self): + from services.browser import locks + from browser.tasks import run_browser_job + locks.acquire_session_lock(999) # another job holds the session + job = self._job() + with mock.patch.object(run_browser_job, "apply_async") as mock_requeue: + run_browser_job.delay(job.pk) + job.refresh_from_db() + self.assertEqual(job.status, JobStatus.PENDING) # bounced back to queue + mock_requeue.assert_called_once() # and re-enqueued + + def test_cancel_flag_before_start_cancels(self): + from services.browser import cancellation + job = self._job() + cancellation.request_cancel(job.pk) + from browser.tasks import run_browser_job + run_browser_job.delay(job.pk) + job.refresh_from_db() + self.assertEqual(job.status, JobStatus.CANCELLED) + + +from django.contrib.auth import get_user_model +from users.models import UserProfile, OnboardingStatus + + +class HealthAndBudgetTests(TestCase): + def setUp(self): + cache.clear() + + def _profile_and_uuid(self, **budget): + uid = uuid.uuid4() + user = get_user_model().objects.create(username=f"u{uuid.uuid4().hex[:8]}") + UserProfile.objects.create( + user=user, supabase_user_id=uid, + daily_action_budget=budget.get("daily_action_budget", 0), + action_budgets=budget.get("action_budgets", {}), + action_cooldown_seconds=0, + ) + return user, uid + + @mock.patch("browser.tasks.BrowserService") + def test_health_error_stops_and_autostops(self, svc): + svc.return_value.run_linkedin_search.side_effect = SessionHealthError("captcha") + user, uid = self._profile_and_uuid() + job = BrowserJob.objects.create(requested_by=uid) + from browser.tasks import run_browser_job + run_browser_job.delay(job.pk) + job.refresh_from_db() + self.assertEqual(job.status, JobStatus.FAILED) + self.assertEqual(job.error.count("captcha") >= 1, True) + self.assertTrue(control.emergency_stop_active()) # captcha escalated + + @override_settings(BROWSER_AUTOMATION_PROVIDER=FIXTURE) + def test_profile_budget_truncates(self): + user, uid = self._profile_and_uuid(action_budgets={"profile_view": 1}) + job = BrowserJob.objects.create(requested_by=uid) + from browser.tasks import run_browser_job + run_browser_job.delay(job.pk) + job.refresh_from_db() + self.assertEqual(job.status, JobStatus.SUCCEEDED) + self.assertEqual(job.result_count, 1) # fixture yields 2, capped to 1 diff --git a/browser/test_tasks.py b/browser/test_tasks.py index ac0779f..f845d42 100644 --- a/browser/test_tasks.py +++ b/browser/test_tasks.py @@ -1,6 +1,7 @@ import uuid from unittest import mock +from django.core.cache import cache from django.test import TestCase, override_settings from companies.models import Company @@ -16,6 +17,14 @@ BROWSER_AUTOMATION_PROVIDER="services.browser.providers.fixture.FixtureBrowserProvider" ) class RunBrowserJobTests(TestCase): + def setUp(self): + # The task now touches process-global cache keys (emergency-stop + # flag, single-session lock) added by the safety controls — clear + # them so an earlier test's state (e.g. a held lock, or a stop + # flag's TTL) can't leak into this class. Same convention as + # services/browser/test_control.py and test_locks.py. + cache.clear() + def test_succeeds_and_creates_people(self): job = BrowserJob.objects.create(requested_by=uuid.uuid4()) From 6b6cf4c0fffb794b2c7ef112af8afbb4a528449e Mon Sep 17 00:00:00 2001 From: Dominik Ledzion Date: Thu, 9 Jul 2026 16:07:16 +0200 Subject: [PATCH 21/34] wyjscie z biura czwartek --- .env.example | 13 ++ config/settings/base.py | 38 ++++ docs/integrations/kimi.md | 51 +++++ frontend/src/app/(app)/dashboard/page.tsx | 6 + frontend/src/app/(app)/search/page.tsx | 12 +- frontend/src/app/(auth)/login/page.tsx | 4 +- frontend/src/app/(auth)/signup/page.tsx | 4 +- frontend/src/app/actions/auth.ts | 18 +- frontend/src/components/providers.tsx | 10 + frontend/src/features/search/hooks.ts | 5 +- frontend/src/lib/types.ts | 7 + pytest.ini | 2 + services/browser/providers/kimi.py | 259 +++++++++++++++++++++- services/browser/providers/tests.py | 178 ++++++++++++++- 14 files changed, 584 insertions(+), 23 deletions(-) diff --git a/.env.example b/.env.example index fd6f7fc..e6927b2 100644 --- a/.env.example +++ b/.env.example @@ -21,6 +21,19 @@ SUPABASE_DB_PASSWORD= SUPABASE_URL= SUPABASE_SERVICE_ROLE_KEY= +# Browser automation (Kimi WebBridge — services/browser/providers/kimi.py). +# The default provider is the safe fixture (no real browser). To drive the +# real LinkedIn Sales Navigator via the Kimi WebBridge daemon on THIS +# machine, set: +# BROWSER_AUTOMATION_PROVIDER=services.browser.providers.kimi.KimiBrowserProvider +BROWSER_AUTOMATION_PROVIDER= +KIMI_WEBBRIDGE_BASE_URL=http://127.0.0.1:10086 +# The dedicated, persistent Chrome profile you keep logged into LinkedIn. +# scripts/launch_kimi_chrome.ps1 starts Chrome with --user-data-dir= +# (never a temporary/incognito profile); log into LinkedIn once and the +# session persists for every future Browser Job. See docs/integrations/kimi.md. +KIMI_CHROME_USER_DATA_DIR=C:\Users\ledzi\AppData\Local\KIMI\ChromeProfile + # Enrichment sources (Phase 5 — services/richapi, services/apollo). # Leave a key empty and that source records a "not configured" attempt # and the waterfall moves on; nothing crashes without paid keys. diff --git a/config/settings/base.py b/config/settings/base.py index 9d10e2d..a3ff462 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -66,6 +66,36 @@ KIMI_WEBBRIDGE_MAX_PAGES = int(os.environ.get('KIMI_WEBBRIDGE_MAX_PAGES', '20')) +# The dedicated, persistent Chrome profile the operator keeps logged into +# LinkedIn. Kimi WebBridge is a daemon + browser-extension architecture: it +# drives whichever Chrome profile hosts its extension via chrome.debugger — +# it does NOT launch Chrome and there is no Playwright userDataDir the code +# can set (docs/integrations/kimi.md). This value therefore serves two +# concrete purposes and is NOT read by the daemon: +# 1. scripts/launch_kimi_chrome.ps1 starts Chrome with +# --user-data-dir= so the profile is fixed and never temporary; +# 2. KimiBrowserProvider names it in the session-invalidated error so a +# logged-out profile is actionable rather than mysterious. +# Log into LinkedIn once in this profile and the session persists (cookies +# on disk) across all future Browser Jobs — re-login is only needed if +# LinkedIn itself invalidates the cookies. +KIMI_CHROME_USER_DATA_DIR = ( + os.environ.get('KIMI_CHROME_USER_DATA_DIR') + or r'C:\Users\ledzi\AppData\Local\KIMI\ChromeProfile' +) + +# Where KimiBrowserProvider drops operator-internal debugging artifacts +# (screenshot + page HTML) when a Sales Navigator extraction finds no +# profiles. This is diagnostic state for the operator/admin only — it is +# NEVER surfaced to a User (docs/integrations/kimi.md: "Never expose +# automation state (screenshots, HTML, selectors) to any User-facing +# surface"). The daemon runs on the same host as Django/Celery, so the +# provider and the daemon share this path. +KIMI_WEBBRIDGE_DEBUG_DIR = ( + os.environ.get('KIMI_WEBBRIDGE_DEBUG_DIR') + or str(BASE_DIR / 'artifacts' / 'browser') +) + # Enrichment sources (docs/workflows/enrichment.md). Keys come from the # environment — never hardcoded (CLAUDE.md). An empty key doesn't crash @@ -226,6 +256,14 @@ 'CONN_MAX_AGE': 60, + # CONN_HEALTH_CHECKS makes Django ping each pooled connection before + # reuse and discard+reconnect dead ones. Supabase reaps idle + # connections on its direct endpoint; without this, the next request + # reuses a dead socket → "server closed the connection unexpectedly" / + # "consuming input failed". (Top-level DATABASES key, not an OPTIONS + # entry — OPTIONS are passed through to libpq/psycopg.) + 'CONN_HEALTH_CHECKS': True, + 'OPTIONS': {'sslmode': 'require'}, } diff --git a/docs/integrations/kimi.md b/docs/integrations/kimi.md index 4a9f1ec..43b8045 100644 --- a/docs/integrations/kimi.md +++ b/docs/integrations/kimi.md @@ -153,8 +153,59 @@ Two live findings worth keeping in mind: - Never expose automation state (screenshots, HTML, selectors) to any User-facing surface. +## Persistent Chrome session (the login only happens once) + +**Kimi WebBridge is a daemon + browser-extension architecture, not a browser +launcher.** The daemon at `127.0.0.1:10086` opens a WebSocket to a Chrome +extension you installed in your real Chrome and drives tabs via the +`chrome.debugger` API (the provider's `cdp` action is a raw +`chrome.debugger` passthrough). Confirm it live with +`kimi-webbridge status` → `extension_connected: true`. + +Consequences that matter for authentication: + +- There is **no Playwright** anywhere in this system, so there is no + `launch()` vs `launch_persistent_context()` choice and **no per-job clean + context**. The provider only ever sends `navigate` / `evaluate` / `cdp`; + it never creates a browser context. A "logged-out" search is not a fresh + context discarding cookies — it means the Chrome profile hosting the + extension is simply not logged into LinkedIn. +- The **`userDataDir` is the profile directory of the Chrome where the Kimi + extension is installed.** The provider cannot set it. To make it a fixed, + dedicated, persistent profile, launch Chrome yourself with a fixed + `--user-data-dir` and install the extension there — that is what + `scripts/launch_kimi_chrome.ps1` does. + +### One-time setup + +1. Run `scripts/launch_kimi_chrome.ps1`. It starts Chrome with + `--user-data-dir=%KIMI_CHROME_USER_DATA_DIR%` (default + `C:\Users\ledzi\AppData\Local\KIMI\ChromeProfile`) — a persistent + on-disk profile, never incognito/guest. +2. Install the Kimi WebBridge extension in that profile. +3. Log into LinkedIn once (open Sales Navigator to confirm the seat). + +Thereafter every Browser Job — and later Apollo / RichAPI browser jobs — +reuses that one authenticated session. Cookies persist to disk +(`cookies.db`, Local/Session Storage); you only log in again if LinkedIn +invalidates the cookies. + +### Preflight session check + +Before it touches Sales Navigator, `KimiBrowserProvider._assert_linkedin_session` +navigates to the LinkedIn feed and checks whether the profile is bounced to +an auth wall (`/login`, `/checkpoint`, `/authwall`, `/uas/login`). A +logged-out profile fails immediately with `KimiWebBridgeSessionError` naming +`KIMI_CHROME_USER_DATA_DIR`, instead of the operator discovering it deep in +results polling. The provider **never re-authenticates** +(docs/workflows/browser.md). + ## Settings - `KIMI_WEBBRIDGE_BASE_URL` (default `http://127.0.0.1:10086`) - `KIMI_WEBBRIDGE_MAX_RESULTS` (default `100`) - `KIMI_WEBBRIDGE_MAX_PAGES` (default `20`) +- `KIMI_CHROME_USER_DATA_DIR` (default + `C:\Users\ledzi\AppData\Local\KIMI\ChromeProfile`) — the dedicated + persistent Chrome profile; used by `scripts/launch_kimi_chrome.ps1` and + named in the session-invalidated error. Not read by the daemon. diff --git a/frontend/src/app/(app)/dashboard/page.tsx b/frontend/src/app/(app)/dashboard/page.tsx index 4e936fd..2085d0d 100644 --- a/frontend/src/app/(app)/dashboard/page.tsx +++ b/frontend/src/app/(app)/dashboard/page.tsx @@ -5,13 +5,17 @@ import { useQuery } from "@tanstack/react-query"; import { Users, BadgeCheck, Radar, Send, ArrowRight } from "lucide-react"; import { supabase } from "@/lib/supabase/queries"; import { useActivity } from "@/features/activity/hooks"; +import { useMyProfile } from "@/features/search/hooks"; import { PageHeader } from "@/components/shared/page-header"; +import { OnboardingBanner } from "@/components/onboarding-banner"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { Skeleton } from "@/components/ui/skeleton"; import { JobStatusBadge } from "@/components/shared/status-badge"; import { formatRelativeTime, formatCompact } from "@/lib/utils"; export default function DashboardPage() { + const profile = useMyProfile(); + const counts = useQuery({ queryKey: ["dashboard", "counts"], queryFn: async () => { @@ -49,6 +53,8 @@ export default function DashboardPage() {
+ +
} label="People" value={counts.data?.people} href="/people" /> } label="Verified" value={counts.data?.verified} href="/enrichment" /> diff --git a/frontend/src/app/(app)/search/page.tsx b/frontend/src/app/(app)/search/page.tsx index af0d9f6..5f52bd8 100644 --- a/frontend/src/app/(app)/search/page.tsx +++ b/frontend/src/app/(app)/search/page.tsx @@ -4,11 +4,12 @@ import { useRouter } from "next/navigation"; import { useMutation } from "@tanstack/react-query"; import { toast } from "sonner"; import { Play, Info } from "lucide-react"; -import { useMyCriteria } from "@/features/search/hooks"; +import { useMyCriteria, useMyProfile } from "@/features/search/hooks"; import { triggerBrowserJob } from "@/lib/api/django"; import { ApiError } from "@/lib/api/django"; import { PageHeader } from "@/components/shared/page-header"; import { ErrorState } from "@/components/shared/empty-state"; +import { OnboardingBanner } from "@/components/onboarding-banner"; import { Button } from "@/components/ui/button"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { Skeleton } from "@/components/ui/skeleton"; @@ -17,6 +18,8 @@ import { Badge } from "@/components/ui/badge"; export default function SearchPage() { const router = useRouter(); const { data: criteria, isLoading, isError } = useMyCriteria(); + const { data: profile } = useMyProfile(); + const isReady = profile?.status === "READY"; const run = useMutation({ mutationFn: () => triggerBrowserJob(), @@ -45,12 +48,17 @@ export default function SearchPage() { title="Search" description="Your assigned Sales Navigator search criteria. Run it to discover new leads." actions={ - } /> + +

diff --git a/frontend/src/app/(auth)/login/page.tsx b/frontend/src/app/(auth)/login/page.tsx index 70562ce..2a91276 100644 --- a/frontend/src/app/(auth)/login/page.tsx +++ b/frontend/src/app/(auth)/login/page.tsx @@ -24,8 +24,8 @@ function LoginForm() {

{notice === "account-created" ? (

- Account created. An Administrator must provision your workspace profile before the - API accepts your session. + Account created. Check your email to confirm your address, then sign in — your + workspace is provisioned automatically on first login.

) : null} {state?.error ? ( diff --git a/frontend/src/app/(auth)/signup/page.tsx b/frontend/src/app/(auth)/signup/page.tsx index 5e91ecf..eb563f7 100644 --- a/frontend/src/app/(auth)/signup/page.tsx +++ b/frontend/src/app/(auth)/signup/page.tsx @@ -45,8 +45,8 @@ export default function SignupPage() {

- After signup, an Administrator provisions your workspace profile before the API accepts - your session. + After signup, confirm your email, then sign in — your workspace is provisioned + automatically on first login.

diff --git a/frontend/src/app/actions/auth.ts b/frontend/src/app/actions/auth.ts index c602c7d..30d2585 100644 --- a/frontend/src/app/actions/auth.ts +++ b/frontend/src/app/actions/auth.ts @@ -10,11 +10,11 @@ import { createServerSupabaseClient } from "@/lib/supabase/server"; * token is later sent to the Django API as a Bearer. Django only verifies * the JWT — it never issues one (users/authentication.py). * - * Note on provisioning: Django requires a UserProfile row matching the - * Supabase user's `sub` (admin-provisioned, docs/workflows/admin.md). A - * freshly signed-up user can authenticate with Supabase but will get - * "No local account provisioned" from Django until an Administrator - * creates their UserProfile. The UI surfaces this honestly. + * Note on provisioning: on the user's first verified login, Django + * auto-provisions a shadow User + UserProfile linked to the Supabase + * `sub` (users/authentication.py). Signup therefore needs no backend + * call — the user confirms their email, signs in, and the workspace is + * created on first API request. */ export type AuthFormState = { error?: string; ok?: boolean } | null; @@ -48,10 +48,10 @@ export async function signupAction( const { error } = await supabase.auth.signUp({ email, password }); if (error) return { error: error.message }; - // After signup, the user exists in Supabase Auth but has no Django - // UserProfile yet — the Administrator must provision it. Send them to - // login; the backend will surface the provisioning gap on first API - // call if the admin hasn't acted yet. + // After signup, the user exists in Supabase Auth and must confirm their + // email. On their first verified login, Django auto-provisions the + // shadow User + UserProfile (users/authentication.py) — no admin action + // required. Send them to login with a notice to check their email. redirect("/login?notice=account-created"); } diff --git a/frontend/src/components/providers.tsx b/frontend/src/components/providers.tsx index e527d93..77f6e86 100644 --- a/frontend/src/components/providers.tsx +++ b/frontend/src/components/providers.tsx @@ -40,6 +40,16 @@ export function Providers({ children }: { children: ReactNode }) { defaultTheme="dark" enableSystem disableTransitionOnChange + // next-themes injects an inline