diff --git a/cmd/e2a-contract-server/main.go b/cmd/e2a-contract-server/main.go index 973ab5307..9c4015888 100644 --- a/cmd/e2a-contract-server/main.go +++ b/cmd/e2a-contract-server/main.go @@ -30,12 +30,13 @@ func main() { } // E2A_TEST_CAPPED_API_KEY authenticates the secondary account seeded with - // testutil.CappedLimits. Scenarios that assert quota enforcement run as - // that account; CI sources this file with `set -a`, so the runners pick it - // up with no workflow change. + // testutil.CappedLimits. E2A_TEST_OVERCAP_API_KEY authenticates the third + // account, seeded already over testutil.OverCapLimits. Scenarios that + // assert quota enforcement run as one of those accounts; CI sources this + // file with `set -a`, so the runners pick both up with no workflow change. envContent := fmt.Sprintf( - "E2A_TEST_BASE_URL=%s\nE2A_TEST_API_KEY=%s\nE2A_TEST_CAPPED_API_KEY=%s\n", - srv.BaseURL, srv.APIKey, srv.CappedAPIKey, + "E2A_TEST_BASE_URL=%s\nE2A_TEST_API_KEY=%s\nE2A_TEST_CAPPED_API_KEY=%s\nE2A_TEST_OVERCAP_API_KEY=%s\n", + srv.BaseURL, srv.APIKey, srv.CappedAPIKey, srv.OverCapAPIKey, ) if envFile != "" { if err := os.WriteFile(envFile, []byte(envContent), 0o600); err != nil { diff --git a/internal/testutil/contract_server.go b/internal/testutil/contract_server.go index eba420de8..190c388fd 100644 --- a/internal/testutil/contract_server.go +++ b/internal/testutil/contract_server.go @@ -2,6 +2,7 @@ package testutil import ( "context" + "fmt" "net" "net/http" "time" @@ -45,6 +46,18 @@ var CappedLimits = limits.Limits{ UpgradeURL: "https://e2a.dev/upgrade", } +// OverCapLimits are applied to the third account (ContractServer.OverCapAPIKey) +// AFTER its domains/agents are already seeded past this cap, so every create +// attempt is refused with Current strictly greater than Limit. +var OverCapLimits = limits.Limits{ + PlanCode: "contract_overcap", + MaxAgents: 2, + MaxDomains: 1, + MaxMessagesMonth: 100000, + MaxStorageBytes: 1 << 40, + UpgradeURL: "https://e2a.dev/upgrade", +} + type ContractServer struct { BaseURL string APIKey string @@ -63,13 +76,16 @@ type ContractServer struct { // is no staleness window for a scenario to race. CappedAPIKey string CappedUserID string - DBPool *pgxpool.Pool - Store *identity.Store - WSHub *ws.Hub - SMTPAddr string - httpServer *http.Server - httpLn net.Listener - smtpServer *relay.Server + // OverCapAPIKey authenticates the third account. See OverCapLimits. + OverCapAPIKey string + OverCapUserID string + DBPool *pgxpool.Pool + Store *identity.Store + WSHub *ws.Hub + SMTPAddr string + httpServer *http.Server + httpLn net.Listener + smtpServer *relay.Server } func StartContractServer(ctx context.Context, dbURL string) (*ContractServer, error) { @@ -253,19 +269,77 @@ func StartContractServer(ctx context.Context, dbURL string) (*ContractServer, er return nil, err } + // The over-cap account: seed unlimited (maxDomains/maxAgents <= 0), then + // apply OverCapLimits below so the downgrade lands on counts already over it. + overCapUser, err := store.CreateOrGetUser(ctx, "overcap@test.dev", "Contract OverCap", "google-contract-overcap") + if err != nil { + _ = smtpServer.Close() + _ = httpServer.Shutdown(context.Background()) + _ = httpLn.Close() + wsHub.Close() + pool.Close() + return nil, err + } + if _, err := store.ClaimOrCreateDomain(ctx, "overcap-1.test.dev", overCapUser.ID); err != nil { + _ = smtpServer.Close() + _ = httpServer.Shutdown(context.Background()) + _ = httpLn.Close() + wsHub.Close() + pool.Close() + return nil, err + } + if _, err := store.ClaimOrCreateDomain(ctx, "overcap-2.test.dev", overCapUser.ID); err != nil { + _ = smtpServer.Close() + _ = httpServer.Shutdown(context.Background()) + _ = httpLn.Close() + wsHub.Close() + pool.Close() + return nil, err + } + for i := 1; i <= 3; i++ { + agentEmail := fmt.Sprintf("overcap-bot-%d@agents.e2a.dev", i) + if _, err := store.CreateAgentWithLimit(ctx, agentEmail, "overcap-1.test.dev", "OverCap Bot", overCapUser.ID, 0); err != nil { + _ = smtpServer.Close() + _ = httpServer.Shutdown(context.Background()) + _ = httpLn.Close() + wsHub.Close() + pool.Close() + return nil, err + } + } + if err := limits.NewStore(pool).Upsert(ctx, overCapUser.ID, OverCapLimits); err != nil { + _ = smtpServer.Close() + _ = httpServer.Shutdown(context.Background()) + _ = httpLn.Close() + wsHub.Close() + pool.Close() + return nil, err + } + overCapKey, err := store.CreateAPIKey(ctx, overCapUser.ID, "contract-overcap-key", nil) + if err != nil { + _ = smtpServer.Close() + _ = httpServer.Shutdown(context.Background()) + _ = httpLn.Close() + wsHub.Close() + pool.Close() + return nil, err + } + return &ContractServer{ - BaseURL: "http://" + httpLn.Addr().String(), - APIKey: key.PlaintextKey, - UserID: user.ID, - CappedAPIKey: cappedKey.PlaintextKey, - CappedUserID: cappedUser.ID, - DBPool: pool, - Store: store, - WSHub: wsHub, - SMTPAddr: smtpAddr, - httpServer: httpServer, - httpLn: httpLn, - smtpServer: smtpServer, + BaseURL: "http://" + httpLn.Addr().String(), + APIKey: key.PlaintextKey, + UserID: user.ID, + CappedAPIKey: cappedKey.PlaintextKey, + CappedUserID: cappedUser.ID, + OverCapAPIKey: overCapKey.PlaintextKey, + OverCapUserID: overCapUser.ID, + DBPool: pool, + Store: store, + WSHub: wsHub, + SMTPAddr: smtpAddr, + httpServer: httpServer, + httpLn: httpLn, + smtpServer: smtpServer, }, nil } diff --git a/sdks/python/tests/test_contract.py b/sdks/python/tests/test_contract.py index d12d4329f..a97e85240 100644 --- a/sdks/python/tests/test_contract.py +++ b/sdks/python/tests/test_contract.py @@ -7,6 +7,10 @@ account, seeded with tiny plan caps. Scenarios asserting quota enforcement run as that account and skip without it (a deployed staging target has no capped account to offer). + E2A_TEST_OVERCAP_API_KEY: optional; key for the contract server's third + account, seeded already over its plan caps. The scenario proving the 402 + envelope's `current` field runs as that account and skips without it (a + deployed staging target has no over-cap account to offer). The runner drives the server over raw HTTP (a thin scenario interpreter, not the ergonomic client): @@ -47,6 +51,7 @@ BASE_URL = os.environ.get("E2A_TEST_BASE_URL", "") API_KEY = os.environ.get("E2A_TEST_API_KEY", "") CAPPED_API_KEY = os.environ.get("E2A_TEST_CAPPED_API_KEY", "") +OVERCAP_API_KEY = os.environ.get("E2A_TEST_OVERCAP_API_KEY", "") # tests/test_contract.py -> sdks/python/tests/ -> sdks/python/ -> sdks/ -> repo root SCENARIOS_PATH = Path(__file__).resolve().parents[3] / "tests" / "contract" / "scenarios.yaml" @@ -136,13 +141,11 @@ def values_equal(json_val: Any, yaml_val: Any) -> bool: STORE_ACTIONS = {"inject_message", "verify_and_retry"} CAPPED_KEY_PLACEHOLDER = "{capped_api_key}" +OVERCAP_KEY_PLACEHOLDER = "{overcap_api_key}" -def scenario_needs_capped_account(sc: dict[str, Any]) -> bool: - """True when the scenario authenticates as the capped-plan account. - - Those scenarios need a cap they can actually reach, which only the contract - server's seeded secondary account provides. +def _scenario_uses_placeholder(sc: dict[str, Any], placeholder: str) -> bool: + """True when the scenario's auth_override anywhere names `placeholder`. Inspects auth_override VALUES specifically. Dumping the whole scenario and substring-matching looks equivalent and is not: the scenario's own @@ -156,9 +159,25 @@ def scenario_needs_capped_account(sc: dict[str, Any]) -> bool: for step in sc.get(key) or []: if isinstance(step, dict): overrides.append(step.get("auth_override")) - return any( - isinstance(o, str) and CAPPED_KEY_PLACEHOLDER in o for o in overrides - ) + return any(isinstance(o, str) and placeholder in o for o in overrides) + + +def scenario_needs_capped_account(sc: dict[str, Any]) -> bool: + """True when the scenario authenticates as the capped-plan account. + + Those scenarios need a cap they can actually reach, which only the contract + server's seeded secondary account provides. + """ + return _scenario_uses_placeholder(sc, CAPPED_KEY_PLACEHOLDER) + + +def scenario_needs_overcap_account(sc: dict[str, Any]) -> bool: + """True when the scenario authenticates as the over-cap account. + + Those scenarios need an account already over its plan caps, which only the + contract server's seeded third account provides. + """ + return _scenario_uses_placeholder(sc, OVERCAP_KEY_PLACEHOLDER) def scenario_needs_store(sc: dict[str, Any]) -> bool: @@ -193,6 +212,8 @@ def __init__(self, base_url: str, api_key: str, scenario: dict[str, Any]): # empty bearer token can never reach the wire as a confusing 401. if CAPPED_API_KEY: self.vars["capped_api_key"] = CAPPED_API_KEY + if OVERCAP_API_KEY: + self.vars["overcap_api_key"] = OVERCAP_API_KEY self._http = httpx.Client(base_url=base_url, timeout=30) def close(self): @@ -1179,6 +1200,8 @@ def test_contract_scenario(scenario): # runs everywhere, so this skip cannot silently become zero coverage. if scenario_needs_capped_account(scenario) and not CAPPED_API_KEY: pytest.skip(f"scenario {scenario['name']}: needs E2A_TEST_CAPPED_API_KEY") + if scenario_needs_overcap_account(scenario) and not OVERCAP_API_KEY: + pytest.skip(f"scenario {scenario['name']}: needs E2A_TEST_OVERCAP_API_KEY") runner = Runner(BASE_URL, API_KEY, scenario) try: diff --git a/sdks/typescript/test/v1/contract.test.ts b/sdks/typescript/test/v1/contract.test.ts index 54eb8cda3..f808959ba 100644 --- a/sdks/typescript/test/v1/contract.test.ts +++ b/sdks/typescript/test/v1/contract.test.ts @@ -8,6 +8,10 @@ * secondary account, seeded with tiny plan caps. Scenarios that assert * quota enforcement run as that account and skip without it (a deployed * staging target has no capped account to offer). + * E2A_TEST_OVERCAP_API_KEY: optional; key for the contract server's third + * account, seeded already over its plan caps. The scenario proving the + * 402 envelope's `current` field runs as that account and skips without + * it (a deployed staging target has no over-cap account to offer). * * The runner drives the server over raw HTTP (a thin scenario interpreter, * not the ergonomic client) plus {@link WSListener} for WebSocket steps. @@ -38,22 +42,34 @@ const SEED = seedEnabled(); // runner is pointed at a deployed server, which has no such account. const CAPPED_API_KEY = process.env.E2A_TEST_CAPPED_API_KEY; +// The contract server's over-cap-account key (see the header). Absent when +// the runner is pointed at a deployed server, which has no such account. +const OVERCAP_API_KEY = process.env.E2A_TEST_OVERCAP_API_KEY; + /** - * True when the scenario authenticates as the capped account anywhere. + * True when the scenario authenticates as `placeholder`'s account anywhere. * * Inspects auth_override VALUES specifically. Stringifying the whole scenario - * looks equivalent and is not: the scenario's own description mentions - * {capped_api_key} in prose, so a blob match stays true even if every + * looks equivalent and is not: the scenario's own description mentions the + * placeholder in prose, so a blob match stays true even if every * auth_override is switched back to the primary account — exactly the * regression this is meant to detect. */ -function scenarioNeedsCappedAccount(sc: Scenario): boolean { +function scenarioUsesPlaceholder(sc: Scenario, placeholder: string): boolean { const overrides = [ sc.auth_override, ...(sc.steps ?? []).map((s) => s.auth_override), ...(sc.cleanup ?? []).map((s) => s.auth_override), ]; - return overrides.some((o) => typeof o === "string" && o.includes("{capped_api_key}")); + return overrides.some((o) => typeof o === "string" && o.includes(placeholder)); +} + +function scenarioNeedsCappedAccount(sc: Scenario): boolean { + return scenarioUsesPlaceholder(sc, "{capped_api_key}"); +} + +function scenarioNeedsOverCapAccount(sc: Scenario): boolean { + return scenarioUsesPlaceholder(sc, "{overcap_api_key}"); } it("parses the generated message lifecycle page contract", () => { @@ -831,6 +847,7 @@ class Runner { // Only bind when present: scenarios needing it are skipped otherwise, so a // silently-empty bearer token can never reach the wire as a confusing 401. if (CAPPED_API_KEY) this.vars.capped_api_key = CAPPED_API_KEY; + if (OVERCAP_API_KEY) this.vars.overcap_api_key = OVERCAP_API_KEY; this.api = new RawApi(apiKey, baseUrl); this.seeder = SEED ? new Seeder(baseUrl, apiKey) : null; } @@ -1249,15 +1266,16 @@ describe.skipIf(!baseUrl || !apiKey)("Contract scenarios", () => { for (const sc of scenarios) { // Store-dependent scenarios run when SEED supplies their preconditions over // the API; otherwise they skip. Account-global scenarios skip regardless. - // Quota scenarios need the capped account; without its key there is no way - // to reach a cap on a live server, so they skip. The Go runner owns the - // contract server in-process and always has it, and the always-on shape - // test below fails if the scenario is ever deleted or defanged — so a skip - // here can never quietly become zero coverage. + // Quota scenarios need the capped or over-cap account; without its key + // there is no way to reach a cap on a live server, so they skip. The Go + // runner owns the contract server in-process and always has it, and the + // always-on shape test below fails if the scenario is ever deleted or + // defanged, so a skip here can never quietly become zero coverage. const skip = ACCOUNT_GLOBAL.has(sc.name) || (scenarioNeedsStore(sc) && !SEED) || - (scenarioNeedsCappedAccount(sc) && !CAPPED_API_KEY); + (scenarioNeedsCappedAccount(sc) && !CAPPED_API_KEY) || + (scenarioNeedsOverCapAccount(sc) && !OVERCAP_API_KEY); (skip ? it.skip : it)( sc.name, diff --git a/tests/contract/contract_test.go b/tests/contract/contract_test.go index 7b278c13d..b12fe21c5 100644 --- a/tests/contract/contract_test.go +++ b/tests/contract/contract_test.go @@ -138,6 +138,10 @@ type testEnv struct { // cappedAPIKey authenticates the contract server's secondary account, // seeded with testutil.CappedLimits, that quota scenarios run as. cappedAPIKey string + // overCapAPIKey authenticates the contract server's third account, + // seeded with testutil.OverCapLimits over resources already exceeding + // it, that the current-field-proof scenario runs as. + overCapAPIKey string } func setupEnv(t *testing.T) *testEnv { @@ -162,7 +166,8 @@ func setupEnv(t *testing.T) *testEnv { apiKey: cs.APIKey, userID: cs.UserID, - cappedAPIKey: cs.CappedAPIKey, + cappedAPIKey: cs.CappedAPIKey, + overCapAPIKey: cs.OverCapAPIKey, } } @@ -357,6 +362,7 @@ func (r *runner) resolve(s string) string { s = strings.ReplaceAll(s, "{base_url}", r.env.baseURL) s = strings.ReplaceAll(s, "{api_key}", r.env.apiKey) s = strings.ReplaceAll(s, "{capped_api_key}", r.env.cappedAPIKey) + s = strings.ReplaceAll(s, "{overcap_api_key}", r.env.overCapAPIKey) for k, v := range r.vars { s = strings.ReplaceAll(s, "{"+k+"}", v) } @@ -1008,6 +1014,35 @@ func scenarioUsesCappedKey(t *testing.T, sc scenario) bool { const cappedKeyPlaceholder = "{capped_api_key}" +// requireOverCapKey is requireCappedKey for the over-cap account (see +// testutil.OverCapLimits): same reasoning, same failure-not-skip rule. +func requireOverCapKey(t *testing.T, env *testEnv, sc scenario) { + t.Helper() + if env.overCapAPIKey == "" && scenarioUsesOverCapKey(t, sc) { + t.Fatalf("scenario %s uses %s but the contract server supplied no over-cap API key", sc.Name, overCapKeyPlaceholder) + } +} + +// scenarioUsesOverCapKey is scenarioUsesCappedKey for the over-cap account. +func scenarioUsesOverCapKey(t *testing.T, sc scenario) bool { + t.Helper() + overrides := []*string{sc.AuthOverride} + for _, s := range sc.Steps { + overrides = append(overrides, s.AuthOverride) + } + for _, s := range sc.Cleanup { + overrides = append(overrides, s.AuthOverride) + } + for _, o := range overrides { + if o != nil && strings.Contains(*o, overCapKeyPlaceholder) { + return true + } + } + return false +} + +const overCapKeyPlaceholder = "{overcap_api_key}" + func TestScenarios(t *testing.T) { scenarios := loadScenarios(t) for _, sc := range scenarios { @@ -1015,6 +1050,7 @@ func TestScenarios(t *testing.T) { t.Run(sc.Name, func(t *testing.T) { env := setupEnv(t) requireCappedKey(t, env, sc) + requireOverCapKey(t, env, sc) r := newRunner(env, sc) t.Cleanup(func() { r.cleanup(t) }) r.executeSetup(t) @@ -1123,3 +1159,65 @@ func TestLimitsScenarioShape(t *testing.T) { } } } + +// TestOverCapScenarioShape is TestLimitsScenarioShape for +// account_limits_current_field_proven: it pins the current-strictly-greater- +// than-limit assertion the scenario exists for. +func TestOverCapScenarioShape(t *testing.T) { + var sc scenario + for _, candidate := range loadScenarios(t) { + if candidate.Name == "account_limits_current_field_proven" { + sc = candidate + break + } + } + if sc.Name == "" { + t.Fatal("scenario account_limits_current_field_proven not found: the 402 envelope's current field would have no independent proof in any runner") + } + if !scenarioUsesOverCapKey(t, sc) { + t.Fatalf("scenario %s no longer authenticates as the over-cap account, so current can no longer differ from limit", sc.Name) + } + + steps := map[string]step{} + for _, s := range sc.Steps { + steps[s.ID] = s + } + matched := func(id string) map[string]interface{} { + t.Helper() + s, ok := steps[id] + if !ok || s.Expect == nil { + t.Fatalf("step %s is missing or has no expect block", id) + } + return s.Expect.BodyMatch + } + + // The whole point: current strictly greater than limit, which only a + // server reading the real resource count can produce. + domainRefusal := matched("domain_create_reports_true_overcap_current") + for path, want := range map[string]interface{}{ + "error.details.resource": "domains", + "error.details.limit": 1, + "error.details.current": 2, + } { + if got := domainRefusal[path]; fmt.Sprint(got) != fmt.Sprint(want) { + t.Errorf("domain_create_reports_true_overcap_current body_match[%q] = %v, want %v", path, got, want) + } + } + if fmt.Sprint(domainRefusal["error.details.current"]) == fmt.Sprint(domainRefusal["error.details.limit"]) { + t.Fatalf("domain_create_reports_true_overcap_current pins current == limit, which cannot distinguish a real count from a hardcoded one") + } + + agentRefusal := matched("agent_create_reports_true_overcap_current") + for path, want := range map[string]interface{}{ + "error.details.resource": "agents", + "error.details.limit": 2, + "error.details.current": 3, + } { + if got := agentRefusal[path]; fmt.Sprint(got) != fmt.Sprint(want) { + t.Errorf("agent_create_reports_true_overcap_current body_match[%q] = %v, want %v", path, got, want) + } + } + if fmt.Sprint(agentRefusal["error.details.current"]) == fmt.Sprint(agentRefusal["error.details.limit"]) { + t.Fatalf("agent_create_reports_true_overcap_current pins current == limit, which cannot distinguish a real count from a hardcoded one") + } +} diff --git a/tests/contract/scenarios.yaml b/tests/contract/scenarios.yaml index 7be514bef..bc87f88da 100644 --- a/tests/contract/scenarios.yaml +++ b/tests/contract/scenarios.yaml @@ -2198,10 +2198,12 @@ scenarios: the upgrade URL — the fields an SDK needs to tell a caller WHICH quota stopped them and how to get out of it. The two resources carry DIFFERENT caps (domains 1, agents 2) so a single hardcoded number cannot satisfy - both. `current` is asserted too, but note it is NOT independently proven - here: at the moment of refusal `current == limit` by definition of the - `count >= cap` check, so only an over-cap account (a downgrade) could - distinguish them. Tracked separately. Everything here runs as a SECOND, permanently + both. `current` is asserted too, but at the moment of refusal + `current == limit` by definition of the `count >= cap` check, so this + scenario alone cannot tell a server that reports the real resource + count apart from one that just echoes the cap back: see + account_limits_current_field_proven below for the over-cap account that + does. Everything here runs as a SECOND, permanently capped account (auth_override with {capped_api_key}) that the contract server seeds at startup with max_domains 1 / max_agents 1. Nothing mutates a cap, so this scenario cannot leak enforcement into the ones @@ -2368,6 +2370,53 @@ scenarios: path: /v1/domains/capped-{scenario_token}.test.dev?confirm=DELETE auth_override: "Bearer {capped_api_key}" + - name: account_limits_current_field_proven + description: > + account_limits_enforced cannot independently prove `current` in the 402 + envelope: at refusal `current == limit` by construction, so a server + that hardcoded Current to the cap would pass every assertion there too. + This scenario runs as a THIRD account (auth_override with + {overcap_api_key}) the contract server seeds with 2 domains and 3 + agents BEFORE applying a downgraded cap of 1 domain / 2 agents, so it + starts already over both caps. A further create attempt is refused with + `current` strictly greater than `limit` (2 > 1 for domains, 3 > 2 for + agents), the only value a server actually reading the resource count + can produce: a hardcoded `current: limit` fails both assertions below. + Nothing here can succeed (the account is over cap on every resource + this touches), so nothing mutates and no cleanup is needed. + steps: + - id: domain_create_reports_true_overcap_current + action: request + method: POST + path: /v1/domains + auth_override: "Bearer {overcap_api_key}" + body: + domain: overcap-blocked-{scenario_token}.test.dev + expect: + status: 402 + body_match: + "error.code": limit_exceeded + "error.details.resource": domains + "error.details.limit": 1 + "error.details.current": 2 + "error.details.plan_code": contract_overcap + + - id: agent_create_reports_true_overcap_current + action: request + method: POST + path: /v1/agents + auth_override: "Bearer {overcap_api_key}" + body: + email: overcap-blocked-bot-{scenario_token}@agents.e2a.dev + expect: + status: 402 + body_match: + "error.code": limit_exceeded + "error.details.resource": agents + "error.details.limit": 2 + "error.details.current": 3 + "error.details.plan_code": contract_overcap + - name: messages_list_filter description: > The beta `filter` query parameter on listMessages narrows results by