Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions cmd/e2a-contract-server/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,13 @@ func main() {
}

// E2A_TEST_CAPPED_API_KEY authenticates the secondary account seeded with
// testutil.CappedLimits. Scenarios that assert quota enforcement run as
// that account; CI sources this file with `set -a`, so the runners pick it
// up with no workflow change.
// testutil.CappedLimits. E2A_TEST_OVERCAP_API_KEY authenticates the third
// account, seeded already over testutil.OverCapLimits. Scenarios that
// assert quota enforcement run as one of those accounts; CI sources this
// file with `set -a`, so the runners pick both up with no workflow change.
envContent := fmt.Sprintf(
"E2A_TEST_BASE_URL=%s\nE2A_TEST_API_KEY=%s\nE2A_TEST_CAPPED_API_KEY=%s\n",
srv.BaseURL, srv.APIKey, srv.CappedAPIKey,
"E2A_TEST_BASE_URL=%s\nE2A_TEST_API_KEY=%s\nE2A_TEST_CAPPED_API_KEY=%s\nE2A_TEST_OVERCAP_API_KEY=%s\n",
srv.BaseURL, srv.APIKey, srv.CappedAPIKey, srv.OverCapAPIKey,
)
if envFile != "" {
if err := os.WriteFile(envFile, []byte(envContent), 0o600); err != nil {
Expand Down
112 changes: 93 additions & 19 deletions internal/testutil/contract_server.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package testutil

import (
"context"
"fmt"
"net"
"net/http"
"time"
Expand Down Expand Up @@ -45,6 +46,18 @@ var CappedLimits = limits.Limits{
UpgradeURL: "https://e2a.dev/upgrade",
}

// OverCapLimits are applied to the third account (ContractServer.OverCapAPIKey)
// AFTER its domains/agents are already seeded past this cap, so every create
// attempt is refused with Current strictly greater than Limit.
var OverCapLimits = limits.Limits{
PlanCode: "contract_overcap",
MaxAgents: 2,
MaxDomains: 1,
MaxMessagesMonth: 100000,
MaxStorageBytes: 1 << 40,
UpgradeURL: "https://e2a.dev/upgrade",
}

type ContractServer struct {
BaseURL string
APIKey string
Expand All @@ -63,13 +76,16 @@ type ContractServer struct {
// is no staleness window for a scenario to race.
CappedAPIKey string
CappedUserID string
DBPool *pgxpool.Pool
Store *identity.Store
WSHub *ws.Hub
SMTPAddr string
httpServer *http.Server
httpLn net.Listener
smtpServer *relay.Server
// OverCapAPIKey authenticates the third account. See OverCapLimits.
OverCapAPIKey string
OverCapUserID string
DBPool *pgxpool.Pool
Store *identity.Store
WSHub *ws.Hub
SMTPAddr string
httpServer *http.Server
httpLn net.Listener
smtpServer *relay.Server
}

func StartContractServer(ctx context.Context, dbURL string) (*ContractServer, error) {
Expand Down Expand Up @@ -253,19 +269,77 @@ func StartContractServer(ctx context.Context, dbURL string) (*ContractServer, er
return nil, err
}

// The over-cap account: seed unlimited (maxDomains/maxAgents <= 0), then
// apply OverCapLimits below so the downgrade lands on counts already over it.
overCapUser, err := store.CreateOrGetUser(ctx, "overcap@test.dev", "Contract OverCap", "google-contract-overcap")
if err != nil {
_ = smtpServer.Close()
_ = httpServer.Shutdown(context.Background())
_ = httpLn.Close()
wsHub.Close()
pool.Close()
return nil, err
}
if _, err := store.ClaimOrCreateDomain(ctx, "overcap-1.test.dev", overCapUser.ID); err != nil {
_ = smtpServer.Close()
_ = httpServer.Shutdown(context.Background())
_ = httpLn.Close()
wsHub.Close()
pool.Close()
return nil, err
}
if _, err := store.ClaimOrCreateDomain(ctx, "overcap-2.test.dev", overCapUser.ID); err != nil {
_ = smtpServer.Close()
_ = httpServer.Shutdown(context.Background())
_ = httpLn.Close()
wsHub.Close()
pool.Close()
return nil, err
}
for i := 1; i <= 3; i++ {
agentEmail := fmt.Sprintf("overcap-bot-%d@agents.e2a.dev", i)
if _, err := store.CreateAgentWithLimit(ctx, agentEmail, "overcap-1.test.dev", "OverCap Bot", overCapUser.ID, 0); err != nil {
_ = smtpServer.Close()
_ = httpServer.Shutdown(context.Background())
_ = httpLn.Close()
wsHub.Close()
pool.Close()
return nil, err
}
}
if err := limits.NewStore(pool).Upsert(ctx, overCapUser.ID, OverCapLimits); err != nil {
_ = smtpServer.Close()
_ = httpServer.Shutdown(context.Background())
_ = httpLn.Close()
wsHub.Close()
pool.Close()
return nil, err
}
overCapKey, err := store.CreateAPIKey(ctx, overCapUser.ID, "contract-overcap-key", nil)
if err != nil {
_ = smtpServer.Close()
_ = httpServer.Shutdown(context.Background())
_ = httpLn.Close()
wsHub.Close()
pool.Close()
return nil, err
}

return &ContractServer{
BaseURL: "http://" + httpLn.Addr().String(),
APIKey: key.PlaintextKey,
UserID: user.ID,
CappedAPIKey: cappedKey.PlaintextKey,
CappedUserID: cappedUser.ID,
DBPool: pool,
Store: store,
WSHub: wsHub,
SMTPAddr: smtpAddr,
httpServer: httpServer,
httpLn: httpLn,
smtpServer: smtpServer,
BaseURL: "http://" + httpLn.Addr().String(),
APIKey: key.PlaintextKey,
UserID: user.ID,
CappedAPIKey: cappedKey.PlaintextKey,
CappedUserID: cappedUser.ID,
OverCapAPIKey: overCapKey.PlaintextKey,
OverCapUserID: overCapUser.ID,
DBPool: pool,
Store: store,
WSHub: wsHub,
SMTPAddr: smtpAddr,
httpServer: httpServer,
httpLn: httpLn,
smtpServer: smtpServer,
}, nil
}

Expand Down
39 changes: 31 additions & 8 deletions sdks/python/tests/test_contract.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@
account, seeded with tiny plan caps. Scenarios asserting quota enforcement
run as that account and skip without it (a deployed staging target has no
capped account to offer).
E2A_TEST_OVERCAP_API_KEY: optional; key for the contract server's third
account, seeded already over its plan caps. The scenario proving the 402
envelope's `current` field runs as that account and skips without it (a
deployed staging target has no over-cap account to offer).

The runner drives the server over raw HTTP (a thin scenario interpreter, not
the ergonomic client):
Expand Down Expand Up @@ -47,6 +51,7 @@
BASE_URL = os.environ.get("E2A_TEST_BASE_URL", "")
API_KEY = os.environ.get("E2A_TEST_API_KEY", "")
CAPPED_API_KEY = os.environ.get("E2A_TEST_CAPPED_API_KEY", "")
OVERCAP_API_KEY = os.environ.get("E2A_TEST_OVERCAP_API_KEY", "")

# tests/test_contract.py -> sdks/python/tests/ -> sdks/python/ -> sdks/ -> repo root
SCENARIOS_PATH = Path(__file__).resolve().parents[3] / "tests" / "contract" / "scenarios.yaml"
Expand Down Expand Up @@ -136,13 +141,11 @@ def values_equal(json_val: Any, yaml_val: Any) -> bool:
STORE_ACTIONS = {"inject_message", "verify_and_retry"}

CAPPED_KEY_PLACEHOLDER = "{capped_api_key}"
OVERCAP_KEY_PLACEHOLDER = "{overcap_api_key}"


def scenario_needs_capped_account(sc: dict[str, Any]) -> bool:
"""True when the scenario authenticates as the capped-plan account.

Those scenarios need a cap they can actually reach, which only the contract
server's seeded secondary account provides.
def _scenario_uses_placeholder(sc: dict[str, Any], placeholder: str) -> bool:
"""True when the scenario's auth_override anywhere names `placeholder`.

Inspects auth_override VALUES specifically. Dumping the whole scenario and
substring-matching looks equivalent and is not: the scenario's own
Expand All @@ -156,9 +159,25 @@ def scenario_needs_capped_account(sc: dict[str, Any]) -> bool:
for step in sc.get(key) or []:
if isinstance(step, dict):
overrides.append(step.get("auth_override"))
return any(
isinstance(o, str) and CAPPED_KEY_PLACEHOLDER in o for o in overrides
)
return any(isinstance(o, str) and placeholder in o for o in overrides)


def scenario_needs_capped_account(sc: dict[str, Any]) -> bool:
"""True when the scenario authenticates as the capped-plan account.

Those scenarios need a cap they can actually reach, which only the contract
server's seeded secondary account provides.
"""
return _scenario_uses_placeholder(sc, CAPPED_KEY_PLACEHOLDER)


def scenario_needs_overcap_account(sc: dict[str, Any]) -> bool:
"""True when the scenario authenticates as the over-cap account.

Those scenarios need an account already over its plan caps, which only the
contract server's seeded third account provides.
"""
return _scenario_uses_placeholder(sc, OVERCAP_KEY_PLACEHOLDER)


def scenario_needs_store(sc: dict[str, Any]) -> bool:
Expand Down Expand Up @@ -193,6 +212,8 @@ def __init__(self, base_url: str, api_key: str, scenario: dict[str, Any]):
# empty bearer token can never reach the wire as a confusing 401.
if CAPPED_API_KEY:
self.vars["capped_api_key"] = CAPPED_API_KEY
if OVERCAP_API_KEY:
self.vars["overcap_api_key"] = OVERCAP_API_KEY
self._http = httpx.Client(base_url=base_url, timeout=30)

def close(self):
Expand Down Expand Up @@ -1179,6 +1200,8 @@ def test_contract_scenario(scenario):
# runs everywhere, so this skip cannot silently become zero coverage.
if scenario_needs_capped_account(scenario) and not CAPPED_API_KEY:
pytest.skip(f"scenario {scenario['name']}: needs E2A_TEST_CAPPED_API_KEY")
if scenario_needs_overcap_account(scenario) and not OVERCAP_API_KEY:
pytest.skip(f"scenario {scenario['name']}: needs E2A_TEST_OVERCAP_API_KEY")

runner = Runner(BASE_URL, API_KEY, scenario)
try:
Expand Down
40 changes: 29 additions & 11 deletions sdks/typescript/test/v1/contract.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@
* secondary account, seeded with tiny plan caps. Scenarios that assert
* quota enforcement run as that account and skip without it (a deployed
* staging target has no capped account to offer).
* E2A_TEST_OVERCAP_API_KEY: optional; key for the contract server's third
* account, seeded already over its plan caps. The scenario proving the
* 402 envelope's `current` field runs as that account and skips without
* it (a deployed staging target has no over-cap account to offer).
*
* The runner drives the server over raw HTTP (a thin scenario interpreter,
* not the ergonomic client) plus {@link WSListener} for WebSocket steps.
Expand Down Expand Up @@ -38,22 +42,34 @@ const SEED = seedEnabled();
// runner is pointed at a deployed server, which has no such account.
const CAPPED_API_KEY = process.env.E2A_TEST_CAPPED_API_KEY;

// The contract server's over-cap-account key (see the header). Absent when
// the runner is pointed at a deployed server, which has no such account.
const OVERCAP_API_KEY = process.env.E2A_TEST_OVERCAP_API_KEY;

/**
* True when the scenario authenticates as the capped account anywhere.
* True when the scenario authenticates as `placeholder`'s account anywhere.
*
* Inspects auth_override VALUES specifically. Stringifying the whole scenario
* looks equivalent and is not: the scenario's own description mentions
* {capped_api_key} in prose, so a blob match stays true even if every
* looks equivalent and is not: the scenario's own description mentions the
* placeholder in prose, so a blob match stays true even if every
* auth_override is switched back to the primary account — exactly the
* regression this is meant to detect.
*/
function scenarioNeedsCappedAccount(sc: Scenario): boolean {
function scenarioUsesPlaceholder(sc: Scenario, placeholder: string): boolean {
const overrides = [
sc.auth_override,
...(sc.steps ?? []).map((s) => s.auth_override),
...(sc.cleanup ?? []).map((s) => s.auth_override),
];
return overrides.some((o) => typeof o === "string" && o.includes("{capped_api_key}"));
return overrides.some((o) => typeof o === "string" && o.includes(placeholder));
}

function scenarioNeedsCappedAccount(sc: Scenario): boolean {
return scenarioUsesPlaceholder(sc, "{capped_api_key}");
}

function scenarioNeedsOverCapAccount(sc: Scenario): boolean {
return scenarioUsesPlaceholder(sc, "{overcap_api_key}");
}

it("parses the generated message lifecycle page contract", () => {
Expand Down Expand Up @@ -831,6 +847,7 @@ class Runner {
// Only bind when present: scenarios needing it are skipped otherwise, so a
// silently-empty bearer token can never reach the wire as a confusing 401.
if (CAPPED_API_KEY) this.vars.capped_api_key = CAPPED_API_KEY;
if (OVERCAP_API_KEY) this.vars.overcap_api_key = OVERCAP_API_KEY;
this.api = new RawApi(apiKey, baseUrl);
this.seeder = SEED ? new Seeder(baseUrl, apiKey) : null;
}
Expand Down Expand Up @@ -1249,15 +1266,16 @@ describe.skipIf(!baseUrl || !apiKey)("Contract scenarios", () => {
for (const sc of scenarios) {
// Store-dependent scenarios run when SEED supplies their preconditions over
// the API; otherwise they skip. Account-global scenarios skip regardless.
// Quota scenarios need the capped account; without its key there is no way
// to reach a cap on a live server, so they skip. The Go runner owns the
// contract server in-process and always has it, and the always-on shape
// test below fails if the scenario is ever deleted or defanged — so a skip
// here can never quietly become zero coverage.
// Quota scenarios need the capped or over-cap account; without its key
// there is no way to reach a cap on a live server, so they skip. The Go
// runner owns the contract server in-process and always has it, and the
// always-on shape test below fails if the scenario is ever deleted or
// defanged, so a skip here can never quietly become zero coverage.
const skip =
ACCOUNT_GLOBAL.has(sc.name) ||
(scenarioNeedsStore(sc) && !SEED) ||
(scenarioNeedsCappedAccount(sc) && !CAPPED_API_KEY);
(scenarioNeedsCappedAccount(sc) && !CAPPED_API_KEY) ||
(scenarioNeedsOverCapAccount(sc) && !OVERCAP_API_KEY);

(skip ? it.skip : it)(
sc.name,
Expand Down
Loading
Loading