From 7a91428806b38a91f6a3a3940545bb09fa9aff51 Mon Sep 17 00:00:00 2001 From: Miguel Lezama Date: Wed, 3 Jun 2026 18:35:34 -0300 Subject: [PATCH 1/3] Bundled agents: model 'auto' + surface silent model substitution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bundled demo agents shipped `model: 'claude-haiku-4-5'`, but no provider exposes that exact id — wp-ai-client silently falls back to the provider's default (Opus on Anthropic), so the demos ran on an expensive model while the config implied Haiku, with no signal. - Switch the bundled agents (example, loop-demo, site-introspection, workflow-drafter, coordinator) to `model: 'auto'` so they adopt whatever the active provider is configured for, instead of a misleading, unresolvable id. - Runner: when an agent pins a model id that the provider substitutes, record it on the turn telemetry and log `[openclawp] model_substitution agent=… requested=… used=…`, so silent fallback to a premium model is visible. - README: document that cloud defaults pick the provider's (often priciest) model, how to pin a specific id, and the substitution log line. Note: 'auto' still defers model choice to the provider (Opus by default on Anthropic). Truly preventing the silent premium fallback is the provider / wp-ai-client's job — filed separately upstream. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 9 ++++++ includes/class-openclawp-agent-registrar.php | 10 +++---- includes/class-openclawp-runner.php | 30 ++++++++++++++++++++ 3 files changed, 44 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 20e0e2e..f270cd6 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,15 @@ studio --path "$SITE_PATH" wp option update \ connectors_ai_anthropic_api_key "$ANTHROPIC_API_KEY" ``` +> **Cost note.** The bundled demo agents use `model: auto`, so the active +> provider picks the model — which on a cloud provider is its *default*, often +> the priciest (e.g. Opus). To control spend, set each agent's +> `default_config['model']` to a specific id the provider exposes (e.g. a dated +> Anthropic Haiku id). If a configured model id doesn't resolve, wp-ai-client +> silently falls back to the provider default; openclaWP logs that substitution +> as `[openclawp] model_substitution …` so you can catch it. Pair with the +> per-site budget caps (`budget_daily_usd`, `budget_daily_turns`). + ### Path B — an existing WordPress site If a site already exists (Pressable, a VPS, [Local](https://localwp.com/), your own Docker, …), drop the plugins in. Anthropic shown to keep the block short — the Ollama recipe in Path A applies verbatim, just call `wp` instead of `studio --path … wp`. diff --git a/includes/class-openclawp-agent-registrar.php b/includes/class-openclawp-agent-registrar.php index e085487..d77ca1c 100644 --- a/includes/class-openclawp-agent-registrar.php +++ b/includes/class-openclawp-agent-registrar.php @@ -53,7 +53,7 @@ public static function maybe_register_site_introspection_agent(): void { 'owner_resolver' => static fn(): int => get_current_user_id(), 'default_config' => array( 'provider' => 'auto', - 'model' => 'claude-haiku-4-5', + 'model' => 'auto', 'tools' => array( 'openclawp/get-recent-posts', 'openclawp/count-comments', @@ -93,7 +93,7 @@ public static function maybe_register_loop_demo_agent(): void { 'owner_resolver' => static fn(): int => get_current_user_id(), 'default_config' => array( 'provider' => 'auto', - 'model' => 'claude-haiku-4-5', + 'model' => 'auto', 'tools' => array( 'openclawp/get-time' ), 'max_turns' => 5, ), @@ -131,7 +131,7 @@ public static function maybe_register_example_agent(): void { 'owner_resolver' => static fn(): int => get_current_user_id(), 'default_config' => array( 'provider' => 'auto', - 'model' => 'claude-haiku-4-5', + 'model' => 'auto', ), 'meta' => array( 'source_plugin' => 'openclawp/openclawp.php', @@ -177,7 +177,7 @@ public static function maybe_register_workflow_drafter_agent(): void { 'owner_resolver' => static fn(): int => get_current_user_id(), 'default_config' => array( 'provider' => 'auto', - 'model' => 'claude-haiku-4-5', + 'model' => 'auto', ), 'meta' => array( 'source_plugin' => 'openclawp/openclawp.php', @@ -305,7 +305,7 @@ public static function maybe_register_coordinator_demo_agent(): void { 'owner_resolver' => static fn(): int => get_current_user_id(), 'default_config' => array( 'provider' => 'auto', - 'model' => 'claude-haiku-4-5', + 'model' => 'auto', 'max_turns' => 6, ), 'subagents' => array( diff --git a/includes/class-openclawp-runner.php b/includes/class-openclawp-runner.php index de6a3db..405e6d8 100644 --- a/includes/class-openclawp-runner.php +++ b/includes/class-openclawp-runner.php @@ -368,6 +368,17 @@ private static function build_turn_runner( WP_Agent $agent, array $session, stri if ( null !== $preference ) { $builder = $builder->using_model_preference( $preference ); } + // The model id the agent asked for (empty when 'auto'/unset). Used + // below to flag silent provider substitution — wp-ai-client falls + // back to the provider's default model (often the priciest) when the + // requested id isn't one the active provider exposes, rather than + // erroring, which has surprised installs with unexpected billing. + $requested_model_id = ''; + if ( is_array( $preference ) ) { + $requested_model_id = (string) ( $preference[1] ?? '' ); + } elseif ( is_string( $preference ) ) { + $requested_model_id = $preference; + } $start_us = microtime( true ); $generated = $builder->generate_text_result(); @@ -390,6 +401,25 @@ private static function build_turn_runner( WP_Agent $agent, array $session, stri $telemetry['model'] = self::extract_model_id( $generated ); $telemetry['token_usage'] = self::extract_token_usage( $generated ); + // Surface silent model substitution: the agent asked for one model + // but the provider answered with another (its default fallback). This + // is otherwise invisible and can mean paying for a premium model + // (e.g. an unrecognised id resolving to Opus). Record it on the turn + // telemetry and warn once per turn. + $used_model = (string) $telemetry['model']; + if ( '' !== $requested_model_id && '' !== $used_model && $requested_model_id !== $used_model ) { + $telemetry['model_requested'] = $requested_model_id; + // phpcs:ignore WordPress.PHP.DevelopmentFunctions.error_log_error_log -- Matches openclaWP's telemetry channel (grep error_log for [openclawp]). + error_log( + sprintf( + '[openclawp] model_substitution agent=%s requested=%s used=%s — the configured model id did not resolve on the active provider and silently fell back. Pin a model id the provider exposes to control cost.', + (string) $agent_obj->get_slug(), + $requested_model_id, + $used_model + ) + ); + } + $tool_calls = self::extract_tool_calls( $generated ); $telemetry['tool_call_count'] = count( $tool_calls ); // `toText()` throws "No text content found in first candidate" From b682e8afb81358637a9c3e571d6607b5bba88c42 Mon Sep 17 00:00:00 2001 From: Miguel Lezama Date: Wed, 3 Jun 2026 18:38:04 -0300 Subject: [PATCH 2/3] Update prompt-assembly snapshots for model 'auto' The bundled agents now declare model 'auto' instead of the unresolvable 'claude-haiku-4-5'; regenerate the five payload snapshots to match. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../prompt-assembly/__snapshots__/coordinator--chat.json | 2 +- .../prompt-assembly/__snapshots__/example--whatsapp.json | 2 +- .../prompt-assembly/__snapshots__/loop-demo--chat.json | 2 +- .../__snapshots__/site-introspection--whatsapp.json | 2 +- .../prompt-assembly/__snapshots__/workflow-drafter--chat.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/prompt-assembly/__snapshots__/coordinator--chat.json b/tests/integration/prompt-assembly/__snapshots__/coordinator--chat.json index 3d3c7b3..6345ecb 100644 --- a/tests/integration/prompt-assembly/__snapshots__/coordinator--chat.json +++ b/tests/integration/prompt-assembly/__snapshots__/coordinator--chat.json @@ -13,7 +13,7 @@ } ], "model_preference": { - "model": "claude-haiku-4-5" + "model": "auto" }, "runtime_context": { "client_context": { diff --git a/tests/integration/prompt-assembly/__snapshots__/example--whatsapp.json b/tests/integration/prompt-assembly/__snapshots__/example--whatsapp.json index 6d0b69c..8df4e65 100644 --- a/tests/integration/prompt-assembly/__snapshots__/example--whatsapp.json +++ b/tests/integration/prompt-assembly/__snapshots__/example--whatsapp.json @@ -10,7 +10,7 @@ } ], "model_preference": { - "model": "claude-haiku-4-5" + "model": "auto" }, "runtime_context": { "client_context": { diff --git a/tests/integration/prompt-assembly/__snapshots__/loop-demo--chat.json b/tests/integration/prompt-assembly/__snapshots__/loop-demo--chat.json index 5e6386e..02f536f 100644 --- a/tests/integration/prompt-assembly/__snapshots__/loop-demo--chat.json +++ b/tests/integration/prompt-assembly/__snapshots__/loop-demo--chat.json @@ -10,7 +10,7 @@ } ], "model_preference": { - "model": "claude-haiku-4-5" + "model": "auto" }, "runtime_context": { "client_context": { diff --git a/tests/integration/prompt-assembly/__snapshots__/site-introspection--whatsapp.json b/tests/integration/prompt-assembly/__snapshots__/site-introspection--whatsapp.json index 238598b..c5d5782 100644 --- a/tests/integration/prompt-assembly/__snapshots__/site-introspection--whatsapp.json +++ b/tests/integration/prompt-assembly/__snapshots__/site-introspection--whatsapp.json @@ -10,7 +10,7 @@ } ], "model_preference": { - "model": "claude-haiku-4-5" + "model": "auto" }, "runtime_context": { "client_context": { diff --git a/tests/integration/prompt-assembly/__snapshots__/workflow-drafter--chat.json b/tests/integration/prompt-assembly/__snapshots__/workflow-drafter--chat.json index 1669017..ad50a5e 100644 --- a/tests/integration/prompt-assembly/__snapshots__/workflow-drafter--chat.json +++ b/tests/integration/prompt-assembly/__snapshots__/workflow-drafter--chat.json @@ -10,7 +10,7 @@ } ], "model_preference": { - "model": "claude-haiku-4-5" + "model": "auto" }, "runtime_context": { "client_context": { From 76bb9fdb77e754fd8503a1270ddfdd6375e0b164 Mon Sep 17 00:00:00 2001 From: Miguel Lezama Date: Wed, 3 Jun 2026 18:41:25 -0300 Subject: [PATCH 3/3] Fix assembly snapshots: model 'auto' resolves model_preference to null resolve_model_preference() returns null for model 'auto', so the payload's model_preference is null (not an object). Correct the five snapshots. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../prompt-assembly/__snapshots__/coordinator--chat.json | 4 +--- .../prompt-assembly/__snapshots__/example--whatsapp.json | 4 +--- .../prompt-assembly/__snapshots__/loop-demo--chat.json | 4 +--- .../__snapshots__/site-introspection--whatsapp.json | 4 +--- .../prompt-assembly/__snapshots__/workflow-drafter--chat.json | 4 +--- 5 files changed, 5 insertions(+), 15 deletions(-) diff --git a/tests/integration/prompt-assembly/__snapshots__/coordinator--chat.json b/tests/integration/prompt-assembly/__snapshots__/coordinator--chat.json index 6345ecb..d90709a 100644 --- a/tests/integration/prompt-assembly/__snapshots__/coordinator--chat.json +++ b/tests/integration/prompt-assembly/__snapshots__/coordinator--chat.json @@ -12,9 +12,7 @@ "role": "user" } ], - "model_preference": { - "model": "auto" - }, + "model_preference": null, "runtime_context": { "client_context": { "client_name": "chat-block", diff --git a/tests/integration/prompt-assembly/__snapshots__/example--whatsapp.json b/tests/integration/prompt-assembly/__snapshots__/example--whatsapp.json index 8df4e65..92140b3 100644 --- a/tests/integration/prompt-assembly/__snapshots__/example--whatsapp.json +++ b/tests/integration/prompt-assembly/__snapshots__/example--whatsapp.json @@ -9,9 +9,7 @@ "role": "user" } ], - "model_preference": { - "model": "auto" - }, + "model_preference": null, "runtime_context": { "client_context": { "client_name": "whatsapp", diff --git a/tests/integration/prompt-assembly/__snapshots__/loop-demo--chat.json b/tests/integration/prompt-assembly/__snapshots__/loop-demo--chat.json index 02f536f..6e4ca67 100644 --- a/tests/integration/prompt-assembly/__snapshots__/loop-demo--chat.json +++ b/tests/integration/prompt-assembly/__snapshots__/loop-demo--chat.json @@ -9,9 +9,7 @@ "role": "user" } ], - "model_preference": { - "model": "auto" - }, + "model_preference": null, "runtime_context": { "client_context": { "client_name": "chat-block", diff --git a/tests/integration/prompt-assembly/__snapshots__/site-introspection--whatsapp.json b/tests/integration/prompt-assembly/__snapshots__/site-introspection--whatsapp.json index c5d5782..3b21a13 100644 --- a/tests/integration/prompt-assembly/__snapshots__/site-introspection--whatsapp.json +++ b/tests/integration/prompt-assembly/__snapshots__/site-introspection--whatsapp.json @@ -9,9 +9,7 @@ "role": "user" } ], - "model_preference": { - "model": "auto" - }, + "model_preference": null, "runtime_context": { "client_context": { "client_name": "whatsapp", diff --git a/tests/integration/prompt-assembly/__snapshots__/workflow-drafter--chat.json b/tests/integration/prompt-assembly/__snapshots__/workflow-drafter--chat.json index ad50a5e..b6198fb 100644 --- a/tests/integration/prompt-assembly/__snapshots__/workflow-drafter--chat.json +++ b/tests/integration/prompt-assembly/__snapshots__/workflow-drafter--chat.json @@ -9,9 +9,7 @@ "role": "user" } ], - "model_preference": { - "model": "auto" - }, + "model_preference": null, "runtime_context": { "client_context": { "client_name": "chat-block",