From 4ce39ee680ab78b6a3f3a3ae6ca4be8570ebd9d6 Mon Sep 17 00:00:00 2001 From: Aleksandar Vucenovic Date: Thu, 4 Jun 2026 10:45:15 +0200 Subject: [PATCH 1/3] Throw when no requested model preference is available. Requesting a specific model id via usingModelPreference() that no resolved provider exposes previously fell through to the first discovered model with no signal, silently routing the request to an arbitrary (potentially far more expensive) model. A typo, stale id, or non-dated alias could route Haiku-priced traffic to Opus with no error. Generation now throws an InvalidArgumentException when explicit preferences were provided but none are available, naming the requested preferences (and provider, if locked) so the cause is obvious. Automatic discovery when no preference is set is unchanged, as is preference fallback among available preferences. Fixes #241. Co-Authored-By: Claude Opus 4.8 --- src/Builders/PromptBuilder.php | 75 +++++++++++++++++++--- tests/unit/Builders/PromptBuilderTest.php | 76 ++++++++++++++++++++++- 2 files changed, 138 insertions(+), 13 deletions(-) diff --git a/src/Builders/PromptBuilder.php b/src/Builders/PromptBuilder.php index 130fc574..59a9d9af 100644 --- a/src/Builders/PromptBuilder.php +++ b/src/Builders/PromptBuilder.php @@ -275,6 +275,11 @@ public function usingModel(ModelInterface $model): self /** * Sets preferred models to evaluate in order. * + * The preferences are evaluated in order, and the first one that is available for the prompt's + * requirements is used. If none of the provided preferences are available, generation throws an + * InvalidArgumentException rather than silently falling back to an arbitrary (potentially far more + * expensive) model. To allow automatic discovery instead, omit this method entirely. + * * @since 0.2.0 * * @param string|ModelInterface|array{0:string,1:string} ...$preferredModels The preferred models as model IDs, @@ -1320,7 +1325,8 @@ protected function appendPartToMessages(MessagePart $part): void * * @param CapabilityEnum $capability The capability the model will be using. * @return ModelInterface The model to use. - * @throws InvalidArgumentException If no suitable model is found or set model doesn't meet requirements. + * @throws InvalidArgumentException If no suitable model is found, the set model doesn't meet requirements, or + * explicit model preferences were provided but none are available. */ private function getConfiguredModel(CapabilityEnum $capability): ModelInterface { @@ -1363,18 +1369,24 @@ private function getConfiguredModel(CapabilityEnum $capability): ModelInterface $candidateMap ); - if (!empty($matchingPreferences)) { - // Get the first matching preference key - $firstMatchKey = key($matchingPreferences); - [$providerId, $modelId] = $candidateMap[$firstMatchKey]; - - $model = $this->registry->getProviderModel($providerId, $modelId, $this->modelConfig); - $this->bindModelRequestOptions($model); - return $model; + if (empty($matchingPreferences)) { + // Explicit model preferences were provided, but none of them are available from the + // resolved provider(s). Silently falling back to an arbitrary discovered model would + // route the request to a different (potentially far more expensive) model than the + // caller asked for, with no signal. Fail loudly instead. See issue #241. + throw new InvalidArgumentException($this->buildUnmatchedPreferenceMessage($capability)); } + + // Get the first matching preference key + $firstMatchKey = key($matchingPreferences); + [$providerId, $modelId] = $candidateMap[$firstMatchKey]; + + $model = $this->registry->getProviderModel($providerId, $modelId, $this->modelConfig); + $this->bindModelRequestOptions($model); + return $model; } - // No preference matched; fall back to the first candidate discovered. + // No preference specified; fall back to the first candidate discovered. [$providerId, $modelId] = reset($candidateMap); $model = $this->registry->getProviderModel($providerId, $modelId, $this->modelConfig); @@ -1519,6 +1531,49 @@ private function createModelPreferenceKey(string $modelId): string return 'model::' . $modelId; } + /** + * Builds the exception message for when no requested model preference is available. + * + * Reconstructs a human-readable list of the requested preferences from their internal + * keys so a typo or stale model identifier is obvious to the caller. + * + * @since n.e.x.t + * + * @param CapabilityEnum $capability The capability the model would be used for. + * @return string The exception message. + */ + private function buildUnmatchedPreferenceMessage(CapabilityEnum $capability): string + { + $requested = []; + foreach ($this->modelPreferenceKeys as $preferenceKey) { + if (strpos($preferenceKey, 'providerModel::') === 0) { + [$providerId, $modelId] = explode('::', substr($preferenceKey, strlen('providerModel::')), 2); + $requested[] = sprintf('"%s" (provider "%s")', $modelId, $providerId); + } else { + $requested[] = sprintf('"%s"', substr($preferenceKey, strlen('model::'))); + } + } + + if ($this->providerIdOrClassName !== null) { + return sprintf( + 'None of the requested model preferences (%s) are available from provider "%s" for %s. ' + . 'Use a model identifier that the provider exposes, or call usingModelPreference() with an ' + . 'available model.', + implode(', ', $requested), + $this->providerIdOrClassName, + $capability->value + ); + } + + return sprintf( + 'None of the requested model preferences (%s) are available from any registered provider for %s. ' + . 'Use a model identifier that a provider exposes, or call usingModelPreference() with an ' + . 'available model.', + implode(', ', $requested), + $capability->value + ); + } + /** * Parses various input types into a Message with the given role. * diff --git a/tests/unit/Builders/PromptBuilderTest.php b/tests/unit/Builders/PromptBuilderTest.php index ce68a223..587994b9 100644 --- a/tests/unit/Builders/PromptBuilderTest.php +++ b/tests/unit/Builders/PromptBuilderTest.php @@ -902,11 +902,82 @@ public function testUsingModelPreferenceSkipsUnavailableModelId(): void } /** - * Tests usingModelPreference falls back to discovery when no preferences are available. + * Tests usingModelPreference throws when none of the requested preferences are available. + * + * Requesting a specific model id that no provider exposes must fail loudly rather than + * silently routing the request to an arbitrary (potentially far more expensive) model. + * See issue #241. + * + * @return void + */ + public function testUsingModelPreferenceThrowsWhenNoPreferenceAvailable(): void + { + $metadata = $this->createTextModelMetadataWithInputSupport('discovered-id'); + $providerMetadata = $this->createTestProviderMetadata(); + $providerModelsMetadata = new ProviderModelsMetadata($providerMetadata, [$metadata]); + + $this->registry->expects($this->once()) + ->method('findModelsMetadataForSupport') + ->with($this->isInstanceOf(ModelRequirements::class)) + ->willReturn([$providerModelsMetadata]); + + // No model must be instantiated when the requested preference is unavailable. + $this->registry->expects($this->never()) + ->method('getProviderModel'); + + $this->registry->expects($this->never()) + ->method('findProviderModelsMetadataForSupport'); + + $builder = new PromptBuilder($this->registry, 'Test prompt'); + $builder->usingModelPreference('unavailable-model'); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('None of the requested model preferences ("unavailable-model")'); + + $builder->generateTextResult(); + } + + /** + * Tests usingModelPreference throws for an unavailable preference when a provider is locked in. + * + * @return void + */ + public function testUsingModelPreferenceThrowsWhenNoPreferenceAvailableForProvider(): void + { + $metadata = $this->createTextModelMetadataWithInputSupport('available-id'); + + $this->registry->expects($this->once()) + ->method('getProviderId') + ->with('test-provider') + ->willReturn('test-provider'); + + $this->registry->expects($this->once()) + ->method('findProviderModelsMetadataForSupport') + ->with('test-provider', $this->isInstanceOf(ModelRequirements::class)) + ->willReturn([$metadata]); + + $this->registry->expects($this->never()) + ->method('getProviderModel'); + + $this->registry->expects($this->never()) + ->method('findModelsMetadataForSupport'); + + $builder = new PromptBuilder($this->registry, 'Test prompt'); + $builder->usingProvider('test-provider'); + $builder->usingModelPreference('unavailable-model'); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('provider "test-provider"'); + + $builder->generateTextResult(); + } + + /** + * Tests automatic discovery is still used when no model preference is set. * * @return void */ - public function testUsingModelPreferenceFallsBackToDiscovery(): void + public function testFallsBackToDiscoveryWhenNoPreferenceSet(): void { $result = $this->createTestResult('Discovered model result'); $metadata = $this->createTextModelMetadataWithInputSupport('discovered-id'); @@ -929,7 +1000,6 @@ public function testUsingModelPreferenceFallsBackToDiscovery(): void ->method('findProviderModelsMetadataForSupport'); $builder = new PromptBuilder($this->registry, 'Test prompt'); - $builder->usingModelPreference('unavailable-model'); $actualResult = $builder->generateTextResult(); From 5447cd4ba6ba4e9e0dd03db9512e1b3da09150c5 Mon Sep 17 00:00:00 2001 From: Aleksandar Vucenovic Date: Thu, 4 Jun 2026 10:57:28 +0200 Subject: [PATCH 2/3] Use polyfilled str_starts_with() to match house style. Co-Authored-By: Claude Opus 4.8 --- src/Builders/PromptBuilder.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Builders/PromptBuilder.php b/src/Builders/PromptBuilder.php index 59a9d9af..e1d71197 100644 --- a/src/Builders/PromptBuilder.php +++ b/src/Builders/PromptBuilder.php @@ -1546,7 +1546,7 @@ private function buildUnmatchedPreferenceMessage(CapabilityEnum $capability): st { $requested = []; foreach ($this->modelPreferenceKeys as $preferenceKey) { - if (strpos($preferenceKey, 'providerModel::') === 0) { + if (str_starts_with($preferenceKey, 'providerModel::')) { [$providerId, $modelId] = explode('::', substr($preferenceKey, strlen('providerModel::')), 2); $requested[] = sprintf('"%s" (provider "%s")', $modelId, $providerId); } else { From 24e2707e9320302cda5016ed1603c0f9c697c7c6 Mon Sep 17 00:00:00 2001 From: Aleksandar Vucenovic Date: Thu, 4 Jun 2026 11:45:26 +0200 Subject: [PATCH 3/3] Address review: harden preference parsing, clarify message, cover tuple branch. - Make buildUnmatchedPreferenceMessage() defensive against unexpected key shapes, falling back to the raw key instead of a misleading substring. - Reword the exception to point callers at removing usingModelPreference() for automatic discovery, instead of the circular "call it with an available model". - Add a unit test for an unavailable provider/model tuple preference, covering the provider-attributed message branch. Co-Authored-By: Claude Opus 4.8 --- src/Builders/PromptBuilder.php | 20 +++++++++------ tests/unit/Builders/PromptBuilderTest.php | 31 +++++++++++++++++++++++ 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/src/Builders/PromptBuilder.php b/src/Builders/PromptBuilder.php index e1d71197..71f4dcb8 100644 --- a/src/Builders/PromptBuilder.php +++ b/src/Builders/PromptBuilder.php @@ -1546,19 +1546,23 @@ private function buildUnmatchedPreferenceMessage(CapabilityEnum $capability): st { $requested = []; foreach ($this->modelPreferenceKeys as $preferenceKey) { - if (str_starts_with($preferenceKey, 'providerModel::')) { - [$providerId, $modelId] = explode('::', substr($preferenceKey, strlen('providerModel::')), 2); - $requested[] = sprintf('"%s" (provider "%s")', $modelId, $providerId); - } else { + $providerModelParts = explode('::', $preferenceKey, 3); + + if ($providerModelParts[0] === 'providerModel' && count($providerModelParts) === 3) { + $requested[] = sprintf('"%s" (provider "%s")', $providerModelParts[2], $providerModelParts[1]); + } elseif ($providerModelParts[0] === 'model' && count($providerModelParts) >= 2) { $requested[] = sprintf('"%s"', substr($preferenceKey, strlen('model::'))); + } else { + // Unexpected key shape; surface the raw value rather than a misleading substring. + $requested[] = sprintf('"%s"', $preferenceKey); } } if ($this->providerIdOrClassName !== null) { return sprintf( 'None of the requested model preferences (%s) are available from provider "%s" for %s. ' - . 'Use a model identifier that the provider exposes, or call usingModelPreference() with an ' - . 'available model.', + . 'Use a model identifier that the provider exposes, or remove the usingModelPreference() ' + . 'call to allow automatic model discovery.', implode(', ', $requested), $this->providerIdOrClassName, $capability->value @@ -1567,8 +1571,8 @@ private function buildUnmatchedPreferenceMessage(CapabilityEnum $capability): st return sprintf( 'None of the requested model preferences (%s) are available from any registered provider for %s. ' - . 'Use a model identifier that a provider exposes, or call usingModelPreference() with an ' - . 'available model.', + . 'Use a model identifier that a provider exposes, or remove the usingModelPreference() ' + . 'call to allow automatic model discovery.', implode(', ', $requested), $capability->value ); diff --git a/tests/unit/Builders/PromptBuilderTest.php b/tests/unit/Builders/PromptBuilderTest.php index 587994b9..64ff1e45 100644 --- a/tests/unit/Builders/PromptBuilderTest.php +++ b/tests/unit/Builders/PromptBuilderTest.php @@ -972,6 +972,37 @@ public function testUsingModelPreferenceThrowsWhenNoPreferenceAvailableForProvid $builder->generateTextResult(); } + /** + * Tests usingModelPreference throws and attributes the provider for an unavailable tuple preference. + * + * @return void + */ + public function testUsingModelPreferenceThrowsWhenTuplePreferenceUnavailable(): void + { + $metadata = $this->createTextModelMetadataWithInputSupport('available-id'); + $providerMetadata = $this->createTestProviderMetadata(); + $providerModelsMetadata = new ProviderModelsMetadata($providerMetadata, [$metadata]); + + $this->registry->expects($this->once()) + ->method('findModelsMetadataForSupport') + ->with($this->isInstanceOf(ModelRequirements::class)) + ->willReturn([$providerModelsMetadata]); + + $this->registry->expects($this->never()) + ->method('getProviderModel'); + + $this->registry->expects($this->never()) + ->method('findProviderModelsMetadataForSupport'); + + $builder = new PromptBuilder($this->registry, 'Test prompt'); + $builder->usingModelPreference(['missing-provider', 'missing-model']); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('"missing-model" (provider "missing-provider")'); + + $builder->generateTextResult(); + } + /** * Tests automatic discovery is still used when no model preference is set. *