diff --git a/src/Builders/PromptBuilder.php b/src/Builders/PromptBuilder.php index 130fc574..71f4dcb8 100644 --- a/src/Builders/PromptBuilder.php +++ b/src/Builders/PromptBuilder.php @@ -275,6 +275,11 @@ public function usingModel(ModelInterface $model): self /** * Sets preferred models to evaluate in order. * + * The preferences are evaluated in order, and the first one that is available for the prompt's + * requirements is used. If none of the provided preferences are available, generation throws an + * InvalidArgumentException rather than silently falling back to an arbitrary (potentially far more + * expensive) model. To allow automatic discovery instead, omit this method entirely. + * * @since 0.2.0 * * @param string|ModelInterface|array{0:string,1:string} ...$preferredModels The preferred models as model IDs, @@ -1320,7 +1325,8 @@ protected function appendPartToMessages(MessagePart $part): void * * @param CapabilityEnum $capability The capability the model will be using. * @return ModelInterface The model to use. - * @throws InvalidArgumentException If no suitable model is found or set model doesn't meet requirements. + * @throws InvalidArgumentException If no suitable model is found, the set model doesn't meet requirements, or + * explicit model preferences were provided but none are available. */ private function getConfiguredModel(CapabilityEnum $capability): ModelInterface { @@ -1363,18 +1369,24 @@ private function getConfiguredModel(CapabilityEnum $capability): ModelInterface $candidateMap ); - if (!empty($matchingPreferences)) { - // Get the first matching preference key - $firstMatchKey = key($matchingPreferences); - [$providerId, $modelId] = $candidateMap[$firstMatchKey]; - - $model = $this->registry->getProviderModel($providerId, $modelId, $this->modelConfig); - $this->bindModelRequestOptions($model); - return $model; + if (empty($matchingPreferences)) { + // Explicit model preferences were provided, but none of them are available from the + // resolved provider(s). Silently falling back to an arbitrary discovered model would + // route the request to a different (potentially far more expensive) model than the + // caller asked for, with no signal. Fail loudly instead. See issue #241. + throw new InvalidArgumentException($this->buildUnmatchedPreferenceMessage($capability)); } + + // Get the first matching preference key + $firstMatchKey = key($matchingPreferences); + [$providerId, $modelId] = $candidateMap[$firstMatchKey]; + + $model = $this->registry->getProviderModel($providerId, $modelId, $this->modelConfig); + $this->bindModelRequestOptions($model); + return $model; } - // No preference matched; fall back to the first candidate discovered. + // No preference specified; fall back to the first candidate discovered. [$providerId, $modelId] = reset($candidateMap); $model = $this->registry->getProviderModel($providerId, $modelId, $this->modelConfig); @@ -1519,6 +1531,53 @@ private function createModelPreferenceKey(string $modelId): string return 'model::' . $modelId; } + /** + * Builds the exception message for when no requested model preference is available. + * + * Reconstructs a human-readable list of the requested preferences from their internal + * keys so a typo or stale model identifier is obvious to the caller. + * + * @since n.e.x.t + * + * @param CapabilityEnum $capability The capability the model would be used for. + * @return string The exception message. + */ + private function buildUnmatchedPreferenceMessage(CapabilityEnum $capability): string + { + $requested = []; + foreach ($this->modelPreferenceKeys as $preferenceKey) { + $providerModelParts = explode('::', $preferenceKey, 3); + + if ($providerModelParts[0] === 'providerModel' && count($providerModelParts) === 3) { + $requested[] = sprintf('"%s" (provider "%s")', $providerModelParts[2], $providerModelParts[1]); + } elseif ($providerModelParts[0] === 'model' && count($providerModelParts) >= 2) { + $requested[] = sprintf('"%s"', substr($preferenceKey, strlen('model::'))); + } else { + // Unexpected key shape; surface the raw value rather than a misleading substring. + $requested[] = sprintf('"%s"', $preferenceKey); + } + } + + if ($this->providerIdOrClassName !== null) { + return sprintf( + 'None of the requested model preferences (%s) are available from provider "%s" for %s. ' + . 'Use a model identifier that the provider exposes, or remove the usingModelPreference() ' + . 'call to allow automatic model discovery.', + implode(', ', $requested), + $this->providerIdOrClassName, + $capability->value + ); + } + + return sprintf( + 'None of the requested model preferences (%s) are available from any registered provider for %s. ' + . 'Use a model identifier that a provider exposes, or remove the usingModelPreference() ' + . 'call to allow automatic model discovery.', + implode(', ', $requested), + $capability->value + ); + } + /** * Parses various input types into a Message with the given role. * diff --git a/tests/unit/Builders/PromptBuilderTest.php b/tests/unit/Builders/PromptBuilderTest.php index ce68a223..64ff1e45 100644 --- a/tests/unit/Builders/PromptBuilderTest.php +++ b/tests/unit/Builders/PromptBuilderTest.php @@ -902,11 +902,113 @@ public function testUsingModelPreferenceSkipsUnavailableModelId(): void } /** - * Tests usingModelPreference falls back to discovery when no preferences are available. + * Tests usingModelPreference throws when none of the requested preferences are available. + * + * Requesting a specific model id that no provider exposes must fail loudly rather than + * silently routing the request to an arbitrary (potentially far more expensive) model. + * See issue #241. + * + * @return void + */ + public function testUsingModelPreferenceThrowsWhenNoPreferenceAvailable(): void + { + $metadata = $this->createTextModelMetadataWithInputSupport('discovered-id'); + $providerMetadata = $this->createTestProviderMetadata(); + $providerModelsMetadata = new ProviderModelsMetadata($providerMetadata, [$metadata]); + + $this->registry->expects($this->once()) + ->method('findModelsMetadataForSupport') + ->with($this->isInstanceOf(ModelRequirements::class)) + ->willReturn([$providerModelsMetadata]); + + // No model must be instantiated when the requested preference is unavailable. + $this->registry->expects($this->never()) + ->method('getProviderModel'); + + $this->registry->expects($this->never()) + ->method('findProviderModelsMetadataForSupport'); + + $builder = new PromptBuilder($this->registry, 'Test prompt'); + $builder->usingModelPreference('unavailable-model'); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('None of the requested model preferences ("unavailable-model")'); + + $builder->generateTextResult(); + } + + /** + * Tests usingModelPreference throws for an unavailable preference when a provider is locked in. + * + * @return void + */ + public function testUsingModelPreferenceThrowsWhenNoPreferenceAvailableForProvider(): void + { + $metadata = $this->createTextModelMetadataWithInputSupport('available-id'); + + $this->registry->expects($this->once()) + ->method('getProviderId') + ->with('test-provider') + ->willReturn('test-provider'); + + $this->registry->expects($this->once()) + ->method('findProviderModelsMetadataForSupport') + ->with('test-provider', $this->isInstanceOf(ModelRequirements::class)) + ->willReturn([$metadata]); + + $this->registry->expects($this->never()) + ->method('getProviderModel'); + + $this->registry->expects($this->never()) + ->method('findModelsMetadataForSupport'); + + $builder = new PromptBuilder($this->registry, 'Test prompt'); + $builder->usingProvider('test-provider'); + $builder->usingModelPreference('unavailable-model'); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('provider "test-provider"'); + + $builder->generateTextResult(); + } + + /** + * Tests usingModelPreference throws and attributes the provider for an unavailable tuple preference. * * @return void */ - public function testUsingModelPreferenceFallsBackToDiscovery(): void + public function testUsingModelPreferenceThrowsWhenTuplePreferenceUnavailable(): void + { + $metadata = $this->createTextModelMetadataWithInputSupport('available-id'); + $providerMetadata = $this->createTestProviderMetadata(); + $providerModelsMetadata = new ProviderModelsMetadata($providerMetadata, [$metadata]); + + $this->registry->expects($this->once()) + ->method('findModelsMetadataForSupport') + ->with($this->isInstanceOf(ModelRequirements::class)) + ->willReturn([$providerModelsMetadata]); + + $this->registry->expects($this->never()) + ->method('getProviderModel'); + + $this->registry->expects($this->never()) + ->method('findProviderModelsMetadataForSupport'); + + $builder = new PromptBuilder($this->registry, 'Test prompt'); + $builder->usingModelPreference(['missing-provider', 'missing-model']); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('"missing-model" (provider "missing-provider")'); + + $builder->generateTextResult(); + } + + /** + * Tests automatic discovery is still used when no model preference is set. + * + * @return void + */ + public function testFallsBackToDiscoveryWhenNoPreferenceSet(): void { $result = $this->createTestResult('Discovered model result'); $metadata = $this->createTextModelMetadataWithInputSupport('discovered-id'); @@ -929,7 +1031,6 @@ public function testUsingModelPreferenceFallsBackToDiscovery(): void ->method('findProviderModelsMetadataForSupport'); $builder = new PromptBuilder($this->registry, 'Test prompt'); - $builder->usingModelPreference('unavailable-model'); $actualResult = $builder->generateTextResult();