From b78555ac8c07a7e4ad7b8d9c28359018794455a8 Mon Sep 17 00:00:00 2001 From: erin2722 <16248113+erin2722@users.noreply.github.com> Date: Tue, 4 Aug 2026 20:04:02 +0000 Subject: [PATCH 1/6] Update google provider types --- .../typescript/src/generated/google/Level.ts | 2 +- .../src/generated/google/MediaResolution.ts | 4 +- .../src/generated/google/ToolCall.ts | 4 + crates/generate-types/src/main.rs | 226 +++++++-- crates/lingua/src/providers/google/convert.rs | 67 ++- .../lingua/src/providers/google/generated.rs | 63 ++- specs/google/discovery.json | 459 +++++++++++++++++- 7 files changed, 776 insertions(+), 49 deletions(-) diff --git a/bindings/typescript/src/generated/google/Level.ts b/bindings/typescript/src/generated/google/Level.ts index 148eab41b..a547786a1 100644 --- a/bindings/typescript/src/generated/google/Level.ts +++ b/bindings/typescript/src/generated/google/Level.ts @@ -1,6 +1,6 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. /** - * The media resolution level. + * The tokenization quality used for given media. for Gemini API support . */ export type Level = "MEDIA_RESOLUTION_HIGH" | "MEDIA_RESOLUTION_LOW" | "MEDIA_RESOLUTION_MEDIUM" | "MEDIA_RESOLUTION_ULTRA_HIGH" | "MEDIA_RESOLUTION_UNSPECIFIED"; diff --git a/bindings/typescript/src/generated/google/MediaResolution.ts b/bindings/typescript/src/generated/google/MediaResolution.ts index dd819789b..27cfeda5d 100644 --- a/bindings/typescript/src/generated/google/MediaResolution.ts +++ b/bindings/typescript/src/generated/google/MediaResolution.ts @@ -4,10 +4,10 @@ import type { Level } from "./Level"; /** * Optional. Media resolution for the input media. * - * Media resolution for the input media. + * Media resolution for tokenization. */ export type MediaResolution = { /** - * The media resolution level. + * The tokenization quality used for given media. for Gemini API support . */ level: Level | null, }; diff --git a/bindings/typescript/src/generated/google/ToolCall.ts b/bindings/typescript/src/generated/google/ToolCall.ts index 8ad477981..07533b05b 100644 --- a/bindings/typescript/src/generated/google/ToolCall.ts +++ b/bindings/typescript/src/generated/google/ToolCall.ts @@ -22,6 +22,10 @@ args: unknown, * the matching `id`. */ id: string | null, +/** + * Optional. The name of the tool that was called. + */ +toolName: string | null, /** * Required. The type of tool that was called. */ diff --git a/crates/generate-types/src/main.rs b/crates/generate-types/src/main.rs index 085de74d8..51aa5b844 100644 --- a/crates/generate-types/src/main.rs +++ b/crates/generate-types/src/main.rs @@ -1747,11 +1747,7 @@ fn add_google_schema_with_dependencies( processed.insert(type_name.to_string()); - let Some(schema) = all_schemas - .get(type_name) - .cloned() - .or_else(|| google_missing_discovery_schema(type_name)) - else { + let Some(schema) = all_schemas.get(type_name).cloned() else { return; }; @@ -1762,7 +1758,18 @@ fn add_google_schema_with_dependencies( if let Some(obj) = cleaned.as_object_mut() { obj.remove("id"); } - essential_schemas.insert(type_name.to_string(), cleaned); + + let public_name = google_public_schema_name(type_name); + if let Some(existing) = essential_schemas.get(public_name) { + assert_eq!( + existing, &cleaned, + "Google Discovery schema `{}` maps to the public name `{}`, which is already \ + defined by a different schema. Resolve the collision in \ + google_public_schema_name before regenerating.", + type_name, public_name + ); + } + essential_schemas.insert(public_name.to_string(), cleaned); // Find and add referenced types (Discovery uses bare $ref names) let mut refs = std::collections::HashSet::new(); @@ -1779,36 +1786,21 @@ fn add_google_schema_with_dependencies( } } -fn google_missing_discovery_schema(type_name: &str) -> Option { - match type_name { - // The live Google Discovery spec references MediaResolution from Part but does not - // currently include a MediaResolution entry in schemas. Preserve the schema shape - // from prior Discovery specs so generation can remain fully typed. - "MediaResolution" => Some(serde_json::json!({ - "description": "Media resolution for the input media.", - "type": "object", - "properties": { - "level": { - "description": "The media resolution level.", - "type": "string", - "enum": [ - "MEDIA_RESOLUTION_UNSPECIFIED", - "MEDIA_RESOLUTION_LOW", - "MEDIA_RESOLUTION_MEDIUM", - "MEDIA_RESOLUTION_HIGH", - "MEDIA_RESOLUTION_ULTRA_HIGH" - ], - "enumDescriptions": [ - "Media resolution has not been set.", - "Media resolution set to low.", - "Media resolution set to medium.", - "Media resolution set to high.", - "Media resolution set to ultra high." - ] - } - } - })), - _ => None, +/// Internal API-surface prefix Google uses on some Discovery schema ids. +const GOOGLE_DISCOVERY_SURFACE_PREFIX: &str = "V1main"; + +/// Map a Google Discovery schema id to the public type name Lingua exposes. +/// +/// Google publishes some schemas under an internal API-surface prefix +/// (`V1mainMediaResolution`, `V1mainTuningSnapshot`). The prefix carries no semantic +/// meaning - `V1mainTuningSnapshot` is byte-identical to `TuningSnapshot` apart from its +/// id - and quicktype derives public Rust and TypeScript names verbatim from the id, so +/// leaving it in place leaks `V1Main...` into Lingua's public surface. Strip it instead. +/// Collisions are rejected by the caller rather than resolved silently. +fn google_public_schema_name(discovery_id: &str) -> &str { + match discovery_id.strip_prefix(GOOGLE_DISCOVERY_SURFACE_PREFIX) { + Some(stripped) if stripped.starts_with(|c: char| c.is_ascii_uppercase()) => stripped, + _ => discovery_id, } } @@ -1846,12 +1838,16 @@ fn convert_discovery_schema_to_json_schema(schema: &serde_json::Value) -> serde_ for (key, value) in obj { if key == "$ref" { - // Convert bare type name refs to JSON Schema #/definitions/ refs + // Convert bare type name refs to JSON Schema #/definitions/ refs, + // targeting the public name the definition was registered under. if let Some(ref_str) = value.as_str() { if !ref_str.starts_with('#') { fixed_obj.insert( key.clone(), - serde_json::Value::String(format!("#/definitions/{}", ref_str)), + serde_json::Value::String(format!( + "#/definitions/{}", + google_public_schema_name(ref_str) + )), ); } else { fixed_obj.insert(key.clone(), value.clone()); @@ -2335,3 +2331,157 @@ mod google_post_process_tests { assert!(!output.contains("ModeNone")); } } + +#[cfg(test)] +mod google_schema_name_tests { + use super::serde_json; + use super::{ + add_google_schema_with_dependencies, convert_discovery_schema_to_json_schema, + google_public_schema_name, + }; + + fn discovery_schemas(schemas: serde_json::Value) -> serde_json::Map { + schemas + .as_object() + .expect("test schemas must be an object") + .clone() + } + + fn collect( + root: &str, + all_schemas: &serde_json::Map, + ) -> serde_json::Map { + let mut essential = serde_json::Map::new(); + let mut processed = std::collections::HashSet::new(); + add_google_schema_with_dependencies(root, all_schemas, &mut essential, &mut processed); + essential + } + + #[test] + fn strips_internal_surface_prefix_from_discovery_ids() { + assert_eq!( + google_public_schema_name("V1mainMediaResolution"), + "MediaResolution" + ); + assert_eq!( + google_public_schema_name("V1mainTuningSnapshot"), + "TuningSnapshot" + ); + // Names without the prefix, and names where the prefix is not followed by a new + // type name, are left exactly as published. + assert_eq!(google_public_schema_name("Part"), "Part"); + assert_eq!(google_public_schema_name("V1main"), "V1main"); + assert_eq!(google_public_schema_name("V1mainly"), "V1mainly"); + } + + #[test] + fn part_media_resolution_ref_targets_unprefixed_public_name() { + let all_schemas = discovery_schemas(serde_json::json!({ + "Part": { + "id": "Part", + "type": "object", + "properties": { + "mediaResolution": { "$ref": "V1mainMediaResolution" } + } + }, + "V1mainMediaResolution": { + "id": "V1mainMediaResolution", + "type": "object", + "properties": { + "level": { "type": "string", "enum": ["MEDIA_RESOLUTION_ULTRA_HIGH"] } + } + } + })); + + let essential = collect("Part", &all_schemas); + + // The definition is registered under the public name, so quicktype derives + // `MediaResolution` rather than leaking Google's `V1main` surface prefix. + assert!(essential.contains_key("MediaResolution")); + assert!(!essential.contains_key("V1mainMediaResolution")); + + let part = convert_discovery_schema_to_json_schema(&essential["Part"]); + assert_eq!( + part["properties"]["mediaResolution"]["$ref"], + serde_json::json!("#/definitions/MediaResolution"), + ); + } + + #[test] + fn dangling_discovery_ref_is_not_backfilled_with_a_hand_authored_schema() { + let all_schemas = discovery_schemas(serde_json::json!({ + "Part": { + "id": "Part", + "type": "object", + "properties": { + "mediaResolution": { "$ref": "MediaResolution" } + } + } + })); + + let essential = collect("Part", &all_schemas); + + // Google now publishes the schema, so the generator must not resurrect a + // hand-authored stand-in; an unpublished $ref has to surface as a generation + // failure instead of silently diverging from upstream. + assert_eq!(essential.keys().collect::>(), vec!["Part"]); + } + + #[test] + fn prefixed_duplicate_of_an_identical_schema_collapses_onto_one_definition() { + let snapshot = serde_json::json!({ + "type": "object", + "properties": { "step": { "type": "integer" } } + }); + let mut prefixed = snapshot.clone(); + prefixed["id"] = serde_json::json!("V1mainTuningSnapshot"); + let mut plain = snapshot.clone(); + plain["id"] = serde_json::json!("TuningSnapshot"); + + let all_schemas = discovery_schemas(serde_json::json!({ + "Root": { + "id": "Root", + "type": "object", + "properties": { + "a": { "$ref": "TuningSnapshot" }, + "b": { "$ref": "V1mainTuningSnapshot" } + } + }, + "TuningSnapshot": plain, + "V1mainTuningSnapshot": prefixed + })); + + let essential = collect("Root", &all_schemas); + + assert!(essential.contains_key("TuningSnapshot")); + assert!(!essential.contains_key("V1mainTuningSnapshot")); + assert_eq!(essential["TuningSnapshot"], snapshot); + } + + #[test] + #[should_panic(expected = "maps to the public name `TuningSnapshot`")] + fn prefixed_duplicate_that_diverges_fails_generation() { + let all_schemas = discovery_schemas(serde_json::json!({ + "Root": { + "id": "Root", + "type": "object", + "properties": { + "a": { "$ref": "TuningSnapshot" }, + "b": { "$ref": "V1mainTuningSnapshot" } + } + }, + "TuningSnapshot": { + "id": "TuningSnapshot", + "type": "object", + "properties": { "step": { "type": "integer" } } + }, + "V1mainTuningSnapshot": { + "id": "V1mainTuningSnapshot", + "type": "object", + "properties": { "step": { "type": "string" } } + } + })); + + collect("Root", &all_schemas); + } +} diff --git a/crates/lingua/src/providers/google/convert.rs b/crates/lingua/src/providers/google/convert.rs index 00c4bb154..875bae48f 100644 --- a/crates/lingua/src/providers/google/convert.rs +++ b/crates/lingua/src/providers/google/convert.rs @@ -2115,7 +2115,8 @@ mod tests { // serde round trip into and out of the generated types. use crate::providers::google::generated::{ - Category, ComputerUse, DisabledSafetyPolicy, Environment, Type, + Category, ComputerUse, DisabledSafetyPolicy, Environment, Level, MediaResolution, + MediaResolutionEnum, Type, }; #[test] @@ -2260,4 +2261,68 @@ mod tests { serde_json::from_value(json!({"enableAffectiveDialog": true})).unwrap(); assert_eq!(parsed.enable_affective_dialog, Some(true)); } + + #[test] + fn test_part_media_resolution_is_a_nested_level_object() { + // Google publishes the part-level schema under the internal Discovery id + // `V1mainMediaResolution`; the generator strips that surface prefix so the public + // name stays `MediaResolution`. Using it here as a struct with a `level` field + // fails to compile if the identifier ever comes to name the scalar enum instead. + let parsed: GooglePart = serde_json::from_value(json!({ + "text": "hi", + "mediaResolution": {"level": "MEDIA_RESOLUTION_ULTRA_HIGH"} + })) + .unwrap(); + assert_eq!( + parsed.media_resolution, + Some(MediaResolution { + level: Some(Level::MediaResolutionUltraHigh), + }) + ); + + let reserialized = serde_json::to_value(&parsed).unwrap(); + assert_eq!( + reserialized["mediaResolution"]["level"], + json!("MEDIA_RESOLUTION_ULTRA_HIGH") + ); + + // Omitting the field must not emit a null. + let bare: GooglePart = serde_json::from_value(json!({"text": "hi"})).unwrap(); + assert_eq!(bare.media_resolution, None); + assert!(!serde_json::to_string(&bare) + .unwrap() + .contains("mediaResolution")); + } + + #[test] + fn test_generation_config_media_resolution_is_a_scalar_enum() { + // The request-level field is a bare string enum, not the part-level object, so it + // must serialize to a scalar. Constructing it positionally as an enum guards the + // `MediaResolution`/`MediaResolutionEnum` name split. + let config = GenerationConfig { + media_resolution: Some(MediaResolutionEnum::MediaResolutionLow), + ..Default::default() + }; + let value = serde_json::to_value(&config).unwrap(); + assert_eq!(value["mediaResolution"], json!("MEDIA_RESOLUTION_LOW")); + + let parsed: GenerationConfig = + serde_json::from_value(json!({"mediaResolution": "MEDIA_RESOLUTION_HIGH"})).unwrap(); + assert_eq!( + parsed.media_resolution, + Some(MediaResolutionEnum::MediaResolutionHigh) + ); + + // The two enums are deliberately not interchangeable: the part-level `Level` has + // ULTRA_HIGH but GenerationConfig's inline enum does not, so any future + // part-to-request normalizer has to confront the asymmetry rather than assume it + // away. + assert_eq!( + serde_json::to_string(&Level::MediaResolutionUltraHigh).unwrap(), + "\"MEDIA_RESOLUTION_ULTRA_HIGH\"" + ); + assert!( + serde_json::from_str::("\"MEDIA_RESOLUTION_ULTRA_HIGH\"").is_err() + ); + } } diff --git a/crates/lingua/src/providers/google/generated.rs b/crates/lingua/src/providers/google/generated.rs index d09034eb5..c4861ecc3 100644 --- a/crates/lingua/src/providers/google/generated.rs +++ b/crates/lingua/src/providers/google/generated.rs @@ -422,16 +422,16 @@ pub struct Blob { /// Optional. Media resolution for the input media. /// -/// Media resolution for the input media. +/// Media resolution for tokenization. #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, TS)] #[ts(export_to = "google/")] pub struct MediaResolution { - /// The media resolution level. + /// The tokenization quality used for given media. for Gemini API support . #[serde(skip_serializing_if = "Option::is_none")] pub level: Option, } -/// The media resolution level. +/// The tokenization quality used for given media. for Gemini API support . #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, TS)] #[serde(rename_all = "SCREAMING_SNAKE_CASE")] #[ts(export_to = "google/")] @@ -469,6 +469,9 @@ pub struct ToolCall { /// the matching `id`. #[serde(skip_serializing_if = "Option::is_none")] pub id: Option, + /// Optional. The name of the tool that was called. + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_name: Option, /// Required. The type of tool that was called. #[serde(skip_serializing_if = "Option::is_none")] pub tool_type: Option, @@ -553,6 +556,9 @@ pub struct GenerationConfig { #[ts(type = "unknown")] #[serde(skip_serializing_if = "Option::is_none")] pub response_json_schema: Option, + /// Optional. Config for audio transcription (speech recognition). + #[serde(skip_serializing_if = "Option::is_none")] + pub audio_transcription_config: Option, /// Optional. Number of generated responses to return. If unset, this will default to 1. /// Please note that this doesn't work for previous generation models (Gemini 1.0 family) #[serde(skip_serializing_if = "Option::is_none")] @@ -675,6 +681,52 @@ pub struct GenerationConfig { pub translation_config: Option, } +/// Optional. Config for audio transcription (speech recognition). +/// +/// The audio transcription configuration. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "google/")] +pub struct AudioTranscriptionConfig { + /// Optional. A list of phrases used for speech adaptation, which biases the ASR model to + /// improve recognition of these specific terms. + #[serde(skip_serializing_if = "Option::is_none")] + pub adaptation_phrases: Option>, + /// Optional. A list of custom vocabulary phrases to bias the speech recognition model toward + /// recognizing specific terms (product names, proper nouns, jargon). + #[serde(skip_serializing_if = "Option::is_none")] + pub custom_vocabulary: Option>, + /// Optional. Configures speaker diarization. + #[serde(skip_serializing_if = "Option::is_none")] + pub diarization: Option, + /// Optional. The model will detect the language automatically. + #[ts(type = "unknown")] + #[serde(skip_serializing_if = "Option::is_none")] + pub language_auto: Option>, + /// Optional. BCP-47 language codes providing hints about the languages present in the audio. + /// If omitted or empty, defaults to automatic language detection. + #[serde(skip_serializing_if = "Option::is_none")] + pub language_codes: Option>, + /// Optional. Specifies one or more languages in the audio. + #[serde(skip_serializing_if = "Option::is_none")] + pub language_hints: Option, + /// Optional. Configures word-level timestamp generation. + #[serde(skip_serializing_if = "Option::is_none")] + pub word_timestamp: Option, +} + +/// Optional. Specifies one or more languages in the audio. +/// +/// Provides hints to the model about possible languages present in the audio. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "google/")] +pub struct LanguageHints { + /// Required. BCP-47 language codes. + #[serde(skip_serializing_if = "Option::is_none")] + pub language_codes: Option>, +} + /// Optional. Config for image generation. An error will be returned if this field is set for /// models that don't support these config options. /// @@ -1034,7 +1086,7 @@ pub enum Type { Number, #[serde(alias = "object")] Object, - #[serde(alias = "string")] + #[serde(rename = "STRING", alias = "string")] String, #[serde(rename = "TYPE_UNSPECIFIED")] TypeUnspecified, @@ -1302,9 +1354,10 @@ pub struct FunctionCallingConfig { pub enum FunctionCallingConfigMode { Any, Auto, + #[serde(rename = "NONE")] + None, #[serde(rename = "MODE_UNSPECIFIED")] ModeUnspecified, - None, Validated, } diff --git a/specs/google/discovery.json b/specs/google/discovery.json index b6d4b5404..3c5302822 100644 --- a/specs/google/discovery.json +++ b/specs/google/discovery.json @@ -435,6 +435,7 @@ "corpora": { "methods": { "create": { + "deprecated": true, "description": "Creates an empty `Corpus`.", "flatPath": "v1beta/corpora", "httpMethod": "POST", @@ -450,6 +451,7 @@ } }, "delete": { + "deprecated": true, "description": "Deletes a `Corpus`.", "flatPath": "v1beta/corpora/{corporaId}", "httpMethod": "DELETE", @@ -477,6 +479,7 @@ } }, "get": { + "deprecated": true, "description": "Gets information about a specific `Corpus`.", "flatPath": "v1beta/corpora/{corporaId}", "httpMethod": "GET", @@ -499,6 +502,7 @@ } }, "list": { + "deprecated": true, "description": "Lists all `Corpora` owned by the user.", "flatPath": "v1beta/corpora", "httpMethod": "GET", @@ -743,6 +747,91 @@ } } }, + "environments": { + "methods": { + "create": { + "description": "Creates an environment.", + "flatPath": "v1beta/environments:create", + "httpMethod": "POST", + "id": "generativelanguage.environments.create", + "parameterOrder": [], + "parameters": {}, + "path": "v1beta/environments:create", + "request": { + "$ref": "CreateEnvironmentRequest" + }, + "response": { + "$ref": "Environment" + } + }, + "delete": { + "description": "Deletes an environment.", + "flatPath": "v1beta/environments/{id}:delete", + "httpMethod": "DELETE", + "id": "generativelanguage.environments.delete", + "parameterOrder": [ + "id" + ], + "parameters": { + "id": { + "description": "Required. The identifier of the environment to delete.", + "location": "path", + "required": true, + "type": "string" + } + }, + "path": "v1beta/environments/{id}:delete", + "response": { + "$ref": "Empty" + } + }, + "get": { + "description": "Gets an environment.", + "flatPath": "v1beta/environments/{id}:get", + "httpMethod": "GET", + "id": "generativelanguage.environments.get", + "parameterOrder": [ + "id" + ], + "parameters": { + "id": { + "description": "Required. The identifier of the environment to retrieve.", + "location": "path", + "required": true, + "type": "string" + } + }, + "path": "v1beta/environments/{id}:get", + "response": { + "$ref": "Environment" + } + }, + "list": { + "description": "Lists environments.", + "flatPath": "v1beta/environments:list", + "httpMethod": "GET", + "id": "generativelanguage.environments.list", + "parameterOrder": [], + "parameters": { + "pageSize": { + "description": "Optional. Maximum number of environments to return. If unspecified, defaults to 50. Maximum is 1000.", + "format": "int32", + "location": "query", + "type": "integer" + }, + "pageToken": { + "description": "Optional. Pagination token.", + "location": "query", + "type": "string" + } + }, + "path": "v1beta/environments:list", + "response": { + "$ref": "ListEnvironmentsResponse" + } + } + } + }, "fileSearchStores": { "methods": { "create": { @@ -1302,6 +1391,7 @@ } }, "batchEmbedText": { + "deprecated": true, "description": "Generates multiple embeddings from the model given input text in a synchronous call.", "flatPath": "v1beta/models/{modelsId}:batchEmbedText", "httpMethod": "POST", @@ -1352,6 +1442,7 @@ } }, "countMessageTokens": { + "deprecated": true, "description": "Runs a model's tokenizer on a string and returns the token count.", "flatPath": "v1beta/models/{modelsId}:countMessageTokens", "httpMethod": "POST", @@ -1377,6 +1468,7 @@ } }, "countTextTokens": { + "deprecated": true, "description": "Runs a model's tokenizer on a text and returns the token count.", "flatPath": "v1beta/models/{modelsId}:countTextTokens", "httpMethod": "POST", @@ -1452,6 +1544,7 @@ } }, "embedText": { + "deprecated": true, "description": "Generates an embedding from the model given an input message.", "flatPath": "v1beta/models/{modelsId}:embedText", "httpMethod": "POST", @@ -1477,6 +1570,7 @@ } }, "generateAnswer": { + "deprecated": true, "description": "Generates a grounded answer from the model given an input `GenerateAnswerRequest`.", "flatPath": "v1beta/models/{modelsId}:generateAnswer", "httpMethod": "POST", @@ -1527,6 +1621,7 @@ } }, "generateMessage": { + "deprecated": true, "description": "Generates a response from the model given an input `MessagePrompt`.", "flatPath": "v1beta/models/{modelsId}:generateMessage", "httpMethod": "POST", @@ -1552,6 +1647,7 @@ } }, "generateText": { + "deprecated": true, "description": "Generates a response from the model given an input message.", "flatPath": "v1beta/models/{modelsId}:generateText", "httpMethod": "POST", @@ -1823,6 +1919,7 @@ } }, "create": { + "deprecated": true, "description": "Creates a tuned model. Check intermediate tuning progress (if any) through the [google.longrunning.Operations] service. Access status and results through the Operations service. Example: GET /v1/tunedModels/az2mb0bpw6i/operations/000-111-222", "flatPath": "v1beta/tunedModels", "httpMethod": "POST", @@ -1844,6 +1941,7 @@ } }, "delete": { + "deprecated": true, "description": "Deletes a tuned model.", "flatPath": "v1beta/tunedModels/{tunedModelsId}", "httpMethod": "DELETE", @@ -1891,6 +1989,7 @@ } }, "generateText": { + "deprecated": true, "description": "Generates a response from the model given an input message.", "flatPath": "v1beta/tunedModels/{tunedModelsId}:generateText", "httpMethod": "POST", @@ -1916,6 +2015,7 @@ } }, "get": { + "deprecated": true, "description": "Gets information about a specific TunedModel.", "flatPath": "v1beta/tunedModels/{tunedModelsId}", "httpMethod": "GET", @@ -1938,6 +2038,7 @@ } }, "list": { + "deprecated": true, "description": "Lists created tuned models.", "flatPath": "v1beta/tunedModels", "httpMethod": "GET", @@ -1967,6 +2068,7 @@ } }, "patch": { + "deprecated": true, "description": "Updates a tuned model.", "flatPath": "v1beta/tunedModels/{tunedModelsId}", "httpMethod": "PATCH", @@ -2258,7 +2360,7 @@ } } }, - "revision": "20260717", + "revision": "20260803", "rootUrl": "https://generativelanguage.googleapis.com/", "schemas": { "AsyncBatchEmbedContentRequest": { @@ -2345,19 +2447,44 @@ "id": "AudioTranscriptionConfig", "properties": { "adaptationPhrases": { + "deprecated": true, "description": "Optional. A list of phrases used for speech adaptation, which biases the ASR model to improve recognition of these specific terms.", "items": { "type": "string" }, "type": "array" }, + "customVocabulary": { + "description": "Optional. A list of custom vocabulary phrases to bias the speech recognition model toward recognizing specific terms (product names, proper nouns, jargon).", + "items": { + "type": "string" + }, + "type": "array" + }, + "diarization": { + "description": "Optional. Configures speaker diarization.", + "type": "boolean" + }, "languageAuto": { "$ref": "LanguageAuto", + "deprecated": true, "description": "Optional. The model will detect the language automatically." }, + "languageCodes": { + "description": "Optional. BCP-47 language codes providing hints about the languages present in the audio. If omitted or empty, defaults to automatic language detection.", + "items": { + "type": "string" + }, + "type": "array" + }, "languageHints": { "$ref": "LanguageHints", + "deprecated": true, "description": "Optional. Specifies one or more languages in the audio." + }, + "wordTimestamp": { + "description": "Optional. Configures word-level timestamp generation.", + "type": "boolean" } }, "type": "object" @@ -2380,6 +2507,10 @@ "format": "google-fieldmask", "type": "string" }, + "interactionId": { + "description": "Optional. Input only. Immutable. The interaction ID that this token is scoped to. Specific to the Live Interactions API.", + "type": "string" + }, "name": { "description": "Output only. Identifier. The token itself.", "readOnly": true, @@ -3373,6 +3504,36 @@ }, "type": "object" }, + "CreateEnvironmentRequest": { + "description": "Request for `CreateEnvironment`.", + "id": "CreateEnvironmentRequest", + "properties": { + "networkAllowlist": { + "$ref": "EnvironmentNetworkEgressAllowlist", + "description": "Allow only specific domains." + }, + "networkMode": { + "description": "Network egress mode.", + "enum": [ + "NETWORK_MODE_UNSPECIFIED", + "DISABLED" + ], + "enumDescriptions": [ + "Default value. Unused.", + "All network egress is blocked." + ], + "type": "string" + }, + "sources": { + "description": "Sources to be mounted into the environment.", + "items": { + "$ref": "Source" + }, + "type": "array" + } + }, + "type": "object" + }, "CreateFileRequest": { "description": "Request for `CreateFile`.", "id": "CreateFileRequest", @@ -3673,6 +3834,24 @@ }, "type": "object" }, + "EgressRule": { + "description": "A network egress rule that controls which external domains the environment is allowed to reach. Each rule identifies a target domain and, optionally, a set of HTTP headers to inject into every matching outbound request.", + "id": "EgressRule", + "properties": { + "domain": { + "description": "The domain pattern to match for this rule. Use an exact hostname (e.g., `github.com`), a wildcard prefix (e.g., `*.googleapis.com`), or `*` to match all domains.", + "type": "string" + }, + "transform": { + "additionalProperties": { + "type": "string" + }, + "description": "Headers to inject into requests matching this rule. Key: header name (e.g., \"Authorization\"). Value: header value (e.g., \"Bearer your-token\").", + "type": "object" + } + }, + "type": "object" + }, "EmbedContentBatch": { "description": "A resource representing a batch of `EmbedContent` requests.", "id": "EmbedContentBatch", @@ -3998,6 +4177,97 @@ "properties": {}, "type": "object" }, + "Environment": { + "description": "An execution environment for an agent.", + "id": "Environment", + "properties": { + "created": { + "description": "Output only. The time at which the environment was created in ISO 8601 format (YYYY-MM-DDThh:mm:ssZ).", + "readOnly": true, + "type": "string" + }, + "fileCount": { + "description": "Output only. The number of files in the environment, output only.", + "format": "int64", + "readOnly": true, + "type": "string" + }, + "id": { + "description": "Required. Output only. The ID of the environment.", + "readOnly": true, + "type": "string" + }, + "lastAccessed": { + "description": "Output only. The time at which the environment was last accessed in ISO 8601 format (YYYY-MM-DDThh:mm:ssZ).", + "readOnly": true, + "type": "string" + }, + "network_allowlist": { + "$ref": "EnvironmentNetworkEgressAllowlist", + "description": "Allow only specific domains." + }, + "network_mode": { + "description": "Network egress mode.", + "enum": [ + "NETWORK_MODE_UNSPECIFIED", + "DISABLED" + ], + "enumDescriptions": [ + "Default value. Unused.", + "All network egress is blocked." + ], + "type": "string" + }, + "sizeBytes": { + "description": "Output only. The total size of the environment files in bytes, output only.", + "format": "int64", + "readOnly": true, + "type": "string" + }, + "sources": { + "description": "Sources to be mounted into the environment.", + "items": { + "$ref": "Source" + }, + "type": "array" + }, + "status": { + "description": "Output only. The status of the environment container.", + "enum": [ + "STATUS_UNSPECIFIED", + "ACTIVE", + "EXPIRED" + ], + "enumDescriptions": [ + "", + "", + "" + ], + "readOnly": true, + "type": "string" + }, + "updated": { + "description": "Output only. The time at which the environment was last updated in ISO 8601 format (YYYY-MM-DDThh:mm:ssZ).", + "readOnly": true, + "type": "string" + } + }, + "type": "object" + }, + "EnvironmentNetworkEgressAllowlist": { + "description": "Network egress configuration for the environment.", + "id": "EnvironmentNetworkEgressAllowlist", + "properties": { + "allowlist": { + "description": "List of allowed domains and their configurations.", + "items": { + "$ref": "EgressRule" + }, + "type": "array" + } + }, + "type": "object" + }, "Example": { "description": "An input/output example used to instruct the Model. It demonstrates how the model should respond or format its response.", "id": "Example", @@ -5047,9 +5317,14 @@ "id": "GenerationConfig", "properties": { "_responseJsonSchema": { + "deprecated": true, "description": "Optional. Output schema of the generated response. This is an alternative to `response_schema` that accepts [JSON Schema](https://json-schema.org/). If set, `response_schema` must be omitted, but `response_mime_type` is required. While the full JSON Schema may be sent, not all features are supported. Specifically, only the following properties are supported: - `$id` - `$defs` - `$ref` - `$anchor` - `type` - `format` - `title` - `description` - `enum` (for strings and numbers) - `items` - `prefixItems` - `minItems` - `maxItems` - `minimum` - `maximum` - `anyOf` - `oneOf` (interpreted the same as `anyOf`) - `properties` - `additionalProperties` - `required` The non-standard `propertyOrdering` property may also be set. Cyclic references are unrolled to a limited degree and, as such, may only be used within non-required properties. (Nullable properties are not sufficient.) If `$ref` is set on a sub-schema, no other properties, except for than those starting as a `$`, may be set.", "type": "any" }, + "audioTranscriptionConfig": { + "$ref": "AudioTranscriptionConfig", + "description": "Optional. Config for audio transcription (speech recognition)." + }, "candidateCount": { "description": "Optional. Number of generated responses to return. If unset, this will default to 1. Please note that this doesn't work for previous generation models (Gemini 1.0 family)", "format": "int32", @@ -5140,6 +5415,7 @@ }, "responseSchema": { "$ref": "Schema", + "deprecated": true, "description": "Optional. Output schema of the generated candidate text. Schemas must be a subset of the [OpenAPI schema](https://spec.openapis.org/oas/v3.0.3#schema) and can be objects, primitives or arrays. If set, a compatible `response_mime_type` must also be set. Compatible MIME types: `application/json`: Schema for JSON response. Refer to the [JSON text generation guide](https://ai.google.dev/gemini-api/docs/json-mode) for more details." }, "seed": { @@ -5465,6 +5741,33 @@ }, "type": "object" }, + "HttpBody": { + "description": "Message that represents an arbitrary HTTP body. It should only be used for payload formats that can't be represented as JSON, such as raw binary or an HTML page. This message can be used both in streaming and non-streaming API methods in the request as well as the response. It can be used as a top-level request field, which is convenient if one wants to extract parameters from either the URL or HTTP template into the request fields and also want access to the raw HTTP body. Example: message GetResourceRequest { // A unique request id. string request_id = 1; // The raw HTTP body is bound to this field. google.api.HttpBody http_body = 2; } service ResourceService { rpc GetResource(GetResourceRequest) returns (google.api.HttpBody); rpc UpdateResource(google.api.HttpBody) returns (google.protobuf.Empty); } Example with streaming methods: service CaldavService { rpc GetCalendar(stream google.api.HttpBody) returns (stream google.api.HttpBody); rpc UpdateCalendar(stream google.api.HttpBody) returns (stream google.api.HttpBody); } Use of this type only changes how the request and response bodies are handled, all other features will continue to work unchanged.", + "id": "HttpBody", + "properties": { + "contentType": { + "description": "The HTTP Content-Type header value specifying the content type of the body.", + "type": "string" + }, + "data": { + "description": "The HTTP request/response body as raw binary.", + "format": "byte", + "type": "string" + }, + "extensions": { + "description": "Application specific response metadata. Must be set in the first response for streaming APIs.", + "items": { + "additionalProperties": { + "description": "Properties of the object. Contains field @type with type URL.", + "type": "any" + }, + "type": "object" + }, + "type": "array" + } + }, + "type": "object" + }, "Hyperparameters": { "description": "Hyperparameters controlling the tuning process. Read more at https://ai.google.dev/docs/model_tuning_guidance", "id": "Hyperparameters", @@ -5871,16 +6174,19 @@ "type": "object" }, "LanguageAuto": { + "deprecated": true, "description": "Indicates the language of the audio should be automatically detected.", "id": "LanguageAuto", "properties": {}, "type": "object" }, "LanguageHints": { + "deprecated": true, "description": "Provides hints to the model about possible languages present in the audio.", "id": "LanguageHints", "properties": { "languageCodes": { + "deprecated": true, "description": "Required. BCP-47 language codes.", "items": { "type": "string" @@ -5961,6 +6267,24 @@ }, "type": "object" }, + "ListEnvironmentsResponse": { + "description": "Response for `ListEnvironments`.", + "id": "ListEnvironmentsResponse", + "properties": { + "environments": { + "description": "Environments belonging to the provided project.", + "items": { + "$ref": "Environment" + }, + "type": "array" + }, + "nextPageToken": { + "description": "Pagination token.", + "type": "string" + } + }, + "type": "object" + }, "ListFileSearchStoresResponse": { "description": "Response from `ListFileSearchStores` containing a paginated list of `FileSearchStores`. The results are sorted by ascending `file_search_store.create_time`.", "id": "ListFileSearchStoresResponse", @@ -6494,7 +6818,7 @@ "description": "Inline media bytes." }, "mediaResolution": { - "$ref": "MediaResolution", + "$ref": "V1mainMediaResolution", "description": "Optional. Media resolution for the input media." }, "partMetadata": { @@ -7273,6 +7597,44 @@ }, "type": "object" }, + "Source": { + "description": "A source to be mounted into the environment.", + "id": "Source", + "properties": { + "content": { + "description": "The inline content if `type` is `INLINE`.", + "type": "string" + }, + "encoding": { + "description": "Optional encoding for inline content (e.g. `base64`).", + "type": "string" + }, + "source": { + "description": "The source of the environment. For GCS, this is the GCS path. For GitHub, this is the GitHub path.", + "type": "string" + }, + "target": { + "description": "Where the source should appear in the environment.", + "type": "string" + }, + "type": { + "enum": [ + "TYPE_UNSPECIFIED", + "GCS", + "INLINE", + "REPOSITORY" + ], + "enumDescriptions": [ + "", + "A GCS bucket.", + "Inline content.", + "A generic repository. The protocol prefix in the source URL identifies the provider (e.g., github://, gcs://)." + ], + "type": "string" + } + }, + "type": "object" + }, "SpeakerVoiceConfig": { "description": "The configuration for a single speaker in a multi speaker setup.", "id": "SpeakerVoiceConfig", @@ -7539,6 +7901,10 @@ "description": "Optional. Unique identifier of the tool call. The server returns the tool response with the matching `id`.", "type": "string" }, + "toolName": { + "description": "Optional. The name of the tool that was called.", + "type": "string" + }, "toolType": { "description": "Required. The type of tool that was called.", "enum": [ @@ -8024,6 +8390,95 @@ }, "type": "object" }, + "V1mainCreateTunedModelMetadata": { + "description": "Metadata about the state and progress of creating a tuned model returned from the long-running operation", + "id": "V1mainCreateTunedModelMetadata", + "properties": { + "completedPercent": { + "description": "The completed percentage for the tuning operation.", + "format": "float", + "type": "number" + }, + "completedSteps": { + "description": "The number of steps completed.", + "format": "int32", + "type": "integer" + }, + "snapshots": { + "description": "Metrics collected during tuning.", + "items": { + "$ref": "V1mainTuningSnapshot" + }, + "type": "array" + }, + "totalSteps": { + "description": "The total number of tuning steps.", + "format": "int32", + "type": "integer" + }, + "tunedModel": { + "description": "Name of the tuned model associated with the tuning operation.", + "type": "string" + } + }, + "type": "object" + }, + "V1mainMediaResolution": { + "description": "Media resolution for tokenization.", + "id": "V1mainMediaResolution", + "properties": { + "level": { + "description": "The tokenization quality used for given media. for Gemini API support .", + "enum": [ + "MEDIA_RESOLUTION_UNSPECIFIED", + "MEDIA_RESOLUTION_LOW", + "MEDIA_RESOLUTION_MEDIUM", + "MEDIA_RESOLUTION_HIGH", + "MEDIA_RESOLUTION_ULTRA_HIGH" + ], + "enumDescriptions": [ + "Media resolution has not been set.", + "Media resolution set to low.", + "Media resolution set to medium.", + "Media resolution set to high.", + "Media resolution set to ultra high." + ], + "type": "string" + } + }, + "type": "object" + }, + "V1mainTuningSnapshot": { + "description": "Record for a single tuning step.", + "id": "V1mainTuningSnapshot", + "properties": { + "computeTime": { + "description": "Output only. The timestamp when this metric was computed.", + "format": "google-datetime", + "readOnly": true, + "type": "string" + }, + "epoch": { + "description": "Output only. The epoch this step was part of.", + "format": "int32", + "readOnly": true, + "type": "integer" + }, + "meanLoss": { + "description": "Output only. The mean loss of the training examples for this step.", + "format": "float", + "readOnly": true, + "type": "number" + }, + "step": { + "description": "Output only. The tuning step.", + "format": "int32", + "readOnly": true, + "type": "integer" + } + }, + "type": "object" + }, "VideoFileMetadata": { "description": "Metadata for a video `File`.", "id": "VideoFileMetadata", From 56c27aef2d51e7d7dbf1540ea631a94366e8a1ed Mon Sep 17 00:00:00 2001 From: Alex Z Date: Wed, 5 Aug 2026 16:54:16 -0700 Subject: [PATCH 2/6] fixes --- .../src/generated/AssistantContentPart.ts | 3 +- .../src/generated/BuiltinToolIdentity.ts | 7 + .../generated/BuiltinToolResultContentPart.ts | 8 + .../src/generated/ToolContentPart.ts | 3 +- .../src/requests_expected_differences.json | 8 + .../src/streaming_expected_differences.json | 4 + crates/lingua/src/processing/dedup.rs | 36 +++ crates/lingua/src/processing/transform.rs | 23 ++ .../lingua/src/providers/anthropic/convert.rs | 36 ++- .../lingua/src/providers/bedrock/convert.rs | 42 ++- crates/lingua/src/providers/google/adapter.rs | 127 +++++++- crates/lingua/src/providers/google/convert.rs | 273 ++++++++++++++++-- crates/lingua/src/providers/google/params.rs | 27 ++ crates/lingua/src/providers/openai/convert.rs | 82 ++++++ crates/lingua/src/universal/message.rs | 40 ++- crates/lingua/src/universal/tools.rs | 13 +- crates/lingua/tests/import_fixtures.rs | 1 + payloads/cases/advanced.ts | 40 ++- payloads/cases/params.ts | 23 ++ payloads/cases/types.ts | 39 ++- .../__snapshots__/transforms.test.ts.snap | 134 +++++++++ .../google/followup-request.json | 41 +++ .../google/followup-response-streaming.json | 184 ++++++++++++ .../google/followup-response.json | 32 ++ .../google/request.json | 24 ++ .../google/response-streaming.json | 94 ++++++ .../google/response.json | 32 ++ .../google/error.json | 3 + .../google/request.json | 33 +++ .../audioTranscriptionConfigParam.json | 27 ++ .../googleProviderExecutedToolRoundtrip.json | 3 + .../audioTranscriptionConfigParam.json | 35 +++ ...oviderExecutedToolRoundtrip-streaming.json | 3 + .../googleProviderExecutedToolRoundtrip.json | 3 + .../audioTranscriptionConfigParam.json | 102 +++++++ .../googleProviderExecutedToolRoundtrip.json | 3 + payloads/transforms/transform_errors.json | 6 + plan.md | 62 ++++ 38 files changed, 1607 insertions(+), 49 deletions(-) create mode 100644 bindings/typescript/src/generated/BuiltinToolIdentity.ts create mode 100644 bindings/typescript/src/generated/BuiltinToolResultContentPart.ts create mode 100644 payloads/snapshots/audioTranscriptionConfigParam/google/followup-request.json create mode 100644 payloads/snapshots/audioTranscriptionConfigParam/google/followup-response-streaming.json create mode 100644 payloads/snapshots/audioTranscriptionConfigParam/google/followup-response.json create mode 100644 payloads/snapshots/audioTranscriptionConfigParam/google/request.json create mode 100644 payloads/snapshots/audioTranscriptionConfigParam/google/response-streaming.json create mode 100644 payloads/snapshots/audioTranscriptionConfigParam/google/response.json create mode 100644 payloads/snapshots/googleProviderExecutedToolRoundtrip/google/error.json create mode 100644 payloads/snapshots/googleProviderExecutedToolRoundtrip/google/request.json create mode 100644 payloads/transforms/google_to_anthropic/audioTranscriptionConfigParam.json create mode 100644 payloads/transforms/google_to_anthropic/googleProviderExecutedToolRoundtrip.json create mode 100644 payloads/transforms/google_to_chat-completions/audioTranscriptionConfigParam.json create mode 100644 payloads/transforms/google_to_chat-completions/googleProviderExecutedToolRoundtrip-streaming.json create mode 100644 payloads/transforms/google_to_chat-completions/googleProviderExecutedToolRoundtrip.json create mode 100644 payloads/transforms/google_to_responses/audioTranscriptionConfigParam.json create mode 100644 payloads/transforms/google_to_responses/googleProviderExecutedToolRoundtrip.json create mode 100644 plan.md diff --git a/bindings/typescript/src/generated/AssistantContentPart.ts b/bindings/typescript/src/generated/AssistantContentPart.ts index b7b029a58..d1b3dce93 100644 --- a/bindings/typescript/src/generated/AssistantContentPart.ts +++ b/bindings/typescript/src/generated/AssistantContentPart.ts @@ -1,4 +1,5 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { BuiltinToolIdentity } from "./BuiltinToolIdentity"; import type { ProviderOptions } from "./ProviderOptions"; import type { TextContentPart } from "./TextContentPart"; import type { ToolCallArguments } from "./ToolCallArguments"; @@ -12,4 +13,4 @@ export type AssistantContentPart = { "type": "text" } & TextContentPart | { "typ * Providers will occasionally return encrypted content for reasoning parts which can * be useful when you send a follow up message. */ -encrypted_content?: string, } | { "type": "tool_call", tool_call_id: string, tool_name: string, arguments: ToolCallArguments, status?: string, caller?: ToolCaller, encrypted_content?: string, provider_options?: ProviderOptions, provider_executed?: boolean, } | { "type": "program", call_id: string, code: string, fingerprint?: string, id?: string, } | { "type": "program_output", call_id: string, result: string, status: string, id?: string, } | { "type": "tool_discovery_call", tool_call_id: string, discovery_tool_name: string, query?: string, arguments?: unknown, status?: string, execution?: string, provider_options?: ProviderOptions, } | { "type": "tool_result", tool_call_id: string, tool_name: string, output: unknown, caller?: ToolCaller, provider_options?: ProviderOptions, }; +encrypted_content?: string, } | { "type": "tool_call", tool_call_id: string, tool_name: string, arguments: ToolCallArguments, status?: string, caller?: ToolCaller, encrypted_content?: string, provider_options?: ProviderOptions, provider_executed?: boolean, } | { "type": "builtin_tool_call", tool_call_id: string, tool_name?: string, builtin_tool: BuiltinToolIdentity, arguments?: ToolCallArguments, status?: string, provider_options?: ProviderOptions, provider_executed?: boolean, } | { "type": "program", call_id: string, code: string, fingerprint?: string, id?: string, } | { "type": "program_output", call_id: string, result: string, status: string, id?: string, } | { "type": "tool_discovery_call", tool_call_id: string, discovery_tool_name: string, query?: string, arguments?: unknown, status?: string, execution?: string, provider_options?: ProviderOptions, } | { "type": "tool_result", tool_call_id: string, tool_name: string, output: unknown, caller?: ToolCaller, provider_options?: ProviderOptions, }; diff --git a/bindings/typescript/src/generated/BuiltinToolIdentity.ts b/bindings/typescript/src/generated/BuiltinToolIdentity.ts new file mode 100644 index 000000000..9128f069a --- /dev/null +++ b/bindings/typescript/src/generated/BuiltinToolIdentity.ts @@ -0,0 +1,7 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { BuiltinToolProvider } from "./BuiltinToolProvider"; + +/** + * Stable identity for a provider-executed built-in tool call or result. + */ +export type BuiltinToolIdentity = { provider: BuiltinToolProvider, builtin_type: string, }; diff --git a/bindings/typescript/src/generated/BuiltinToolResultContentPart.ts b/bindings/typescript/src/generated/BuiltinToolResultContentPart.ts new file mode 100644 index 000000000..f505f67f7 --- /dev/null +++ b/bindings/typescript/src/generated/BuiltinToolResultContentPart.ts @@ -0,0 +1,8 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { BuiltinToolIdentity } from "./BuiltinToolIdentity"; +import type { ProviderOptions } from "./ProviderOptions"; + +/** + * Reusable result for a provider-executed built-in tool. + */ +export type BuiltinToolResultContentPart = { tool_call_id: string, tool_name?: string, builtin_tool: BuiltinToolIdentity, output: any, provider_options?: ProviderOptions, }; diff --git a/bindings/typescript/src/generated/ToolContentPart.ts b/bindings/typescript/src/generated/ToolContentPart.ts index c5e32a948..5845bab95 100644 --- a/bindings/typescript/src/generated/ToolContentPart.ts +++ b/bindings/typescript/src/generated/ToolContentPart.ts @@ -1,8 +1,9 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { BuiltinToolResultContentPart } from "./BuiltinToolResultContentPart"; import type { ToolDiscoveryResultContentPart } from "./ToolDiscoveryResultContentPart"; import type { ToolResultContentPart } from "./ToolResultContentPart"; /** * Tool content parts - only tool results allowed */ -export type ToolContentPart = { "type": "tool_result" } & ToolResultContentPart | { "type": "tool_discovery_result" } & ToolDiscoveryResultContentPart; +export type ToolContentPart = { "type": "tool_result" } & ToolResultContentPart | { "type": "builtin_tool_result" } & BuiltinToolResultContentPart | { "type": "tool_discovery_result" } & ToolDiscoveryResultContentPart; diff --git a/crates/coverage-report/src/requests_expected_differences.json b/crates/coverage-report/src/requests_expected_differences.json index 2d5172758..b5deaa815 100644 --- a/crates/coverage-report/src/requests_expected_differences.json +++ b/crates/coverage-report/src/requests_expected_differences.json @@ -196,6 +196,14 @@ } ], "perTestCase": [ + { + "testCase": "googleProviderExecutedToolRoundtrip", + "source": "Google", + "target": "*", + "errors": [ + { "pattern": "built-in tool", "reason": "Google server-side ToolCall/ToolResponse history carries a Google ToolType identity that other provider request formats cannot represent losslessly" } + ] + }, { "testCase": "openaiServiceTierFastParam", "source": "Responses", diff --git a/crates/coverage-report/src/streaming_expected_differences.json b/crates/coverage-report/src/streaming_expected_differences.json index b306dc389..f45a89942 100644 --- a/crates/coverage-report/src/streaming_expected_differences.json +++ b/crates/coverage-report/src/streaming_expected_differences.json @@ -187,6 +187,10 @@ } ], "perTestCase": [ + { + "testCase": "audioTranscriptionConfigParam", "source": "Google", "target": "*", + "fields": [{ "pattern": "served_service_tier", "reason": "Anthropic-compatible content stream events have no message metadata envelope for Google service-tier metadata" }] + }, { "testCase": "toolCallRequest", "source": "ChatCompletions", "target": "Responses", "fields": [{ "pattern": "served_service_tier", "reason": "Responses tool-call stream events have no response metadata envelope" }] diff --git a/crates/lingua/src/processing/dedup.rs b/crates/lingua/src/processing/dedup.rs index bb8d64758..73d02dd48 100644 --- a/crates/lingua/src/processing/dedup.rs +++ b/crates/lingua/src/processing/dedup.rs @@ -185,6 +185,35 @@ fn hash_assistant_content(content: &AssistantContent, hasher: &mut DefaultHasher } } } + AssistantContentPart::BuiltinToolCall { + tool_call_id, + tool_name, + builtin_tool, + arguments, + status, + .. + } => { + "builtin_tool_call".hash(hasher); + tool_call_id.hash(hasher); + tool_name.hash(hasher); + builtin_tool.hash(hasher); + status.hash(hasher); + match arguments { + Some(crate::universal::ToolCallArguments::Valid(map)) => { + "valid".hash(hasher); + map.hash(hasher); + } + Some(crate::universal::ToolCallArguments::Invalid(s)) => { + "invalid".hash(hasher); + s.hash(hasher); + } + Some(crate::universal::ToolCallArguments::Custom(s)) => { + "custom".hash(hasher); + s.hash(hasher); + } + None => "none".hash(hasher), + } + } AssistantContentPart::ToolDiscoveryCall { tool_call_id, discovery_tool_name, @@ -283,6 +312,13 @@ fn hash_tool_content(content: &ToolContent, hasher: &mut DefaultHasher) { result.output.hash(hasher); result.caller.hash(hasher); } + crate::universal::ToolContentPart::BuiltinToolResult(result) => { + "builtin_tool_result".hash(hasher); + result.tool_call_id.hash(hasher); + result.tool_name.hash(hasher); + result.builtin_tool.hash(hasher); + result.output.hash(hasher); + } crate::universal::ToolContentPart::ToolDiscoveryResult(result) => { "tool_discovery_result".hash(hasher); result.tool_call_id.hash(hasher); diff --git a/crates/lingua/src/processing/transform.rs b/crates/lingua/src/processing/transform.rs index 0fc3006aa..0169b4a95 100644 --- a/crates/lingua/src/processing/transform.rs +++ b/crates/lingua/src/processing/transform.rs @@ -747,6 +747,29 @@ fn assistant_content_to_stream_delta(content: &AssistantContent) -> UniversalStr }), }); } + AssistantContentPart::BuiltinToolCall { + tool_call_id, + tool_name, + builtin_tool, + arguments, + .. + } => { + let tool_call_index = tool_calls.len() as u32; + tool_calls.push(UniversalToolCallDelta { + index: Some(tool_call_index), + id: Some(tool_call_id.clone()), + call_type: Some(format!( + "builtin:{}:{}", + builtin_tool.provider.label(), + builtin_tool.builtin_type + )), + custom_tool_call: None, + function: Some(UniversalToolFunctionDelta { + name: tool_name.clone(), + arguments: arguments.as_ref().map(ToString::to_string), + }), + }); + } AssistantContentPart::File { .. } | AssistantContentPart::ToolResult { .. } | AssistantContentPart::ToolDiscoveryCall { .. } diff --git a/crates/lingua/src/providers/anthropic/convert.rs b/crates/lingua/src/providers/anthropic/convert.rs index 5748dbec5..02dcb0c75 100644 --- a/crates/lingua/src/providers/anthropic/convert.rs +++ b/crates/lingua/src/providers/anthropic/convert.rs @@ -1265,6 +1265,18 @@ impl TryFromLLM for generated::InputMessage { file_id: None, }) }, + AssistantContentPart::BuiltinToolCall { + builtin_tool, .. + } => { + return Err(ConvertError::UnsupportedMapping { + from: format!( + "{} built-in tool call `{}`", + builtin_tool.provider.label(), + builtin_tool.builtin_type + ), + to: "Anthropic assistant content", + }); + } AssistantContentPart::ToolDiscoveryCall { tool_call_id, discovery_tool_name, @@ -1422,6 +1434,16 @@ impl TryFromLLM for generated::InputMessage { file_id: None, }); } + ToolContentPart::BuiltinToolResult(result) => { + return Err(ConvertError::UnsupportedMapping { + from: format!( + "{} built-in tool result `{}`", + result.builtin_tool.provider.label(), + result.builtin_tool.builtin_type + ), + to: "Anthropic tool_result", + }); + } } } @@ -1815,7 +1837,9 @@ fn split_mixed_tool_discovery_message(message: Message) -> Vec { for part in content { match part { - ToolContentPart::ToolResult(_) => tool_results.push(part), + ToolContentPart::ToolResult(_) | ToolContentPart::BuiltinToolResult(_) => { + tool_results.push(part) + } ToolContentPart::ToolDiscoveryResult(_) => discovery_results.push(part), } } @@ -2184,6 +2208,16 @@ impl TryFromLLM> for Vec { file_id: None, }); } + AssistantContentPart::BuiltinToolCall { builtin_tool, .. } => { + return Err(ConvertError::UnsupportedMapping { + from: format!( + "{} built-in tool call `{}`", + builtin_tool.provider.label(), + builtin_tool.builtin_type + ), + to: "Anthropic response content", + }); + } AssistantContentPart::ToolDiscoveryCall { tool_call_id, discovery_tool_name, diff --git a/crates/lingua/src/providers/bedrock/convert.rs b/crates/lingua/src/providers/bedrock/convert.rs index 69376fc04..0b8ea8bb8 100644 --- a/crates/lingua/src/providers/bedrock/convert.rs +++ b/crates/lingua/src/providers/bedrock/convert.rs @@ -229,6 +229,9 @@ impl TryFromLLM for BedrockMessage { (BedrockConversationRole::User, blocks) } Message::Assistant { content, .. } => { + if let AssistantContent::Array(parts) = &content { + reject_builtin_tool_calls(parts, "Bedrock Converse toolUse")?; + } let blocks = match content { AssistantContent::String(s) => vec![BedrockContentBlock::Text { text: s }], AssistantContent::Array(parts) => parts @@ -292,11 +295,25 @@ impl TryFromLLM for BedrockMessage { fn tool_result_blocks_from_content( content: Vec, ) -> Result, ConvertError> { + if let Some(result) = content.iter().find_map(|part| match part { + ToolContentPart::BuiltinToolResult(result) => Some(result), + _ => None, + }) { + return Err(ConvertError::UnsupportedMapping { + from: format!( + "{} built-in tool result `{}`", + result.builtin_tool.provider.label(), + result.builtin_tool.builtin_type + ), + to: "Bedrock Converse toolResult", + }); + } + content .into_iter() .filter_map(|part| match part { ToolContentPart::ToolResult(result) => Some(result), - ToolContentPart::ToolDiscoveryResult(_) => None, + ToolContentPart::BuiltinToolResult(_) | ToolContentPart::ToolDiscoveryResult(_) => None, }) .map(|result| { let content_text = match result.output { @@ -319,6 +336,26 @@ fn tool_result_blocks_from_content( .collect() } +fn reject_builtin_tool_calls( + parts: &[AssistantContentPart], + target: &'static str, +) -> Result<(), ConvertError> { + if let Some(builtin_tool) = parts.iter().find_map(|part| match part { + AssistantContentPart::BuiltinToolCall { builtin_tool, .. } => Some(builtin_tool), + _ => None, + }) { + return Err(ConvertError::UnsupportedMapping { + from: format!( + "{} built-in tool call `{}`", + builtin_tool.provider.label(), + builtin_tool.builtin_type + ), + to: target, + }); + } + Ok(()) +} + fn is_tool_result_message(message: &BedrockMessage) -> bool { message.role == BedrockConversationRole::User && !message.content.is_empty() @@ -588,6 +625,9 @@ impl TryFromLLM for BedrockOutputMessage { fn try_from(message: Message) -> Result { match message { Message::Assistant { content, .. } => { + if let AssistantContent::Array(parts) = &content { + reject_builtin_tool_calls(parts, "Bedrock output toolUse")?; + } let blocks = match content { AssistantContent::String(s) => { vec![BedrockOutputContentBlock::Text { text: s }] diff --git a/crates/lingua/src/providers/google/adapter.rs b/crates/lingua/src/providers/google/adapter.rs index c9204267f..3b680ad0e 100644 --- a/crates/lingua/src/providers/google/adapter.rs +++ b/crates/lingua/src/providers/google/adapter.rs @@ -65,6 +65,43 @@ struct GoogleResponseFormatView { config: Option, } +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +struct GoogleProviderExtrasView { + generation_config: Option, +} + +fn unmapped_generation_config(config: &GenerationConfig) -> Option { + let mut residual = config.clone(); + + residual.temperature = None; + residual.top_p = None; + residual.top_k = None; + residual.max_output_tokens = None; + residual.stop_sequences = None; + residual.thinking_config = None; + residual.response_mime_type = None; + residual.generation_config_response_json_schema = None; + residual.response_schema = Box::new(None); + + (residual != GenerationConfig::default()).then_some(residual) +} + +fn generation_config_from_extras( + params: &UniversalParams, +) -> Result, TransformError> { + let Some(extras) = params.extras.get(&ProviderFormat::Google) else { + return Ok(None); + }; + let view: GoogleProviderExtrasView = serde_json::from_value(Value::Object(extras.clone())) + .map_err(|e| { + TransformError::FromUniversalFailed(format!( + "Google provider extras must contain a typed generationConfig: {e}" + )) + })?; + Ok(view.generation_config) +} + impl GoogleResponseFormatView { fn response_format(&self) -> Option { self.generation_config @@ -152,6 +189,11 @@ impl ProviderAdapter for GoogleAdapter { let messages = as TryFromLLM>>::try_from(contents) .map_err(|e| TransformError::ToUniversalFailed(e.to_string()))?; + let residual_generation_config = typed_params + .generation_config + .as_ref() + .and_then(unmapped_generation_config); + // Extract params from generationConfig (now typed in params struct) let (temperature, top_p, top_k, max_tokens, stop, reasoning) = if let Some(config) = &typed_params.generation_config { @@ -244,13 +286,20 @@ impl ProviderAdapter for GoogleAdapter { extras: Default::default(), }; - // Use extras captured automatically via #[serde(flatten)] - if !typed_params.extras.is_empty() { - params.extras.insert( - ProviderFormat::Google, - typed_params.extras.into_iter().collect(), + // Preserve the typed, unmapped generationConfig remainder alongside unknown + // top-level fields. The key is the real Google field name, not a marker, and the + // canonical fields above are removed so there is only one source of truth. + let mut google_extras: Map = typed_params.extras.into_iter().collect(); + if let Some(config) = residual_generation_config { + google_extras.insert( + "generationConfig".to_string(), + serde_json::to_value(config) + .map_err(|e| TransformError::SerializationFailed(e.to_string()))?, ); } + if !google_extras.is_empty() { + params.extras.insert(ProviderFormat::Google, google_extras); + } Ok(UniversalRequest { model, @@ -373,13 +422,15 @@ impl ProviderAdapter for GoogleAdapter { .as_ref() .map(|rf| rf.format_type.is_some()) .unwrap_or(false); + let residual_generation_config = generation_config_from_extras(&req.params)?; let has_params = req.params.temperature.is_some() || req.params.top_p.is_some() || req.params.top_k.is_some() || req.params.output_token_budget().is_some() || req.params.stop.is_some() || has_reasoning - || has_response_format; + || has_response_format + || residual_generation_config.is_some(); if has_params { // Convert ReasoningConfig to Google's thinkingConfig @@ -445,15 +496,13 @@ impl ProviderAdapter for GoogleAdapter { let stop_sequences = req.params.stop.clone(); - let mut config = GenerationConfig { - temperature: req.params.temperature, - top_p: req.params.top_p, - top_k: req.params.top_k, - max_output_tokens: req.params.output_token_budget(), - stop_sequences, - thinking_config, - ..Default::default() - }; + let mut config = residual_generation_config.unwrap_or_default(); + config.temperature = req.params.temperature; + config.top_p = req.params.top_p; + config.top_k = req.params.top_k; + config.max_output_tokens = req.params.output_token_budget(); + config.stop_sequences = stop_sequences; + config.thinking_config = thinking_config; // Apply response format to generationConfig if let Some(format) = &req.params.response_format { @@ -1076,6 +1125,7 @@ fn add_dummy_thought_signatures_for_transferred_function_call_history(messages: #[cfg(test)] mod tests { use super::*; + use crate::providers::google::generated::{AudioTranscriptionConfig, MediaResolutionEnum}; use crate::providers::google::GenerateContentRequest; use crate::serde_json::json; use crate::universal::request::ToolChoiceMode; @@ -1272,6 +1322,53 @@ mod tests { assert!(reconstructed.contents.is_some()); } + #[test] + fn test_google_preserves_unmapped_typed_generation_config_fields() { + let adapter = GoogleAdapter; + let payload = json!({ + "contents": [{ + "role": "user", + "parts": [{"text": "Transcribe this."}] + }], + "generationConfig": { + "temperature": 0.7, + "mediaResolution": "MEDIA_RESOLUTION_LOW", + "audioTranscriptionConfig": { + "customVocabulary": ["Lingua"], + "diarization": true, + "languageCodes": ["en-US"], + "wordTimestamp": true + } + } + }); + + let mut universal = adapter.request_to_universal(payload).unwrap(); + universal.params.temperature = Some(0.2); + + let reconstructed = adapter.request_from_universal(&universal).unwrap(); + let reconstructed: GenerateContentRequest = + serde_json::from_value(reconstructed).expect("request should deserialize"); + let config = reconstructed + .generation_config + .expect("generationConfig should be present"); + + assert_eq!(config.temperature, Some(0.2)); + assert_eq!( + config.media_resolution, + Some(MediaResolutionEnum::MediaResolutionLow) + ); + assert_eq!( + config.audio_transcription_config, + Some(AudioTranscriptionConfig { + custom_vocabulary: Some(vec!["Lingua".to_string()]), + diarization: Some(true), + language_codes: Some(vec!["en-US".to_string()]), + word_timestamp: Some(true), + ..Default::default() + }) + ); + } + #[test] fn test_google_tool_choice_to_universal() { let adapter = GoogleAdapter; diff --git a/crates/lingua/src/providers/google/convert.rs b/crates/lingua/src/providers/google/convert.rs index c845c2d47..8d7248c48 100644 --- a/crates/lingua/src/providers/google/convert.rs +++ b/crates/lingua/src/providers/google/convert.rs @@ -17,14 +17,16 @@ use crate::providers::google::generated::{ FunctionCallingConfigMode, FunctionDeclaration, FunctionResponse as GoogleFunctionResponse, GenerateContentRequest, GenerateContentResponse, GenerationConfig, Modality as GoogleTokenModality, ModalityTokenCount as GoogleModalityTokenCount, - Part as GooglePart, Tool as GoogleTool, ToolConfig, UsageMetadata, + Part as GooglePart, Tool as GoogleTool, ToolCall as GoogleToolCall, ToolConfig, + ToolResponse as GoogleToolResponse, ToolType as GoogleToolType, UsageMetadata, }; use crate::serde_json::{self, Map, Value}; use crate::universal::convert::TryFromLLM; use crate::universal::defaults::DEFAULT_MIME_TYPE; use crate::universal::message::{ - AssistantContent, AssistantContentPart, Message, ProviderOptions, TextContentPart, - ToolCallArguments, ToolContentPart, ToolResultContentPart, UserContent, UserContentPart, + AssistantContent, AssistantContentPart, BuiltinToolIdentity, BuiltinToolResultContentPart, + Message, ProviderOptions, TextContentPart, ToolCallArguments, ToolContentPart, + ToolResultContentPart, UserContent, UserContentPart, }; use crate::universal::request::{ JsonSchemaConfig, ResponseFormatConfig, ResponseFormatType, ToolChoiceConfig, ToolChoiceMode, @@ -75,6 +77,49 @@ fn value_to_map(value: &Value) -> Option> { } } +fn builtin_identity_from_google_tool_type(tool_type: &GoogleToolType) -> BuiltinToolIdentity { + let builtin_type = match tool_type { + GoogleToolType::FileSearch => "FILE_SEARCH", + GoogleToolType::GoogleMaps => "GOOGLE_MAPS", + GoogleToolType::GoogleSearchImage => "GOOGLE_SEARCH_IMAGE", + GoogleToolType::GoogleSearchWeb => "GOOGLE_SEARCH_WEB", + GoogleToolType::ToolTypeUnspecified => "TOOL_TYPE_UNSPECIFIED", + GoogleToolType::UrlContext => "URL_CONTEXT", + }; + BuiltinToolIdentity { + provider: BuiltinToolProvider::Google, + builtin_type: builtin_type.to_string(), + } +} + +fn google_tool_type_from_builtin_identity( + identity: &BuiltinToolIdentity, +) -> Result { + if identity.provider != BuiltinToolProvider::Google { + return Err(ConvertError::UnsupportedMapping { + from: format!( + "{} built-in tool `{}`", + identity.provider.label(), + identity.builtin_type + ), + to: "Google server-side toolCall", + }); + } + + match identity.builtin_type.as_ref() { + "FILE_SEARCH" => Ok(GoogleToolType::FileSearch), + "GOOGLE_MAPS" => Ok(GoogleToolType::GoogleMaps), + "GOOGLE_SEARCH_IMAGE" => Ok(GoogleToolType::GoogleSearchImage), + "GOOGLE_SEARCH_WEB" => Ok(GoogleToolType::GoogleSearchWeb), + "TOOL_TYPE_UNSPECIFIED" => Ok(GoogleToolType::ToolTypeUnspecified), + "URL_CONTEXT" => Ok(GoogleToolType::UrlContext), + other => Err(ConvertError::UnsupportedMapping { + from: format!("Google built-in tool `{other}`"), + to: "Google ToolType", + }), + } +} + fn provider_options_from_google_assistant_part( executable_code: Option, code_execution_result: Option, @@ -228,6 +273,26 @@ impl TryFromLLM for Message { Some(code_execution_result.clone()), )?, })); + } else if let Some(tool_call) = &part.tool_call { + let tool_call_id = tool_call.id.clone().ok_or_else(|| { + ConvertError::MissingRequiredField { + field: "Part.toolCall.id".to_string(), + } + })?; + let tool_type = tool_call.tool_type.as_ref().ok_or_else(|| { + ConvertError::MissingRequiredField { + field: "Part.toolCall.toolType".to_string(), + } + })?; + assistant_parts.push(AssistantContentPart::BuiltinToolCall { + tool_call_id, + tool_name: tool_call.tool_name.clone(), + builtin_tool: builtin_identity_from_google_tool_type(tool_type), + arguments: tool_call.args.clone().map(ToolCallArguments::Valid), + status: None, + provider_options: None, + provider_executed: Some(true), + }); } else if let Some(fc) = &part.function_call { if let Some(tool_name) = &fc.name { let args_value = match fc.args.as_ref() { @@ -337,6 +402,30 @@ impl TryFromLLM for Message { }); } } + } else if let Some(tool_response) = &part.tool_response { + let tool_call_id = tool_response.id.clone().ok_or_else(|| { + ConvertError::MissingRequiredField { + field: "Part.toolResponse.id".to_string(), + } + })?; + let tool_type = tool_response.tool_type.as_ref().ok_or_else(|| { + ConvertError::MissingRequiredField { + field: "Part.toolResponse.toolType".to_string(), + } + })?; + tool_parts.push(ToolContentPart::BuiltinToolResult( + BuiltinToolResultContentPart { + tool_call_id, + tool_name: None, + builtin_tool: builtin_identity_from_google_tool_type(tool_type), + output: tool_response + .response + .clone() + .map(Value::Object) + .unwrap_or(Value::Null), + provider_options: None, + }, + )); } else if let Some(fr) = &part.function_response { if let Some(tool_name) = &fr.name { let output = match fr.response.as_ref() { @@ -578,6 +667,46 @@ impl TryFromLLM for GoogleContent { ..Default::default() }); } + AssistantContentPart::BuiltinToolCall { + tool_call_id, + tool_name, + builtin_tool, + arguments, + provider_executed, + .. + } => { + if provider_executed != Some(true) { + return Err(ConvertError::UnsupportedMapping { + from: "universal built-in tool call without provider_executed=true".to_string(), + to: "Google server-side toolCall", + }); + } + let args = match arguments { + Some(ToolCallArguments::Valid(map)) => Some(map), + Some(ToolCallArguments::Invalid(_)) + | Some(ToolCallArguments::Custom(_)) => { + return Err(ConvertError::UnsupportedMapping { + from: "non-object built-in tool arguments" + .to_string(), + to: "Google ToolCall.args", + }); + } + None => None, + }; + converted.push(GooglePart { + tool_call: Some(GoogleToolCall { + args, + id: Some(tool_call_id), + tool_name, + tool_type: Some( + google_tool_type_from_builtin_identity( + &builtin_tool, + )?, + ), + }), + ..Default::default() + }); + } AssistantContentPart::Reasoning { text, encrypted_content, @@ -613,28 +742,43 @@ impl TryFromLLM for GoogleContent { ("model".to_string(), parts) } Message::Tool { content } => { - let parts: Vec = content - .into_iter() - .filter_map(|part| match part { - ToolContentPart::ToolResult(result) => Some(result), - ToolContentPart::ToolDiscoveryResult(_) => None, - }) - .map(|result| { - let response = value_to_map(&result.output); - - Ok(GooglePart { - function_response: Some(GoogleFunctionResponse { - id: Some(result.tool_call_id).filter(|s| { - !s.is_empty() && !s.starts_with(SYNTHETIC_CALL_ID_PREFIX) + let mut parts = Vec::new(); + for part in content { + match part { + ToolContentPart::ToolResult(result) => { + parts.push(GooglePart { + function_response: Some(GoogleFunctionResponse { + id: Some(result.tool_call_id).filter(|s| { + !s.is_empty() && !s.starts_with(SYNTHETIC_CALL_ID_PREFIX) + }), + name: Some(result.tool_name), + response: value_to_map(&result.output), + ..Default::default() }), - name: Some(result.tool_name), - response, ..Default::default() - }), - ..Default::default() - }) - }) - .collect::, ConvertError>>()?; + }); + } + ToolContentPart::BuiltinToolResult(result) => { + if result.tool_name.is_some() { + return Err(ConvertError::UnsupportedMapping { + from: "named built-in tool result".to_string(), + to: "Google ToolResponse (which has no toolName field)", + }); + } + parts.push(GooglePart { + tool_response: Some(GoogleToolResponse { + id: Some(result.tool_call_id), + response: value_to_map(&result.output), + tool_type: Some(google_tool_type_from_builtin_identity( + &result.builtin_tool, + )?), + }), + ..Default::default() + }); + } + ToolContentPart::ToolDiscoveryResult(_) => {} + } + } ("user".to_string(), parts) } Message::AdditionalTools { .. } => { @@ -1878,6 +2022,89 @@ mod tests { assert_eq!(fc.name.as_deref(), Some("get_weather")); } + #[test] + fn test_google_provider_executed_tool_call_roundtrips_without_name() { + let original = GoogleContent { + role: Some("model".to_string()), + parts: Some(vec![GooglePart { + tool_call: Some(GoogleToolCall { + args: Some(Map::from_iter([( + "query".to_string(), + Value::String("Lingua".to_string()), + )])), + id: Some("google-search-1".to_string()), + tool_name: None, + tool_type: Some(GoogleToolType::GoogleSearchWeb), + }), + ..Default::default() + }]), + }; + + let universal = >::try_from(original.clone()) + .expect("Google toolCall should import"); + let Message::Assistant { + content: AssistantContent::Array(parts), + .. + } = &universal + else { + panic!("expected assistant content parts"); + }; + let AssistantContentPart::BuiltinToolCall { + tool_call_id, + tool_name, + builtin_tool, + provider_executed, + .. + } = &parts[0] + else { + panic!("expected a built-in tool call"); + }; + assert_eq!(tool_call_id, "google-search-1"); + assert_eq!(tool_name, &None); + assert_eq!(builtin_tool.provider, BuiltinToolProvider::Google); + assert_eq!(builtin_tool.builtin_type, "GOOGLE_SEARCH_WEB"); + assert_eq!(*provider_executed, Some(true)); + + let roundtrip = >::try_from(universal) + .expect("built-in tool call should export"); + assert_eq!(roundtrip, original); + } + + #[test] + fn test_google_provider_executed_tool_response_roundtrips() { + let original = GoogleContent { + role: Some("user".to_string()), + parts: Some(vec![GooglePart { + tool_response: Some(GoogleToolResponse { + id: Some("google-search-1".to_string()), + response: Some(Map::from_iter([( + "result".to_string(), + Value::String("Lingua".to_string()), + )])), + tool_type: Some(GoogleToolType::GoogleSearchWeb), + }), + ..Default::default() + }]), + }; + + let universal = >::try_from(original.clone()) + .expect("Google toolResponse should import"); + let Message::Tool { content } = &universal else { + panic!("expected tool content"); + }; + let ToolContentPart::BuiltinToolResult(result) = &content[0] else { + panic!("expected a built-in tool result"); + }; + assert_eq!(result.tool_call_id, "google-search-1"); + assert_eq!(result.tool_name, None); + assert_eq!(result.builtin_tool.provider, BuiltinToolProvider::Google); + assert_eq!(result.builtin_tool.builtin_type, "GOOGLE_SEARCH_WEB"); + + let roundtrip = >::try_from(universal) + .expect("built-in tool result should export"); + assert_eq!(roundtrip, original); + } + #[test] fn test_google_to_universal_simple() { let request = GenerateContentRequest { diff --git a/crates/lingua/src/providers/google/params.rs b/crates/lingua/src/providers/google/params.rs index 4ad1d4843..07e3fecac 100644 --- a/crates/lingua/src/providers/google/params.rs +++ b/crates/lingua/src/providers/google/params.rs @@ -99,4 +99,31 @@ mod tests { // Custom field should be preserved assert_eq!(back.get("customField"), json.get("customField")); } + + #[test] + fn test_google_params_roundtrip_preserves_audio_transcription_config() { + let config_json = json!({ + "audioTranscriptionConfig": { + "customVocabulary": ["Lingua"], + "diarization": true, + "languageCodes": ["en-US"], + "wordTimestamp": true + } + }); + let expected_config: GenerationConfig = + serde_json::from_value(config_json.clone()).unwrap(); + let params: GoogleParams = serde_json::from_value(json!({ + "contents": [{"role": "user", "parts": [{"text": "Transcribe this."}]}], + "generationConfig": config_json + })) + .unwrap(); + + assert!(params.extras.is_empty()); + assert_eq!(params.generation_config.as_ref(), Some(&expected_config)); + + let roundtrip: GoogleParams = + serde_json::from_value(serde_json::to_value(¶ms).unwrap()).unwrap(); + assert!(roundtrip.extras.is_empty()); + assert_eq!(roundtrip.generation_config, Some(expected_config)); + } } diff --git a/crates/lingua/src/providers/openai/convert.rs b/crates/lingua/src/providers/openai/convert.rs index e7eaa8c4f..d80f2928b 100644 --- a/crates/lingua/src/providers/openai/convert.rs +++ b/crates/lingua/src/providers/openai/convert.rs @@ -1977,6 +1977,16 @@ impl TryFromLLM for openai::InputContent { logprobs: Some(vec![]), ..Default::default() }, + AssistantContentPart::BuiltinToolCall { builtin_tool, .. } => { + return Err(ConvertError::UnsupportedMapping { + from: format!( + "{} built-in tool call `{}`", + builtin_tool.provider.label(), + builtin_tool.builtin_type + ), + to: "OpenAI Responses input content", + }); + } AssistantContentPart::ToolDiscoveryCall { .. } => { return Err(ConvertError::UnsupportedInputType { type_info: "AssistantContentPart::ToolDiscoveryCall must be converted as a Responses input item".to_string(), @@ -2562,6 +2572,16 @@ impl TryFromLLM for openai::InputItem { discovery_result, )?); } + ToolContentPart::BuiltinToolResult(tool_result) => { + return Err(ConvertError::UnsupportedMapping { + from: format!( + "{} built-in tool result `{}`", + tool_result.builtin_tool.provider.label(), + tool_result.builtin_tool.builtin_type + ), + to: "OpenAI Responses input item", + }); + } } } @@ -2836,6 +2856,16 @@ pub fn universal_to_responses_input( discovery_result.clone(), )?); } + ToolContentPart::BuiltinToolResult(tool_result) => { + return Err(ConvertError::UnsupportedMapping { + from: format!( + "{} built-in tool result `{}`", + tool_result.builtin_tool.provider.label(), + tool_result.builtin_tool.builtin_type + ), + to: "OpenAI Responses input item", + }); + } } } } @@ -3772,6 +3802,16 @@ impl TryFromLLM> for Vec { ..Default::default() }); } + ToolContentPart::BuiltinToolResult(tool_result) => { + return Err(ConvertError::UnsupportedMapping { + from: format!( + "{} built-in tool result `{}`", + tool_result.builtin_tool.provider.label(), + tool_result.builtin_tool.builtin_type + ), + to: "OpenAI Responses output item", + }); + } } } } @@ -4261,6 +4301,18 @@ impl TryFromLLM> for Vec { ..Default::default() }); } + AssistantContentPart::BuiltinToolCall { + builtin_tool, .. + } => { + return Err(ConvertError::UnsupportedMapping { + from: format!( + "{} built-in tool call `{}`", + builtin_tool.provider.label(), + builtin_tool.builtin_type + ), + to: "OpenAI Responses output item", + }); + } AssistantContentPart::ToolDiscoveryCall { tool_call_id, discovery_tool_name: _, @@ -4864,6 +4916,16 @@ impl TryFromLLM for ChatCompletionRequestMessageExt { ToolContentPart::ToolDiscoveryResult(result) => { tool_discovery_result_to_chat_completion_message(result) } + ToolContentPart::BuiltinToolResult(result) => { + Err(ConvertError::UnsupportedMapping { + from: format!( + "{} built-in tool result `{}`", + result.builtin_tool.provider.label(), + result.builtin_tool.builtin_type + ), + to: "OpenAI Chat Completions tool message", + }) + } } } Message::AdditionalTools { .. } => Err(ConvertError::UnsupportedMapping { @@ -4897,6 +4959,16 @@ pub(crate) fn messages_to_chat_completion_messages( discovery_result, )?); } + ToolContentPart::BuiltinToolResult(tool_result) => { + return Err(ConvertError::UnsupportedMapping { + from: format!( + "{} built-in tool result `{}`", + tool_result.builtin_tool.provider.label(), + tool_result.builtin_tool.builtin_type + ), + to: "OpenAI Chat Completions tool message", + }); + } } } } @@ -5153,6 +5225,16 @@ fn extract_content_tool_calls_and_reasoning( custom: None, }); } + AssistantContentPart::BuiltinToolCall { builtin_tool, .. } => { + return Err(ConvertError::UnsupportedMapping { + from: format!( + "{} built-in tool call `{}`", + builtin_tool.provider.label(), + builtin_tool.builtin_type + ), + to: "OpenAI Chat Completions assistant message", + }); + } AssistantContentPart::ToolDiscoveryCall { tool_call_id, discovery_tool_name, diff --git a/crates/lingua/src/universal/message.rs b/crates/lingua/src/universal/message.rs index e08753960..dca2c6de5 100644 --- a/crates/lingua/src/universal/message.rs +++ b/crates/lingua/src/universal/message.rs @@ -1,5 +1,5 @@ use crate::serde_json; -use crate::universal::tools::UniversalTool; +use crate::universal::tools::{BuiltinToolProvider, UniversalTool}; use serde::{Deserialize, Serialize}; use serde_with::skip_serializing_none; use ts_rs::TS; @@ -116,6 +116,21 @@ pub enum AssistantContentPart { #[ts(optional)] provider_executed: Option, }, + /// A provider-executed built-in tool call whose free-form name may be absent. + BuiltinToolCall { + tool_call_id: String, + #[ts(optional)] + tool_name: Option, + builtin_tool: BuiltinToolIdentity, + #[ts(optional)] + arguments: Option, + #[ts(optional)] + status: Option, + #[ts(optional)] + provider_options: Option, + #[ts(optional)] + provider_executed: Option, + }, Program { call_id: String, code: String, @@ -226,6 +241,28 @@ pub struct ToolResultContentPart { pub provider_options: Option, } +/// Reusable result for a provider-executed built-in tool. +#[skip_serializing_none] +#[derive(Debug, Clone, Serialize, Deserialize, TS)] +#[ts(export, rename_all = "snake_case", optional_fields)] +pub struct BuiltinToolResultContentPart { + pub tool_call_id: String, + pub tool_name: Option, + pub builtin_tool: BuiltinToolIdentity, + #[ts(type = "any")] + pub output: serde_json::Value, + pub provider_options: Option, +} + +/// Stable identity for a provider-executed built-in tool call or result. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, TS)] +#[ts(export, rename_all = "snake_case")] +#[serde(rename_all = "snake_case")] +pub struct BuiltinToolIdentity { + pub provider: BuiltinToolProvider, + pub builtin_type: String, +} + /// Reusable text content part for tagged unions #[skip_serializing_none] #[derive(Debug, Clone, Serialize, Deserialize, TS)] @@ -269,6 +306,7 @@ pub enum CacheControlTtl { #[serde(tag = "type", rename_all = "snake_case")] pub enum ToolContentPart { ToolResult(ToolResultContentPart), + BuiltinToolResult(BuiltinToolResultContentPart), ToolDiscoveryResult(ToolDiscoveryResultContentPart), } diff --git a/crates/lingua/src/universal/tools.rs b/crates/lingua/src/universal/tools.rs index 4b2f1e1d3..243f23a06 100644 --- a/crates/lingua/src/universal/tools.rs +++ b/crates/lingua/src/universal/tools.rs @@ -178,7 +178,7 @@ pub enum UniversalToolType { } /// Provider identity for built-in tool passthrough. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, TS)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, TS)] #[ts(export)] #[serde(rename_all = "snake_case")] pub enum BuiltinToolProvider { @@ -188,6 +188,17 @@ pub enum BuiltinToolProvider { Converse, } +impl BuiltinToolProvider { + pub fn label(self) -> &'static str { + match self { + Self::Anthropic => "anthropic", + Self::Responses => "responses", + Self::Google => "google", + Self::Converse => "converse", + } + } +} + // ============================================================================= // UniversalTool Constructors // ============================================================================= diff --git a/crates/lingua/tests/import_fixtures.rs b/crates/lingua/tests/import_fixtures.rs index 9afadd312..bb64ce7d8 100644 --- a/crates/lingua/tests/import_fixtures.rs +++ b/crates/lingua/tests/import_fixtures.rs @@ -170,6 +170,7 @@ fn assistant_content_part_type(part: &AssistantContentPart) -> &'static str { AssistantContentPart::File { .. } => "file", AssistantContentPart::Reasoning { .. } => "reasoning", AssistantContentPart::ToolCall { .. } => "tool_call", + AssistantContentPart::BuiltinToolCall { .. } => "builtin_tool_call", AssistantContentPart::Program { .. } => "program", AssistantContentPart::ProgramOutput { .. } => "program_output", AssistantContentPart::ToolDiscoveryCall { .. } => "tool_discovery_call", diff --git a/payloads/cases/advanced.ts b/payloads/cases/advanced.ts index 945b434d4..f5c84212c 100644 --- a/payloads/cases/advanced.ts +++ b/payloads/cases/advanced.ts @@ -1,4 +1,4 @@ -import { Type, FunctionCallingConfigMode } from "@google/genai"; +import { Type, FunctionCallingConfigMode, ToolType } from "@google/genai"; import { AnthropicMessageCreateParams, TestCaseCollection } from "./types"; import { OPENAI_CHAT_COMPLETIONS_MODEL, @@ -499,6 +499,44 @@ export const advancedCases: TestCaseCollection = { }, }, + googleProviderExecutedToolRoundtrip: { + "chat-completions": null, + responses: null, + anthropic: null, + google: { + contents: [ + { + role: "model", + parts: [ + { + toolCall: { + id: "google-search-1", + toolName: "search_the_web", + toolType: ToolType.GOOGLE_SEARCH_WEB, + args: { query: "Lingua message format" }, + }, + }, + ], + }, + { + role: "user", + parts: [ + { + toolResponse: { + id: "google-search-1", + toolType: ToolType.GOOGLE_SEARCH_WEB, + response: { + result: "Lingua is a universal LLM message format.", + }, + }, + }, + ], + }, + ], + }, + bedrock: null, + }, + glmToolCallWithLeadingTextRequest: { "chat-completions": null, responses: null, diff --git a/payloads/cases/params.ts b/payloads/cases/params.ts index 23139493c..c32948387 100644 --- a/payloads/cases/params.ts +++ b/payloads/cases/params.ts @@ -3132,6 +3132,29 @@ export const paramsCases: TestCaseCollection = { bedrock: null, }, + audioTranscriptionConfigParam: { + "chat-completions": null, + responses: null, + anthropic: null, + google: { + contents: [ + { + role: "user", + parts: [{ text: "Transcribe the attached audio." }], + }, + ], + generationConfig: { + audioTranscriptionConfig: { + customVocabulary: ["Lingua"], + diarization: true, + languageCodes: ["en-US"], + wordTimestamp: true, + }, + }, + }, + bedrock: null, + }, + googleToolSchemaNumericInt64Param: (() => { const indexNameSchema: Record = { type: Type.STRING, diff --git a/payloads/cases/types.ts b/payloads/cases/types.ts index 7851f5682..7f285a19c 100644 --- a/payloads/cases/types.ts +++ b/payloads/cases/types.ts @@ -1,16 +1,47 @@ import OpenAI from "openai"; import Anthropic from "@anthropic-ai/sdk"; -import type { Content, GenerateContentConfig, Tool } from "@google/genai"; +import type { + AudioTranscriptionConfig, + Content, + GenerateContentConfig, + Part, + Tool, + ToolCall, +} from "@google/genai"; import type { ConverseCommandInput } from "@aws-sdk/client-bedrock-runtime"; +// Compatibility types for fields present in the latest Google API schema but +// not yet represented completely by the installed @google/genai SDK types. +export interface GoogleProviderToolCall extends ToolCall { + toolName?: string; +} + +export interface GoogleProviderPart extends Omit { + toolCall?: GoogleProviderToolCall; +} + +export interface GoogleProviderContent extends Omit { + parts?: GoogleProviderPart[]; +} + +export interface GoogleAudioTranscriptionConfig extends AudioTranscriptionConfig { + customVocabulary?: string[]; + diarization?: boolean; + wordTimestamp?: boolean; +} + +export interface GoogleGenerateContentConfig extends GenerateContentConfig { + audioTranscriptionConfig?: GoogleAudioTranscriptionConfig; +} + // Google Gemini API request type (matching the js-genai library) export interface GoogleGenerateContentRequest { model?: string; - contents: Content[]; - generationConfig?: GenerateContentConfig; + contents: GoogleProviderContent[]; + generationConfig?: GoogleGenerateContentConfig; tools?: Tool[]; toolConfig?: Record; - systemInstruction?: Content; + systemInstruction?: GoogleProviderContent; } // Re-export Bedrock type for convenience diff --git a/payloads/scripts/transforms/__snapshots__/transforms.test.ts.snap b/payloads/scripts/transforms/__snapshots__/transforms.test.ts.snap index dfbf6b86a..b81531d61 100644 --- a/payloads/scripts/transforms/__snapshots__/transforms.test.ts.snap +++ b/payloads/scripts/transforms/__snapshots__/transforms.test.ts.snap @@ -21243,6 +21243,48 @@ exports[`chat-completions → responses > topPReasoningModelParam > response 1`] } `; +exports[`google → anthropic > audioTranscriptionConfigParam > request 1`] = ` +{ + "max_tokens": 4096, + "messages": [ + { + "content": "Transcribe the attached audio.", + "role": "user", + }, + ], + "model": "claude-sonnet-4-5-20250929", +} +`; + +exports[`google → anthropic > audioTranscriptionConfigParam > response 1`] = ` +{ + "candidates": [ + { + "content": { + "parts": [ + { + "text": "I apologize, but I don't see any audio file attached to your message. Could you please try uploading the audio file again? + +Once you've attached the audio file, I'll be happy to transcribe it for you.", + }, + ], + "role": "model", + }, + "finishReason": "STOP", + "index": 0, + }, + ], + "modelVersion": "claude-sonnet-4-5-20250929", + "usageMetadata": { + "cachedContentTokenCount": 0, + "candidatesTokenCount": 52, + "promptTokenCount": 14, + "serviceTier": "standard", + "totalTokenCount": 66, + }, +} +`; + exports[`google → anthropic > complexReasoningRequest > request 1`] = ` { "max_tokens": 20000, @@ -24012,6 +24054,53 @@ exports[`google → anthropic > webSearchToolParam > response 1`] = ` } `; +exports[`google → chat-completions > audioTranscriptionConfigParam > request 1`] = ` +{ + "messages": [ + { + "content": "Transcribe the attached audio.", + "role": "user", + }, + ], + "model": "gpt-5-nano", +} +`; + +exports[`google → chat-completions > audioTranscriptionConfigParam > response 1`] = ` +{ + "candidates": [ + { + "content": { + "parts": [ + { + "text": "I can transcribe it, but I don’t see an attached audio file yet. Please either upload the audio file here (MP3, WAV, M4A, etc.) or share a link to it. + +If you upload, please also specify: +- Language and any dialects +- Transcript style: verbatim (including filler words like um, ah) or clean (remove most fillers) +- Timestamps: include every 30 seconds, or at each speaker change, or none +- Speaker labeling: use “Speaker 1 / Speaker 2” or provide names if known + +If you can’t upload, you can also paste a short excerpt or provide a shareable link and I’ll transcribe from that.", + }, + ], + "role": "model", + }, + "finishReason": "STOP", + "index": 0, + }, + ], + "modelVersion": "gpt-5-nano-2025-08-07", + "usageMetadata": { + "cachedContentTokenCount": 0, + "candidatesTokenCount": 155, + "promptTokenCount": 12, + "thoughtsTokenCount": 512, + "totalTokenCount": 679, + }, +} +`; + exports[`google → chat-completions > complexReasoningRequest > request 1`] = ` { "max_completion_tokens": 20000, @@ -26506,6 +26595,51 @@ If you want, I can also prepare a short, current-news brief for you once you pas } `; +exports[`google → responses > audioTranscriptionConfigParam > request 1`] = ` +{ + "input": [ + { + "content": "Transcribe the attached audio.", + "role": "user", + "type": "message", + }, + ], + "model": "gpt-5.6-terra", +} +`; + +exports[`google → responses > audioTranscriptionConfigParam > response 1`] = ` +{ + "candidates": [ + { + "content": { + "parts": [ + { + "text": "", + "thought": true, + "thoughtSignature": "gAAAAABqc8ekkMe4TGiPbVH5y865n6Vjjo1kNqU-e11_Ak5jCBUHNW-6slzouWAYAcZaRyREsO1SSlJlH_CFjAz_LT-pyGJvqAUL0Aa5V6pVUOu9N2Rggnq2IrvxJ_LNALAOTgLQO2u_zsNlocU6uWAJtbg3wdzlHKXEyaFoipKo9LIj5hqjeJVGe3Fs1TppwJXccaGXNRJUBD0HesazGzGcA9n4IfIyk5GJhBSe8_31RvxPsHueG0CxlDD_U9zBrwPasdvuyiHSg81xurEOFK6MlUdnglKcHJx4mcDamnjRu6KxhH2Bv_LV8vIChRnCpzfYvIdA19zqQHMS8HzGVN2JLbi3_d6zZkp6M46qXbnLGAUMqc5HsfeliP5x8KjnaYeguxRmQyPSiqsDrR1xZCrNkm6pyfWZfl_Vdh3zpJxCZOJCKvEBNALs2Ki-on3t4WSnX243KnyYKDY639CNdxrkM0_hUJSsVxKwV3A_6rUZeW1dCjUhINOj7qu8Jg0_7QdjbC5XaiGkgPSvjGoPCx4EOeICl4ljqA8R_OciLvoi2YrOF42FzqmeIOEB1hNX__bhhs_mMRyQsKNlAgT7YplDMVu3nG-2f1auJBwmYNIsb5Ky4herTB3jwVpDjll2GFoBGa6ALZKKl5Eczw2jhIdGe38nZYigioZ5tJD5SqU1u382i_i_8n9N8ncz9VA742Yca4LHwIe0KQ43G1s52F1ADRSZYVfGsZUv0PHVU12dMgeES6BHsi3LZNd3tZiUZIJD3AAVlfTvaon9esHJHwqFW8u6nne3yH27oDu0WHLXYXLLCaima2TQvOBo_qG8puJyGlw4fIh-9jRHvtLL9iXNfRzCrmV7eM20XMp3e9Vdylte-Z7Ycil4bgxIEQHlOhwoNYKS91lzLfOIzO62wKGx3iO4Newp6P2hnArs9YLKBrtPM_4gvENGXrIx7nvfi7lrOeHyyDihczRsGWDuhmZenCHE88mhH3TeloqLYS36-hUR4O4st2RAXGIbcbuLghNaM_AETJtV1RuiKkmrQ1KgGgeAy84DuaHeaniBpdTqGW9FqSzwOQyQoL3IvhMfPHUg4UlCnLdqadfDcRcq263dJkjYmGt_mfcsWppP13r1X6RZYLTUB3INGGIPkXwoJDX-VgyRqUHzp2pXRgaBZ179BvNpqFq3Qw==", + }, + { + "text": "I don’t see an audio attachment available here. Please upload the audio file, and I’ll transcribe it.", + }, + ], + "role": "model", + }, + "finishReason": "STOP", + "index": 0, + }, + ], + "modelVersion": "gpt-5.6-terra", + "usageMetadata": { + "cachedContentTokenCount": 0, + "candidatesTokenCount": 29, + "promptTokenCount": 12, + "thoughtsTokenCount": 37, + "totalTokenCount": 78, + }, +} +`; + exports[`google → responses > complexReasoningRequest > request 1`] = ` { "input": [ diff --git a/payloads/snapshots/audioTranscriptionConfigParam/google/followup-request.json b/payloads/snapshots/audioTranscriptionConfigParam/google/followup-request.json new file mode 100644 index 000000000..2c4334277 --- /dev/null +++ b/payloads/snapshots/audioTranscriptionConfigParam/google/followup-request.json @@ -0,0 +1,41 @@ +{ + "contents": [ + { + "role": "user", + "parts": [ + { + "text": "Transcribe the attached audio." + } + ] + }, + { + "role": "model", + "parts": [ + { + "text": "I will explain this thing one time and one time only.", + "thoughtSignature": "EukHCuYHARFNMg9EgA3yC4+TAi0LEEEzPFAxKBiHHK6PZ9HQTtUm2PoFj5YeXr2gTs+B9NyIhBKcO7Fou2bXTG6jF5sj2yqmx07WsGT75MlQRDJ8bE1bax3JUDlRg9C+6rilI3YQHPkCmNcPImsyCUuGSA9kn886BU/szbMISNVQOmnzvUe8TRlu9g6j5c9QiQkk+mhtgbv6HXPltLzMinvx9v8+/CmNuC5GaohdaR8s786WRj0U7tEib7LP9ijbMIsR7fL9RE3z1n43OuBzYkhZBHWDSalZnpVvQOJ/Ufvwv/DcRhVKIoJ3LBinu6A9ZnK/dyLcokxvjwgSuVEfCYR7rr9A69TFFmZshIhRtcBFQZ4fKp7sMg3PlCz8d12J/f5ueB9rWYcXP4Wxx76b/Lo/nHAqMex8hfKPRk414jujttNN+xGGg+rOVGwbAkGxSM/aHhBAeqGWoleX2mmPRKAu7uPCrKjlF++TMXIXDYT4ROpk8t0mjVa5wQjdoSi4siY50ITJqX9C3GVLs6OpBH4eImyuZ9vLElYP/Ep7dDCEO5QU58CBvNF7dWqT5jBbMi1toYeE0Qx4+5W8OY3KPP5F7M8e0CLnQfd4i7yIlawAmfn7xy++X+06IpNyBtAN2W2vhrF2PpcA0+wW2VeJmnqFDCvJNUVeIsddBzMq4wrcaXQWP3A0cSkll12LOfgX7v4Z61ZioC9gr2FznCx5eRavDrCavcJvaPR8rgsRM0Izw7LpzOsTe9r5ubxtn2ELvJlWXkrE3W3x+8ldK3yj/HIsD39gV8r1eEiZs0AaKqzcRlzMDs3DNO5ao2NCerHrHDrRi/Z7S7xiZ+gyHFajj7EcijF5eLnYMUOVkqi536iPGKaUSAg4wKfFjUi1+XwX5PTYBLcd/qRjPT3XTiBt+LjQejqkfEND9NlbW0lx68A1Vd0rQhJ8qXHkJ3DRvi7LSx8AjMOvuvwqE6UH0tU5VJdHbpMbicvbsVDaabc8LEb3nDmEZoVhw5Xiw5ciMwfPCW8Jhgi85BYnJuG+XEZJEwCOfpTM6aoUU7HpXHKVPN46PiC/o3wivOsNvCghhsQY0fYSCA8X/EoNBvWxxLfetI6vYXVh7Dw6G/PbZMxQYVEvXVYN7y8nlRznCTs7jUFysRHXkMCGJrTPfqkrD53094j9G/kATvyR2+PzHFE388q31vuMBGKKhe1Nrw0gcFsf5xqZn0Ym/l7e6umd6jtQjGSELnjTD0FBydR8m6f67Aq59xzQeJ8Ygy6c9RTxwHhZ26NZcEGgsveBanUPYuTOMYdG0FNwSuM6LO0yxytU9b0Okhmomoqm4J5FAEw=" + } + ] + }, + { + "role": "user", + "parts": [ + { + "text": "What should I do next?" + } + ] + } + ], + "generationConfig": { + "audioTranscriptionConfig": { + "customVocabulary": [ + "Lingua" + ], + "diarization": true, + "languageCodes": [ + "en-US" + ], + "wordTimestamp": true + } + } +} \ No newline at end of file diff --git a/payloads/snapshots/audioTranscriptionConfigParam/google/followup-response-streaming.json b/payloads/snapshots/audioTranscriptionConfigParam/google/followup-response-streaming.json new file mode 100644 index 000000000..4c8d35304 --- /dev/null +++ b/payloads/snapshots/audioTranscriptionConfigParam/google/followup-response-streaming.json @@ -0,0 +1,184 @@ +[ + { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "Listen very closely, because there won't be a second" + } + ], + "role": "model" + }, + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 27, + "candidatesTokenCount": 12, + "totalTokenCount": 486, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 27 + } + ], + "thoughtsTokenCount": 447, + "serviceTier": "standard" + }, + "modelVersion": "gemini-3.5-flash", + "responseId": "mcdzaqWPAvSEz7IPh7rOEQ" + }, + { + "candidates": [ + { + "content": { + "parts": [ + { + "text": " explanation! 😉 \n\nIn all seriousness, what would you like to do next? You can:\n\n1. **" + } + ], + "role": "model" + }, + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 27, + "candidatesTokenCount": 36, + "totalTokenCount": 510, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 27 + } + ], + "thoughtsTokenCount": 447, + "serviceTier": "standard" + }, + "modelVersion": "gemini-3.5-flash", + "responseId": "mcdzaqWPAvSEz7IPh7rOEQ" + }, + { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "Upload another audio file** for me to transcribe or translate.\n2. **Analyze this clip** (e.g., identify" + } + ], + "role": "model" + }, + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 27, + "candidatesTokenCount": 62, + "totalTokenCount": 536, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 27 + } + ], + "thoughtsTokenCount": 447, + "serviceTier": "standard" + }, + "modelVersion": "gemini-3.5-flash", + "responseId": "mcdzaqWPAvSEz7IPh7rOEQ" + }, + { + "candidates": [ + { + "content": { + "parts": [ + { + "text": " the voice, context, or tone).\n3. **Move on to something completely different** (ask a question, write" + } + ], + "role": "model" + }, + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 27, + "candidatesTokenCount": 87, + "totalTokenCount": 561, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 27 + } + ], + "thoughtsTokenCount": 447, + "serviceTier": "standard" + }, + "modelVersion": "gemini-3.5-flash", + "responseId": "mcdzaqWPAvSEz7IPh7rOEQ" + }, + { + "candidates": [ + { + "content": { + "parts": [ + { + "text": " some code, brainstorm, etc.).\n\nHow can I help you from here?" + } + ], + "role": "model" + }, + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 27, + "candidatesTokenCount": 103, + "totalTokenCount": 577, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 27 + } + ], + "thoughtsTokenCount": 447, + "serviceTier": "standard" + }, + "modelVersion": "gemini-3.5-flash", + "responseId": "mcdzaqWPAvSEz7IPh7rOEQ" + }, + { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "", + "thoughtSignature": "Eu8QCuwQARFNMg+D0/vSGN5H7WjTbjPaxSgXmCQEOpXrtXlKOMgeBLGUKUI1AnIBxC9x5QSI6vLqqFtV8W97oCa1Y5xSA0TJL7IevFy+2Ggvb7fLUF5mmO93I2GTervGU/i2UrTZyUGpWpI9QXkEqkbkcamC5uJKwSY4464J1OexuPXdmenMl/ai+qdpEb93tSH3JnF6zyMZlWfyyUIdb8+fWZgtW45o2qZorciQYKtsvnitdJFzeyQqd3+P7ub1PQnzMiWufkNpnnZbipHiz+XERiuJTmhEDwa7HBMWcXvhbMDSQ0VPjmhTHH5VhNKLBcBn7XkEKNzzvRvGLPKFUHO2nkl/KGSG+elZRwBQ24KTXFnBqAj0uPFQTGFrlIL3c/KXHC9WRNid3N8Q819jY+TcIG1Sg5LX+SKhsBPBlhdjxD/YGvAbMzKawMbSXyMUUKTyi6G/IhEp3CI/xKR2ZM/ReOB/WVwonq6NT6ODkiACybDNylkeLjY3Qj7q1qO2nm6c79MlD2rTICgzpWMxxQog27MVo3kxwF3O9zApXB4IDgiq0ay5oY5jxS10L85nfUwx3iLaGhri5L6yk5gLUtg939y3z0TEPU09Tfck46wSBpg3O2THE6ABEzpa7POGyLEifzUl6MeeTOfkC7KWYCYb3jCuhHcetOfC76H7dqBSUajNky26zbB3z4I2kqcOwxHHbklW3u4BSDG4AbEuZMbdWrOxSCKv37wcZuicfUcl/BLgcvUDo9RhlihWIXqYatGfoRbQcucbnh5vCUR1HgOdg6GJl4o0iE3wjDWnav3jr44xzsG1/WnhcHhBQdmQXprzSameBUtEKTk/+/9OrPuX0iguhro0X2d2kJqSmiDlvs2vItyFdGZqSWnAbnkcL97ypDPAugZ1OZzbmJRis1akg8V5lYCLTym1CGBX4jMiTw/WBrOID0ogrRs8hEU1b+ykrDTdFZ4j3TyEoUPPnCa85UZYxcQQk43oQyJDMfF0u+3RlLo5gK5JAPDpKVEZ8E4VpQSWYZk+uLIF475auMbG8YzQcbLrQ5tthFdZiX6J1zF4w0utn2uOPUu8kQJ+AGhFGSf/e/R+WUkJhTWbt3EMT5ZiIQ3qg5/srZFdGBtOYiTLvH6cKN2e9chXrIU5EVsSDMropux/z8Jt4ODujMBCFL2z+i+0/0wMp6fTYW85GFbds/SqkrwY8E2MAtLdLMloWAGtnxwEoZgOyQqeRhtEM7LsBlKlzZ8o615U/LTmmoYKL1vYQAab+yxi5OcQTk5ptVdmIcggpfXiaxoqaIwCIpEw2XCI2VzM7yUDdwy2V4sbopZ6ly4/wX2+0FB/PdNZptQRO5ziLVxyAfS9egGtl6PUiJJ2EjUIiW1E8IC4druzyfhNYCgOIPeHcKAqPLZ45IllSgVx9+rI6kAkZJ6MB2ff6Z9pyXvz9KMXhqCA2sFy6uTHlEkgNc3NacHqbPHcZJKlHsY2JjwDoHTMu+FIkEinEn7htiUEsfcGCrVag2GmKkWx4ha2fk6Pv6FJWpnPcxNotbk57Th3RZ10HKjRpae6EtD+Juxdjdk/xv2hW11v43zPngDw4uFwXbY1Oxv8enTii9GksH+YJOoiXBjtEmImyRNta1bompK1UTGhDeXfPpelOFU7D2UYqhmYOhTsKKdKmD6fpXfKZcz08avdQQBmAaD/JrVtBo0A3iV2Za3UYqCtA46pDHOyZfplME2lfOKt0Xn5xz9eepUIeVVoc3G/zz8wR1kHPupk4iYOmodrIPpHperz2WMtH/4cWEUkwnXKOkiekw6i94gZZ2D7+c2ZwWWKdXquOYHMORzcFg+Pg33FlH8Xt9GWEKpucM8fHnkA1gL77zdSOyue+XmKgRpbC1jZVZWUN2A0Zfvc3H6G5x7jb5sBUd41QjUJBmqj1FSNgL+8SAW6y9KdvxKfSLX8MaUbuus9V5EyJgYYOlZ+W05mHBFbZMjNYmJBNxWMXkZObXVwXpGAZ1Ngq3Lvs/Fu7Xrg+9M9OLMizWbfPJSwIcn1SzPoM9odzSF0WAcZANlpHCG4QAx5VFbGaIfjy3Zy+NYa9O5dvDebKOrPeckBfhD2+wxTWvmgR+kONF0cA68qAZeJ2J8z2Lejgb6HVGKH0VZI49HlaCukOpGkjDFBXBvxTdYLnGopOv7kaYkhZvRKTE//RROSh56YwmkBXmlNsumPCtMTwZd/1dQSsBrlZ/oafcbxP9pZG8kjBISed9NF1PaJ93IdOSAyn3EUdd0lWFEQN4L0G6jBbB0jFuV04UyTH6IV3NkeKMCxjynSbUChWFjIc6rwWlYiXkg8DwbLyCyUUExBcC3OmIZaTokPv2lTI5BF/6CKw21uD2jmI9BuHtRuZ4tlLqZZVZpj0wdot1Uuc6YA+ca9j1y1owO0d/QZTBWMUnZQb0bdUgpdscrH6ztHxQWd0MLPIGzj0X0QOuJtI44zHLDaz7clFL1Rb62OJFJpC5zLJH40JzBxyiE8h8YAMR6xHuEzG05GGjO0DodiGpeTgBUfCSIKOJfauqOASBSEdUFyU/Jp9YvEb2YyvcU9olRmucGofaYSQuTNhqNOq2jRJcaaodfa3lJpcBfF1Nar8YiEGevfn2T3pBvNvnz9cFAVCzVgbCMA8wEMqWYYWFqtvoGTpXeHECJipLmPtryo+POwQdu7i79DrQK6VQCvaaVKehUo+UWsojILk0RzPmQC5DPPAt5VC3xhn69Sq+hSr9z1cROuFqfJl4ZdyUtudMcXJsjay3OEJ5nqjLzMJ4YdPOxNCDffDA01be7FUwEznH8jGEX36fHe+SJ3KVTqfxIoCjzIWdYcYaqBnePNhCCuLrE4pSB8yRVvY3OXhOntBZsKGVTV804=" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 256, + "candidatesTokenCount": 103, + "totalTokenCount": 806, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 256 + } + ], + "thoughtsTokenCount": 447, + "serviceTier": "standard" + }, + "modelVersion": "gemini-3.5-flash", + "responseId": "mcdzaqWPAvSEz7IPh7rOEQ" + } +] \ No newline at end of file diff --git a/payloads/snapshots/audioTranscriptionConfigParam/google/followup-response.json b/payloads/snapshots/audioTranscriptionConfigParam/google/followup-response.json new file mode 100644 index 000000000..b93978375 --- /dev/null +++ b/payloads/snapshots/audioTranscriptionConfigParam/google/followup-response.json @@ -0,0 +1,32 @@ +{ + "candidates": [ + { + "content": { + "parts": [ + { + "text": "I'm locked in and paying close attention—since you're only explaining it once! \n\nWhat would you like to do next? Here are a few options:\n\n1. **Explain \"the thing\":** Tell me what it is you are explaining, and we can write a guide, a script, or a story around it.\n2. **Transcribe more:** If you have another audio file you need transcribed, go ahead and upload it.\n3. **Translate or edit:** I can translate this transcription into another language, or format it however you need for a project.\n\nLet me know how you'd like to proceed!", + "thoughtSignature": "ErcVCrQVARFNMg/F2cpPQBrXwqnuRgLdIBiwuR5lpzn/tEFspCvKr42OEbHmJQNVSENtYjSmh0IrNggEIitXkeTcykVJDgl87zvLIRfI9vIhI2t5VbsUxIDBClalZMelFunbMF6c1kBU3SwyRXV+l5NpUyWB28B6v75NPzvQ9h08uXIsf6uxoU5T3baXWxQXgMVO/F8eZGzQQV2H6euj3q06tWICfNAqnESWU3MhqZVX7YBhmOeCTySpEufUNEUHPyKTX3HeQj7cPPuTO8DfC4tokELHYW/RHg9Hjt66K/sNlzkcWS7YysW/K1skA95RCU/eGHhnvF1LZpVntrhR7ebGVK/ocDEFnNgDxuIZYHKEmZN/7oqrLvfJPrhYTDnffqyKktkvGPt2j2cMw3Qv8co31E3MqxBcs1KsAbYNkzZcRGMyveyYisM8aI/zqTOhms7qXANIE3hERIwa+I4FB3+LQB7kYLu3K8Jgg2fYbEMb27ts1MMJkbB5uoqN0jujDMx7ohZyljDs9bdzCuE08fD2uN2bPxsdU/KAuLLMXNK+F5qogbTEJp1qo1WoNUsegXDsjvIIUzw0AJyucYTf4s/R7NSQt5D7uxPugLodfIjKVeelXjX7pOCf7UxurJZFUNn6F46XcMDrz8VG4cl8FysBmaZyUIXwhEngL4cSawUF9UrVCNzPi1RzmOkJ4L3cDSc/lvjPBrEL8MYY5XV17OtzEd1kGEVLSzBXeRGiRWOVkbB3Qnx45dGrgNha3uU2WKVz0U69B5ISKGOEume0ygNlAYAMw6YcFcKp1awqygip0nd6Gh5LWlTHrcb7ZM+XDX2Rtk00f8dY9XppPZ5a9Pf9wMGRvWaQxy2ZRa3QahtDdlIgmiSzHcFdPbQM1g5JYNltS8XWHZd71bAlW1z4ecCOyvbKM8KVCXfL6vdcRbprEo1xtXGZfInMEcsyVwI7zXQf1/Cv9xjRXLs9Ymj932Qt8wEy+CIjwJINH3U01mfjCP3DG2Sxy5US2VAaR1O7pc7z8Y6D4reoZ/F6Gp5DIqkHTCqBIvAC0QYP+jUn674f+W2VyJFM3W21LyOPmuidb+6MqfHp886Ay5hT+Id+M6i+gUOtoscEaCHm28x9EnIP3lnFGdDDpiJagm2WT1EVbKRc9Jf3WWvSzZacTzR8qN694eSRn4SWvSFO61/jsJIUEskweGxazjn+OFO5PpMJ6OJLmrRMaZvdscYFTARgx2n6di0cjGiLfdPivyj11pWddfQRAusrvGTMBqUU45XXzMsY3Ns88M9aOIYPX1uliK79QTDxFHnMvFYGRVWlkXvsMf21+zEAMZq8RS1eT+KE2cWAX+dmN5SrTwH2Y19RvNNPRlA0w45ddB1w8DMflrRIptukWwspTJ3HRLC2CIKZjpK04Rz1ZSzyEe5zq9JVdpvK0uJQoaV97EUKDhQAIdWpfoyh1B4vzPPZa2Fp748HNc3MFDfNESzHf+XVVt76IGfvxeztrh9+jlVTuCp7V2jlCLSfvEgdk+Ne3zB9f9c7Tfb7kPL7KDZV3z/Mun2GATHkTCOZ2lLZ9YcDNa5DgrWGG7RAilr2cUFjKW3OIwnx83YlWg94EHMaaJmRx3fq222AAZPdRnmgOuvOm6CvDR+ftrlW5T/mvk9mnZj/AAYcjOjEurTaGwieWcGGO/Bj/xsM3bJhkuaz6QMsypTPD4XlpRDzwGj2UWkicOoJ7ShUKsvvmm62LZRa3/k2agcCW0ggel9phPXRWmkvQ4DH0n2Pnv0d9ktWLcKahrFrZXqz/nJmuS3sNv6l7K2+l2ioNvF8vKSDJbuYhxGCtMbT6/ReZNUvEEcDMPyxvTKtwb8FEgoZBcjh78a2XI/qoyLxrpqFjTuH1MgHbElQnrDjppyyabiG+sBV2Zf2CorrLhH1+qCQ08yc+xWEatRR6pyKTuW5vwnD7VLvGXFd5v11KUcU1tEf2Uj0571eDNtUQsfYxWmkAevKdHV7NzWhL9Ohg5CGVuRQQPx/wj66IzSkuX0z5hSkdegXOWxNICypOBSqLScD8OiB3mxRZpJBEuKnemfk2edL6iQUhiK6s9GUdjjyfalZqenlbBQAu7uV4FNJd4Yx3xbk8RzzRqGCXKZokV9COaEyiz9QVBfSU7tnhkOQZQfIo6qz5CPXP17s9FTpsOo9Pw9MO0urATW31SywYZfoMt+TGgcNHE+82uHVGZ+mvTeepSVbI4/iV9XHKPjKYmld+UHqdMFeTjCCkhmgeVytuf8wGlSX2SBQWZowWU2omhnpvX9L6cLY1Mb/CWb826CCHy0rvmpBYs9WSXqq283u/s+DT/01ceOvDPe4ooQMdUxy/AgeBVeVaKiXfIK1daqIp4+p423v6ko5xuIWNMxd0T5QtKYXA+z7JxarU4bgpoa8xPMEp+LPSshN7ODoQotUyT8x15oTWuCxosZBksDor6kmtDd4Pv9w8dB6d4H4qG6OtCMB0v5kWZDb8B9vuQAemHd5vSqBBlzkkEZiyDsxwIlGE1eTWQ5DtJ62/JPXu+ZRfOx8lGIakynBrvJ87oWUyPlyPFei/4PZd7nQcjjNo942ZJ508N2Yh6rVLsI+QtFAYzsgDoKvj4Lu/Xufg4tckCokiV/viOpb43lTu/K6GjUtks91cBURJdY4M0IajjJZeT0dt1SzqxpmozyfkbApSEYWBnH55k8YRXwh989Jy3HeohHOA39M/piZRfaolgulTowSAq95rvcHmCJvi2PHLluQVOhrsPHJJaeMMiFsMyFv1D9xkDkPG5d0krsnWvFV8iibeG5uZz9ltX6OCccF4uTrQZepy0nhi+P0dfnEDoWgSufgWFRmhsQ90TUDaktOg2j3oPk8o6oUt8W7z6oo0TsV5dxmpuPL6D5gjglOsgiT9CfyHz3JlRX0RugZNggYHzedGYmqjIwvNzQYJ+VSW2p7Z3Flg65xCOZ425DWNs8l1wFXLlA0nXt6x8Eja3uE28gxvICBlRNvFHZhBpLP4utPyUyzO9cRv4DIe8IbKcxO1LO5p7e6Y1Cns+lfPGWM2OaOnT0ywA8CZR4fkytrQmp0sUyWsKCzjTUN6nESIL6XneIkVP3GJZ8n7S25xzALds7T20DB66zWp6EXPjWQzjgc6K5CFslCOFzGvlc/nCCLeshiWMXQ7WwjsOenouvxbE9QB7GwZsfynkERHcAeVcIlCBL9sKp13vvK+AwfScDNmBB7rCGOqdykNv9w+FnkUgxQFTw0/ojV1G+SS+ZBHMTpuy0H8YkYqpnYaZGLFFAZMCOfseQ2D+SvYo5vCn9HbXFZklY0Yt9ljcgCrkkKTKRPGb+/iualry6w2wcdQbXPc/me4ZsI9DJ4HkdyJi5DJKdXlu590JcLFcL57MzTfppEIlStxAGsEY9jeuQaYBHbfykWGSCgV7FkL6psXkjKDkbbU+Pqlsb1KFnBQSASVA2DiZ5quoMJF6x0eu0m2FO33/JFWDKCkEIHt7HExr3wT7Vzt6nOwv3O7SYwxg2pI1L7M0L+Y4v6/zUQbV3gEQyGmEQDwmh5dTkeuwR8tyJpOu04TOxhQRzXVIVjr2VRY5Vj2LV9Jc4/7fHN7M+Ws4d1ZRDY/yZYDAqFSk1j599IyQESs751fl9edAfYGesIbJPC9oXG8Q==" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 256, + "candidatesTokenCount": 133, + "totalTokenCount": 1070, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 256 + } + ], + "thoughtsTokenCount": 681, + "serviceTier": "standard" + }, + "modelVersion": "gemini-3.5-flash", + "responseId": "mMdzauvkPKSCz7IPzOiz6Qg" +} \ No newline at end of file diff --git a/payloads/snapshots/audioTranscriptionConfigParam/google/request.json b/payloads/snapshots/audioTranscriptionConfigParam/google/request.json new file mode 100644 index 000000000..4dc75ed71 --- /dev/null +++ b/payloads/snapshots/audioTranscriptionConfigParam/google/request.json @@ -0,0 +1,24 @@ +{ + "contents": [ + { + "role": "user", + "parts": [ + { + "text": "Transcribe the attached audio." + } + ] + } + ], + "generationConfig": { + "audioTranscriptionConfig": { + "customVocabulary": [ + "Lingua" + ], + "diarization": true, + "languageCodes": [ + "en-US" + ], + "wordTimestamp": true + } + } +} \ No newline at end of file diff --git a/payloads/snapshots/audioTranscriptionConfigParam/google/response-streaming.json b/payloads/snapshots/audioTranscriptionConfigParam/google/response-streaming.json new file mode 100644 index 000000000..75bcdcf26 --- /dev/null +++ b/payloads/snapshots/audioTranscriptionConfigParam/google/response-streaming.json @@ -0,0 +1,94 @@ +[ + { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "What" + } + ], + "role": "model" + }, + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 7, + "candidatesTokenCount": 1, + "totalTokenCount": 179, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 7 + } + ], + "thoughtsTokenCount": 171, + "serviceTier": "standard" + }, + "modelVersion": "gemini-3.5-flash", + "responseId": "l8dzaomcCrWez7IP677v6A8" + }, + { + "candidates": [ + { + "content": { + "parts": [ + { + "text": " do you call a developer who doesn't like to work in teams? \n\nAn independent variable." + } + ], + "role": "model" + }, + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 7, + "candidatesTokenCount": 22, + "totalTokenCount": 200, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 7 + } + ], + "thoughtsTokenCount": 171, + "serviceTier": "standard" + }, + "modelVersion": "gemini-3.5-flash", + "responseId": "l8dzaomcCrWez7IP677v6A8" + }, + { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "", + "thoughtSignature": "EsoGCscGARFNMg+AHL3b7UKBC1wwPkenWpCQgPhANmQ0ExTdOa/HBJX8R9Vtge8veQwOCBrVPv5VGaxpQrVjFGdPIg6+mJjJed4GxAgIuOqQRGoC1pMwbd/uZD5hAMnNt0sDCqnKo/sJAQ1WsG3K8y9VHZrvFp6bv/AWZbyM9+jCvX5mPOLID/ZxYJXqEk2bYFGjyC+syMomiRhu6ddQi4Aj73JT8ZtTUix7qZmZ9lHK6LVkqgFr5Peq0SwSK6mYrdQACljL1dGrQFXviiD3faOUUIlSb1B5oemtglOb8gpoj+LDZrWfPlYOWiMutufpsAb3rxFLEvNuEK0zowBoqnfRLKAgrfA5lVMrKgU9RdkqoO6cDqPc/SpKl+UZjj0rZxtBQBRt9wigw2GMlr+CxZjFliR1APWs8LSzAqT32k1maGcegZwru95wIEAMOO0Vn88dMxDzq7rrnwznMxfr+ox57jM6INPbcuIzQO2c6/2r2QQx7FAlWduzS9kLP8juRyoxm4oTDJmIU+1tbpyZ/UP35ThqWotsJfueU/z5Z2Cff+MPyvbd/YSIRW7VIhMQin3VXzHeYuSHBn/Oa40MB22eHnjsw7l5RaOhSGIbquBmA5Ym18+0ia3jdD4yMIYV+kHZldTov87nnAXUmysjP/5uQlKpYVUFMnF5jhOrXJxF0XPtD4/QYwtsxs8S84jdR3gwNEtf7XZ9SQpwP9DbhbGFA2clpbTr5132D0MACC3C0lWSXSxZv4ScK3rSJTcCgGKa9/9/K5nmVYxpM+TNrbA0B0+15930AW80iKs0EuqXDRP2l4yUmU3N6m1qvaxP9f/sY4QvUjc9AcRZ/aPbKXAjKtc5iZHuna/UsnNc+lWee6YywkbduD4J0paZFlco+tPQKKk/MNU75NtG6ba1iv3mvwiP7HDj0QVDcsqL4EA32t3893e/KWPZVkCwRFcUTETJsDLP3h+OqJOr47kaCYj7gA+Yut3xsLjm5e6lYM7c6Yj736ZxIvEOUKt9l0Gcn6UNw75XKTGavX5UI5dmZF0akbpRi+Ybiroj7k9R1Vq3TUch5mGCy1EGoT+JsMgGiFMlSgDfLFNB2Oc+eJqMk3yKUJfrVP4s3MZ22U4=" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 7, + "candidatesTokenCount": 22, + "totalTokenCount": 200, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 7 + } + ], + "thoughtsTokenCount": 171, + "serviceTier": "standard" + }, + "modelVersion": "gemini-3.5-flash", + "responseId": "l8dzaomcCrWez7IP677v6A8" + } +] \ No newline at end of file diff --git a/payloads/snapshots/audioTranscriptionConfigParam/google/response.json b/payloads/snapshots/audioTranscriptionConfigParam/google/response.json new file mode 100644 index 000000000..2f0dc5662 --- /dev/null +++ b/payloads/snapshots/audioTranscriptionConfigParam/google/response.json @@ -0,0 +1,32 @@ +{ + "candidates": [ + { + "content": { + "parts": [ + { + "text": "I will explain this thing one time and one time only.", + "thoughtSignature": "EukHCuYHARFNMg9EgA3yC4+TAi0LEEEzPFAxKBiHHK6PZ9HQTtUm2PoFj5YeXr2gTs+B9NyIhBKcO7Fou2bXTG6jF5sj2yqmx07WsGT75MlQRDJ8bE1bax3JUDlRg9C+6rilI3YQHPkCmNcPImsyCUuGSA9kn886BU/szbMISNVQOmnzvUe8TRlu9g6j5c9QiQkk+mhtgbv6HXPltLzMinvx9v8+/CmNuC5GaohdaR8s786WRj0U7tEib7LP9ijbMIsR7fL9RE3z1n43OuBzYkhZBHWDSalZnpVvQOJ/Ufvwv/DcRhVKIoJ3LBinu6A9ZnK/dyLcokxvjwgSuVEfCYR7rr9A69TFFmZshIhRtcBFQZ4fKp7sMg3PlCz8d12J/f5ueB9rWYcXP4Wxx76b/Lo/nHAqMex8hfKPRk414jujttNN+xGGg+rOVGwbAkGxSM/aHhBAeqGWoleX2mmPRKAu7uPCrKjlF++TMXIXDYT4ROpk8t0mjVa5wQjdoSi4siY50ITJqX9C3GVLs6OpBH4eImyuZ9vLElYP/Ep7dDCEO5QU58CBvNF7dWqT5jBbMi1toYeE0Qx4+5W8OY3KPP5F7M8e0CLnQfd4i7yIlawAmfn7xy++X+06IpNyBtAN2W2vhrF2PpcA0+wW2VeJmnqFDCvJNUVeIsddBzMq4wrcaXQWP3A0cSkll12LOfgX7v4Z61ZioC9gr2FznCx5eRavDrCavcJvaPR8rgsRM0Izw7LpzOsTe9r5ubxtn2ELvJlWXkrE3W3x+8ldK3yj/HIsD39gV8r1eEiZs0AaKqzcRlzMDs3DNO5ao2NCerHrHDrRi/Z7S7xiZ+gyHFajj7EcijF5eLnYMUOVkqi536iPGKaUSAg4wKfFjUi1+XwX5PTYBLcd/qRjPT3XTiBt+LjQejqkfEND9NlbW0lx68A1Vd0rQhJ8qXHkJ3DRvi7LSx8AjMOvuvwqE6UH0tU5VJdHbpMbicvbsVDaabc8LEb3nDmEZoVhw5Xiw5ciMwfPCW8Jhgi85BYnJuG+XEZJEwCOfpTM6aoUU7HpXHKVPN46PiC/o3wivOsNvCghhsQY0fYSCA8X/EoNBvWxxLfetI6vYXVh7Dw6G/PbZMxQYVEvXVYN7y8nlRznCTs7jUFysRHXkMCGJrTPfqkrD53094j9G/kATvyR2+PzHFE388q31vuMBGKKhe1Nrw0gcFsf5xqZn0Ym/l7e6umd6jtQjGSELnjTD0FBydR8m6f67Aq59xzQeJ8Ygy6c9RTxwHhZ26NZcEGgsveBanUPYuTOMYdG0FNwSuM6LO0yxytU9b0Okhmomoqm4J5FAEw=" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 7, + "candidatesTokenCount": 12, + "totalTokenCount": 248, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 7 + } + ], + "thoughtsTokenCount": 229, + "serviceTier": "standard" + }, + "modelVersion": "gemini-3.5-flash", + "responseId": "l8dzauGSCousz7IPzZbu-Ac" +} \ No newline at end of file diff --git a/payloads/snapshots/googleProviderExecutedToolRoundtrip/google/error.json b/payloads/snapshots/googleProviderExecutedToolRoundtrip/google/error.json new file mode 100644 index 000000000..47aa4bdaf --- /dev/null +++ b/payloads/snapshots/googleProviderExecutedToolRoundtrip/google/error.json @@ -0,0 +1,3 @@ +{ + "error": "Error: ApiError: {\n \"error\": {\n \"code\": 400,\n \"message\": \"* GenerateContentRequest.contents: contents is not specified\\n\",\n \"status\": \"INVALID_ARGUMENT\"\n }\n}\n" +} \ No newline at end of file diff --git a/payloads/snapshots/googleProviderExecutedToolRoundtrip/google/request.json b/payloads/snapshots/googleProviderExecutedToolRoundtrip/google/request.json new file mode 100644 index 000000000..ba9c7d947 --- /dev/null +++ b/payloads/snapshots/googleProviderExecutedToolRoundtrip/google/request.json @@ -0,0 +1,33 @@ +{ + "contents": [ + { + "role": "model", + "parts": [ + { + "toolCall": { + "id": "google-search-1", + "toolName": "search_the_web", + "toolType": "GOOGLE_SEARCH_WEB", + "args": { + "query": "Lingua message format" + } + } + } + ] + }, + { + "role": "user", + "parts": [ + { + "toolResponse": { + "id": "google-search-1", + "toolType": "GOOGLE_SEARCH_WEB", + "response": { + "result": "Lingua is a universal LLM message format." + } + } + } + ] + } + ] +} \ No newline at end of file diff --git a/payloads/transforms/google_to_anthropic/audioTranscriptionConfigParam.json b/payloads/transforms/google_to_anthropic/audioTranscriptionConfigParam.json new file mode 100644 index 000000000..1590225b7 --- /dev/null +++ b/payloads/transforms/google_to_anthropic/audioTranscriptionConfigParam.json @@ -0,0 +1,27 @@ +{ + "model": "claude-sonnet-4-5-20250929", + "id": "msg_011CdkUM3nyF7wivEAbqkkNS", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "text", + "text": "I apologize, but I don't see any audio file attached to your message. Could you please try uploading the audio file again?\n\nOnce you've attached the audio file, I'll be happy to transcribe it for you." + } + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "stop_details": null, + "usage": { + "input_tokens": 14, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": { + "ephemeral_5m_input_tokens": 0, + "ephemeral_1h_input_tokens": 0 + }, + "output_tokens": 52, + "service_tier": "standard", + "inference_geo": "not_available" + } +} diff --git a/payloads/transforms/google_to_anthropic/googleProviderExecutedToolRoundtrip.json b/payloads/transforms/google_to_anthropic/googleProviderExecutedToolRoundtrip.json new file mode 100644 index 000000000..26fd3b0a0 --- /dev/null +++ b/payloads/transforms/google_to_anthropic/googleProviderExecutedToolRoundtrip.json @@ -0,0 +1,3 @@ +{ + "error": "Conversion from universal format failed: Unsupported mapping: cannot convert google built-in tool result `GOOGLE_SEARCH_WEB` to Anthropic tool_result" +} \ No newline at end of file diff --git a/payloads/transforms/google_to_chat-completions/audioTranscriptionConfigParam.json b/payloads/transforms/google_to_chat-completions/audioTranscriptionConfigParam.json new file mode 100644 index 000000000..ccf5da0c6 --- /dev/null +++ b/payloads/transforms/google_to_chat-completions/audioTranscriptionConfigParam.json @@ -0,0 +1,35 @@ +{ + "id": "chatcmpl-E9fME38UeB5EZLCHn4x3d7rTsaPZN", + "object": "chat.completion", + "created": 1785972638, + "model": "gpt-5-nano-2025-08-07", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "I can transcribe it, but I don’t see an attached audio file yet. Please either upload the audio file here (MP3, WAV, M4A, etc.) or share a link to it.\n\nIf you upload, please also specify:\n- Language and any dialects\n- Transcript style: verbatim (including filler words like um, ah) or clean (remove most fillers)\n- Timestamps: include every 30 seconds, or at each speaker change, or none\n- Speaker labeling: use “Speaker 1 / Speaker 2” or provide names if known\n\nIf you can’t upload, you can also paste a short excerpt or provide a shareable link and I’ll transcribe from that.", + "refusal": null, + "annotations": [] + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 12, + "completion_tokens": 667, + "total_tokens": 679, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 512, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": null +} \ No newline at end of file diff --git a/payloads/transforms/google_to_chat-completions/googleProviderExecutedToolRoundtrip-streaming.json b/payloads/transforms/google_to_chat-completions/googleProviderExecutedToolRoundtrip-streaming.json new file mode 100644 index 000000000..1079be584 --- /dev/null +++ b/payloads/transforms/google_to_chat-completions/googleProviderExecutedToolRoundtrip-streaming.json @@ -0,0 +1,3 @@ +{ + "error": "Conversion from universal format failed: Unsupported mapping: cannot convert google built-in tool result `GOOGLE_SEARCH_WEB` to OpenAI Chat Completions tool message" +} \ No newline at end of file diff --git a/payloads/transforms/google_to_chat-completions/googleProviderExecutedToolRoundtrip.json b/payloads/transforms/google_to_chat-completions/googleProviderExecutedToolRoundtrip.json new file mode 100644 index 000000000..1079be584 --- /dev/null +++ b/payloads/transforms/google_to_chat-completions/googleProviderExecutedToolRoundtrip.json @@ -0,0 +1,3 @@ +{ + "error": "Conversion from universal format failed: Unsupported mapping: cannot convert google built-in tool result `GOOGLE_SEARCH_WEB` to OpenAI Chat Completions tool message" +} \ No newline at end of file diff --git a/payloads/transforms/google_to_responses/audioTranscriptionConfigParam.json b/payloads/transforms/google_to_responses/audioTranscriptionConfigParam.json new file mode 100644 index 000000000..90a338f2a --- /dev/null +++ b/payloads/transforms/google_to_responses/audioTranscriptionConfigParam.json @@ -0,0 +1,102 @@ +{ + "id": "resp_0dcc0d0837988a9f006a73c7a1f300819898ab073b840bb398", + "object": "response", + "created_at": 1785972642, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1785972644, + "error": null, + "frequency_penalty": 0, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-5.6-terra", + "moderation": null, + "output": [ + { + "id": "rs_0dcc0d0837988a9f006a73c7a2cf8081988eab86972621055b", + "type": "reasoning", + "content": [], + "encrypted_content": "gAAAAABqc8ekkMe4TGiPbVH5y865n6Vjjo1kNqU-e11_Ak5jCBUHNW-6slzouWAYAcZaRyREsO1SSlJlH_CFjAz_LT-pyGJvqAUL0Aa5V6pVUOu9N2Rggnq2IrvxJ_LNALAOTgLQO2u_zsNlocU6uWAJtbg3wdzlHKXEyaFoipKo9LIj5hqjeJVGe3Fs1TppwJXccaGXNRJUBD0HesazGzGcA9n4IfIyk5GJhBSe8_31RvxPsHueG0CxlDD_U9zBrwPasdvuyiHSg81xurEOFK6MlUdnglKcHJx4mcDamnjRu6KxhH2Bv_LV8vIChRnCpzfYvIdA19zqQHMS8HzGVN2JLbi3_d6zZkp6M46qXbnLGAUMqc5HsfeliP5x8KjnaYeguxRmQyPSiqsDrR1xZCrNkm6pyfWZfl_Vdh3zpJxCZOJCKvEBNALs2Ki-on3t4WSnX243KnyYKDY639CNdxrkM0_hUJSsVxKwV3A_6rUZeW1dCjUhINOj7qu8Jg0_7QdjbC5XaiGkgPSvjGoPCx4EOeICl4ljqA8R_OciLvoi2YrOF42FzqmeIOEB1hNX__bhhs_mMRyQsKNlAgT7YplDMVu3nG-2f1auJBwmYNIsb5Ky4herTB3jwVpDjll2GFoBGa6ALZKKl5Eczw2jhIdGe38nZYigioZ5tJD5SqU1u382i_i_8n9N8ncz9VA742Yca4LHwIe0KQ43G1s52F1ADRSZYVfGsZUv0PHVU12dMgeES6BHsi3LZNd3tZiUZIJD3AAVlfTvaon9esHJHwqFW8u6nne3yH27oDu0WHLXYXLLCaima2TQvOBo_qG8puJyGlw4fIh-9jRHvtLL9iXNfRzCrmV7eM20XMp3e9Vdylte-Z7Ycil4bgxIEQHlOhwoNYKS91lzLfOIzO62wKGx3iO4Newp6P2hnArs9YLKBrtPM_4gvENGXrIx7nvfi7lrOeHyyDihczRsGWDuhmZenCHE88mhH3TeloqLYS36-hUR4O4st2RAXGIbcbuLghNaM_AETJtV1RuiKkmrQ1KgGgeAy84DuaHeaniBpdTqGW9FqSzwOQyQoL3IvhMfPHUg4UlCnLdqadfDcRcq263dJkjYmGt_mfcsWppP13r1X6RZYLTUB3INGGIPkXwoJDX-VgyRqUHzp2pXRgaBZ179BvNpqFq3Qw==", + "summary": [] + }, + { + "id": "msg_0dcc0d0837988a9f006a73c7a33490819887b473713b965b83", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "I don’t see an audio attachment available here. Please upload the audio file, and I’ll transcribe it." + } + ], + "phase": "final_answer", + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": "24h", + "reasoning": { + "context": "all_turns", + "effort": "medium", + "mode": "standard", + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 0.98, + "truncation": "disabled", + "usage": { + "input_tokens": 12, + "input_tokens_details": { + "cache_write_tokens": 0, + "cached_tokens": 0 + }, + "output_tokens": 66, + "output_tokens_details": { + "reasoning_tokens": 37 + }, + "total_tokens": 78 + }, + "user": null, + "metadata": {}, + "output_text": "I don’t see an audio attachment available here. Please upload the audio file, and I’ll transcribe it." +} \ No newline at end of file diff --git a/payloads/transforms/google_to_responses/googleProviderExecutedToolRoundtrip.json b/payloads/transforms/google_to_responses/googleProviderExecutedToolRoundtrip.json new file mode 100644 index 000000000..ff5235a70 --- /dev/null +++ b/payloads/transforms/google_to_responses/googleProviderExecutedToolRoundtrip.json @@ -0,0 +1,3 @@ +{ + "error": "Conversion from universal format failed: Unsupported mapping: cannot convert google built-in tool call `GOOGLE_SEARCH_WEB` to OpenAI Responses input content" +} \ No newline at end of file diff --git a/payloads/transforms/transform_errors.json b/payloads/transforms/transform_errors.json index f4d654de4..85dfae164 100644 --- a/payloads/transforms/transform_errors.json +++ b/payloads/transforms/transform_errors.json @@ -65,16 +65,22 @@ }, "google_to_anthropic": { "codeInterpreterToolParam": "Expected: Google code_execution is provider-specific and is not a lossless equivalent of Anthropic bash", + "googleProviderExecutedToolRoundtrip": "Expected: Google server-side ToolCall/ToolResponse history carries a Google ToolType identity that Anthropic cannot represent losslessly", "urlContextToolParam": "Tool 'url_context' of type 'url_context' is not supported by anthropic", "reasoningSummaryParam": "Anthropic requires max_tokens > budget_tokens; min budget (1024) exceeds max_tokens (100)", "reasoningEffortLowParam": "Anthropic requires max_tokens > budget_tokens; min budget (1024) exceeds max_tokens (100)" }, "google_to_chat-completions": { "codeInterpreterToolParam": "Expected: Google code_execution is provider-specific and is not supported by OpenAI Chat Completions", + "googleProviderExecutedToolRoundtrip": "Expected: Google server-side ToolCall/ToolResponse history carries a Google ToolType identity that OpenAI Chat Completions cannot represent losslessly", "urlContextToolParam": "Tool 'url_context' of type 'url_context' is not supported by openai" }, "google_to_responses": { "codeInterpreterToolParam": "Expected: Google code_execution is provider-specific and is not a lossless equivalent of OpenAI Responses code_interpreter", + "googleProviderExecutedToolRoundtrip": "Expected: Google server-side ToolCall/ToolResponse history carries a Google ToolType identity that OpenAI Responses cannot represent losslessly", "urlContextToolParam": "Tool 'url_context' of type 'url_context' is not supported by responses" + }, + "google_to_chat-completions_streaming": { + "googleProviderExecutedToolRoundtrip": "Expected: Google server-side ToolCall/ToolResponse history carries a Google ToolType identity that OpenAI Chat Completions streaming cannot represent losslessly" } } diff --git a/plan.md b/plan.md new file mode 100644 index 000000000..8cdb8271a --- /dev/null +++ b/plan.md @@ -0,0 +1,62 @@ +# Google provider type follow-up plan + +## Root cause + +- Google Discovery schema ids can move between public and internal `V1main` names, allowing quicktype collision resolution to silently swap the public `MediaResolution` struct and enum names. +- `Part.toolCall` and `Part.toolResponse` are omitted by the Google content converter. The universal tool-call/result shapes require a function name and have no typed provider-executed builtin identity, so preserving Google’s optional `toolName` and `toolType` is impossible today. +- The Google adapter lifts a small canonical subset of `generationConfig` and discards every other typed field, including `audioTranscriptionConfig` and request-level `mediaResolution`. + +## Target files + +- `crates/generate-types/src/main.rs` +- `crates/lingua/src/universal/message.rs` +- Provider adapters and import helpers that construct or consume universal tool calls/results +- `crates/lingua/src/providers/google/convert.rs` +- `crates/lingua/src/providers/google/adapter.rs` +- `crates/lingua/src/providers/google/params.rs` +- `payloads/cases/advanced.ts` +- `payloads/cases/params.ts` +- Generated TypeScript universal bindings produced by `make generate-types` +- Narrow transform expectations only if an intentional cross-provider limitation remains + +## Expected behavior + +- `V1main` Discovery ids normalize to stable public names, while the GenerationConfig scalar enum remains `MediaResolutionEnum`; generation fails loudly on real normalized-name collisions. +- Dedicated universal builtin-tool call and result parts carry an optional free-form name plus a typed identity (`provider` and `builtin_type`). Google server-side tool calls/results round-trip with `provider_executed: true` without fabricating a function name, while ordinary function-tool parts remain source-compatible. +- Providers that cannot represent a provider-executed builtin return an explicit unsupported-mapping error instead of silently dropping it. +- Google-to-universal preserves only the unmapped, typed remainder of `generationConfig` in Google-scoped extras. Universal-to-Google starts from that typed remainder and lets canonical fields override it, avoiding duplicate sources of truth. +- The accepted REST `audioTranscriptionConfig` subtree and request-level `mediaResolution` survive Google round trips byte-for-byte at the semantic JSON level. + +## Tests to add or update + +- Keep the existing generator normalization/collision tests and media-resolution compile-time serialization guards. +- Add Google converter tests for named and unnamed provider-executed builtin calls and responses. +- Add Google params/adapter tests proving unmapped `generationConfig` fields survive while canonical temperature/reasoning/response-format values take precedence. +- Keep payload cases `googleProviderExecutedToolRoundtrip` and `audioTranscriptionConfigParam`; recapture after the logic fix. +- Update existing universal/provider tests for optional tool names and builtin identities. + +## Expected-diff impact + +- Google same-provider request coverage should stop reporting loss of `generationConfig.audioTranscriptionConfig` and `generationConfig.mediaResolution`. +- Google provider-executed tool call/response parts should stop disappearing. Cross-provider transforms may produce explicit unsupported errors where no equivalent builtin exists; any expectation entry must be case-specific. +- Generated Google provider files should change only through the generator. Generated universal TypeScript bindings will reflect optional tool names and builtin identity fields. + +## Validation commands + +1. `make capture FILTER=audioTranscriptionConfigParam` +2. `make capture FILTER=googleProviderExecutedToolRoundtrip` +3. `cargo test -p generate-types google_post_process_tests` +4. `cargo test -p generate-types google_schema_name_tests` +5. `cargo test -p lingua providers::google::convert::tests` +6. `cargo test -p lingua providers::google::params::tests` +7. `cargo test -p lingua providers::google::` +8. `make capture FILTER=audioTranscriptionConfigParam` +9. `make capture FILTER=googleProviderExecutedToolRoundtrip` +10. `make test-payloads` +11. `cargo test -p coverage-report --test cross_provider_test cross_provider_transformations_have_no_unexpected_failures` +12. `make typed-boundary-check` +13. `make typed-boundary-check-branch BASE=main` +14. `make generate-types PROVIDER=google` +15. `git diff --exit-code crates/lingua/src/providers/google/generated.rs bindings/typescript/src/generated/google` +16. `cargo check -p lingua` +17. `cd bindings/typescript && pnpm run typecheck` From ce425364f7f306816aa17c5a16ff879413e619fe Mon Sep 17 00:00:00 2001 From: Alex Z Date: Thu, 6 Aug 2026 10:29:27 -0700 Subject: [PATCH 3/6] fixes --- .../src/generated/AssistantContentPart.ts | 2 +- .../generated/BuiltinToolResultContentPart.ts | 2 +- crates/lingua/src/processing/stream.rs | 7 +- crates/lingua/src/processing/transform.rs | 31 ++- crates/lingua/src/providers/google/adapter.rs | 254 +++++++++++++++--- crates/lingua/src/providers/google/convert.rs | 97 +++++-- crates/lingua/src/universal/message.rs | 5 +- payloads/cases/advanced.ts | 2 - .../google/request.json | 2 - .../googleProviderExecutedToolRoundtrip.json | 2 +- ...oviderExecutedToolRoundtrip-streaming.json | 2 +- .../googleProviderExecutedToolRoundtrip.json | 2 +- plan.md | 6 + 13 files changed, 342 insertions(+), 72 deletions(-) diff --git a/bindings/typescript/src/generated/AssistantContentPart.ts b/bindings/typescript/src/generated/AssistantContentPart.ts index d1b3dce93..9489753bd 100644 --- a/bindings/typescript/src/generated/AssistantContentPart.ts +++ b/bindings/typescript/src/generated/AssistantContentPart.ts @@ -13,4 +13,4 @@ export type AssistantContentPart = { "type": "text" } & TextContentPart | { "typ * Providers will occasionally return encrypted content for reasoning parts which can * be useful when you send a follow up message. */ -encrypted_content?: string, } | { "type": "tool_call", tool_call_id: string, tool_name: string, arguments: ToolCallArguments, status?: string, caller?: ToolCaller, encrypted_content?: string, provider_options?: ProviderOptions, provider_executed?: boolean, } | { "type": "builtin_tool_call", tool_call_id: string, tool_name?: string, builtin_tool: BuiltinToolIdentity, arguments?: ToolCallArguments, status?: string, provider_options?: ProviderOptions, provider_executed?: boolean, } | { "type": "program", call_id: string, code: string, fingerprint?: string, id?: string, } | { "type": "program_output", call_id: string, result: string, status: string, id?: string, } | { "type": "tool_discovery_call", tool_call_id: string, discovery_tool_name: string, query?: string, arguments?: unknown, status?: string, execution?: string, provider_options?: ProviderOptions, } | { "type": "tool_result", tool_call_id: string, tool_name: string, output: unknown, caller?: ToolCaller, provider_options?: ProviderOptions, }; +encrypted_content?: string, } | { "type": "tool_call", tool_call_id: string, tool_name: string, arguments: ToolCallArguments, status?: string, caller?: ToolCaller, encrypted_content?: string, provider_options?: ProviderOptions, provider_executed?: boolean, } | { "type": "builtin_tool_call", tool_call_id?: string, tool_name?: string, builtin_tool: BuiltinToolIdentity, arguments?: ToolCallArguments, status?: string, provider_options?: ProviderOptions, provider_executed?: boolean, } | { "type": "program", call_id: string, code: string, fingerprint?: string, id?: string, } | { "type": "program_output", call_id: string, result: string, status: string, id?: string, } | { "type": "tool_discovery_call", tool_call_id: string, discovery_tool_name: string, query?: string, arguments?: unknown, status?: string, execution?: string, provider_options?: ProviderOptions, } | { "type": "tool_result", tool_call_id: string, tool_name: string, output: unknown, caller?: ToolCaller, provider_options?: ProviderOptions, }; diff --git a/bindings/typescript/src/generated/BuiltinToolResultContentPart.ts b/bindings/typescript/src/generated/BuiltinToolResultContentPart.ts index f505f67f7..2b889d430 100644 --- a/bindings/typescript/src/generated/BuiltinToolResultContentPart.ts +++ b/bindings/typescript/src/generated/BuiltinToolResultContentPart.ts @@ -5,4 +5,4 @@ import type { ProviderOptions } from "./ProviderOptions"; /** * Reusable result for a provider-executed built-in tool. */ -export type BuiltinToolResultContentPart = { tool_call_id: string, tool_name?: string, builtin_tool: BuiltinToolIdentity, output: any, provider_options?: ProviderOptions, }; +export type BuiltinToolResultContentPart = { tool_call_id?: string, tool_name?: string, builtin_tool: BuiltinToolIdentity, output: any, provider_options?: ProviderOptions, }; diff --git a/crates/lingua/src/processing/stream.rs b/crates/lingua/src/processing/stream.rs index 2bfa31ae4..4db9ab937 100644 --- a/crates/lingua/src/processing/stream.rs +++ b/crates/lingua/src/processing/stream.rs @@ -5,7 +5,8 @@ use std::collections::BTreeMap; use crate::capabilities::ProviderFormat; use crate::processing::adapters::adapter_for_format; use crate::processing::transform::{ - serialize_stream_value, transform_stream_chunk_step, TransformError, TransformResult, + ensure_stream_builtin_tools_supported, serialize_stream_value, transform_stream_chunk_step, + TransformError, TransformResult, }; #[cfg(feature = "openai")] use crate::providers::openai::responses_adapter::{ @@ -1703,6 +1704,10 @@ pub fn parse_stream_event( }); } + if let Some(universal) = &universal_opt { + ensure_stream_builtin_tools_supported(universal, target_format)?; + } + let target_adapter = adapter_for_format(target_format) .ok_or(TransformError::UnsupportedTargetFormat(target_format))?; diff --git a/crates/lingua/src/processing/transform.rs b/crates/lingua/src/processing/transform.rs index 0169b4a95..302d9941d 100644 --- a/crates/lingua/src/processing/transform.rs +++ b/crates/lingua/src/processing/transform.rs @@ -699,6 +699,31 @@ pub(crate) fn serialize_stream_value(value: &Value) -> Result Result<(), TransformError> { + if target_format == ProviderFormat::Google { + return Ok(()); + } + + let builtin_call_type = chunk + .choices + .iter() + .filter_map(UniversalStreamChoice::delta_view) + .flat_map(|delta| delta.tool_calls) + .filter_map(|tool_call| tool_call.call_type) + .find(|call_type| call_type.starts_with("builtin:")); + + if let Some(call_type) = builtin_call_type { + return Err(TransformError::FromUniversalFailed(format!( + "target format {target_format:?} cannot represent streaming built-in tool call `{call_type}`" + ))); + } + + Ok(()) +} + fn assistant_content_to_stream_delta(content: &AssistantContent) -> UniversalStreamDelta { match content { AssistantContent::String(text) => UniversalStreamDelta { @@ -757,7 +782,7 @@ fn assistant_content_to_stream_delta(content: &AssistantContent) -> UniversalStr let tool_call_index = tool_calls.len() as u32; tool_calls.push(UniversalToolCallDelta { index: Some(tool_call_index), - id: Some(tool_call_id.clone()), + id: tool_call_id.clone(), call_type: Some(format!( "builtin:{}:{}", builtin_tool.provider.label(), @@ -902,6 +927,10 @@ pub(crate) fn transform_stream_chunk_step( }); } + if let Some(universal_chunk) = &universal { + ensure_stream_builtin_tools_supported(universal_chunk, target_format)?; + } + let target_adapter = adapter_for_format(target_format) .ok_or(TransformError::UnsupportedTargetFormat(target_format))?; let bytes = match &universal { diff --git a/crates/lingua/src/providers/google/adapter.rs b/crates/lingua/src/providers/google/adapter.rs index 3b680ad0e..0df0f4c7d 100644 --- a/crates/lingua/src/providers/google/adapter.rs +++ b/crates/lingua/src/providers/google/adapter.rs @@ -14,19 +14,25 @@ use crate::processing::transform::TransformError; use crate::providers::google::capabilities::{ effort_to_thinking_level, thinking_level_to_effort, GoogleCapabilities, GoogleThinkingStyle, }; -use crate::providers::google::convert::SYNTHETIC_CALL_ID_PREFIX; +use crate::providers::google::convert::{ + builtin_identity_from_google_tool_type, google_tool_type_from_builtin_identity, + SYNTHETIC_CALL_ID_PREFIX, +}; use crate::providers::google::detect::try_parse_google; use crate::providers::google::generated::{ Content as GoogleContent, GenerateContentResponse, GenerationConfig, ServiceTier, - ThinkingConfig, ThinkingLevel, Tool as GoogleTool, ToolConfig, UsageMetadata, + ThinkingConfig, ThinkingLevel, Tool as GoogleTool, ToolCall as GoogleToolCall, ToolConfig, + UsageMetadata, }; use crate::providers::google::params::GoogleParams; use crate::serde_json::{self, Map, Value}; use crate::universal::convert::TryFromLLM; -use crate::universal::message::{AssistantContent, AssistantContentPart, Message}; +use crate::universal::message::{ + AssistantContent, AssistantContentPart, BuiltinToolIdentity, Message, +}; use crate::universal::reasoning::{budget_to_effort, effort_to_budget, MIN_THINKING_BUDGET}; use crate::universal::request::ToolChoiceConfig; -use crate::universal::tools::UniversalTool; +use crate::universal::tools::{BuiltinToolProvider, UniversalTool}; use crate::universal::ToolContentPart; use crate::universal::{ extract_system_messages, flatten_consecutive_messages, FinishReason, ReasoningCanonical, @@ -144,6 +150,41 @@ struct GoogleStreamPart { thought_signature: Option, #[serde(skip_serializing_if = "Option::is_none")] function_call: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + tool_call: Option, +} + +fn builtin_stream_call_type(identity: &BuiltinToolIdentity) -> String { + format!( + "builtin:{}:{}", + identity.provider.label(), + identity.builtin_type + ) +} + +fn builtin_identity_from_stream_call_type( + call_type: Option<&str>, +) -> Result, TransformError> { + let Some(call_type) = call_type else { + return Ok(None); + }; + let Some(encoded_identity) = call_type.strip_prefix("builtin:") else { + return Ok(None); + }; + let Some((provider, builtin_type)) = encoded_identity.split_once(':') else { + return Err(TransformError::FromUniversalFailed(format!( + "invalid built-in stream tool-call type `{call_type}`" + ))); + }; + if provider != BuiltinToolProvider::Google.label() { + return Err(TransformError::FromUniversalFailed(format!( + "Google streaming cannot represent {provider} built-in tool `{builtin_type}`" + ))); + } + Ok(Some(BuiltinToolIdentity { + provider: BuiltinToolProvider::Google, + builtin_type: builtin_type.to_string(), + })) } impl ProviderAdapter for GoogleAdapter { @@ -775,35 +816,55 @@ impl ProviderAdapter for GoogleAdapter { let response_id = typed_payload.response_id.as_deref(); let mut tool_call_index = 0_u32; - let tool_calls: Vec = parts - .iter() - .filter_map(|part| { - part.function_call.as_ref().map(|function_call| { - let index = tool_call_index; - tool_call_index += 1; - UniversalToolCallDelta { - index: Some(index), - id: function_call.id.clone().or_else(|| { - Some(match response_id { - Some(response_id) => { - format!("{SYNTHETIC_CALL_ID_PREFIX}{response_id}_{index}") - } - None => format!("{SYNTHETIC_CALL_ID_PREFIX}{index}"), - }) - }), - call_type: Some("function".to_string()), - custom_tool_call: None, - function: Some(UniversalToolFunctionDelta { - name: function_call.name.clone(), - arguments: function_call - .args - .as_ref() - .map(|args| Value::Object(args.clone()).to_string()), - }), - } - }) - }) - .collect(); + let mut tool_calls = Vec::new(); + for part in &parts { + if let Some(function_call) = &part.function_call { + let index = tool_call_index; + tool_call_index += 1; + tool_calls.push(UniversalToolCallDelta { + index: Some(index), + id: function_call.id.clone().or_else(|| { + Some(match response_id { + Some(response_id) => { + format!("{SYNTHETIC_CALL_ID_PREFIX}{response_id}_{index}") + } + None => format!("{SYNTHETIC_CALL_ID_PREFIX}{index}"), + }) + }), + call_type: Some("function".to_string()), + custom_tool_call: None, + function: Some(UniversalToolFunctionDelta { + name: function_call.name.clone(), + arguments: function_call + .args + .as_ref() + .map(|args| Value::Object(args.clone()).to_string()), + }), + }); + } else if let Some(tool_call) = &part.tool_call { + let tool_type = tool_call.tool_type.as_ref().ok_or_else(|| { + TransformError::ToUniversalFailed( + "Google streaming Part.toolCall is missing toolType".to_string(), + ) + })?; + let identity = builtin_identity_from_google_tool_type(tool_type); + let index = tool_call_index; + tool_call_index += 1; + tool_calls.push(UniversalToolCallDelta { + index: Some(index), + id: tool_call.id.clone(), + call_type: Some(builtin_stream_call_type(&identity)), + custom_tool_call: None, + function: Some(UniversalToolFunctionDelta { + name: tool_call.tool_name.clone(), + arguments: tool_call + .args + .as_ref() + .map(|args| Value::Object(args.clone()).to_string()), + }), + }); + } + } let finish_reason = candidate .finish_reason @@ -868,17 +929,15 @@ impl ProviderAdapter for GoogleAdapter { .as_ref() .and_then(|d| d.content.as_deref()) .unwrap_or(""); - let has_function_call_part = delta.as_ref().is_some_and(|d| { - d.tool_calls - .iter() - .any(|tool_call| tool_call.function.is_some()) - }); + let has_tool_call_part = delta + .as_ref() + .is_some_and(|d| !d.tool_calls.is_empty()); let text_reasoning_signature = delta .as_ref() .and_then(|d| d.reasoning_signature.as_deref()) .filter(|_| { delta.as_ref().is_none_or(|d| { - !has_function_call_part && (!text.is_empty() || d.reasoning.is_empty()) + !has_tool_call_part && (!text.is_empty() || d.reasoning.is_empty()) }) }); @@ -894,7 +953,7 @@ impl ProviderAdapter for GoogleAdapter { d.reasoning_signature.as_deref().filter(|_| { !reasoning_texts.is_empty() && text.is_empty() - && !has_function_call_part + && !has_tool_call_part }); for (index, text) in reasoning_texts.iter().enumerate() { @@ -924,9 +983,46 @@ impl ProviderAdapter for GoogleAdapter { parts.push(text_part); } - // Add functionCall parts from tool_calls + // Add functionCall or provider-executed toolCall parts from tool_calls. if let Some(ref d) = delta { for tc in &d.tool_calls { + if let Some(identity) = builtin_identity_from_stream_call_type( + tc.call_type.as_deref(), + )? { + let args = tc + .function + .as_ref() + .and_then(|function| function.arguments.as_deref()) + .map(|arguments| { + serde_json::from_str::>(arguments).map_err( + |error| { + TransformError::FromUniversalFailed(format!( + "Google built-in stream tool arguments must be an object: {error}" + )) + }, + ) + }) + .transpose()?; + let mut part = GoogleStreamPart { + tool_call: Some(GoogleToolCall { + args, + id: tc.id.clone(), + tool_name: tc + .function + .as_ref() + .and_then(|function| function.name.clone()), + tool_type: Some(google_tool_type_from_builtin_identity( + &identity, + )?), + }), + ..Default::default() + }; + if let Some(ref signature) = d.reasoning_signature { + part.thought_signature = Some(signature.clone()); + } + parts.push(part); + continue; + } if let Some(ref func) = tc.function { let mut function_call = Map::new(); if let Some(ref name) = func.name { @@ -1727,6 +1823,82 @@ mod tests { assert_eq!(tool_call.id.as_deref(), Some("call_response_123_0")); } + #[test] + fn test_google_stream_builtin_tool_call_roundtrips_without_id() { + let adapter = GoogleAdapter; + let payload = json!({ + "responseId": "response_builtin_123", + "candidates": [{ + "index": 0, + "content": { + "role": "model", + "parts": [{ + "toolCall": { + "toolName": "search_the_web", + "toolType": "GOOGLE_SEARCH_WEB", + "args": {"query": "Lingua"} + } + }] + }, + "finishReason": "STOP" + }] + }); + + let chunk = adapter + .stream_to_universal(payload.clone()) + .unwrap() + .expect("stream chunk should be present"); + let choice = chunk.choices.first().expect("choice should be present"); + let delta = choice.delta_view().expect("delta should be present"); + let tool_call = delta + .tool_calls + .first() + .expect("built-in tool call should be present"); + + assert_eq!(choice.finish_reason.as_deref(), Some("tool_calls")); + assert_eq!(tool_call.index, Some(0)); + assert_eq!(tool_call.id, None); + assert_eq!( + tool_call.call_type.as_deref(), + Some("builtin:google:GOOGLE_SEARCH_WEB") + ); + let function = tool_call + .function + .as_ref() + .expect("built-in call metadata should be present"); + assert_eq!(function.name.as_deref(), Some("search_the_web")); + assert_eq!(function.arguments.as_deref(), Some(r#"{"query":"Lingua"}"#)); + + let roundtrip = adapter + .stream_from_universal(&chunk) + .expect("built-in stream tool call should export to Google"); + let roundtrip: GenerateContentResponse = + serde_json::from_value(roundtrip).expect("stream response should deserialize"); + let roundtrip_candidates = roundtrip.candidates.unwrap(); + let roundtrip_call = roundtrip_candidates[0] + .content + .as_ref() + .and_then(|content| content.parts.as_ref()) + .and_then(|parts| parts.first()) + .and_then(|part| part.tool_call.as_ref()) + .expect("roundtrip should contain toolCall"); + assert_eq!(roundtrip_call.id, None); + assert_eq!(roundtrip_call.tool_name.as_deref(), Some("search_the_web")); + assert_eq!( + roundtrip_call.tool_type, + Some(crate::providers::google::generated::ToolType::GoogleSearchWeb) + ); + + let error = crate::processing::transform::transform_stream_chunk( + bytes::Bytes::from(serde_json::to_vec(&payload).unwrap()), + ProviderFormat::ChatCompletions, + ) + .expect_err("non-Google stream target must reject Google built-in identity"); + assert!(error + .to_string() + .contains("cannot represent streaming built-in tool call")); + } + #[test] fn test_google_stream_tool_call_indexes_are_tool_call_relative() { let adapter = GoogleAdapter; diff --git a/crates/lingua/src/providers/google/convert.rs b/crates/lingua/src/providers/google/convert.rs index 8d7248c48..768d14d87 100644 --- a/crates/lingua/src/providers/google/convert.rs +++ b/crates/lingua/src/providers/google/convert.rs @@ -77,7 +77,9 @@ fn value_to_map(value: &Value) -> Option> { } } -fn builtin_identity_from_google_tool_type(tool_type: &GoogleToolType) -> BuiltinToolIdentity { +pub(super) fn builtin_identity_from_google_tool_type( + tool_type: &GoogleToolType, +) -> BuiltinToolIdentity { let builtin_type = match tool_type { GoogleToolType::FileSearch => "FILE_SEARCH", GoogleToolType::GoogleMaps => "GOOGLE_MAPS", @@ -92,7 +94,7 @@ fn builtin_identity_from_google_tool_type(tool_type: &GoogleToolType) -> Builtin } } -fn google_tool_type_from_builtin_identity( +pub(super) fn google_tool_type_from_builtin_identity( identity: &BuiltinToolIdentity, ) -> Result { if identity.provider != BuiltinToolProvider::Google { @@ -274,18 +276,13 @@ impl TryFromLLM for Message { )?, })); } else if let Some(tool_call) = &part.tool_call { - let tool_call_id = tool_call.id.clone().ok_or_else(|| { - ConvertError::MissingRequiredField { - field: "Part.toolCall.id".to_string(), - } - })?; let tool_type = tool_call.tool_type.as_ref().ok_or_else(|| { ConvertError::MissingRequiredField { field: "Part.toolCall.toolType".to_string(), } })?; assistant_parts.push(AssistantContentPart::BuiltinToolCall { - tool_call_id, + tool_call_id: tool_call.id.clone(), tool_name: tool_call.tool_name.clone(), builtin_tool: builtin_identity_from_google_tool_type(tool_type), arguments: tool_call.args.clone().map(ToolCallArguments::Valid), @@ -403,11 +400,6 @@ impl TryFromLLM for Message { } } } else if let Some(tool_response) = &part.tool_response { - let tool_call_id = tool_response.id.clone().ok_or_else(|| { - ConvertError::MissingRequiredField { - field: "Part.toolResponse.id".to_string(), - } - })?; let tool_type = tool_response.tool_type.as_ref().ok_or_else(|| { ConvertError::MissingRequiredField { field: "Part.toolResponse.toolType".to_string(), @@ -415,7 +407,7 @@ impl TryFromLLM for Message { })?; tool_parts.push(ToolContentPart::BuiltinToolResult( BuiltinToolResultContentPart { - tool_call_id, + tool_call_id: tool_response.id.clone(), tool_name: None, builtin_tool: builtin_identity_from_google_tool_type(tool_type), output: tool_response @@ -696,7 +688,7 @@ impl TryFromLLM for GoogleContent { converted.push(GooglePart { tool_call: Some(GoogleToolCall { args, - id: Some(tool_call_id), + id: tool_call_id, tool_name, tool_type: Some( google_tool_type_from_builtin_identity( @@ -767,7 +759,7 @@ impl TryFromLLM for GoogleContent { } parts.push(GooglePart { tool_response: Some(GoogleToolResponse { - id: Some(result.tool_call_id), + id: result.tool_call_id, response: value_to_map(&result.output), tool_type: Some(google_tool_type_from_builtin_identity( &result.builtin_tool, @@ -2059,7 +2051,7 @@ mod tests { else { panic!("expected a built-in tool call"); }; - assert_eq!(tool_call_id, "google-search-1"); + assert_eq!(tool_call_id.as_deref(), Some("google-search-1")); assert_eq!(tool_name, &None); assert_eq!(builtin_tool.provider, BuiltinToolProvider::Google); assert_eq!(builtin_tool.builtin_type, "GOOGLE_SEARCH_WEB"); @@ -2095,7 +2087,7 @@ mod tests { let ToolContentPart::BuiltinToolResult(result) = &content[0] else { panic!("expected a built-in tool result"); }; - assert_eq!(result.tool_call_id, "google-search-1"); + assert_eq!(result.tool_call_id.as_deref(), Some("google-search-1")); assert_eq!(result.tool_name, None); assert_eq!(result.builtin_tool.provider, BuiltinToolProvider::Google); assert_eq!(result.builtin_tool.builtin_type, "GOOGLE_SEARCH_WEB"); @@ -2105,6 +2097,75 @@ mod tests { assert_eq!(roundtrip, original); } + #[test] + fn test_google_provider_executed_tool_call_roundtrips_without_id() { + let original = GoogleContent { + role: Some("model".to_string()), + parts: Some(vec![GooglePart { + tool_call: Some(GoogleToolCall { + args: Some(Map::from_iter([( + "query".to_string(), + Value::String("Lingua".to_string()), + )])), + id: None, + tool_name: Some("search_the_web".to_string()), + tool_type: Some(GoogleToolType::GoogleSearchWeb), + }), + ..Default::default() + }]), + }; + + let universal = >::try_from(original.clone()) + .expect("Google toolCall without an ID should import"); + let Message::Assistant { + content: AssistantContent::Array(parts), + .. + } = &universal + else { + panic!("expected assistant content parts"); + }; + let AssistantContentPart::BuiltinToolCall { tool_call_id, .. } = &parts[0] else { + panic!("expected a built-in tool call"); + }; + assert_eq!(tool_call_id, &None); + + let roundtrip = >::try_from(universal) + .expect("built-in tool call without an ID should export"); + assert_eq!(roundtrip, original); + } + + #[test] + fn test_google_provider_executed_tool_response_roundtrips_without_id() { + let original = GoogleContent { + role: Some("user".to_string()), + parts: Some(vec![GooglePart { + tool_response: Some(GoogleToolResponse { + id: None, + response: Some(Map::from_iter([( + "result".to_string(), + Value::String("Lingua".to_string()), + )])), + tool_type: Some(GoogleToolType::GoogleSearchWeb), + }), + ..Default::default() + }]), + }; + + let universal = >::try_from(original.clone()) + .expect("Google toolResponse without an ID should import"); + let Message::Tool { content } = &universal else { + panic!("expected tool content"); + }; + let ToolContentPart::BuiltinToolResult(result) = &content[0] else { + panic!("expected a built-in tool result"); + }; + assert_eq!(result.tool_call_id, None); + + let roundtrip = >::try_from(universal) + .expect("built-in tool result without an ID should export"); + assert_eq!(roundtrip, original); + } + #[test] fn test_google_to_universal_simple() { let request = GenerateContentRequest { diff --git a/crates/lingua/src/universal/message.rs b/crates/lingua/src/universal/message.rs index dca2c6de5..932b31168 100644 --- a/crates/lingua/src/universal/message.rs +++ b/crates/lingua/src/universal/message.rs @@ -118,7 +118,8 @@ pub enum AssistantContentPart { }, /// A provider-executed built-in tool call whose free-form name may be absent. BuiltinToolCall { - tool_call_id: String, + #[ts(optional)] + tool_call_id: Option, #[ts(optional)] tool_name: Option, builtin_tool: BuiltinToolIdentity, @@ -246,7 +247,7 @@ pub struct ToolResultContentPart { #[derive(Debug, Clone, Serialize, Deserialize, TS)] #[ts(export, rename_all = "snake_case", optional_fields)] pub struct BuiltinToolResultContentPart { - pub tool_call_id: String, + pub tool_call_id: Option, pub tool_name: Option, pub builtin_tool: BuiltinToolIdentity, #[ts(type = "any")] diff --git a/payloads/cases/advanced.ts b/payloads/cases/advanced.ts index f5c84212c..4134f52aa 100644 --- a/payloads/cases/advanced.ts +++ b/payloads/cases/advanced.ts @@ -510,7 +510,6 @@ export const advancedCases: TestCaseCollection = { parts: [ { toolCall: { - id: "google-search-1", toolName: "search_the_web", toolType: ToolType.GOOGLE_SEARCH_WEB, args: { query: "Lingua message format" }, @@ -523,7 +522,6 @@ export const advancedCases: TestCaseCollection = { parts: [ { toolResponse: { - id: "google-search-1", toolType: ToolType.GOOGLE_SEARCH_WEB, response: { result: "Lingua is a universal LLM message format.", diff --git a/payloads/snapshots/googleProviderExecutedToolRoundtrip/google/request.json b/payloads/snapshots/googleProviderExecutedToolRoundtrip/google/request.json index ba9c7d947..b1cfe0d4e 100644 --- a/payloads/snapshots/googleProviderExecutedToolRoundtrip/google/request.json +++ b/payloads/snapshots/googleProviderExecutedToolRoundtrip/google/request.json @@ -5,7 +5,6 @@ "parts": [ { "toolCall": { - "id": "google-search-1", "toolName": "search_the_web", "toolType": "GOOGLE_SEARCH_WEB", "args": { @@ -20,7 +19,6 @@ "parts": [ { "toolResponse": { - "id": "google-search-1", "toolType": "GOOGLE_SEARCH_WEB", "response": { "result": "Lingua is a universal LLM message format." diff --git a/payloads/transforms/google_to_anthropic/googleProviderExecutedToolRoundtrip.json b/payloads/transforms/google_to_anthropic/googleProviderExecutedToolRoundtrip.json index 26fd3b0a0..d784dab11 100644 --- a/payloads/transforms/google_to_anthropic/googleProviderExecutedToolRoundtrip.json +++ b/payloads/transforms/google_to_anthropic/googleProviderExecutedToolRoundtrip.json @@ -1,3 +1,3 @@ { - "error": "Conversion from universal format failed: Unsupported mapping: cannot convert google built-in tool result `GOOGLE_SEARCH_WEB` to Anthropic tool_result" + "error": "Conversion from universal format failed: Unsupported mapping: cannot convert google built-in tool call `GOOGLE_SEARCH_WEB` to Anthropic assistant content" } \ No newline at end of file diff --git a/payloads/transforms/google_to_chat-completions/googleProviderExecutedToolRoundtrip-streaming.json b/payloads/transforms/google_to_chat-completions/googleProviderExecutedToolRoundtrip-streaming.json index 1079be584..2c8d3ddec 100644 --- a/payloads/transforms/google_to_chat-completions/googleProviderExecutedToolRoundtrip-streaming.json +++ b/payloads/transforms/google_to_chat-completions/googleProviderExecutedToolRoundtrip-streaming.json @@ -1,3 +1,3 @@ { - "error": "Conversion from universal format failed: Unsupported mapping: cannot convert google built-in tool result `GOOGLE_SEARCH_WEB` to OpenAI Chat Completions tool message" + "error": "Conversion from universal format failed: Unsupported mapping: cannot convert google built-in tool call `GOOGLE_SEARCH_WEB` to OpenAI Chat Completions assistant message" } \ No newline at end of file diff --git a/payloads/transforms/google_to_chat-completions/googleProviderExecutedToolRoundtrip.json b/payloads/transforms/google_to_chat-completions/googleProviderExecutedToolRoundtrip.json index 1079be584..2c8d3ddec 100644 --- a/payloads/transforms/google_to_chat-completions/googleProviderExecutedToolRoundtrip.json +++ b/payloads/transforms/google_to_chat-completions/googleProviderExecutedToolRoundtrip.json @@ -1,3 +1,3 @@ { - "error": "Conversion from universal format failed: Unsupported mapping: cannot convert google built-in tool result `GOOGLE_SEARCH_WEB` to OpenAI Chat Completions tool message" + "error": "Conversion from universal format failed: Unsupported mapping: cannot convert google built-in tool call `GOOGLE_SEARCH_WEB` to OpenAI Chat Completions assistant message" } \ No newline at end of file diff --git a/plan.md b/plan.md index 8cdb8271a..bc2fcb536 100644 --- a/plan.md +++ b/plan.md @@ -5,6 +5,8 @@ - Google Discovery schema ids can move between public and internal `V1main` names, allowing quicktype collision resolution to silently swap the public `MediaResolution` struct and enum names. - `Part.toolCall` and `Part.toolResponse` are omitted by the Google content converter. The universal tool-call/result shapes require a function name and have no typed provider-executed builtin identity, so preserving Google’s optional `toolName` and `toolType` is impossible today. - The Google adapter lifts a small canonical subset of `generationConfig` and discards every other typed field, including `audioTranscriptionConfig` and request-level `mediaResolution`. +- Google streaming conversion only inspects `Part.functionCall`, so native `Part.toolCall` chunks are silently reduced to empty assistant deltas and may receive the wrong finish reason. +- Google declares built-in `ToolCall.id` and `ToolResponse.id` optional, but the universal built-in parts currently require a string and the converter rejects valid provider payloads with no ID. ## Target files @@ -23,6 +25,8 @@ - `V1main` Discovery ids normalize to stable public names, while the GenerationConfig scalar enum remains `MediaResolutionEnum`; generation fails loudly on real normalized-name collisions. - Dedicated universal builtin-tool call and result parts carry an optional free-form name plus a typed identity (`provider` and `builtin_type`). Google server-side tool calls/results round-trip with `provider_executed: true` without fabricating a function name, while ordinary function-tool parts remain source-compatible. +- Built-in call/result correlation IDs are optional so a missing Google ID round-trips as absent rather than being rejected or synthesized. Real IDs remain unchanged. +- Native Google streaming `toolCall` parts become typed universal built-in tool-call deltas, set the `tool_calls` finish reason, and round-trip back to Google. Streaming targets that cannot represent the built-in identity fail explicitly instead of treating it as a function call. - Providers that cannot represent a provider-executed builtin return an explicit unsupported-mapping error instead of silently dropping it. - Google-to-universal preserves only the unmapped, typed remainder of `generationConfig` in Google-scoped extras. Universal-to-Google starts from that typed remainder and lets canonical fields override it, avoiding duplicate sources of truth. - The accepted REST `audioTranscriptionConfig` subtree and request-level `mediaResolution` survive Google round trips byte-for-byte at the semantic JSON level. @@ -31,6 +35,8 @@ - Keep the existing generator normalization/collision tests and media-resolution compile-time serialization guards. - Add Google converter tests for named and unnamed provider-executed builtin calls and responses. +- Add Google converter tests for built-in calls and responses with absent IDs. +- Add Google streaming tests for typed built-in call conversion, absent-ID preservation, Google roundtrip, finish-reason handling, and explicit rejection by non-Google targets. - Add Google params/adapter tests proving unmapped `generationConfig` fields survive while canonical temperature/reasoning/response-format values take precedence. - Keep payload cases `googleProviderExecutedToolRoundtrip` and `audioTranscriptionConfigParam`; recapture after the logic fix. - Update existing universal/provider tests for optional tool names and builtin identities. From 197e417813cf4734f8d5b4deb5f33672691f0003 Mon Sep 17 00:00:00 2001 From: Alex Z Date: Thu, 6 Aug 2026 10:46:55 -0700 Subject: [PATCH 4/6] stuff --- .../src/generated/AssistantContentPart.ts | 2 +- crates/lingua/src/processing/dedup.rs | 2 + crates/lingua/src/processing/transform.rs | 4 ++ crates/lingua/src/providers/google/adapter.rs | 52 +++++++++++++++++-- crates/lingua/src/providers/google/convert.rs | 9 ++++ crates/lingua/src/universal/message.rs | 2 + plan.md | 6 +++ 7 files changed, 73 insertions(+), 4 deletions(-) diff --git a/bindings/typescript/src/generated/AssistantContentPart.ts b/bindings/typescript/src/generated/AssistantContentPart.ts index 9489753bd..ed85ee849 100644 --- a/bindings/typescript/src/generated/AssistantContentPart.ts +++ b/bindings/typescript/src/generated/AssistantContentPart.ts @@ -13,4 +13,4 @@ export type AssistantContentPart = { "type": "text" } & TextContentPart | { "typ * Providers will occasionally return encrypted content for reasoning parts which can * be useful when you send a follow up message. */ -encrypted_content?: string, } | { "type": "tool_call", tool_call_id: string, tool_name: string, arguments: ToolCallArguments, status?: string, caller?: ToolCaller, encrypted_content?: string, provider_options?: ProviderOptions, provider_executed?: boolean, } | { "type": "builtin_tool_call", tool_call_id?: string, tool_name?: string, builtin_tool: BuiltinToolIdentity, arguments?: ToolCallArguments, status?: string, provider_options?: ProviderOptions, provider_executed?: boolean, } | { "type": "program", call_id: string, code: string, fingerprint?: string, id?: string, } | { "type": "program_output", call_id: string, result: string, status: string, id?: string, } | { "type": "tool_discovery_call", tool_call_id: string, discovery_tool_name: string, query?: string, arguments?: unknown, status?: string, execution?: string, provider_options?: ProviderOptions, } | { "type": "tool_result", tool_call_id: string, tool_name: string, output: unknown, caller?: ToolCaller, provider_options?: ProviderOptions, }; +encrypted_content?: string, } | { "type": "tool_call", tool_call_id: string, tool_name: string, arguments: ToolCallArguments, status?: string, caller?: ToolCaller, encrypted_content?: string, provider_options?: ProviderOptions, provider_executed?: boolean, } | { "type": "builtin_tool_call", tool_call_id?: string, tool_name?: string, builtin_tool: BuiltinToolIdentity, arguments?: ToolCallArguments, status?: string, encrypted_content?: string, provider_options?: ProviderOptions, provider_executed?: boolean, } | { "type": "program", call_id: string, code: string, fingerprint?: string, id?: string, } | { "type": "program_output", call_id: string, result: string, status: string, id?: string, } | { "type": "tool_discovery_call", tool_call_id: string, discovery_tool_name: string, query?: string, arguments?: unknown, status?: string, execution?: string, provider_options?: ProviderOptions, } | { "type": "tool_result", tool_call_id: string, tool_name: string, output: unknown, caller?: ToolCaller, provider_options?: ProviderOptions, }; diff --git a/crates/lingua/src/processing/dedup.rs b/crates/lingua/src/processing/dedup.rs index 73d02dd48..4ea465bcd 100644 --- a/crates/lingua/src/processing/dedup.rs +++ b/crates/lingua/src/processing/dedup.rs @@ -191,6 +191,7 @@ fn hash_assistant_content(content: &AssistantContent, hasher: &mut DefaultHasher builtin_tool, arguments, status, + encrypted_content, .. } => { "builtin_tool_call".hash(hasher); @@ -198,6 +199,7 @@ fn hash_assistant_content(content: &AssistantContent, hasher: &mut DefaultHasher tool_name.hash(hasher); builtin_tool.hash(hasher); status.hash(hasher); + encrypted_content.hash(hasher); match arguments { Some(crate::universal::ToolCallArguments::Valid(map)) => { "valid".hash(hasher); diff --git a/crates/lingua/src/processing/transform.rs b/crates/lingua/src/processing/transform.rs index 302d9941d..0100631fa 100644 --- a/crates/lingua/src/processing/transform.rs +++ b/crates/lingua/src/processing/transform.rs @@ -777,6 +777,7 @@ fn assistant_content_to_stream_delta(content: &AssistantContent) -> UniversalStr tool_name, builtin_tool, arguments, + encrypted_content, .. } => { let tool_call_index = tool_calls.len() as u32; @@ -794,6 +795,9 @@ fn assistant_content_to_stream_delta(content: &AssistantContent) -> UniversalStr arguments: arguments.as_ref().map(ToString::to_string), }), }); + if reasoning_signature.is_none() { + reasoning_signature = encrypted_content.clone(); + } } AssistantContentPart::File { .. } | AssistantContentPart::ToolResult { .. } diff --git a/crates/lingua/src/providers/google/adapter.rs b/crates/lingua/src/providers/google/adapter.rs index 0df0f4c7d..04a87bb68 100644 --- a/crates/lingua/src/providers/google/adapter.rs +++ b/crates/lingua/src/providers/google/adapter.rs @@ -645,9 +645,13 @@ impl ProviderAdapter for GoogleAdapter { .. } = m { - parts - .iter() - .any(|p| matches!(p, AssistantContentPart::ToolCall { .. })) + parts.iter().any(|p| { + matches!( + p, + AssistantContentPart::ToolCall { .. } + | AssistantContentPart::BuiltinToolCall { .. } + ) + }) } else { false } @@ -1781,6 +1785,48 @@ mod tests { assert_eq!(back_typed.model_version.as_deref(), Some("gemini-1.5")); } + #[test] + fn test_google_response_builtin_tool_call_sets_tool_calls_finish_reason() { + let adapter = GoogleAdapter; + let payload = json!({ + "candidates": [{ + "content": { + "role": "model", + "parts": [{ + "thoughtSignature": "google_builtin_signature", + "toolCall": { + "toolType": "GOOGLE_SEARCH_WEB", + "args": {"query": "Lingua"} + } + }] + }, + "finishReason": "STOP" + }] + }); + + let universal = adapter.response_to_universal(payload).unwrap(); + + assert_eq!(universal.finish_reason, Some(FinishReason::ToolCalls)); + assert_eq!(universal.finish_reasons, vec![FinishReason::Stop]); + let Message::Assistant { + content: AssistantContent::Array(parts), + .. + } = &universal.messages[0] + else { + panic!("expected assistant content parts"); + }; + let AssistantContentPart::BuiltinToolCall { + encrypted_content, .. + } = &parts[0] + else { + panic!("expected built-in tool call"); + }; + assert_eq!( + encrypted_content.as_deref(), + Some("google_builtin_signature") + ); + } + #[test] fn test_google_stream_tool_call_sets_tool_calls_finish_reason() { let adapter = GoogleAdapter; diff --git a/crates/lingua/src/providers/google/convert.rs b/crates/lingua/src/providers/google/convert.rs index 768d14d87..4d3648372 100644 --- a/crates/lingua/src/providers/google/convert.rs +++ b/crates/lingua/src/providers/google/convert.rs @@ -287,6 +287,7 @@ impl TryFromLLM for Message { builtin_tool: builtin_identity_from_google_tool_type(tool_type), arguments: tool_call.args.clone().map(ToolCallArguments::Valid), status: None, + encrypted_content: part.thought_signature.clone(), provider_options: None, provider_executed: Some(true), }); @@ -664,6 +665,7 @@ impl TryFromLLM for GoogleContent { tool_name, builtin_tool, arguments, + encrypted_content, provider_executed, .. } => { @@ -696,6 +698,7 @@ impl TryFromLLM for GoogleContent { )?, ), }), + thought_signature: encrypted_content, ..Default::default() }); } @@ -2028,6 +2031,7 @@ mod tests { tool_name: None, tool_type: Some(GoogleToolType::GoogleSearchWeb), }), + thought_signature: Some("google_builtin_signature".to_string()), ..Default::default() }]), }; @@ -2045,6 +2049,7 @@ mod tests { tool_call_id, tool_name, builtin_tool, + encrypted_content, provider_executed, .. } = &parts[0] @@ -2055,6 +2060,10 @@ mod tests { assert_eq!(tool_name, &None); assert_eq!(builtin_tool.provider, BuiltinToolProvider::Google); assert_eq!(builtin_tool.builtin_type, "GOOGLE_SEARCH_WEB"); + assert_eq!( + encrypted_content.as_deref(), + Some("google_builtin_signature") + ); assert_eq!(*provider_executed, Some(true)); let roundtrip = >::try_from(universal) diff --git a/crates/lingua/src/universal/message.rs b/crates/lingua/src/universal/message.rs index 932b31168..d9b6a2575 100644 --- a/crates/lingua/src/universal/message.rs +++ b/crates/lingua/src/universal/message.rs @@ -128,6 +128,8 @@ pub enum AssistantContentPart { #[ts(optional)] status: Option, #[ts(optional)] + encrypted_content: Option, + #[ts(optional)] provider_options: Option, #[ts(optional)] provider_executed: Option, diff --git a/plan.md b/plan.md index bc2fcb536..ebcaa2b31 100644 --- a/plan.md +++ b/plan.md @@ -7,6 +7,8 @@ - The Google adapter lifts a small canonical subset of `generationConfig` and discards every other typed field, including `audioTranscriptionConfig` and request-level `mediaResolution`. - Google streaming conversion only inspects `Part.functionCall`, so native `Part.toolCall` chunks are silently reduced to empty assistant deltas and may receive the wrong finish reason. - Google declares built-in `ToolCall.id` and `ToolResponse.id` optional, but the universal built-in parts currently require a string and the converter rejects valid provider payloads with no ID. +- Non-streaming Google built-in tool calls discard `Part.thoughtSignature`, so replaying provider-executed call history can fail Gemini signature validation. +- Non-streaming response finish-reason detection recognizes only ordinary function calls, so a built-in call paired with Google `STOP` is incorrectly classified as a completed turn. ## Target files @@ -27,6 +29,8 @@ - Dedicated universal builtin-tool call and result parts carry an optional free-form name plus a typed identity (`provider` and `builtin_type`). Google server-side tool calls/results round-trip with `provider_executed: true` without fabricating a function name, while ordinary function-tool parts remain source-compatible. - Built-in call/result correlation IDs are optional so a missing Google ID round-trips as absent rather than being rejected or synthesized. Real IDs remain unchanged. - Native Google streaming `toolCall` parts become typed universal built-in tool-call deltas, set the `tool_calls` finish reason, and round-trip back to Google. Streaming targets that cannot represent the built-in identity fail explicitly instead of treating it as a function call. +- Built-in calls carry optional opaque `encrypted_content`, allowing Google `thoughtSignature` values to survive non-streaming and full-response streaming roundtrips. +- Non-streaming responses containing either ordinary or built-in tool calls use the canonical `ToolCalls` finish reason. - Providers that cannot represent a provider-executed builtin return an explicit unsupported-mapping error instead of silently dropping it. - Google-to-universal preserves only the unmapped, typed remainder of `generationConfig` in Google-scoped extras. Universal-to-Google starts from that typed remainder and lets canonical fields override it, avoiding duplicate sources of truth. - The accepted REST `audioTranscriptionConfig` subtree and request-level `mediaResolution` survive Google round trips byte-for-byte at the semantic JSON level. @@ -37,6 +41,8 @@ - Add Google converter tests for named and unnamed provider-executed builtin calls and responses. - Add Google converter tests for built-in calls and responses with absent IDs. - Add Google streaming tests for typed built-in call conversion, absent-ID preservation, Google roundtrip, finish-reason handling, and explicit rejection by non-Google targets. +- Add a Google built-in call test with `thoughtSignature` and assert exact non-streaming roundtrip preservation. +- Add a Google response test proving a built-in call overrides provider `STOP` with canonical `ToolCalls`. - Add Google params/adapter tests proving unmapped `generationConfig` fields survive while canonical temperature/reasoning/response-format values take precedence. - Keep payload cases `googleProviderExecutedToolRoundtrip` and `audioTranscriptionConfigParam`; recapture after the logic fix. - Update existing universal/provider tests for optional tool names and builtin identities. From c37aea21299c02866144686c763a06558be7dc21 Mon Sep 17 00:00:00 2001 From: Alex Z Date: Fri, 7 Aug 2026 11:47:38 -0700 Subject: [PATCH 5/6] fixes --- crates/coverage-report/src/runner.rs | 59 ++++++ crates/lingua/src/processing/transform.rs | 28 +-- .../lingua/src/providers/anthropic/adapter.rs | 2 + .../lingua/src/providers/bedrock/adapter.rs | 1 + crates/lingua/src/providers/google/adapter.rs | 176 ++++++++++++------ .../src/providers/openai/responses_adapter.rs | 2 + crates/lingua/src/universal/stream.rs | 50 +++++ plan.md | 8 + 8 files changed, 253 insertions(+), 73 deletions(-) diff --git a/crates/coverage-report/src/runner.rs b/crates/coverage-report/src/runner.rs index f132b8243..1e482f4de 100644 --- a/crates/coverage-report/src/runner.rs +++ b/crates/coverage-report/src/runner.rs @@ -799,6 +799,12 @@ fn merge_tool_call_deltas( if incoming_tool_call.call_type.is_some() { existing_tool_call.call_type = incoming_tool_call.call_type; } + if incoming_tool_call.builtin_tool.is_some() { + existing_tool_call.builtin_tool = incoming_tool_call.builtin_tool; + } + if incoming_tool_call.encrypted_content.is_some() { + existing_tool_call.encrypted_content = incoming_tool_call.encrypted_content; + } match ( &mut existing_tool_call.function, incoming_tool_call.function, @@ -1491,4 +1497,57 @@ mod tests { Some("{\"q\":\"camera\"}") ); } + + #[test] + fn merge_stream_chunks_preserves_builtin_identity_and_per_call_signature() { + let merged = merge_universal_stream_chunks(vec![ + stream_chunk( + 0, + Some(json!({ + "tool_calls": [{ + "index": 0, + "type": "builtin_tool_call", + "builtin_tool": { + "provider": "google", + "builtin_type": "GOOGLE_SEARCH_WEB" + }, + "encrypted_content": "search_signature" + }] + })), + None, + ), + stream_chunk( + 0, + Some(json!({ + "tool_calls": [{ + "index": 0, + "function": { + "arguments": "{\"query\":\"Lingua\"}" + } + }] + })), + Some("tool_calls"), + ), + ]) + .expect("chunks should merge"); + + let delta: UniversalStreamDelta = + lingua::serde_json::from_value(merged.choices[0].delta.clone().unwrap()).unwrap(); + let tool_call = &delta.tool_calls[0]; + assert_eq!( + tool_call.builtin_tool.as_ref().unwrap().provider, + lingua::universal::BuiltinToolProvider::Google + ); + assert_eq!( + tool_call.encrypted_content.as_deref(), + Some("search_signature") + ); + assert_eq!( + tool_call + .function + .as_ref() + .and_then(|function| function.arguments.as_deref()), + Some("{\"query\":\"Lingua\"}") + ); + } } diff --git a/crates/lingua/src/processing/transform.rs b/crates/lingua/src/processing/transform.rs index 0100631fa..f11b1e639 100644 --- a/crates/lingua/src/processing/transform.rs +++ b/crates/lingua/src/processing/transform.rs @@ -707,17 +707,18 @@ pub(crate) fn ensure_stream_builtin_tools_supported( return Ok(()); } - let builtin_call_type = chunk + let builtin_tool = chunk .choices .iter() .filter_map(UniversalStreamChoice::delta_view) .flat_map(|delta| delta.tool_calls) - .filter_map(|tool_call| tool_call.call_type) - .find(|call_type| call_type.starts_with("builtin:")); + .find_map(|tool_call| tool_call.builtin_tool); - if let Some(call_type) = builtin_call_type { + if let Some(builtin_tool) = builtin_tool { return Err(TransformError::FromUniversalFailed(format!( - "target format {target_format:?} cannot represent streaming built-in tool call `{call_type}`" + "target format {target_format:?} cannot represent streaming {} built-in tool `{}`", + builtin_tool.provider.label(), + builtin_tool.builtin_type ))); } @@ -757,6 +758,7 @@ fn assistant_content_to_stream_delta(content: &AssistantContent) -> UniversalStr tool_call_id, tool_name, arguments, + encrypted_content, .. } => { let tool_call_index = tool_calls.len() as u32; @@ -766,11 +768,16 @@ fn assistant_content_to_stream_delta(content: &AssistantContent) -> UniversalStr call_type: Some("function".to_string()), custom_tool_call: matches!(arguments, ToolCallArguments::Custom(_)) .then_some(true), + builtin_tool: None, + encrypted_content: None, function: Some(UniversalToolFunctionDelta { name: Some(tool_name.clone()), arguments: Some(arguments.to_string()), }), }); + if reasoning_signature.is_none() { + reasoning_signature = encrypted_content.clone(); + } } AssistantContentPart::BuiltinToolCall { tool_call_id, @@ -784,20 +791,15 @@ fn assistant_content_to_stream_delta(content: &AssistantContent) -> UniversalStr tool_calls.push(UniversalToolCallDelta { index: Some(tool_call_index), id: tool_call_id.clone(), - call_type: Some(format!( - "builtin:{}:{}", - builtin_tool.provider.label(), - builtin_tool.builtin_type - )), + call_type: Some("builtin_tool_call".to_string()), custom_tool_call: None, + builtin_tool: Some(builtin_tool.clone()), + encrypted_content: encrypted_content.clone(), function: Some(UniversalToolFunctionDelta { name: tool_name.clone(), arguments: arguments.as_ref().map(ToString::to_string), }), }); - if reasoning_signature.is_none() { - reasoning_signature = encrypted_content.clone(); - } } AssistantContentPart::File { .. } | AssistantContentPart::ToolResult { .. } diff --git a/crates/lingua/src/providers/anthropic/adapter.rs b/crates/lingua/src/providers/anthropic/adapter.rs index 706979f57..83138fd78 100644 --- a/crates/lingua/src/providers/anthropic/adapter.rs +++ b/crates/lingua/src/providers/anthropic/adapter.rs @@ -1156,6 +1156,7 @@ impl ProviderAdapter for AnthropicAdapter { name: part.name, arguments: Some(arguments), }), + ..Default::default() }], ..Default::default() }) @@ -1223,6 +1224,7 @@ impl ProviderAdapter for AnthropicAdapter { name: Some(name.to_string()), arguments: Some(String::new()), }), + ..Default::default() }], ..Default::default() })), diff --git a/crates/lingua/src/providers/bedrock/adapter.rs b/crates/lingua/src/providers/bedrock/adapter.rs index 63e8cfada..a18a0fd8d 100644 --- a/crates/lingua/src/providers/bedrock/adapter.rs +++ b/crates/lingua/src/providers/bedrock/adapter.rs @@ -534,6 +534,7 @@ impl ProviderAdapter for BedrockAdapter { name: Some(tool_use.name), arguments: Some(String::new()), }), + ..Default::default() }], ..Default::default() })), diff --git a/crates/lingua/src/providers/google/adapter.rs b/crates/lingua/src/providers/google/adapter.rs index 04a87bb68..07540e0de 100644 --- a/crates/lingua/src/providers/google/adapter.rs +++ b/crates/lingua/src/providers/google/adapter.rs @@ -27,12 +27,10 @@ use crate::providers::google::generated::{ use crate::providers::google::params::GoogleParams; use crate::serde_json::{self, Map, Value}; use crate::universal::convert::TryFromLLM; -use crate::universal::message::{ - AssistantContent, AssistantContentPart, BuiltinToolIdentity, Message, -}; +use crate::universal::message::{AssistantContent, AssistantContentPart, Message}; use crate::universal::reasoning::{budget_to_effort, effort_to_budget, MIN_THINKING_BUDGET}; use crate::universal::request::ToolChoiceConfig; -use crate::universal::tools::{BuiltinToolProvider, UniversalTool}; +use crate::universal::tools::UniversalTool; use crate::universal::ToolContentPart; use crate::universal::{ extract_system_messages, flatten_consecutive_messages, FinishReason, ReasoningCanonical, @@ -154,39 +152,6 @@ struct GoogleStreamPart { tool_call: Option, } -fn builtin_stream_call_type(identity: &BuiltinToolIdentity) -> String { - format!( - "builtin:{}:{}", - identity.provider.label(), - identity.builtin_type - ) -} - -fn builtin_identity_from_stream_call_type( - call_type: Option<&str>, -) -> Result, TransformError> { - let Some(call_type) = call_type else { - return Ok(None); - }; - let Some(encoded_identity) = call_type.strip_prefix("builtin:") else { - return Ok(None); - }; - let Some((provider, builtin_type)) = encoded_identity.split_once(':') else { - return Err(TransformError::FromUniversalFailed(format!( - "invalid built-in stream tool-call type `{call_type}`" - ))); - }; - if provider != BuiltinToolProvider::Google.label() { - return Err(TransformError::FromUniversalFailed(format!( - "Google streaming cannot represent {provider} built-in tool `{builtin_type}`" - ))); - } - Ok(Some(BuiltinToolIdentity { - provider: BuiltinToolProvider::Google, - builtin_type: builtin_type.to_string(), - })) -} - impl ProviderAdapter for GoogleAdapter { fn format(&self) -> ProviderFormat { ProviderFormat::Google @@ -816,7 +781,10 @@ impl ProviderAdapter for GoogleAdapter { } } let text = text_segments.join(""); - let reasoning_signature = parts.iter().find_map(|part| part.thought_signature.clone()); + let reasoning_signature = parts + .iter() + .filter(|part| part.tool_call.is_none()) + .find_map(|part| part.thought_signature.clone()); let response_id = typed_payload.response_id.as_deref(); let mut tool_call_index = 0_u32; @@ -837,6 +805,8 @@ impl ProviderAdapter for GoogleAdapter { }), call_type: Some("function".to_string()), custom_tool_call: None, + builtin_tool: None, + encrypted_content: None, function: Some(UniversalToolFunctionDelta { name: function_call.name.clone(), arguments: function_call @@ -857,8 +827,10 @@ impl ProviderAdapter for GoogleAdapter { tool_calls.push(UniversalToolCallDelta { index: Some(index), id: tool_call.id.clone(), - call_type: Some(builtin_stream_call_type(&identity)), + call_type: Some("builtin_tool_call".to_string()), custom_tool_call: None, + builtin_tool: Some(identity), + encrypted_content: part.thought_signature.clone(), function: Some(UniversalToolFunctionDelta { name: tool_call.tool_name.clone(), arguments: tool_call @@ -933,17 +905,10 @@ impl ProviderAdapter for GoogleAdapter { .as_ref() .and_then(|d| d.content.as_deref()) .unwrap_or(""); - let has_tool_call_part = delta - .as_ref() - .is_some_and(|d| !d.tool_calls.is_empty()); let text_reasoning_signature = delta .as_ref() .and_then(|d| d.reasoning_signature.as_deref()) - .filter(|_| { - delta.as_ref().is_none_or(|d| { - !has_tool_call_part && (!text.is_empty() || d.reasoning.is_empty()) - }) - }); + .filter(|_| delta.as_ref().is_none_or(|d| !text.is_empty() || d.reasoning.is_empty())); if let Some(ref d) = delta { let reasoning_texts: Vec<&str> = d @@ -955,9 +920,7 @@ impl ProviderAdapter for GoogleAdapter { .collect(); let thought_reasoning_signature = d.reasoning_signature.as_deref().filter(|_| { - !reasoning_texts.is_empty() - && text.is_empty() - && !has_tool_call_part + !reasoning_texts.is_empty() && text.is_empty() }); for (index, text) in reasoning_texts.iter().enumerate() { @@ -990,9 +953,7 @@ impl ProviderAdapter for GoogleAdapter { // Add functionCall or provider-executed toolCall parts from tool_calls. if let Some(ref d) = delta { for tc in &d.tool_calls { - if let Some(identity) = builtin_identity_from_stream_call_type( - tc.call_type.as_deref(), - )? { + if let Some(identity) = tc.builtin_tool.as_ref() { let args = tc .function .as_ref() @@ -1016,12 +977,12 @@ impl ProviderAdapter for GoogleAdapter { .as_ref() .and_then(|function| function.name.clone()), tool_type: Some(google_tool_type_from_builtin_identity( - &identity, + identity, )?), }), ..Default::default() }; - if let Some(ref signature) = d.reasoning_signature { + if let Some(ref signature) = tc.encrypted_content { part.thought_signature = Some(signature.clone()); } parts.push(part); @@ -1867,6 +1828,7 @@ mod tests { ); assert_eq!(tool_call.index, Some(0)); assert_eq!(tool_call.id.as_deref(), Some("call_response_123_0")); + assert_eq!(tool_call.encrypted_content, None); } #[test] @@ -1879,6 +1841,7 @@ mod tests { "content": { "role": "model", "parts": [{ + "thoughtSignature": "builtin_signature", "toolCall": { "toolName": "search_the_web", "toolType": "GOOGLE_SEARCH_WEB", @@ -1904,9 +1867,17 @@ mod tests { assert_eq!(choice.finish_reason.as_deref(), Some("tool_calls")); assert_eq!(tool_call.index, Some(0)); assert_eq!(tool_call.id, None); + assert_eq!(tool_call.call_type.as_deref(), Some("builtin_tool_call")); assert_eq!( - tool_call.call_type.as_deref(), - Some("builtin:google:GOOGLE_SEARCH_WEB") + tool_call.builtin_tool.as_ref(), + Some(&crate::universal::message::BuiltinToolIdentity { + provider: crate::universal::tools::BuiltinToolProvider::Google, + builtin_type: "GOOGLE_SEARCH_WEB".to_string(), + }) + ); + assert_eq!( + tool_call.encrypted_content.as_deref(), + Some("builtin_signature") ); let function = tool_call .function @@ -1921,13 +1892,20 @@ mod tests { let roundtrip: GenerateContentResponse = serde_json::from_value(roundtrip).expect("stream response should deserialize"); let roundtrip_candidates = roundtrip.candidates.unwrap(); - let roundtrip_call = roundtrip_candidates[0] + let roundtrip_part = roundtrip_candidates[0] .content .as_ref() .and_then(|content| content.parts.as_ref()) .and_then(|parts| parts.first()) - .and_then(|part| part.tool_call.as_ref()) + .expect("roundtrip should contain a part"); + let roundtrip_call = roundtrip_part + .tool_call + .as_ref() .expect("roundtrip should contain toolCall"); + assert_eq!( + roundtrip_part.thought_signature.as_deref(), + Some("builtin_signature") + ); assert_eq!(roundtrip_call.id, None); assert_eq!(roundtrip_call.tool_name.as_deref(), Some("search_the_web")); assert_eq!( @@ -1942,7 +1920,85 @@ mod tests { .expect_err("non-Google stream target must reject Google built-in identity"); assert!(error .to_string() - .contains("cannot represent streaming built-in tool call")); + .contains("cannot represent streaming google built-in tool")); + } + + #[test] + fn test_google_stream_preserves_signatures_per_builtin_tool_call() { + let adapter = GoogleAdapter; + let payload = json!({ + "responseId": "response_builtin_signatures", + "candidates": [{ + "index": 0, + "content": { + "role": "model", + "parts": [ + { + "text": "Calling provider tools", + "thoughtSignature": "text_signature" + }, + { + "thoughtSignature": "search_signature", + "toolCall": { + "toolType": "GOOGLE_SEARCH_WEB", + "args": {"query": "Lingua"} + } + }, + { + "thoughtSignature": "maps_signature", + "toolCall": { + "toolType": "GOOGLE_MAPS", + "args": {"query": "San Francisco"} + } + } + ] + }, + "finishReason": "STOP" + }] + }); + + let chunk = adapter + .stream_to_universal(payload) + .unwrap() + .expect("stream chunk should be present"); + let delta = chunk.choices[0] + .delta_view() + .expect("delta should be present"); + + assert_eq!(delta.reasoning_signature.as_deref(), Some("text_signature")); + assert_eq!(delta.tool_calls.len(), 2); + assert_eq!( + delta.tool_calls[0].encrypted_content.as_deref(), + Some("search_signature") + ); + assert_eq!( + delta.tool_calls[1].encrypted_content.as_deref(), + Some("maps_signature") + ); + + let roundtrip = adapter + .stream_from_universal(&chunk) + .expect("signed built-in calls should export to Google"); + let typed: GenerateContentResponse = + serde_json::from_value(roundtrip).expect("stream response should deserialize"); + let candidates = typed.candidates.unwrap(); + let parts = candidates[0] + .content + .as_ref() + .and_then(|content| content.parts.as_ref()) + .expect("roundtrip should contain parts"); + let signatures: Vec<_> = parts + .iter() + .map(|part| part.thought_signature.as_deref()) + .collect(); + assert_eq!( + signatures, + vec![ + Some("text_signature"), + Some("search_signature"), + Some("maps_signature") + ] + ); } #[test] diff --git a/crates/lingua/src/providers/openai/responses_adapter.rs b/crates/lingua/src/providers/openai/responses_adapter.rs index 06818dcc3..baab1ec7f 100644 --- a/crates/lingua/src/providers/openai/responses_adapter.rs +++ b/crates/lingua/src/providers/openai/responses_adapter.rs @@ -562,6 +562,7 @@ fn responses_tool_call_start_chunk( name: Some(name.to_string()), arguments: Some(String::new()), }), + ..Default::default() }] })), finish_reason: None, @@ -591,6 +592,7 @@ fn responses_tool_call_arguments_delta_chunk( name: None, arguments: Some(arguments), }), + ..Default::default() }] })), finish_reason: None, diff --git a/crates/lingua/src/universal/stream.rs b/crates/lingua/src/universal/stream.rs index 3f5ea4fbf..3959488ec 100644 --- a/crates/lingua/src/universal/stream.rs +++ b/crates/lingua/src/universal/stream.rs @@ -7,6 +7,7 @@ structure as the canonical representation. */ use crate::serde_json::{self, Value}; +use crate::universal::message::BuiltinToolIdentity; use crate::universal::response::{ServedServiceTier, UniversalUsage}; use serde::{Deserialize, Serialize}; @@ -51,6 +52,12 @@ pub struct UniversalToolCallDelta { pub call_type: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub custom_tool_call: Option, + /// Typed identity for provider-executed built-in calls. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub builtin_tool: Option, + /// Opaque provider data that must be replayed with this specific call. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub encrypted_content: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub function: Option, } @@ -405,4 +412,47 @@ mod tests { assert_eq!(parsed.reasoning[0].content.as_deref(), Some("thought")); assert_eq!(parsed.reasoning_signature.as_deref(), Some("sig_123")); } + + #[test] + fn test_stream_delta_builtin_tool_identity_and_signature_from_into_value() { + use crate::universal::tools::BuiltinToolProvider; + + let delta = UniversalStreamDelta { + tool_calls: vec![UniversalToolCallDelta { + index: Some(0), + call_type: Some("builtin_tool_call".to_string()), + builtin_tool: Some(BuiltinToolIdentity { + provider: BuiltinToolProvider::Google, + builtin_type: "GOOGLE_SEARCH_WEB".to_string(), + }), + encrypted_content: Some("call_signature".to_string()), + ..Default::default() + }], + ..Default::default() + }; + + let value = Value::from(delta); + assert_eq!( + value["tool_calls"][0]["builtin_tool"], + crate::serde_json::json!({ + "provider": "google", + "builtin_type": "GOOGLE_SEARCH_WEB" + }) + ); + assert_eq!( + value["tool_calls"][0]["encrypted_content"], + "call_signature" + ); + + let parsed: UniversalStreamDelta = serde_json::from_value(value).unwrap(); + let tool_call = &parsed.tool_calls[0]; + assert_eq!( + tool_call.builtin_tool.as_ref().unwrap().provider, + BuiltinToolProvider::Google + ); + assert_eq!( + tool_call.encrypted_content.as_deref(), + Some("call_signature") + ); + } } diff --git a/plan.md b/plan.md index ebcaa2b31..b35f49da1 100644 --- a/plan.md +++ b/plan.md @@ -9,11 +9,15 @@ - Google declares built-in `ToolCall.id` and `ToolResponse.id` optional, but the universal built-in parts currently require a string and the converter rejects valid provider payloads with no ID. - Non-streaming Google built-in tool calls discard `Part.thoughtSignature`, so replaying provider-executed call history can fail Gemini signature validation. - Non-streaming response finish-reason detection recognizes only ordinary function calls, so a built-in call paired with Google `STOP` is incorrectly classified as a completed turn. +- Universal streaming encodes provider-executed built-in identity inside the open-ended tool-call `type` string, which leaks an internal marker and can misclassify a legitimate type with the same prefix. +- Google streaming collapses every part-level `thoughtSignature` into one chunk-level signature, so multiple signed tool calls can receive the wrong or duplicated signature on export. ## Target files - `crates/generate-types/src/main.rs` - `crates/lingua/src/universal/message.rs` +- `crates/lingua/src/universal/stream.rs` +- `crates/lingua/src/processing/transform.rs` - Provider adapters and import helpers that construct or consume universal tool calls/results - `crates/lingua/src/providers/google/convert.rs` - `crates/lingua/src/providers/google/adapter.rs` @@ -29,6 +33,8 @@ - Dedicated universal builtin-tool call and result parts carry an optional free-form name plus a typed identity (`provider` and `builtin_type`). Google server-side tool calls/results round-trip with `provider_executed: true` without fabricating a function name, while ordinary function-tool parts remain source-compatible. - Built-in call/result correlation IDs are optional so a missing Google ID round-trips as absent rather than being rejected or synthesized. Real IDs remain unchanged. - Native Google streaming `toolCall` parts become typed universal built-in tool-call deltas, set the `tool_calls` finish reason, and round-trip back to Google. Streaming targets that cannot represent the built-in identity fail explicitly instead of treating it as a function call. +- Universal tool-call deltas carry optional typed built-in identity and per-call encrypted content. The open-ended `type` field contains only a stable discriminator, never encoded provider semantics. +- Google streaming preserves each function or built-in call's `thoughtSignature` on that call. Text/reasoning signatures remain chunk-level and are never copied onto tool calls. - Built-in calls carry optional opaque `encrypted_content`, allowing Google `thoughtSignature` values to survive non-streaming and full-response streaming roundtrips. - Non-streaming responses containing either ordinary or built-in tool calls use the canonical `ToolCalls` finish reason. - Providers that cannot represent a provider-executed builtin return an explicit unsupported-mapping error instead of silently dropping it. @@ -41,6 +47,8 @@ - Add Google converter tests for named and unnamed provider-executed builtin calls and responses. - Add Google converter tests for built-in calls and responses with absent IDs. - Add Google streaming tests for typed built-in call conversion, absent-ID preservation, Google roundtrip, finish-reason handling, and explicit rejection by non-Google targets. +- Add Google streaming tests for multiple independently signed built-in calls and a separately signed text part. +- Add universal stream serialization and stream-merge tests for typed built-in identity and per-call encrypted content. - Add a Google built-in call test with `thoughtSignature` and assert exact non-streaming roundtrip preservation. - Add a Google response test proving a built-in call overrides provider `STOP` with canonical `ToolCalls`. - Add Google params/adapter tests proving unmapped `generationConfig` fields survive while canonical temperature/reasoning/response-format values take precedence. From ba58896b90bd7c63b690bc8ba90f6202a3e9b0e5 Mon Sep 17 00:00:00 2001 From: Alex Z Date: Fri, 7 Aug 2026 16:36:13 -0700 Subject: [PATCH 6/6] maybe this time --- crates/lingua/src/processing/transform.rs | 70 ++++- crates/lingua/src/providers/google/adapter.rs | 256 +++++++++++++++++- crates/lingua/src/providers/google/convert.rs | 37 +++ crates/lingua/src/providers/openai/adapter.rs | 32 ++- plan.md | 14 +- 5 files changed, 390 insertions(+), 19 deletions(-) diff --git a/crates/lingua/src/processing/transform.rs b/crates/lingua/src/processing/transform.rs index f11b1e639..aee14550f 100644 --- a/crates/lingua/src/processing/transform.rs +++ b/crates/lingua/src/processing/transform.rs @@ -769,15 +769,12 @@ fn assistant_content_to_stream_delta(content: &AssistantContent) -> UniversalStr custom_tool_call: matches!(arguments, ToolCallArguments::Custom(_)) .then_some(true), builtin_tool: None, - encrypted_content: None, + encrypted_content: encrypted_content.clone(), function: Some(UniversalToolFunctionDelta { name: Some(tool_name.clone()), arguments: Some(arguments.to_string()), }), }); - if reasoning_signature.is_none() { - reasoning_signature = encrypted_content.clone(); - } } AssistantContentPart::BuiltinToolCall { tool_call_id, @@ -1139,7 +1136,9 @@ where mod tests { use super::*; use crate::serde_json::json; - use crate::universal::{ResponseRequirement, ServedServiceTier, UserContent, UserContentPart}; + use crate::universal::{ + FinishReason, ResponseRequirement, ServedServiceTier, UserContent, UserContentPart, + }; fn to_bytes(value: &Value) -> Bytes { Bytes::from(crate::serde_json::to_vec(value).unwrap()) @@ -2416,6 +2415,67 @@ mod tests { assert_eq!(output["item"]["name"], json!("exec")); } + #[test] + fn test_response_to_stream_chunk_preserves_tool_call_signatures_per_call() { + let response = UniversalResponse { + id: None, + id_format: None, + model: Some("gemini-3.5-flash".to_string()), + messages: vec![Message::Assistant { + id: None, + content: AssistantContent::Array(vec![ + AssistantContentPart::Reasoning { + text: "Choosing tools".to_string(), + encrypted_content: Some("reasoning-signature".to_string()), + }, + AssistantContentPart::ToolCall { + tool_call_id: "call_1".to_string(), + tool_name: "first".to_string(), + arguments: ToolCallArguments::from("{}".to_string()), + status: None, + caller: None, + encrypted_content: Some("first-signature".to_string()), + provider_options: None, + provider_executed: None, + }, + AssistantContentPart::ToolCall { + tool_call_id: "call_2".to_string(), + tool_name: "second".to_string(), + arguments: ToolCallArguments::from("{}".to_string()), + status: None, + caller: None, + encrypted_content: Some("second-signature".to_string()), + provider_options: None, + provider_executed: None, + }, + ]), + }], + usage: None, + served_service_tier: None, + finish_reason: Some(FinishReason::ToolCalls), + finish_reasons: vec![FinishReason::ToolCalls], + }; + + let chunk = response_to_stream_chunk(response, ProviderFormat::Google); + let delta = chunk.choices[0] + .delta_view() + .expect("delta should be present"); + + assert_eq!( + delta.reasoning_signature.as_deref(), + Some("reasoning-signature") + ); + let signatures: Vec<_> = delta + .tool_calls + .iter() + .map(|tool_call| tool_call.encrypted_content.as_deref()) + .collect(); + assert_eq!( + signatures, + vec![Some("first-signature"), Some("second-signature")] + ); + } + #[test] #[cfg(feature = "openai")] fn test_transform_response_passthrough() { diff --git a/crates/lingua/src/providers/google/adapter.rs b/crates/lingua/src/providers/google/adapter.rs index 07540e0de..25bb80b14 100644 --- a/crates/lingua/src/providers/google/adapter.rs +++ b/crates/lingua/src/providers/google/adapter.rs @@ -783,7 +783,7 @@ impl ProviderAdapter for GoogleAdapter { let text = text_segments.join(""); let reasoning_signature = parts .iter() - .filter(|part| part.tool_call.is_none()) + .filter(|part| part.function_call.is_none() && part.tool_call.is_none()) .find_map(|part| part.thought_signature.clone()); let response_id = typed_payload.response_id.as_deref(); @@ -806,7 +806,7 @@ impl ProviderAdapter for GoogleAdapter { call_type: Some("function".to_string()), custom_tool_call: None, builtin_tool: None, - encrypted_content: None, + encrypted_content: part.thought_signature.clone(), function: Some(UniversalToolFunctionDelta { name: function_call.name.clone(), arguments: function_call @@ -908,7 +908,11 @@ impl ProviderAdapter for GoogleAdapter { let text_reasoning_signature = delta .as_ref() .and_then(|d| d.reasoning_signature.as_deref()) - .filter(|_| delta.as_ref().is_none_or(|d| !text.is_empty() || d.reasoning.is_empty())); + .filter(|_| { + delta.as_ref().is_none_or(|d| { + !text.is_empty() || (d.reasoning.is_empty() && d.tool_calls.is_empty()) + }) + }); if let Some(ref d) = delta { let reasoning_texts: Vec<&str> = d @@ -952,6 +956,18 @@ impl ProviderAdapter for GoogleAdapter { // Add functionCall or provider-executed toolCall parts from tool_calls. if let Some(ref d) = delta { + let ordinary_function_call_count = d + .tool_calls + .iter() + .filter(|tool_call| { + tool_call.builtin_tool.is_none() && tool_call.function.is_some() + }) + .count(); + let shared_function_signature = (text.is_empty() + && d.reasoning.is_empty() + && ordinary_function_call_count == 1) + .then_some(d.reasoning_signature.as_ref()) + .flatten(); for tc in &d.tool_calls { if let Some(identity) = tc.builtin_tool.as_ref() { let args = tc @@ -1007,7 +1023,9 @@ impl ProviderAdapter for GoogleAdapter { function_call: Some(function_call), ..Default::default() }; - if let Some(ref signature) = d.reasoning_signature { + if let Some(signature) = + tc.encrypted_content.as_ref().or(shared_function_signature) + { part.thought_signature = Some(signature.clone()); } parts.push(part); @@ -1822,13 +1840,235 @@ mod tests { .expect("tool call should be present"); assert_eq!(choice.finish_reason.as_deref(), Some("tool_calls")); + assert_eq!(delta.reasoning_signature.as_deref(), None); + assert_eq!(tool_call.index, Some(0)); + assert_eq!(tool_call.id.as_deref(), Some("call_response_123_0")); assert_eq!( - delta.reasoning_signature.as_deref(), + tool_call.encrypted_content.as_deref(), Some("thought_signature_123") ); - assert_eq!(tool_call.index, Some(0)); - assert_eq!(tool_call.id.as_deref(), Some("call_response_123_0")); - assert_eq!(tool_call.encrypted_content, None); + } + + #[test] + fn test_google_stream_preserves_signatures_for_text_and_function_call() { + let adapter = GoogleAdapter; + let payload = json!({ + "responseId": "response_ambiguous_text_signature", + "candidates": [{ + "index": 0, + "content": { + "role": "model", + "parts": [ + { + "text": "Calling a function", + "thoughtSignature": "text_signature" + }, + { + "thoughtSignature": "function_signature", + "functionCall": { + "name": "lookup", + "args": {"query": "Lingua"} + } + } + ] + }, + "finishReason": "STOP" + }] + }); + + let chunk = adapter + .stream_to_universal(payload) + .unwrap() + .expect("stream chunk should be present"); + let delta = chunk.choices[0] + .delta_view() + .expect("delta should be present"); + assert_eq!(delta.reasoning_signature.as_deref(), Some("text_signature")); + assert_eq!( + delta.tool_calls[0].encrypted_content.as_deref(), + Some("function_signature") + ); + + let roundtrip = adapter + .stream_from_universal(&chunk) + .expect("signed parts should export to Google"); + let typed: GenerateContentResponse = + serde_json::from_value(roundtrip).expect("stream response should deserialize"); + let candidates = typed.candidates.unwrap(); + let parts = candidates[0] + .content + .as_ref() + .and_then(|content| content.parts.as_ref()) + .expect("roundtrip should contain parts"); + let signatures: Vec<_> = parts + .iter() + .map(|part| part.thought_signature.as_deref()) + .collect(); + assert_eq!( + signatures, + vec![Some("text_signature"), Some("function_signature")] + ); + } + + #[test] + fn test_google_stream_preserves_signatures_for_parallel_function_calls() { + let adapter = GoogleAdapter; + let payload = json!({ + "responseId": "response_ambiguous_function_signature", + "candidates": [{ + "index": 0, + "content": { + "role": "model", + "parts": [ + { + "thoughtSignature": "first_function_signature", + "functionCall": { + "name": "first_lookup", + "args": {"query": "Lingua"} + } + }, + { + "thoughtSignature": "second_function_signature", + "functionCall": { + "name": "second_lookup", + "args": {"query": "Braintrust"} + } + } + ] + }, + "finishReason": "STOP" + }] + }); + + let chunk = adapter + .stream_to_universal(payload) + .unwrap() + .expect("stream chunk should be present"); + let delta = chunk.choices[0] + .delta_view() + .expect("delta should be present"); + assert_eq!(delta.reasoning_signature, None); + let signatures: Vec<_> = delta + .tool_calls + .iter() + .map(|tool_call| tool_call.encrypted_content.as_deref()) + .collect(); + assert_eq!( + signatures, + vec![ + Some("first_function_signature"), + Some("second_function_signature") + ] + ); + + let roundtrip = adapter + .stream_from_universal(&chunk) + .expect("parallel signed calls should export to Google"); + let typed: GenerateContentResponse = + serde_json::from_value(roundtrip).expect("stream response should deserialize"); + let candidates = typed.candidates.unwrap(); + let parts = candidates[0] + .content + .as_ref() + .and_then(|content| content.parts.as_ref()) + .expect("roundtrip should contain parts"); + let signatures: Vec<_> = parts + .iter() + .map(|part| part.thought_signature.as_deref()) + .collect(); + assert_eq!( + signatures, + vec![ + Some("first_function_signature"), + Some("second_function_signature") + ] + ); + } + + #[test] + fn test_google_stream_promotes_shared_signature_only_for_lone_function_call() { + let adapter = GoogleAdapter; + let function_call = UniversalToolCallDelta { + index: Some(0), + id: Some("call_1".to_string()), + call_type: Some("function".to_string()), + function: Some(UniversalToolFunctionDelta { + name: Some("lookup".to_string()), + arguments: Some("{}".to_string()), + }), + ..Default::default() + }; + let chunk = UniversalStreamChunk::new( + None, + None, + vec![UniversalStreamChoice { + index: 0, + delta: Some(Value::from(UniversalStreamDelta { + role: Some("assistant".to_string()), + content: Some(String::new()), + tool_calls: vec![function_call.clone()], + reasoning_signature: Some("shared_signature".to_string()), + ..Default::default() + })), + finish_reason: None, + }], + None, + None, + ); + + let payload = adapter + .stream_from_universal(&chunk) + .expect("lone signed function call should export"); + let typed: GenerateContentResponse = + serde_json::from_value(payload).expect("stream response should deserialize"); + let candidates = typed.candidates.unwrap(); + let parts = candidates[0] + .content + .as_ref() + .and_then(|content| content.parts.as_ref()) + .expect("roundtrip should contain parts"); + assert_eq!(parts.len(), 1); + assert!(parts[0].function_call.is_some()); + assert_eq!( + parts[0].thought_signature.as_deref(), + Some("shared_signature") + ); + + let mixed_chunk = UniversalStreamChunk::new( + None, + None, + vec![UniversalStreamChoice { + index: 0, + delta: Some(Value::from(UniversalStreamDelta { + role: Some("assistant".to_string()), + content: Some("Calling a function".to_string()), + tool_calls: vec![function_call], + reasoning_signature: Some("text_signature".to_string()), + ..Default::default() + })), + finish_reason: None, + }], + None, + None, + ); + + let payload = adapter + .stream_from_universal(&mixed_chunk) + .expect("mixed signed text and function call should export"); + let typed: GenerateContentResponse = + serde_json::from_value(payload).expect("stream response should deserialize"); + let candidates = typed.candidates.unwrap(); + let parts = candidates[0] + .content + .as_ref() + .and_then(|content| content.parts.as_ref()) + .expect("roundtrip should contain parts"); + assert_eq!(parts.len(), 2); + assert_eq!( + parts[0].thought_signature.as_deref(), + Some("text_signature") + ); + assert_eq!(parts[1].thought_signature, None); } #[test] diff --git a/crates/lingua/src/providers/google/convert.rs b/crates/lingua/src/providers/google/convert.rs index 4d3648372..28e86f76f 100644 --- a/crates/lingua/src/providers/google/convert.rs +++ b/crates/lingua/src/providers/google/convert.rs @@ -442,6 +442,14 @@ impl TryFromLLM for Message { } } + if !user_parts.is_empty() && !tool_parts.is_empty() { + return Err(ConvertError::UnsupportedMapping { + from: "mixed Google user content containing ordinary user parts and tool responses" + .to_string(), + to: "a single universal message", + }); + } + if !tool_parts.is_empty() { Ok(Message::Tool { content: tool_parts, @@ -2106,6 +2114,35 @@ mod tests { assert_eq!(roundtrip, original); } + #[test] + fn test_google_mixed_user_and_tool_response_is_rejected_without_dropping_content() { + let original = GoogleContent { + role: Some("user".to_string()), + parts: Some(vec![ + text_part("Use this provider result when answering.".to_string()), + GooglePart { + tool_response: Some(GoogleToolResponse { + id: Some("google-search-1".to_string()), + response: Some(Map::from_iter([( + "result".to_string(), + Value::String("Lingua".to_string()), + )])), + tool_type: Some(GoogleToolType::GoogleSearchWeb), + }), + ..Default::default() + }, + ]), + }; + + let error = >::try_from(original) + .expect_err("mixed user and tool-response parts must not be silently truncated"); + + assert!(matches!(error, ConvertError::UnsupportedMapping { .. })); + assert!(error.to_string().contains( + "mixed Google user content containing ordinary user parts and tool responses" + )); + } + #[test] fn test_google_provider_executed_tool_call_roundtrips_without_id() { let original = GoogleContent { diff --git a/crates/lingua/src/providers/openai/adapter.rs b/crates/lingua/src/providers/openai/adapter.rs index b159050c2..580967d95 100644 --- a/crates/lingua/src/providers/openai/adapter.rs +++ b/crates/lingua/src/providers/openai/adapter.rs @@ -41,7 +41,8 @@ use crate::universal::request::{ use crate::universal::tools::{tools_to_openai_chat_value, BuiltinToolProvider, UniversalTool}; use crate::universal::{ parse_stop_sequences, ServedServiceTier, UniversalParams, UniversalRequest, UniversalResponse, - UniversalStreamChoice, UniversalStreamChunk, UniversalUsage, PLACEHOLDER_MODEL, + UniversalStreamChoice, UniversalStreamChunk, UniversalStreamDelta, UniversalUsage, + PLACEHOLDER_MODEL, }; use serde::{de::IgnoredAny, Deserialize}; use std::collections::BTreeMap; @@ -869,7 +870,9 @@ impl ProviderAdapter for OpenAIAdapter { "delta".into(), c.delta .clone() - .map(chat_stream_delta_from_universal) + .map(|delta| { + chat_stream_delta_from_universal(delta, c.delta_view().as_ref()) + }) .unwrap_or(Value::Object(Map::new())), ); let finish_reason_val = match &c.finish_reason { @@ -925,7 +928,24 @@ impl ProviderAdapter for OpenAIAdapter { } } -fn chat_stream_delta_from_universal(mut delta: Value) -> Value { +fn chat_stream_delta_from_universal( + mut delta: Value, + typed_delta: Option<&UniversalStreamDelta>, +) -> Value { + let promoted_signature = typed_delta.and_then(|delta| { + (delta.reasoning_signature.is_none() + && delta.content.as_deref().is_none_or(str::is_empty) + && delta.reasoning.is_empty() + && delta.tool_calls.len() == 1 + && delta.tool_calls[0].builtin_tool.is_none()) + .then(|| delta.tool_calls[0].encrypted_content.clone()) + .flatten() + }); + + if let (Some(delta), Some(signature)) = (delta.as_object_mut(), promoted_signature) { + delta.insert("reasoning_signature".to_string(), Value::String(signature)); + } + if let Some(tool_calls) = delta .as_object_mut() .and_then(|delta| delta.get_mut("tool_calls")) @@ -934,6 +954,8 @@ fn chat_stream_delta_from_universal(mut delta: Value) -> Value { for tool_call in tool_calls { if let Some(tool_call) = tool_call.as_object_mut() { tool_call.remove("custom_tool_call"); + tool_call.remove("builtin_tool"); + tool_call.remove("encrypted_content"); } } } @@ -1179,7 +1201,7 @@ mod tests { } #[test] - fn test_openai_stream_from_universal_omits_custom_tool_call_marker() { + fn test_openai_stream_from_universal_omits_universal_tool_metadata() { let adapter = OpenAIAdapter; let chunk = UniversalStreamChunk::new( None, @@ -1194,6 +1216,7 @@ mod tests { "id": "call_1", "type": "function", "custom_tool_call": true, + "encrypted_content": "provider-signature", "function": { "name": "exec", "arguments": "" @@ -1217,6 +1240,7 @@ mod tests { "delta": { "role": "assistant", "content": null, + "reasoning_signature": "provider-signature", "tool_calls": [{ "index": 0, "id": "call_1", diff --git a/plan.md b/plan.md index b35f49da1..aef17a589 100644 --- a/plan.md +++ b/plan.md @@ -11,6 +11,8 @@ - Non-streaming response finish-reason detection recognizes only ordinary function calls, so a built-in call paired with Google `STOP` is incorrectly classified as a completed turn. - Universal streaming encodes provider-executed built-in identity inside the open-ended tool-call `type` string, which leaks an internal marker and can misclassify a legitimate type with the same prefix. - Google streaming collapses every part-level `thoughtSignature` into one chunk-level signature, so multiple signed tool calls can receive the wrong or duplicated signature on export. +- A Google `user` content containing both ordinary user parts and tool responses is collapsed to only `Message::Tool`, silently deleting the text/image/file prompt parts. +- Ordinary Google streaming function calls still share chunk-level signature storage, so a signed mixed text/reasoning + function-call chunk cannot preserve which original part owned the signature. ## Target files @@ -22,6 +24,8 @@ - `crates/lingua/src/providers/google/convert.rs` - `crates/lingua/src/providers/google/adapter.rs` - `crates/lingua/src/providers/google/params.rs` +- `crates/lingua/src/processing/transform.rs` +- `crates/lingua/src/providers/openai/adapter.rs` - `payloads/cases/advanced.ts` - `payloads/cases/params.ts` - Generated TypeScript universal bindings produced by `make generate-types` @@ -34,7 +38,10 @@ - Built-in call/result correlation IDs are optional so a missing Google ID round-trips as absent rather than being rejected or synthesized. Real IDs remain unchanged. - Native Google streaming `toolCall` parts become typed universal built-in tool-call deltas, set the `tool_calls` finish reason, and round-trip back to Google. Streaming targets that cannot represent the built-in identity fail explicitly instead of treating it as a function call. - Universal tool-call deltas carry optional typed built-in identity and per-call encrypted content. The open-ended `type` field contains only a stable discriminator, never encoded provider semantics. -- Google streaming preserves each function or built-in call's `thoughtSignature` on that call. Text/reasoning signatures remain chunk-level and are never copied onto tool calls. +- Google streaming preserves each ordinary function or built-in call's `thoughtSignature` on that call. Text/reasoning signatures remain chunk-level and are never copied onto tool calls. +- Mixed Google user/tool-response content fails explicitly at conversion instead of silently dropping user parts until the universal model supports an order-preserving mixed representation. +- Chat Completions streaming strips universal-only per-call signature and built-in identity fields before emitting provider wire output. +- A lone signed function call with no text/reasoning may promote between chunk-level and per-call storage for compatibility; mixed or parallel chunks never share that fallback. - Built-in calls carry optional opaque `encrypted_content`, allowing Google `thoughtSignature` values to survive non-streaming and full-response streaming roundtrips. - Non-streaming responses containing either ordinary or built-in tool calls use the canonical `ToolCalls` finish reason. - Providers that cannot represent a provider-executed builtin return an explicit unsupported-mapping error instead of silently dropping it. @@ -48,11 +55,14 @@ - Add Google converter tests for built-in calls and responses with absent IDs. - Add Google streaming tests for typed built-in call conversion, absent-ID preservation, Google roundtrip, finish-reason handling, and explicit rejection by non-Google targets. - Add Google streaming tests for multiple independently signed built-in calls and a separately signed text part. +- Add a Google converter test proving mixed user parts and built-in results return a descriptive unsupported-mapping error. +- Add Google streaming tests proving signed text + ordinary function calls and parallel signed ordinary calls preserve independent signatures. +- Add stream-synthesis and Chat Completions serialization tests for per-call signature ownership without provider-wire leakage. - Add universal stream serialization and stream-merge tests for typed built-in identity and per-call encrypted content. - Add a Google built-in call test with `thoughtSignature` and assert exact non-streaming roundtrip preservation. - Add a Google response test proving a built-in call overrides provider `STOP` with canonical `ToolCalls`. - Add Google params/adapter tests proving unmapped `generationConfig` fields survive while canonical temperature/reasoning/response-format values take precedence. -- Keep payload cases `googleProviderExecutedToolRoundtrip` and `audioTranscriptionConfigParam`; recapture after the logic fix. +- Keep payload cases `googleProviderExecutedToolRoundtrip` and `audioTranscriptionConfigParam`; recapture after the logic fix. The intentionally unsupported mixed user/tool-response shape remains a converter unit regression because payload cases must round-trip through the universal model. - Update existing universal/provider tests for optional tool names and builtin identities. ## Expected-diff impact