From 98acab1f8da1783f3b4a5c5b0455acb3293e7884 Mon Sep 17 00:00:00 2001 From: Kailash Date: Sat, 29 Aug 2026 14:41:26 +0530 Subject: [PATCH] Google VTON Integration --- CHANGELOG.md | 7 + README.md | 2 +- ROADMAP.md | 6 +- SECURITY.md | 2 + docs/docs/agents/planner-agent.md | 2 +- docs/docs/agents/vton-agent.md | 2 +- docs/docs/api-reference/google-vton.md | 73 ++++++ docs/docs/api-reference/nano-banana.md | 2 + docs/docs/api-reference/nvidia-nim.md | 2 +- docs/docs/api-reference/overview.md | 27 ++ docs/docs/community/integrate-next.md | 95 +++++-- docs/docs/community/roadmap.md | 2 +- docs/docs/getting-started/cli.md | 6 +- docs/docs/getting-started/configuration.md | 4 + docs/docs/getting-started/installation.md | 3 + docs/docs/getting-started/mcp.md | 10 + docs/docs/intro.md | 2 +- docs/sidebars.ts | 1 + env.template | 8 + mcp-server/README.md | 1 + mcp-server/config.py | 15 +- mcp-server/test_server.py | 8 + tests/test_cli.py | 19 ++ tests/test_planner_agent.py | 24 ++ tryon/agents/planner/bind.py | 6 + tryon/api/__init__.py | 1 + tryon/api/vton/__init__.py | 2 + tryon/api/vton/google_vton.py | 282 +++++++++++++++++++++ tryon/cli/registry.py | 35 +++ 29 files changed, 622 insertions(+), 27 deletions(-) create mode 100644 docs/docs/api-reference/google-vton.md create mode 100644 tryon/api/vton/google_vton.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0aef778..7213bb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +#### πŸ‘— Virtual try-on β€” Google Vertex (`virtual-try-on-001`) +- **Google Virtual Try-On** (`GoogleVTONAdapter` / CLI `--model google-vton`): first-party Vertex / Gemini Enterprise `recontext_image` + - Dedicated person + product try-on; **not** Nano Banana and **not** `GEMINI_API_KEY` + - Auth: `GOOGLE_CLOUD_PROJECT` + Application Default Credentials (`gcloud auth application-default login` or `GOOGLE_APPLICATION_CREDENTIALS`) + - 1–4 samples; C2PA/SynthID watermark on by default; MCP tool `vton_google_vton` + - Docs: `docs/docs/api-reference/google-vton.md` + #### 🧠 Understand + 🎬 Video β€” NVIDIA NIM (Nemotron Omni, Cosmos 3) - **Nemotron 3 Nano Omni** (`NemotronOmniUnderstandAdapter` / CLI `--model nemotron-omni`): hosted NIM chat (`nvidia/nemotron-3-nano-omni-30b-a3b-reasoning`) - Image, video, and audio understand; `NVIDIA_API_KEY`; thinking on by default diff --git a/README.md b/README.md index b8d7413..55626ba 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ API tutorials, configuration, examples, and agent guides live there β€” not in t | Category | Highlights | |---|---| -| **Virtual try-on** | FLUX VTO, Nova Canvas, Kling AI, Segmind, Pruna P-Image-Try-On, FASHN, Nano Banana 2 Lite, **Qwen-Image** (API + local), **Muse Image** (composition) | +| **Virtual try-on** | FLUX VTO, **Google Vertex VTO**, Nova Canvas, Kling AI, Segmind, Pruna P-Image-Try-On, FASHN, Nano Banana 2 Lite, **Qwen-Image** (API + local), **Muse Image** (composition) | | **Generate / edit** | Nano Banana family, FLUX.2, GPT Image, Luma Photon, Seedream 5.0 Pro, Ideogram 4.0, Grok Imagine Image, Pruna P-Image / **P-Image-Ideogram** / Edit / Upscale, **Qwen-Image** (API + local), **Muse Image**; local FLUX.2-dev Turbo | | **Understand** | Kimi K2.6 / K2.7 Code / K3 (API), Kimi-VL & LLaVA-NeXT (local), **Qwen3.8-Max** (API) + **Qwen3.8-27B** (local) | | **Video** | Veo, Sora, Luma Ray 2 + Ray 3.2, Seedance 2.5, Kling 3.0 / Omni / Turbo, Grok Imagine Video 1.5, Gemini Omni Flash, Pruna P-Video / Replace / Avatar / Animate, **LTX-2.5** (API + local), **Hailuo 2.3**, **MiniMax H3** (API + local), **Wan** (API + local 2.2), **Runway Gen-4.5** | diff --git a/ROADMAP.md b/ROADMAP.md index b8aabe9..3eaa0d0 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -31,7 +31,7 @@ This roadmap tracks what shipped and what comes next. Product strategy: [`VISION β”‚ β€’ Fashion prompt packs + train-pack schema / datasets β”‚ β”‚ β€’ Train / LoRA recipes (`opentryon train`) + notebooks β”‚ β”‚ β€’ Eval + Fashion Bench v0 (`opentryon eval`) β”‚ -β”‚ β€’ One productized local OSS VTON path (CatVTON / IDM / OOT β€” pick one) β”‚ +β”‚ β€’ One productized local OSS VTON path (Leffa / CatVTON / IDM / OOT β€” pick one) β”‚ β”‚ β€’ Fashion agentic workflows via MCP (Try-On QA or Fine-Tune Coach) β”‚ β”‚ β€’ Efficiency card for that local path (VRAM / latency table) β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ @@ -98,7 +98,7 @@ CLI/MCP remain the distribution spine; new capabilities register the same way. - [ ] CLI: `opentryon eval run|report` ### Slice D β€” Local OSS VTON (productize one) -- [ ] Ship **one** of CatVTON / IDM-VTON / OOTDiffusion under `tryon.models` + `opentryon[local]` +- [ ] Ship **one** of Leffa / CatVTON / IDM-VTON / OOTDiffusion under `tryon.models` + `opentryon[local]` (candidates + licenses: [integrate-next](docs/docs/community/integrate-next.md)) - [ ] Same invoke path as cloud adapters (agents don’t care where it runs) - [ ] Docs: install, VRAM, dry-run, known limits @@ -179,7 +179,7 @@ tryon/ **High-value next:** LoRA recipe hardening, Fashion Bench metrics, first local VTON adapter, Try-On QA or Fine-Tune Coach workflow. -**Vendor / model queue** (not a v0.1.0 commitment): [docs/docs/community/integrate-next.md](docs/docs/community/integrate-next.md) β€” NVIDIA Nemotron Omni + Cosmos 3 first, then CatVTON, then new services (LipSync, TRELLIS). +**Vendor / model queue** (not a v0.1.0 commitment): [docs/docs/community/integrate-next.md](docs/docs/community/integrate-next.md) β€” NVIDIA Wave 1 and Google Vertex VTO shipped; next is OutfitAnyone-Plus and one local VTON weight, then new services (LipSync, TRELLIS). See [Contributing](CONTRIBUTING.md), [new-model checklist](docs/docs/advanced/new-model-checklist.md), and [VISION.md](VISION.md). diff --git a/SECURITY.md b/SECURITY.md index eb6c50b..a01d12a 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -58,6 +58,8 @@ KLING_AI_API_KEY KLING_AI_SECRET_KEY SEGMIND_API_KEY GEMINI_API_KEY +GOOGLE_CLOUD_PROJECT +GOOGLE_APPLICATION_CREDENTIALS BFL_API_KEY LUMA_AI_API_KEY OPENAI_API_KEY diff --git a/docs/docs/agents/planner-agent.md b/docs/docs/agents/planner-agent.md index 3f2ca51..b5643a0 100644 --- a/docs/docs/agents/planner-agent.md +++ b/docs/docs/agents/planner-agent.md @@ -41,7 +41,7 @@ Each capability has a **default model**. The planner uses that default unless th | Video generation | `video` | `video-generate` / `sora` | | Background remove | `bg_remove` | `bg-remove` / `ben2` | -- **User named a model** (for example `wan-3.0`, `flux2-pro`, `p-image-ideogram`, `nemotron-omni`, `cosmos3`) β†’ that registry id **only**, even if it lives in another capability. An unknown name is **not** replaced by the default; the planner asks you to pick a real id. `p-image-ideogram` pins Pruna’s model; a bare `ideogram` still pins Ideogram 4.0. `cosmos3-reasoner` pins the VLM, not the Generator. +- **User named a model** (for example `wan-3.0`, `google-vton`, `flux2-pro`, `p-image-ideogram`, `nemotron-omni`, `cosmos3`) β†’ that registry id **only**, even if it lives in another capability. An unknown name is **not** replaced by the default; the planner asks you to pick a real id. `p-image-ideogram` pins Pruna’s model; a bare `ideogram` still pins Ideogram 4.0. `cosmos3-reasoner` pins the VLM, not the Generator. `google-vton` / `virtual-try-on-001` pin Vertex dedicated try-on, not Nano Banana. - **No model named** β†’ the default in the table. - The classifier must leave `model` empty unless the user named one. A leaked default in `plan.model` is ignored unless that id also appears in the prompt. diff --git a/docs/docs/agents/vton-agent.md b/docs/docs/agents/vton-agent.md index 7f78bce..03888aa 100644 --- a/docs/docs/agents/vton-agent.md +++ b/docs/docs/agents/vton-agent.md @@ -12,7 +12,7 @@ keywords: `VTOnAgent` is a **thin Python facade** over the live registry. It is not a LangChain tool loop. For Studio chat, call [`PlannerAgent`](./planner-agent.md) / MCP `planner_agent`. -Default model: **`kling-ai`**. A named model in the prompt (FASHN, FLUX VTO, Segmind, Qwen-Image, …) pins that VTON registry id. +Default model: **`kling-ai`**. A named model in the prompt (Google VTO, FASHN, FLUX VTO, Segmind, Qwen-Image, …) pins that VTON registry id. ```python from tryon.agents.vton import VTOnAgent diff --git a/docs/docs/api-reference/google-vton.md b/docs/docs/api-reference/google-vton.md new file mode 100644 index 0000000..aa29538 --- /dev/null +++ b/docs/docs/api-reference/google-vton.md @@ -0,0 +1,73 @@ +--- +sidebar_position: 8 +title: Google Virtual Try-On +description: Vertex AI virtual-try-on-001 β€” dedicated person + product try-on (not Gemini API / Nano Banana) +--- + +# Google Virtual Try-On + +First-party Vertex / Gemini Enterprise model **`virtual-try-on-001`**. Person photo + product photo β†’ try-on stills (1–4 samples). Output aspect and resolution match the person image. PNG/JPEG, 10MB max. C2PA / SynthID watermark on by default. + +This is **not** Nano Banana composition (`nano-banana-2-lite` / `GEMINI_API_KEY`). The Gemini Developer API does not host this model. + +| CLI `--model` | MCP tool | Adapter | Upstream id | +|---|---|---|---| +| `google-vton` | `vton_google_vton` | `GoogleVTONAdapter` | `virtual-try-on-001` | + +Official docs: + +- [Virtual Try-On 001 (Vertex)](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/imagen/virtual-try-on-001) +- [Generate Virtual Try-On Images](https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/capabilities/generate-virtual-try-on-images) + +GA 20 January 2026. Google lists a discontinuation date of 20 January 2027 for this model id β€” check Vertex docs before relying on it past that. + +## Auth + +Vertex **Application Default Credentials** plus a GCP project. Studio Connect stores the project id only. + +```bash +gcloud auth application-default login +export GOOGLE_CLOUD_PROJECT=your-gcp-project +# optional; default is global +# export GOOGLE_CLOUD_LOCATION=global +# or a service account: +# export GOOGLE_APPLICATION_CREDENTIALS=/path/to/sa.json +``` + +Enable billing and the Vertex / Agent Platform APIs on that project. If `global` 404s in your region, set `GOOGLE_CLOUD_LOCATION=us-central1`. + +## CLI + +No text prompt β€” the API rejects styling instructions. Shopper photos need `--person-generation allow_adult` (the default). + +```bash +opentryon vton --model google-vton \ + --person-image person.jpg \ + --garment-image sweater.jpg + +opentryon vton --model google-vton \ + --person-image person.jpg \ + --garment-image sweater.jpg \ + --num-images 2 \ + --seed 7 \ + --dry-run +``` + +## Python + +```python +from tryon.api import GoogleVTONAdapter + +adapter = GoogleVTONAdapter() # GOOGLE_CLOUD_PROJECT +images = adapter.generate_and_decode( + person="person.jpg", + garment="sweater.jpg", + number_of_images=1, + person_generation="allow_adult", +) +images[0].save("worn.png") +``` + +## Planner / Studio + +Name **`google-vton`** or **`virtual-try-on-001`** in chat to pin this model. The VTON default stays `kling-ai`. After MCP restart, Connect lists **Google Vertex Virtual Try-On** (`GOOGLE_CLOUD_PROJECT`). ADC still lives on the MCP host, not in Studio. diff --git a/docs/docs/api-reference/nano-banana.md b/docs/docs/api-reference/nano-banana.md index 8bb18eb..c08366f 100644 --- a/docs/docs/api-reference/nano-banana.md +++ b/docs/docs/api-reference/nano-banana.md @@ -376,6 +376,8 @@ images[0].save("result.png") Also available from the `opentryon` CLI / MCP server as `vton --model nano-banana-2-lite`. +This is **composition** try-on via the Gemini Developer API (`GEMINI_API_KEY`). Dedicated Google Cloud Virtual Try-On is `--model google-vton` (Vertex `virtual-try-on-001`, ADC + `GOOGLE_CLOUD_PROJECT`). See [Google Virtual Try-On](google-vton). + ## Command Line Usage Use the `image_gen.py` script for command-line image generation: diff --git a/docs/docs/api-reference/nvidia-nim.md b/docs/docs/api-reference/nvidia-nim.md index 24df026..f00c290 100644 --- a/docs/docs/api-reference/nvidia-nim.md +++ b/docs/docs/api-reference/nvidia-nim.md @@ -14,7 +14,7 @@ One key, `NVIDIA_API_KEY` from [build.nvidia.com](https://build.nvidia.com), unl | `cosmos3-reasoner` | `understand` | `Cosmos3ReasonerAdapter` | `nvidia/cosmos3-nano-reasoner` | | `cosmos3` | `video-generate` | `Cosmos3VideoAdapter` | Cosmos 3 Generator nano (`POST` infer, `b64_video`) | -Nemotron is **understanding / agents**, not T2I or VTON. Cosmos 3 Generator is **physics-aware T2V / I2V**. There is no NVIDIA VTON NIM β€” local OSS VTON stays on the [integrate-next](../community/integrate-next.md) Wave 2 list. +Nemotron is **understanding / agents**, not T2I or VTON. Cosmos 3 Generator is **physics-aware T2V / I2V**. There is no NVIDIA VTON NIM β€” dedicated try-on APIs and local weights stay on the [integrate-next](../community/integrate-next.md) Wave 2 list. Official docs: diff --git a/docs/docs/api-reference/overview.md b/docs/docs/api-reference/overview.md index 351a981..4ee2ee9 100644 --- a/docs/docs/api-reference/overview.md +++ b/docs/docs/api-reference/overview.md @@ -196,6 +196,33 @@ See [Kling AI API Documentation](kling-ai) for complete details. --- +### `GoogleVTONAdapter` + +Dedicated Vertex AI Virtual Try-On (`virtual-try-on-001`). Person + product images; **not** Gemini API / Nano Banana. + +```python +from tryon.api import GoogleVTONAdapter + +adapter = GoogleVTONAdapter() # GOOGLE_CLOUD_PROJECT + ADC + +images = adapter.generate_and_decode( + person="person.jpg", + garment="sweater.jpg", + number_of_images=1, +) +``` + +**Parameters:** +- `project` (str, optional): GCP project. Defaults to `GOOGLE_CLOUD_PROJECT` +- `location` (str, optional): Vertex location. Defaults to `GOOGLE_CLOUD_LOCATION` or `global` + +**Methods:** +- `generate_and_decode(person, garment, ...)` - Generate and decode to PIL Images + +See [Google Virtual Try-On](google-vton) for auth, CLI, and MCP. + +--- + ### `AmazonNovaCanvasVTONAdapter` Adapter for Amazon Nova Canvas Virtual Try-On through AWS Bedrock. diff --git a/docs/docs/community/integrate-next.md b/docs/docs/community/integrate-next.md index 5857f9c..7e59550 100644 --- a/docs/docs/community/integrate-next.md +++ b/docs/docs/community/integrate-next.md @@ -1,7 +1,7 @@ --- sidebar_position: 4 title: Integrate next -description: Living backlog of models to add to OpenTryOn β€” NVIDIA Nemotron/NIM first, then VTON and new modalities +description: Living backlog of models to add to OpenTryOn β€” NVIDIA NIM, virtual try-on APIs and local weights, then new modalities --- # Integrate next @@ -14,7 +14,7 @@ Living **candidate queue** for new adapters. This is not a commitment and it is | [`ROADMAP.md`](https://github.com/tryonlabs/opentryon/blob/main/ROADMAP.md) | Product slices (train / eval / one local VTON / agents) | | [`.cursor/skills/integrate-model/`](https://github.com/tryonlabs/opentryon/tree/main/.cursor/skills/integrate-model) | How to integrate once you pick a row | -**Surveyed:** 29 August 2026 Β· Sources: [build.nvidia.com/models](https://build.nvidia.com/models), [Nemotron](https://www.nvidia.com/en-us/ai-data-science/foundation-models/nemotron/), [Cosmos 3](https://docs.nvidia.com/cosmos/latest/cosmos3/index.html), [NIM Cosmos WFM](https://docs.nvidia.com/nim/cosmos/latest/introduction.html). +**Surveyed:** 29 August 2026 Β· Sources: [build.nvidia.com/models](https://build.nvidia.com/models), [Google `virtual-try-on-001`](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/imagen/virtual-try-on-001), [Alibaba OutfitAnyone-Plus](https://www.alibabacloud.com/help/en/model-studio/aitryon-plus-api), [CatVTON](https://github.com/Zheng-Chong/CatVTON/), [Leffa](https://github.com/franciszzj/Leffa), [Nemotron](https://www.nvidia.com/en-us/ai-data-science/foundation-models/nemotron/), [Cosmos 3](https://docs.nvidia.com/cosmos/latest/cosmos3/index.html). **How to use:** pick a `next` row β†’ follow the [integrate-model skill](https://github.com/tryonlabs/opentryon/blob/main/.cursor/skills/integrate-model/SKILL.md) (Path A first-party API, Path B local). After ship, move the row to **Shipped** and bump the date. @@ -61,16 +61,75 @@ Highest leverage: one NIM provider key unlocks understand + video. Nemotron is * --- -## Wave 2 β€” Local OSS VTON (already on the product roadmap) +## Wave 2 β€” Virtual try-on (fashion / D2C / marketplace) -Pick **one** for v0.1.0. These are Path B under `tryon.models` + `opentryon[local]`. NVIDIA has **no** dedicated VTON NIM. +Compile-only as of 29 Aug 2026 β€” **do not integrate until asked.** Developers building fitting rooms, PDP/catalog on-model shots, and marketplace listing tools need **dedicated** person+garment try-on, not another general I2I compose. -| Candidate | Category | Path | Suggested id | Why | Status | +**Trust order for Path A:** hyperscalers and durable public platforms first (Google, Amazon, Alibaba, BFL, Kuaishou, Meta). Fashion specialists we already ship (FASHN, Pruna) stay. Smaller photo-API vendors are `watch` unless a customer names them. Do **not** add Fal / Replicate / PiAPI wrappers of models we already call first-party. + +NVIDIA has **no** dedicated VTON NIM. Product roadmap Slice D is still: pick **one** local OSS path for v0.1.0. + +### Already in OpenTryOn (do not re-add) + +| Registry id | Vendor | Kind | Notes | +|---|---|---|---| +| `flux-vto` | Black Forest Labs | Dedicated VTON API | First-party FLUX VTO | +| `google-vton` | Google Cloud Vertex | Dedicated VTON API | `virtual-try-on-001`; ADC + `GOOGLE_CLOUD_PROJECT`, not `GEMINI_API_KEY` | +| `nova-canvas` | Amazon Bedrock | Dedicated VTON API | Garment classes incl. footwear | +| `kling-ai` | Kuaishou (Kling / Kolors) | Dedicated VTON API | First-party Kolors v1 / v1.5 | +| `fashn-tryon-max` / `fashn-tryon-v1.6` | FASHN | Dedicated VTON API | Fashion suite; v1.6 is the fast e-comm path | +| `p-image-tryon` | Pruna | Dedicated VTON API | Multi-garment (up to 11 refs) | +| `segmind` | Segmind | Hosted try-on diffusion | Third-party hoster; keep, do not add more hosters | +| `nano-banana-2-lite` | Google Gemini | Composition I2I | **Not** Vertex `virtual-try-on-001` | +| `qwen-image` / `qwen-image-local` | Alibaba Qwen | Composition I2I | **Not** OutfitAnyone `aitryon-plus` | +| `muse-image` | Meta | Composition I2I | Multi-ref edit, not a garment-fit model | + +### Path A β€” dedicated VTON APIs to add + +Prefer first-party APIs. Related catalog jobs (productβ†’model, model-swap, parsing) are listed only when they are that vendor’s try-on product. + +| Candidate | Vendor durability | Task | Suggested id | Why | Status | +|---|---|---|---|---|---| +| **Google `virtual-try-on-001`** | Google Cloud (GA 20 Jan 2026; listed discontinue 20 Jan 2027) | Shopper / catalog image VTON | `google-vton` | Dedicated Vertex / Gemini Enterprise predict API. Person + product image, 1–4 samples, C2PA watermark. Auth is **ADC / GCP project**, not `GEMINI_API_KEY`. Distinct from Nano Banana compose. [Docs](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/imagen/virtual-try-on-001) | shipped | +| **Alibaba OutfitAnyone-Plus (`aitryon-plus`)** | Alibaba Cloud Model Studio | Image VTON + combo top/bottom | `outfitanyone` | Dedicated DashScope try-on (async). Top, bottoms, dress, face restore, parsing companion `aitryon-parsing-v1`. Same company as Qwen; **Beijing-region key**, not the Qwen-Image compose path. [Docs](https://www.alibabacloud.com/help/en/model-studio/aitryon-plus-api) | `next` | +| **Photoroom Virtual Try-On / Virtual Model** | Photoroom (widely used e-comm photo API) | Fitting room **or** garmentβ†’lifestyle model | `photoroom-vton` | API-first catalog/shopper flows. Virtual Model is β€œflat-lay in, on-model out” (no person photo). Complements dedicated person+SKU VTON. [Product](https://www.photoroom.com/tools/virtual-try-on) | `next` | +| Pixelcut Try-On | Pixelcut (e-comm photo; Shopify-heavy) | Image VTON + garment transfer | `pixelcut-vton` | REST `/v1/try-on`; upper/lower/full. Smaller than Google/Alibaba. [API](https://www.pixelcut.ai/api/try-on) | `watch` | +| Fitroom | Specialist startup | Combo top+bottom in one request | `fitroom` | Strong e-comm DX; weaker long-term vendor signal. | `watch` | +| Claid | Specialist | Catalog try-on | β€” | Photo-API vendor; overlap with Photoroom. | `watch` | +| BytePlus Effects / live AR try-on | ByteDance | Real-time AR, not still VTON | β€” | SDK/effects stack. Seedream image/edit is already in the registry (`seedream`). | `watch` | +| Tencent Cloud FitDiT (hosted) | Tencent | Commercial FitDiT | β€” | Open weights are NC; Tencent Cloud is the commercial door. Confirm a public REST API before Path A. | `watch` | +| Adobe Firefly Services | Adobe | General gen/edit | β€” | Commercially durable Creative Cloud; **no** dedicated person+garment VTON API found. | `skip` | +| Shopify / Google Shopping / Walmart Zeekit | Platform lock-in | In-app try-on | β€” | Not a developer API we can register. | `skip` | +| Kling via Fal / PiAPI / Replicate | Aggregators | Same Kolors VTON | β€” | We already have first-party `kling-ai`. | `skip` | +| Snap Camera Kit / glasses AR | Snap | Accessory AR | β€” | Different modality (mesh/AR), not image VTON. | `watch` | + +**Related try-on jobs (same developers, not a second `vton` clone):** + +| Job | What they call | Prefer | +|---|---|---| +| Shopper fitting room | Person selfie + SKU photo | Google VTO, FASHN, Kling, BFL, Amazon, OutfitAnyone | +| Catalog on-model | Flat-lay β†’ generated model | Photoroom Virtual Model; FASHN product-to-model (vendor suite β€” do not invent a parallel adapter until asked) | +| Multi-SKU outfit | Top + bottoms one call | OutfitAnyone combo, Fitroom combo, Pruna multi-ref (shipped) | +| Model swap / consistent model | Face/body swap, keep garment | FASHN model-swap (vendor suite) | +| Video try-on | Temporal garment on a clip | CatV2TON local; no durable first-party video-VTON API picked yet | +| Parsing / hotspots | Garment masks, bboxes | Alibaba `aitryon-parsing-v1`; OpenTryOn already has preprocess helpers | + +### Path B β€” local / open-weight VTON + +Pick **one** for v0.1.0 Slice D (`tryon.models` + `opentryon[local]`). Many research checkpoints are **CC BY-NC-SA** β€” fine for OSS demos, a problem for D2C/marketplace production. Confirm license before making one the default. + +| Candidate | Origin | Suggested id | VRAM / notes | License (typical) | Status | |---|---|---|---|---|---| -| **CatVTON** (or CatVTON-FLUX LoRA) | Virtual try-on | B | `catvton` | Roadmap Slice D default; <8GB path; LoRA story for Slice B. | `next` | -| IDM-VTON | Virtual try-on | B | `idm-vton` | Alternate if CatVTON quality/license fails. | `watch` | -| OOTDiffusion | Virtual try-on | B | `ootdiffusion` | Alternate; setup scripts already exist under `tryon/`. | `watch` | -| FLUX-fill LoRA | VTON / local LoRA | B | β€” | Train-slice path, not a third cloud VTON. | `watch` | +| **Leffa** | CVPR 2025; HF `franciszzj/Leffa` | `leffa` | Diffusers; VITON-HD + DressCode try-on + pose transfer; strong **detail/logo** story | Code **MIT**; confirm weight card for commercial D2C | `next` | +| **CatVTON** + **CatVTON-FLUX** LoRA | ICLR 2025; FLUX.1-Fill LoRA ~37M | `catvton` | <8GB @ 1024Γ—768; roadmap Slice D default for efficiency; LoRA story for Slice B | **CC BY-NC-SA 4.0** (code + checkpoints); FLUX-Fill base has its own terms | `next` | +| IDM-VTON | ECCV 2024; `yisol/IDM-VTON` | `idm-vton` | Higher fidelity; ~18–24GB typical | **CC BY-NC-SA** | `watch` | +| OOTDiffusion | `levihsu/OOTDiffusion` | `ootdiffusion` | Community baseline; setup scripts already under `tryon/` | Check repo | `watch` | +| FitDiT | Tencent-affiliated DiT; `BoyuanJiang/FitDiT` | `fitdit` | High garment-detail DiT; ComfyUI exists | **CC BY-NC-SA**; commercial via Tencent Cloud | `watch` | +| CatV2TON | Same lab as CatVTON | `catv2ton` | **Video** try-on; needs a video-VTON design pass | Check repo (likely NC like CatVTON) | `watch` | +| FLUX-fill LoRA (train slice) | BFL Fill + brand LoRA | β€” | Not a third cloud VTON; `opentryon train` path | FLUX terms | `watch` | +| OutfitAnyone **weights** | HumanAIGC / Alibaba paper | β€” | Demos lock person upload; use **`aitryon-plus` API** instead | Restricted demos | `skip` | +| VITON-HD / StableVITON | 2022–2023 warping/diffusion | β€” | Superseded for new work | Mixed | `skip` | +| Qwen-Image-Edit-2511 local | Already `qwen-image-local` | β€” | Composition I2I, not a VTON specialist | β€” | shipped | --- @@ -98,7 +157,7 @@ Pick **one** for v0.1.0. These are Path B under `tryon.models` + `opentryon[loca ### Virtual try-on -No NVIDIA / Nemotron VTON. Use Wave 2 (CatVTON first). Cloud VTON is already broad (FLUX VTO, FASHN, Kling, Pruna, Qwen-Image, …). +NVIDIA / Nemotron has no VTON NIM. Cloud dedicated VTON is already broad (FLUX VTO, **Google Vertex**, Amazon Nova, Kling, FASHN, Pruna). Next **APIs:** Alibaba `aitryon-plus`, Photoroom. Next **local weights:** Leffa (MIT, commercial-friendlier) or CatVTON (Slice D efficiency pick, CC BY-NC-SA). Full tables: Wave 2 above. ### Text-to-video @@ -146,16 +205,19 @@ Biology (AlphaFold, Evo2), CFD, weather, routing, chip sim, protein design β€” o ## Suggested integration order 1. ~~`nemotron-omni` / `cosmos3` / `cosmos3-reasoner`~~ **shipped** (Path A, 29 Aug 2026). -2. **`catvton`** (Path B) β€” v0.1.0 Slice D; product, not vendor chasing. -3. **New services** only after CatVTON: LipSync (A2V), TRELLIS (3D). -4. **SANA-Sprint** if we want a fast local T2I that is not another FLUX/Qwen clone. -5. Optional: `nemotron-omni-local` if someone will run 30B-A3B. +2. ~~`google-vton`~~ **shipped** (Path A Vertex `virtual-try-on-001`, 29 Aug 2026). +3. **VTON Path A (when asked):** `outfitanyone` β€” dedicated DashScope try-on, not composition I2I. +4. **VTON Path B (v0.1.0 Slice D):** pick **one** β€” `leffa` if commercial D2C self-host matters (MIT); `catvton` if the goal is <8GB OSS demos (CC BY-NC-SA). Do not ship both in the first local VTON slice. +5. Optional catalog/on-model: Photoroom Virtual Model if a customer asks for flat-lay β†’ lifestyle without a shopper photo. +6. **New services** only after one local VTON: LipSync (A2V), TRELLIS (3D). +7. **SANA-Sprint** if we want a fast local T2I that is not another FLUX/Qwen clone. +8. Optional: `nemotron-omni-local` if someone will run 30B-A3B. --- ## Shipped (do not re-add) -Invoke-layer highlights already in the registry: FLUX.2 (+ Turbo local), Nano Banana family, GPT Image, Muse Image, Ideogram 4.0, P-Image-Ideogram, Qwen-Image API+local, Veo, Sora, LTX-2.5, Hailuo 2.3, MiniMax H3, Wan, Runway Gen-4.5, **Nemotron Omni**, **Cosmos 3 Reasoner**, **Cosmos 3 Generator**, Kimi K2.6/K2.7/K3, Qwen3.8, BEN2, cloud VTON set. Full table: CLI `--help` / registry. +Invoke-layer highlights already in the registry: FLUX.2 (+ Turbo local), Nano Banana family, GPT Image, Muse Image, Ideogram 4.0, P-Image-Ideogram, Qwen-Image API+local, Veo, Sora, LTX-2.5, Hailuo 2.3, MiniMax H3, Wan, Runway Gen-4.5, **Nemotron Omni**, **Cosmos 3 Reasoner**, **Cosmos 3 Generator**, Kimi K2.6/K2.7/K3, Qwen3.8, BEN2, dedicated cloud VTON (`flux-vto`, `google-vton`, `nova-canvas`, `kling-ai`, FASHN, `p-image-tryon`, Segmind) plus composition try-on (`nano-banana-2-lite`, `qwen-image`, `muse-image`). Full table: CLI `--help` / registry. **No dedicated local VTON weights yet** β€” see Wave 2 Path B. --- @@ -163,4 +225,5 @@ Invoke-layer highlights already in the registry: FLUX.2 (+ Turbo local), Nano Ba When adding a row: vendor, modality, Path A/B, proposed registry id, license/API URL, status. When shipping: move to **Shipped**, delete the `next` row, note the registry id. -Re-survey NVIDIA: [build.nvidia.com/models](https://build.nvidia.com/models) + Cosmos / Nemotron blogs. Do not paste the entire NIM catalog. +Re-survey NVIDIA: [build.nvidia.com/models](https://build.nvidia.com/models) + Cosmos / Nemotron blogs. Do not paste the entire NIM catalog. +Re-survey VTON: Vertex Imagen try-on, DashScope OutfitAnyone, Photoroom/Pixelcut, Hugging Face CatVTON / Leffa / IDM-VTON. Prefer first-party APIs over aggregators. diff --git a/docs/docs/community/roadmap.md b/docs/docs/community/roadmap.md index 56c386c..c3d6a51 100644 --- a/docs/docs/community/roadmap.md +++ b/docs/docs/community/roadmap.md @@ -34,7 +34,7 @@ Canonical file: [`ROADMAP.md`](https://github.com/tryonlabs/opentryon/blob/main/ | **A β€” Data & prompts** | Versioned prompt packs, train-pack schema, `data` CLI helpers | | **B β€” Train** | Brand-style LoRA + one VTON/local LoRA path; `opentryon train` | | **C β€” Eval** | Fashion Bench v0, side-by-side reports; `opentryon eval` | -| **D β€” Local VTON** | Productize **one** of CatVTON / IDM-VTON / OOTDiffusion | +| **D β€” Local VTON** | Productize **one** of Leffa / CatVTON / IDM-VTON / OOTDiffusion β€” candidate queue: [Integrate next](integrate-next#wave-2--virtual-try-on-fashion--d2c--marketplace) | | **E β€” Workflows** | MCP-native Try-On QA *or* Fine-Tune Coach (task agent, not chatbot) | | **F β€” Efficiency card** | VRAM/latency table for the path we actually ship | diff --git a/docs/docs/getting-started/cli.md b/docs/docs/getting-started/cli.md index e956439..72f4c38 100644 --- a/docs/docs/getting-started/cli.md +++ b/docs/docs/getting-started/cli.md @@ -28,7 +28,7 @@ opentryon --model [params...] | Service | What it does | Models | |---|---|---| -| `vton` | Virtual try-on: compose a garment onto a person image | `flux-vto`, `nova-canvas`, `kling-ai`, `segmind`, `p-image-tryon`, `fashn-tryon-max`, `fashn-tryon-v1.6`, `nano-banana-2-lite`, `qwen-image`, `qwen-image-local`, `muse-image` | +| `vton` | Virtual try-on: compose a garment onto a person image | `flux-vto`, `google-vton`, `nova-canvas`, `kling-ai`, `segmind`, `p-image-tryon`, `fashn-tryon-max`, `fashn-tryon-v1.6`, `nano-banana-2-lite`, `qwen-image`, `qwen-image-local`, `muse-image` | | `generate` | Text-to-image generation | `nano-banana`, `nano-banana-pro`, `nano-banana-2`, `flux2-pro`, `flux2-flex`, `flux2-turbo` (local), `gpt-image`, `luma-image`, `seedream`, `ideogram`, `grok-imagine-image`, `p-image`, `p-image-ideogram`, `qwen-image`, `qwen-image-local` (local), `muse-image` | | `edit` | Image editing (image + instruction → image) | `nano-banana`, `nano-banana-pro`, `nano-banana-2`, `flux2-pro`, `flux2-flex`, `flux2-turbo` (local), `gpt-image`, `seedream`, `p-image-edit`, `p-image-upscale`, `qwen-image`, `qwen-image-local` (local), `muse-image` | | `understand` | Image/video understanding | `kimi-k2.6`, `kimi-k2.7-code`, `kimi-k3`, `kimi-vl` (local), `qwen3.8-max`, `qwen3.8` (local), `nemotron-omni`, `cosmos3-reasoner`, `llava-next` (local) | @@ -57,6 +57,10 @@ opentryon understand --model kimi-k2.6 --help # list that model's parameters opentryon vton --model flux-vto \ --person-image model.png --garment-image garment.png +# Google Vertex dedicated try-on (ADC + GOOGLE_CLOUD_PROJECT; not GEMINI_API_KEY) +opentryon vton --model google-vton \ + --person-image model.png --garment-image garment.png + # Text-to-image opentryon generate --model nano-banana-pro \ --prompt "A fashion model wearing elegant evening wear" --resolution 4K diff --git a/docs/docs/getting-started/configuration.md b/docs/docs/getting-started/configuration.md index ef2ef00..efba5c2 100644 --- a/docs/docs/getting-started/configuration.md +++ b/docs/docs/getting-started/configuration.md @@ -40,6 +40,10 @@ AMAZON_NOVA_MODEL_ID=amazon.nova-canvas-v1:0 # Optional # Google Gemini (Nano Banana Image Generation) GEMINI_API_KEY=your_gemini_api_key +# Google Vertex Virtual Try-On (virtual-try-on-001) β€” not GEMINI_API_KEY +GOOGLE_CLOUD_PROJECT=your_gcp_project_id +# GOOGLE_CLOUD_LOCATION=global + # BFL AI (FLUX.2 Image Generation) BFL_API_KEY=your_bfl_api_key diff --git a/docs/docs/getting-started/installation.md b/docs/docs/getting-started/installation.md index e6314f3..2b12813 100644 --- a/docs/docs/getting-started/installation.md +++ b/docs/docs/getting-started/installation.md @@ -118,6 +118,9 @@ AMAZON_NOVA_REGION=us-east-1 # Google Gemini (Nano Banana Image Generation) GEMINI_API_KEY=your_gemini_api_key +# Google Vertex Virtual Try-On (not GEMINI_API_KEY) +GOOGLE_CLOUD_PROJECT=your_gcp_project_id + # BFL AI (FLUX.2 Image Generation) BFL_API_KEY=your_bfl_api_key ``` diff --git a/docs/docs/getting-started/mcp.md b/docs/docs/getting-started/mcp.md index c280438..06dea81 100644 --- a/docs/docs/getting-started/mcp.md +++ b/docs/docs/getting-started/mcp.md @@ -87,6 +87,16 @@ Same `NVIDIA_API_KEY` for Nemotron Omni, Cosmos 3 Reasoner, and Cosmos 3 Generat See [NVIDIA NIM](../api-reference/nvidia-nim.md). +## Google Virtual Try-On (Vertex) + +Dedicated person + product try-on. **Not** `GEMINI_API_KEY` / Nano Banana. + +| MCP tool | Backend | Needs | +|---|---|---| +| `vton_google_vton` | Vertex `virtual-try-on-001` | `GOOGLE_CLOUD_PROJECT` + ADC | + +See [Google Virtual Try-On](../api-reference/google-vton.md). + ## Muse Image tools (generate / edit / VTON) First-party Meta Model API (`MODEL_API_KEY`). **No local twin.** Muse Video is not on the API yet. diff --git a/docs/docs/intro.md b/docs/docs/intro.md index f39cb2b..ea006d9 100644 --- a/docs/docs/intro.md +++ b/docs/docs/intro.md @@ -76,7 +76,7 @@ Plus preprocessing, datasets, Gradio demos, and TryOnDiffusion research code. - **Model integration guidelines** for Path A (API) vs Path B (local) ### Virtual try-on -Cloud adapters including FLUX VTO, Nova Canvas, Kling AI, Segmind, **Pruna P-Image-Try-On**, **FASHN**, Nano Banana 2 Lite composition, **Qwen-Image** (API + local), and **Muse Image** composition. +Cloud adapters including FLUX VTO, **Google Vertex Virtual Try-On**, Nova Canvas, Kling AI, Segmind, **Pruna P-Image-Try-On**, **FASHN**, Nano Banana 2 Lite composition, **Qwen-Image** (API + local), and **Muse Image** composition. ### Image generate / edit Nano Banana family, FLUX.2, GPT Image, Luma Photon, **Seedream 5.0 Pro**, **Ideogram 4.0**, **Grok Imagine Image**, **Pruna P-Image / P-Image-Ideogram / Edit / Upscale**, **Qwen-Image** (DashScope 3.0 + local 2512/Edit-2511), **Muse Image**, plus local FLUX.2-dev Turbo. diff --git a/docs/sidebars.ts b/docs/sidebars.ts index 4b93363..ca9ac44 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -57,6 +57,7 @@ const sidebars: SidebarsConfig = { 'api-reference/preprocessing', 'api-reference/diffusion', 'api-reference/kling-ai', + 'api-reference/google-vton', 'api-reference/nova-canvas', 'api-reference/segmind', 'api-reference/pruna', diff --git a/env.template b/env.template index c2b8b47..b3add28 100644 --- a/env.template +++ b/env.template @@ -39,6 +39,14 @@ PRUNA_API_KEY=your_pruna_api_key_here FASHN_API_KEY=your_fashn_api_key_here # FASHN_BASE_URL=https://api.fashn.ai # Optional override +# Google Vertex Virtual Try-On (virtual-try-on-001) β€” not GEMINI_API_KEY +# Docs: https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/imagen/virtual-try-on-001 +# Required for: opentryon vton --model google-vton +# Auth: gcloud auth application-default login (or GOOGLE_APPLICATION_CREDENTIALS) +GOOGLE_CLOUD_PROJECT=your_gcp_project_id +# GOOGLE_CLOUD_LOCATION=global +# GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account.json + # ======================================== # Image Generation Services # ======================================== diff --git a/mcp-server/README.md b/mcp-server/README.md index 9a6b18c..4ece630 100644 --- a/mcp-server/README.md +++ b/mcp-server/README.md @@ -125,6 +125,7 @@ Every tool returns a structured dict: `{"success": true/false, ...}` -- never ra | Tool | Model | Requires | |---|---|---| | `vton_flux_vto` | Black Forest Labs FLUX VTO | `BFL_API_KEY` | +| `vton_google_vton` | Google Vertex Virtual Try-On (`virtual-try-on-001`) | `GOOGLE_CLOUD_PROJECT` + ADC | | `vton_nova_canvas` | Amazon Nova Canvas | `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` | | `vton_kling_ai` | Kling AI (Kolors Virtual Try-On) | `KLING_AI_API_KEY` / `KLING_AI_SECRET_KEY` | | `vton_segmind` | Segmind Try-On Diffusion | `SEGMIND_API_KEY` | diff --git a/mcp-server/config.py b/mcp-server/config.py index 06c167d..4f3d52c 100644 --- a/mcp-server/config.py +++ b/mcp-server/config.py @@ -27,6 +27,8 @@ "OPENTRYON_AGENT_LLM_API_KEY", "ANTHROPIC_API_KEY", "GOOGLE_API_KEY", + "GOOGLE_CLOUD_LOCATION", + "GOOGLE_APPLICATION_CREDENTIALS", "BYTEPLUS_ARK_API_KEY", "WAN_API_KEY", "RUNWAY_API_KEY", @@ -68,7 +70,18 @@ "Google Gemini", "https://aistudio.google.com/app/apikey", ("GEMINI_API_KEY",), - "Also used by Studio chat when OPENTRYON_AGENT_LLM_PROVIDER=google.", + "Also used by Studio chat when OPENTRYON_AGENT_LLM_PROVIDER=google. " + "Does not unlock Vertex Virtual Try-On (google-vton).", + ), + ( + "vertex", + "Google Vertex Virtual Try-On", + "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/imagen/virtual-try-on-001", + ("GOOGLE_CLOUD_PROJECT",), + "GCP project id, not GEMINI_API_KEY. Also run " + "`gcloud auth application-default login` (or set " + "GOOGLE_APPLICATION_CREDENTIALS). Optional GOOGLE_CLOUD_LOCATION " + "(default global).", ), ( "meta", diff --git a/mcp-server/test_server.py b/mcp-server/test_server.py index 25b42a7..dcd4a5d 100644 --- a/mcp-server/test_server.py +++ b/mcp-server/test_server.py @@ -240,6 +240,12 @@ async def check_list_and_set_api_keys() -> None: ) assert "nvidia" in by_id assert by_id["nvidia"]["vars"][0]["name"] == "NVIDIA_API_KEY" + assert "vertex" in by_id + assert by_id["vertex"]["vars"][0]["name"] == "GOOGLE_CLOUD_PROJECT" + assert any( + u["service"] == "vton" and u["model"] == "google-vton" + for u in by_id["vertex"]["unlocks"] + ) assert any( u["service"] == "understand" and u["model"] == "nemotron-omni" for u in by_id["nvidia"]["unlocks"] @@ -251,6 +257,8 @@ async def check_list_and_set_api_keys() -> None: allowed = server.config.allowed_env_names() assert "MODEL_API_KEY" in allowed and "MINIMAX_API_KEY" in allowed assert "NVIDIA_API_KEY" in allowed + assert "GOOGLE_CLOUD_PROJECT" in allowed + assert "GOOGLE_CLOUD_LOCATION" in allowed assert "META_MODEL_API_KEY" in allowed and "MUSE_API_KEY" in allowed gemini = next(p for p in providers if p["id"] == "gemini") assert gemini["vars"][0]["name"] == "GEMINI_API_KEY" diff --git a/tests/test_cli.py b/tests/test_cli.py index 86f0e39..8aaee45 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -173,6 +173,24 @@ def check_fashn_dry_runs(): print("\u2713 vton fashn-tryon-max / fashn-tryon-v1.6 --dry-run resolve the expected calls") +def check_google_vton_dry_run(): + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + code = cli_main([ + "vton", "--model", "google-vton", + "--person-image", "data/model-1.jpg", + "--garment-image", "data/garment.png", + "--num-images", "2", + "--dry-run", + ]) + printed = buf.getvalue() + print(printed, end="") + assert code == 0, printed + assert "GoogleVTONAdapter" in printed and "generate_and_decode" in printed, printed + assert "'number_of_images': 2" in printed, printed + print("\u2713 vton google-vton --dry-run resolves the expected call") + + def check_gemini_omni_dry_runs(): cases = [ ( @@ -627,6 +645,7 @@ def check_new_media_models_dry_runs(): check_p_image_tryon_dry_run() check_nano_banana_2_lite_dry_runs() check_fashn_dry_runs() + check_google_vton_dry_run() check_gemini_omni_dry_runs() check_new_media_models_dry_runs() check_kimi_dry_runs() diff --git a/tests/test_planner_agent.py b/tests/test_planner_agent.py index 9daa3a1..39d568d 100644 --- a/tests/test_planner_agent.py +++ b/tests/test_planner_agent.py @@ -130,6 +130,29 @@ def check_named_model_nvidia_nim_dry_run(): print("\u2713 named-model chat dry-runs cosmos3 and pins cosmos3-reasoner / nemotron-omni") +def check_named_model_google_vton_dry_run(): + agent = PlannerAgent( + classifier=lambda **kwargs: Plan(intent="vton", task=kwargs["prompt"], reason="try-on") + ) + result = agent.run( + "Try the garment using google-vton", + person_image="person.jpg", + garment_image="shirt.jpg", + dry_run=True, + ) + assert result["success"] is True + assert result["service"] == "vton" + assert result["model"] == "google-vton" + assert "GoogleVTONAdapter" in (result.get("call") or "") + + from tryon.agents.planner.bind import match_named_model, slice_for_intent + + vton = slice_for_intent("vton") + pinned = match_named_model("use virtual-try-on-001 on this look", vton) + assert pinned is not None and pinned.model == "google-vton" + print("\u2713 named-model chat dry-runs google-vton and pins virtual-try-on-001") + + def check_out_of_scope_does_not_delegate(): agent = PlannerAgent( classifier=lambda **kwargs: Plan(intent="out_of_scope", reason="not fashion") @@ -528,6 +551,7 @@ def main(): check_model_swap_and_fashion_dry_run() check_named_model_wan_30_dry_run() check_named_model_nvidia_nim_dry_run() + check_named_model_google_vton_dry_run() check_out_of_scope_does_not_delegate() check_help_answers_without_specialist() check_normalize_help_markdown() diff --git a/tryon/agents/planner/bind.py b/tryon/agents/planner/bind.py index 42d0f34..ad04e0e 100644 --- a/tryon/agents/planner/bind.py +++ b/tryon/agents/planner/bind.py @@ -53,6 +53,12 @@ # needle, model_id, optional service constraint ("kling ai", "kling-ai", "vton"), ("kolors", "kling-ai", "vton"), + ("virtual-try-on-001", "google-vton", "vton"), + ("imagen virtual try-on", "google-vton", "vton"), + ("vertex try-on", "google-vton", "vton"), + ("vertex vton", "google-vton", "vton"), + ("google vton", "google-vton", "vton"), + ("google-vton", "google-vton", "vton"), ("fashn", "fashn-tryon-max", "vton"), ("flux vto", "flux-vto", "vton"), ("nova canvas", "nova-canvas", "vton"), diff --git a/tryon/api/__init__.py b/tryon/api/__init__.py index 3625f63..8587d1a 100644 --- a/tryon/api/__init__.py +++ b/tryon/api/__init__.py @@ -22,6 +22,7 @@ "NanoBanana2LiteAdapter": ".nano_banana", "PImageTryOnAdapter": ".vton", "FashnVTONAdapter": ".vton", + "GoogleVTONAdapter": ".vton", "PImageAdapter": ".pruna", "PImageEditAdapter": ".pruna", "PImageIdeogramAdapter": ".pruna", diff --git a/tryon/api/vton/__init__.py b/tryon/api/vton/__init__.py index 63a95ed..3668c3b 100644 --- a/tryon/api/vton/__init__.py +++ b/tryon/api/vton/__init__.py @@ -10,9 +10,11 @@ from .flux_vto import FluxVTONAdapter from .p_image_tryon import PImageTryOnAdapter from .fashn import FashnVTONAdapter +from .google_vton import GoogleVTONAdapter __all__ = [ "FluxVTONAdapter", "PImageTryOnAdapter", "FashnVTONAdapter", + "GoogleVTONAdapter", ] diff --git a/tryon/api/vton/google_vton.py b/tryon/api/vton/google_vton.py new file mode 100644 index 0000000..7cae35f --- /dev/null +++ b/tryon/api/vton/google_vton.py @@ -0,0 +1,282 @@ +"""Google Vertex AI Virtual Try-On (``virtual-try-on-001``). + +Dedicated person + product try-on via the first-party Vertex / Gemini +Enterprise ``recontext_image`` API. This is **not** Gemini Developer API +composition (Nano Banana) and does **not** use ``GEMINI_API_KEY``. + +Official docs: + https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/imagen/virtual-try-on-001 + https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/capabilities/generate-virtual-try-on-images + +Auth: + Application Default Credentials + a GCP project. + ``gcloud auth application-default login`` or ``GOOGLE_APPLICATION_CREDENTIALS``. + Set ``GOOGLE_CLOUD_PROJECT`` (required) and optionally ``GOOGLE_CLOUD_LOCATION`` + (default ``global``). + +Example: + >>> import os + >>> os.environ["GOOGLE_CLOUD_PROJECT"] = "my-gcp-project" + >>> from tryon.api.vton.google_vton import GoogleVTONAdapter + >>> adapter = GoogleVTONAdapter() + >>> images = adapter.generate_and_decode( + ... person="person.jpg", + ... garment="sweater.jpg", + ... ) + >>> images[0].save("worn.png") +""" + +from __future__ import annotations + +import io +import os +from typing import List, Optional, Union + +import requests +from PIL import Image as PILImage + +try: + from google import genai + from google.genai.types import ( + Image as GenaiImage, + ProductImage, + RecontextImageConfig, + RecontextImageSource, + ) + + GOOGLE_GENAI_AVAILABLE = True +except ImportError: # pragma: no cover - optional at import time + GOOGLE_GENAI_AVAILABLE = False + +ImageInput = Union[str, io.BytesIO, PILImage.Image, bytes] + +DEFAULT_MODEL = "virtual-try-on-001" +DEFAULT_LOCATION = "global" +MAX_IMAGE_BYTES = 10 * 1024 * 1024 +VALID_MIME = {"image/jpeg", "image/png"} +PERSON_GENERATION = ("dont_allow", "allow_adult", "allow_all") +SAFETY_LEVELS = ( + "block_low_and_above", + "block_medium_and_above", + "block_only_high", + "block_none", +) + + +class GoogleVTONAdapter: + """Vertex AI Virtual Try-On adapter (``virtual-try-on-001``).""" + + def __init__( + self, + project: Optional[str] = None, + location: Optional[str] = None, + model: Optional[str] = None, + ): + """ + Args: + project: GCP project id. Defaults to ``GOOGLE_CLOUD_PROJECT``. + location: Vertex location. Defaults to ``GOOGLE_CLOUD_LOCATION`` + or ``global``. + model: Upstream model id. Defaults to ``virtual-try-on-001``. + """ + if not GOOGLE_GENAI_AVAILABLE: + raise ImportError( + "google-genai is required for Google Virtual Try-On. " + "Install with: pip install google-genai" + ) + + self.project = project or os.getenv("GOOGLE_CLOUD_PROJECT") + if not self.project: + raise ValueError( + "GOOGLE_CLOUD_PROJECT is required for Google Virtual Try-On " + "(Vertex AI). This is not GEMINI_API_KEY. Set the GCP project " + "id and authenticate with `gcloud auth application-default login` " + "or GOOGLE_APPLICATION_CREDENTIALS." + ) + self.location = ( + location + or os.getenv("GOOGLE_CLOUD_LOCATION") + or DEFAULT_LOCATION + ) + self.model = model or DEFAULT_MODEL + # Force Vertex. A stray GEMINI_API_KEY must not send this to the + # Gemini Developer API, which does not host virtual-try-on-001. + self.client = genai.Client( + vertexai=True, + project=self.project, + location=self.location, + ) + + def _load_bytes(self, image_input: ImageInput) -> tuple[bytes, str]: + """Return PNG/JPEG (bytes, mime_type) for a path, URL, PIL image, or raw bytes.""" + if isinstance(image_input, PILImage.Image): + return self._pil_to_png_or_jpeg(image_input) + + if isinstance(image_input, (bytes, bytearray)): + return self._as_png_or_jpeg(bytes(image_input)) + + if hasattr(image_input, "read"): + image_input.seek(0) + data = image_input.read() + image_input.seek(0) + return self._as_png_or_jpeg(data) + + if isinstance(image_input, str): + if image_input.startswith("gs://"): + raise ValueError( + "Pass GCS URIs through a downloaded path or https URL. " + "Local files and http(s) URLs are supported." + ) + if image_input.startswith(("http://", "https://")): + response = requests.get(image_input, timeout=60) + response.raise_for_status() + return self._as_png_or_jpeg(response.content) + with open(image_input, "rb") as fh: + return self._as_png_or_jpeg(fh.read()) + + raise ValueError( + "Invalid image input: must be a file path, URL, PIL Image, " + "bytes, or file-like object." + ) + + @staticmethod + def _pil_to_png_or_jpeg(image: PILImage.Image) -> tuple[bytes, str]: + fmt = (image.format or "PNG").upper() + buf = io.BytesIO() + if fmt in ("JPEG", "JPG"): + image.convert("RGB").save(buf, format="JPEG") + return buf.getvalue(), "image/jpeg" + image.save(buf, format="PNG") + return buf.getvalue(), "image/png" + + @classmethod + def _as_png_or_jpeg(cls, data: bytes) -> tuple[bytes, str]: + if data.startswith(b"\x89PNG"): + return data, "image/png" + if data.startswith(b"\xff\xd8"): + return data, "image/jpeg" + return cls._pil_to_png_or_jpeg(PILImage.open(io.BytesIO(data))) + + def _to_genai_image(self, image_input: ImageInput) -> GenaiImage: + data, mime = self._load_bytes(image_input) + if len(data) > MAX_IMAGE_BYTES: + raise ValueError( + f"Image is {len(data):,} bytes; Vertex Virtual Try-On allows " + f"at most {MAX_IMAGE_BYTES:,} bytes (10MB) as PNG or JPEG." + ) + return GenaiImage(image_bytes=data, mime_type=mime) + + def generate_and_decode( + self, + person: Optional[ImageInput] = None, + garment: Optional[ImageInput] = None, + *, + source_image: Optional[ImageInput] = None, + reference_image: Optional[ImageInput] = None, + model_image: Optional[ImageInput] = None, + cloth_image: Optional[ImageInput] = None, + person_image: Optional[ImageInput] = None, + garment_image: Optional[ImageInput] = None, + number_of_images: int = 1, + seed: Optional[int] = None, + person_generation: str = "allow_adult", + safety_filter_level: Optional[str] = None, + add_watermark: bool = True, + output_mime_type: str = "image/png", + output_gcs_uri: Optional[str] = None, + **kwargs, + ) -> List[PILImage.Image]: + """Generate try-on images and return PIL Images. + + Vertex Virtual Try-On does **not** accept a text prompt. Person and + product images are the only inputs. + + Args: + person / garment: Person and product images (path, URL, PIL, bytes). + Aliases match other VTON adapters (source_image, model_image, …). + number_of_images: 1–4 samples. + seed: Optional reproducibility seed. + person_generation: ``dont_allow``, ``allow_adult`` (default), + or ``allow_all``. Shopper photos need ``allow_adult``. + safety_filter_level: Optional Vertex safety enum (lowercase). + add_watermark: SynthID / C2PA watermark (default True). + output_mime_type: ``image/png`` or ``image/jpeg``. + output_gcs_uri: Optional ``gs://`` prefix to also store outputs. + """ + resolved_person = person or source_image or person_image or model_image + resolved_garment = garment or reference_image or garment_image or cloth_image + if resolved_person is None: + raise ValueError( + "Person image is required. Pass person, source_image, " + "person_image, or model_image." + ) + if resolved_garment is None: + raise ValueError( + "Garment/product image is required. Pass garment, " + "reference_image, garment_image, or cloth_image." + ) + if not 1 <= int(number_of_images) <= 4: + raise ValueError("number_of_images must be between 1 and 4.") + pg = (person_generation or "allow_adult").lower() + if pg not in PERSON_GENERATION: + raise ValueError( + f"person_generation must be one of {list(PERSON_GENERATION)}" + ) + if output_mime_type not in VALID_MIME: + raise ValueError("output_mime_type must be image/png or image/jpeg.") + + config_kwargs = { + "number_of_images": int(number_of_images), + "person_generation": pg.upper(), + "add_watermark": bool(add_watermark), + "output_mime_type": output_mime_type, + } + if seed is not None: + config_kwargs["seed"] = seed + if safety_filter_level: + level = safety_filter_level.lower() + if level not in SAFETY_LEVELS: + raise ValueError( + f"safety_filter_level must be one of {list(SAFETY_LEVELS)}" + ) + config_kwargs["safety_filter_level"] = level.upper() + if output_gcs_uri: + config_kwargs["output_gcs_uri"] = output_gcs_uri + + try: + response = self.client.models.recontext_image( + model=self.model, + source=RecontextImageSource( + person_image=self._to_genai_image(resolved_person), + product_images=[ + ProductImage( + product_image=self._to_genai_image(resolved_garment) + ) + ], + ), + config=RecontextImageConfig(**config_kwargs), + ) + except Exception as exc: + raise ValueError( + f"Google Virtual Try-On failed ({self.model} @ " + f"{self.project}/{self.location}): {exc}" + ) from exc + + images: List[PILImage.Image] = [] + for generated in response.generated_images or []: + gimg = getattr(generated, "image", None) + if gimg is None: + continue + raw = getattr(gimg, "image_bytes", None) + if raw: + images.append(PILImage.open(io.BytesIO(raw))) + continue + pil = getattr(gimg, "_pil_image", None) + if pil is not None: + images.append(pil) + if not images: + raise ValueError( + "Google Virtual Try-On returned no images. " + f"Response: {response!r}" + ) + return images diff --git a/tryon/cli/registry.py b/tryon/cli/registry.py index 40f73ca..1e54ac6 100644 --- a/tryon/cli/registry.py +++ b/tryon/cli/registry.py @@ -220,6 +220,41 @@ def _qwen_image_local_sample_args(*, t2i: bool) -> List[Arg]: Arg(("--output-format",), "output_format", default="webp", choices=["jpeg", "png", "webp"], help="Output image format"), ], ), + "google-vton": ModelSpec( + id="google-vton", + label="Google Virtual Try-On (Vertex virtual-try-on-001)", + import_path="tryon.api.vton.google_vton", + class_name="GoogleVTONAdapter", + method="generate_and_decode", + output_kind="images", + env_hint="GOOGLE_CLOUD_PROJECT", + notes=( + "Dedicated Vertex / Gemini Enterprise try-on. Not Nano Banana and not " + "GEMINI_API_KEY. Requires ADC (`gcloud auth application-default login` " + "or GOOGLE_APPLICATION_CREDENTIALS). Optional GOOGLE_CLOUD_LOCATION " + "(default global)." + ), + args=[ + _img(("--person-image", "--model-image"), "person", "Person/model image (path or URL)", required=True), + _img(("--garment-image", "--cloth-image", "--product-image"), "garment", + "Garment/product image (path or URL)", required=True), + Arg(("--project",), "project", target="init", help="GCP project id (default: GOOGLE_CLOUD_PROJECT)"), + Arg(("--location",), "location", target="init", help="Vertex location (default: GOOGLE_CLOUD_LOCATION or global)"), + Arg(("--num-images",), "number_of_images", type=int, default=1, help="Number of outputs 1-4"), + Arg(("--seed",), "seed", type=int, help="Seed for reproducibility"), + Arg(("--person-generation",), "person_generation", default="allow_adult", + choices=["dont_allow", "allow_adult", "allow_all"], + help="Person policy (shopper photos: allow_adult)"), + Arg(("--safety-filter-level",), "safety_filter_level", + choices=["block_low_and_above", "block_medium_and_above", "block_only_high", "block_none"], + help="Vertex safety filter"), + Arg(("--no-watermark",), "add_watermark", action="store_false", default=True, + help="Disable SynthID / C2PA watermark (on by default)"), + Arg(("--output-mime-type",), "output_mime_type", default="image/png", + choices=["image/png", "image/jpeg"], help="Output MIME type"), + Arg(("--output-gcs-uri",), "output_gcs_uri", help="Optional gs:// prefix to store outputs"), + ], + ), "nova-canvas": ModelSpec( id="nova-canvas", label="Amazon Nova Canvas",