From 330b7c0e82d85526bc2e5a81760cb48a6ad3bf3e Mon Sep 17 00:00:00 2001 From: jjohare Date: Sat, 18 Jul 2026 17:10:49 +0100 Subject: [PATCH 1/2] =?UTF-8?q?feat(objects):=20R&D=20frontier=20deliverab?= =?UTF-8?q?les=20=E2=80=94=20native=20service,=20R10=20orientation,=20R7?= =?UTF-8?q?=20image-edit,=20R8=20Pixal3D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four Fable R&D teammates, distinct new modules (each hermetically tested): - scripts/trellis2_native_service.py (R5): native TRELLIS.2 HTTP service — /health vs /ready, JSON error taxonomy (400/413/503/500), param clamp/default hardening, thread-safe lazy load, 64MB cap, full lineage. The durable fix for the ComfyUI drtk ABI fragility (native path uses nvdiffrast). 15 tests. - src/pipeline/object_orientation.py (R10): upright-object YAW solve from the crop's COLMAP camera pose (canonical mesh front +Z ↔ observing-camera ray); frame math verified vs colmap_parser + assembler conventions. 17 tests + research/2026-07-orientation-solve.md derivation. - src/pipeline/image_edit_view.py + qwen_image_edit_view.json (R7 rung b): Qwen-Image-Edit 'show the back' alternate view as an extra single-image best-of-N candidate (NOT panel-stitching); self-gates on UNET staging. 18 tests. - src/pipeline/pixal3d_client.py (R8): MIT TRELLIS.2-backbone successor as a drop-in single-image generator scaffold (VERIFY-ON-ENV-BUILD; weights not staged) + research/2026-07-pixal3d-eval.md head-to-head plan. 13 tests. All new files additive; pipeline wiring is a separate commit. Co-Authored-By: claude-flow --- research/2026-07-orientation-solve.md | 334 +++++++++++ research/2026-07-pixal3d-eval.md | 183 ++++++ scripts/trellis2_native_service.py | 287 ++++++++-- src/pipeline/image_edit_view.py | 512 +++++++++++++++++ src/pipeline/object_orientation.py | 313 +++++++++++ src/pipeline/pixal3d_client.py | 527 ++++++++++++++++++ .../workflows/qwen_image_edit_view.json | 107 ++++ tests/python/test_image_edit_view.py | 331 +++++++++++ tests/python/test_object_orientation.py | 254 +++++++++ tests/python/test_pixal3d_client.py | 287 ++++++++++ tests/python/test_trellis2_native_service.py | 248 +++++++++ 11 files changed, 3337 insertions(+), 46 deletions(-) create mode 100644 research/2026-07-orientation-solve.md create mode 100644 research/2026-07-pixal3d-eval.md create mode 100644 src/pipeline/image_edit_view.py create mode 100644 src/pipeline/object_orientation.py create mode 100644 src/pipeline/pixal3d_client.py create mode 100644 src/pipeline/workflows/qwen_image_edit_view.json create mode 100644 tests/python/test_image_edit_view.py create mode 100644 tests/python/test_object_orientation.py create mode 100644 tests/python/test_pixal3d_client.py create mode 100644 tests/python/test_trellis2_native_service.py diff --git a/research/2026-07-orientation-solve.md b/research/2026-07-orientation-solve.md new file mode 100644 index 00000000..71acee83 --- /dev/null +++ b/research/2026-07-orientation-solve.md @@ -0,0 +1,334 @@ +# R10 orientation solve — upright yaw from the crop camera pose + +**Status:** R&D complete — pure module + tests landed (`src/pipeline/object_orientation.py`, +`tests/python/test_object_orientation.py`); integration diff below, not yet applied. +**Extends:** ADR-025 D3 (placement from splat + crop camera pose), PRD v4 R10 +(position + uniform scale shipped; orientation was left `"unsolved"`). +**Date:** 2026-07-10 + +--- + +## 1. Problem + +`object_placement.solve_placement` puts a generated object at the right place +(Gaussian-subset centroid) at the right size (extent ratio), but leaves the +rotation identity and honestly flags `orientation: "unsolved"`. TRELLIS.2 +emits every mesh in its own canonical frame, so identity rotation means every +object in the assembled scene faces the same arbitrary direction — chairs +facing the wall, a bust facing away from the room. + +## 2. The assumption: canonical front ↔ observing-camera ray + +Single-image 3D generators canonicalize their output **relative to the +conditioning image**: the surface visible in the input photo becomes the +mesh's canonical *front*, exported front-facing **+Z, Y-up** (glTF +convention). ADR-025 conditions the generator on exactly one crop from one +known source frame, and `object_crops` records that frame's COLMAP pose in +the crop provenance (`camera_pose.quaternion_wxyz`, `camera_pose.translation`). + +So the imaged side of the object — the mesh's +Z — was, in reality, the side +facing the crop camera. **Rotate the mesh so its canonical +Z points from the +object's position back toward the crop camera, and the generated front lines +up with the photographed front.** One view cannot pin all three rotational +DoF (see §5), but objects in captured scenes overwhelmingly rest upright, so +we pin pitch/roll with an upright prior and solve **yaw only**. + +This is principled, not heuristic: the crop selector already biases toward +frontal, centred, un-clipped observations (centrality + edge-clearance +scores), i.e. toward exactly the views where this assumption is strongest. + +## 3. Derivation + +### 3.1 Conventions (verified against the pipeline sources) + +**COLMAP** (`colmap_parser.ColmapImage`, images.txt/bin fields) stores +world→camera: + +``` +p_cam = R(q) · p_world + t camera frame: +X right, +Y down, +Z forward (RDF) +``` + +with `q = (qw, qx, qy, qz)`. Two standard consequences: + +``` +camera center (world): C = −Rᵀ · t (p_cam = 0) +optical axis (world): fwd = Rᵀ · (0,0,1)ᵀ = third ROW of R +``` + +(`assemble_usd_scene.colmap_camera_world_position` already computes exactly +`−Rᵀt`; the module mirrors it.) + +**COLMAP → USD** (`assemble_usd_scene.py`, Y-up stage): points map through + +``` +colmap_to_usd_position: (x, y, z) → (x·s, −y·s, −z·s), s = SCENE_SCALE = 0.5 +``` + +The linear part is a 180° rotation about X (the assembler's +`_RDF_TO_YUP_QUAT = (0,1,0,0)`). **Directions** transform with the linear +part only — scale never applies to a direction: + +``` +d_usd = (d_x, −d_y, −d_z) +``` + +Therefore scene **up** is +Y in USD ⇔ **−Y in COLMAP world**. This is the +same gravity assumption the whole assembler already bakes in; the yaw solve +adds no new one. + +**Mesh local frame:** the generator GLB is glTF Y-up/front +Z; the assembler +inlines it via trimesh into the Y-up stage without an up-axis conversion, so +mesh-local up already equals stage up. The upright prior is a *no-op +constraint* on pitch/roll — we simply never rotate about anything but +Y. + +### 3.2 The desired front direction + +The front axis must point from the object toward the observing camera, in +COLMAP world: + +* **Exact (method `camera-ray`)** — we know where the object is (the + Gaussian-subset centroid `p_obj` the position solve already uses): + + ``` + f_colmap = normalize(C − p_obj) = normalize(−Rᵀt − p_obj) + ``` + +* **Proxy (method `optical-axis`)** — no centroid available; the object was + approximately centred in the crop frame (the selector's centrality bias), + so the viewing ray ≈ the optical axis: + + ``` + f_colmap = −fwd = −(third row of R) + ``` + + (minus: the camera looks *at* the object; the object's front points *back*.) + +`camera-ray` is strictly better whenever the centroid exists — the object +need not be centred in frame — and the integration below always passes it. + +### 3.3 Map to USD and apply the upright constraint + +``` +f_usd = (f_x, −f_y, −f_z) (direction map, §3.1) +f_h = (f_usd.x, 0, f_usd.z) (project onto horizontal XZ plane) +``` + +If `‖f_h‖ ≈ 0` the camera looked straight down (or up) at the object: yaw is +unobservable from this view → return identity, `method: "degenerate"` (the +placement stays honestly `"unsolved"`). + +### 3.4 Yaw and quaternion + +A yaw of θ about +Y takes the canonical front +Z to `(sin θ, 0, cos θ)`: + +``` +R_y(θ)·(0,0,1)ᵀ = (sin θ, 0, cos θ)ᵀ ⇒ θ = atan2(f_usd.x, f_usd.z) +``` + +As a wxyz quaternion (axis +Y): + +``` +q_obj = (cos(θ/2), 0, sin(θ/2), 0) — expressed in the USD stage frame +``` + +Because the rotation axis is the stage/mesh up-axis and the placement scale +is uniform, the orient op commutes with the scale op; order in the Xform +stack is translate → orient → scale (standard TRS). + +### 3.5 Worked check (also a unit test) + +Level camera on the +X axis in USD, looking at the origin: COLMAP +`R = R_y(90°)` ⇒ `q_cam = (√½, 0, √½, 0)`, `t = (0,0,2)` (center `C=(2,0,0)`). +Optical axis (row 3) = `(−1,0,0)`; front `= (1,0,0)`; USD `= (1,0,0)`; +`θ = atan2(1,0) = 90°`; `q_obj = (√½, 0, √½, 0)`. Rotating +Z by `q_obj` +gives +X — the object faces the camera. ✔ (16 more cases in the test file, +including elevated, off-centre, behind, top-down-degenerate, malformed.) + +## 4. The module + +`src/pipeline/object_orientation.py` — pure, deterministic, stdlib-`math` +only (no numpy, no pxr, no I/O), GPL-3.0, mirroring `object_placement`'s +testability contract: + +```python +solve_yaw(camera_quaternion_wxyz, camera_translation, + object_centroid=None) -> { + "quat_wxyz": [w, x, y, z], # USD-frame rotation, pure yaw about +Y + "yaw_deg": float, + "method": "camera-ray" | "optical-axis" | "degenerate", + "elevation_deg": float, # confidence signal (see §5) + "front_usd": [x, y, z], # diagnostic, pre-projection +} +``` + +Helpers exposed for reuse/tests: `camera_center_colmap`, `camera_forward_colmap`, +`colmap_dir_to_usd`, `yaw_from_front_usd`, `quat_wxyz_from_yaw`, +`quat_multiply`, `rotate_vec_by_quat`, `normalize_quat_wxyz`, +`quat_to_rotation_matrix`, plus `FRONT_AXIS_USD` / `UP_AXIS_USD` constants. + +Failure posture: malformed input (zero quaternion, wrong arity) raises +`ValueError`; *geometric* degeneracy (top-down view, centroid == camera +center) degrades gracefully with an explicit `method` flag — a wrong +orientation is worse than an unsolved one, same ethos as the scale fallback. + +## 5. Limitations (stated, not hidden) + +1. **Roll is unobservable, pitch is confounded** — one view fixes one axis + correspondence only. The upright prior resolves both; it is wrong for + wall-mounted, leaning, or toppled objects. `method` + `elevation_deg` + in the lineage let downstream consumers judge. +2. **Gravity assumption** — "up" = USD +Y = COLMAP −Y is inherited from the + assembler, not established by it. If a capture's COLMAP frame is not + gravity-aligned, *everything* in the scene (env mesh, cameras, objects) + is tilted by the same global rotation, so the yaw solve remains + self-consistent within the scene; it does not fix the global tilt. +3. **Canonical-front assumption needs one empirical check** — if TRELLIS.2's + export fronts −Z rather than +Z (a fixed 180° offset), every solved yaw + is off by exactly 180°. That is a one-constant fix (`FRONT_AXIS_USD`). + Validation: assemble a run with a hero object whose front is obvious + (bust, chair), render the crop camera's view of the USD scene, compare. +4. **High-elevation crops are weak evidence** — a crop shot from 70° above + mostly shows the top; the generator's "front" choice gets noisy. + `elevation_deg` is reported precisely so an escalation rung (e.g. R7 + best-of-N with a silhouette scorer) can gate on it. A future refinement: + score N yaw hypotheses against the SAM silhouette from a second observing + frame (the "+ silhouette" half of the R&D ticket; out of scope here). +5. **Rotation about the local origin** — the GLB is only *near*-centred; + rotating a slightly off-centre mesh about its origin displaces it + slightly. The same is already true of the scale op; magnitude is bounded + by the centring error × extent, negligible next to centroid noise. + +## 6. Integration (exact diff — NOT applied; shared files untouched) + +### 6.1 placements.json schema additions + +New per-object fields (absent ⇒ legacy placement, assembler behaves as today): + +```json +{ + "vase": { + "label": "vase", + "world_centroid": [...], + "scale_ratio": 1.23, + "orientation": "yaw-solved", // was always "unsolved" + "orientation_quat": [0.707, 0.0, 0.707, 0.0], // wxyz, USD frame + "orientation_yaw_deg": 90.0, + "orientation_method": "camera-ray", // camera-ray | optical-axis + "orientation_elevation_deg": 12.4, + "world_extent": [...], "glb_extent": [...] + } +} +``` + +`orientation` stays `"unsolved"` (and no quat is written) when the pose is +missing/degenerate — consumers keep a single honest flag to key off. + +### 6.2 `src/pipeline/stages.py` — thread the crop camera pose to the mesh info + +The crops manifest already carries `camera_pose` per crop entry; +`_generate_object_from_crop._persist` just needs to forward it: + +```diff +@@ def _persist(glb_data, method, lineage, glb_low=None, mesh=None): # stages.py ~L2628 + info: dict[str, Any] = { + "label": label, + "mesh": str(mesh_glb_path), + "ply": ply_path, + "vertex_count": 0 if mesh is None else len(mesh.vertices), + "method": method, + "generator": True, + "glb_sha256": sha, + "crop": str(crop_path), + "placement": placement, + "glb_extent": glb_extent, ++ # R10 orientation solve input: the crop frame's COLMAP pose. ++ "camera_pose": (crop_entry or {}).get("camera_pose"), + } +``` + +### 6.3 `src/pipeline/object_placement.py` — solve yaw in `build_placements` + +```diff +@@ def build_placements(meshes: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + out: dict[str, dict[str, Any]] = {} + for m in meshes: + placement = m.get("placement") + if not placement: + continue + label = m.get("label", "object") + p = solve_placement( + label, + placement.get("centroid", [0, 0, 0]), + placement.get("extent", [0, 0, 0]), + m.get("glb_extent"), + ) +- out[label] = p.to_dict() ++ d = p.to_dict() ++ # R10 orientation: upright yaw from the crop camera pose (ADR-025 D3). ++ pose = m.get("camera_pose") or {} ++ if pose.get("quaternion_wxyz") and pose.get("translation"): ++ from pipeline.object_orientation import solve_yaw ++ try: ++ o = solve_yaw(pose["quaternion_wxyz"], pose["translation"], ++ object_centroid=placement.get("centroid")) ++ except ValueError: ++ o = None # malformed pose provenance -> stay "unsolved" ++ if o is not None and o["method"] != "degenerate": ++ d["orientation"] = "yaw-solved" ++ d["orientation_quat"] = o["quat_wxyz"] ++ d["orientation_yaw_deg"] = o["yaw_deg"] ++ d["orientation_method"] = o["method"] ++ d["orientation_elevation_deg"] = o["elevation_deg"] ++ out[label] = d + return out +``` + +(If keeping `object_placement` import-free of siblings is preferred, hoist +the import to module top — `object_orientation` is equally pure.) + +### 6.4 `scripts/assemble_usd_scene.py` — apply the quat between translate and scale + +```diff +@@ def assemble_scene(...): # object placement branch, ~L648 + placement = placements.get(label) + if placement is not None: + wc = placement.get("world_centroid", [0.0, 0.0, 0.0]) + ux, uy, uz = colmap_to_usd_position(wc[0], wc[1], wc[2]) + xform.AddTranslateOp().Set(Gf.Vec3d(ux, uy, uz)) ++ # R10 orientation: upright yaw solved from the crop camera pose. ++ # quat is authored in the USD stage frame; TRS order (orient ++ # between translate and scale; commutes with the uniform scale). ++ oq = placement.get("orientation_quat") ++ if placement.get("orientation") == "yaw-solved" and oq and len(oq) == 4: ++ xform.AddOrientOp().Set(Gf.Quatf( ++ float(oq[0]), float(oq[1]), float(oq[2]), float(oq[3]))) + # scale_ratio is in raw world units; USD positions carry SCENE_SCALE, + # so the object's size must be scaled by the same factor to match. + s = float(placement.get("scale_ratio", 1.0)) * SCENE_SCALE + xform.AddScaleOp().Set(Gf.Vec3f(s, s, s)) + prim.SetCustomDataByKey("v2g:placement_orientation", + placement.get("orientation", "unsolved")) ++ if placement.get("orientation_method"): ++ prim.SetCustomDataByKey("v2g:placement_orientation_method", ++ placement.get("orientation_method")) ++ prim.SetCustomDataByKey("v2g:placement_yaw_deg", ++ float(placement.get("orientation_yaw_deg", 0.0))) + prim.SetCustomDataByKey("v2g:placement_scale_ratio", + float(placement.get("scale_ratio", 1.0))) +``` + +(`Gf.Quatf(w, x, y, z)` — real part first, matching our wxyz.) + +## 7. Validation plan (runtime, next e2e run) + +1. Unit: `pytest tests/python/test_object_orientation.py` — 17 tests, green + (known poses → exact yaw/quaternion, upright constraint, ray-vs-axis, + degenerate + malformed inputs). +2. Hermetic: extend the object-arc suite so `build_placements` on a mesh + entry carrying `camera_pose` emits `orientation: "yaw-solved"` + a unit + pure-yaw quat. +3. Empirical (the §5.3 flip check): dreamlab hero object (bust) — render the + assembled USD from the crop camera's USD pose; the generated front must + face the viewport. If it shows its back, flip `FRONT_AXIS_USD` once. +4. Regression guard: legacy placements.json (no `orientation_quat`) must + assemble identically to today — the diff is additive-only. diff --git a/research/2026-07-pixal3d-eval.md b/research/2026-07-pixal3d-eval.md new file mode 100644 index 00000000..da904331 --- /dev/null +++ b/research/2026-07-pixal3d-eval.md @@ -0,0 +1,183 @@ +# Pixal3D vs TRELLIS.2 — head-to-head evaluation plan (PRD v4 R8) + +**Date**: 2026-07-10 +**Status**: Plan — client scaffold landed (`src/pipeline/pixal3d_client.py`), +weights NOT staged, eval not yet run +**Traces to**: PRD v4 R8 (gated evaluation), ADR-025 amendment 2026-07-09 +(licence correction), audit §B.1 + recommendation 6 +(`docs/audit/object-pipeline-audit-2026-07-09.md`) + +--- + +## 1. Why + +Pixal3D (TencentARC, May 2026, SIGGRAPH 2026) is pixel-aligned generation **on +the TRELLIS.2 backbone**: it back-projects pixel features from the conditioning +image into 3D, targeting near-reconstruction fidelity for the *observed* side +of the object. That is precisely Vitrine's weak axis — ADR-025 conditions the +generator on one clean source-frame crop, and everything the crop shows should +survive into the asset. If Pixal3D delivers its claim, front-face geometry and +texture fidelity improve with zero change to the pipeline architecture +(single-image in, PBR GLB out). + +- Repo/weights: `https://huggingface.co/TencentARC/Pixal3D` +- Paper: arXiv 2605.10922 + +## 2. Licence gate — PASSES + +ADR-025 originally rejected Pixal3D as default partly on a "custom non-MIT +licence". The 2026-07-09 amendment **corrected this: Pixal3D is MIT** (GitHub +LICENSE + HF model card, both SPDX MIT). MIT is compatible with Vitrine's +GPL-3.0 and with the non-commercial deployment context — no Tencent-UK-style +territorial carve-out applies (that was the Hunyuan3D community licence, a +different instrument). The remaining rationale for gating was community soak; +Pixal3D has now had ~2 months (May → July 2026). **The gate is quality +evidence only. Re-verify the licence file at the pinned revision when +staging** — model cards have changed licence between revisions before. + +## 3. What is compared + +Both generators run the identical contract: one matted RGBA crop → +single-image generation → PBR GLB persisted verbatim. Same seeds, same crops, +same machine, serial (never concurrent — both want the full GPU). + +### 3.1 Numeric (from `eval/objects/run_eval.py` `mesh_stats`, no GPU to grade) + +| Metric | Source | What it tells us | +|---|---|---| +| faces / vertices | mesh_stats | Topology budget parity (gate: within ±30% of TRELLIS.2 unless quality justifies) | +| watertight | mesh_stats | Downstream Nanite/collision sanity | +| bbox_extents | mesh_stats | Gross proportion sanity | +| has_material / material_type / has_uv | mesh_stats | PBR pipe intact (must be PBRMaterial + UVs — hard gate) | +| GLB bytes + sha256 | mesh_stats | Artifact size; hash pins the exact bytes reviewed | +| duration_s | harness | Wall-time cost per object (gate: ≤1.5× TRELLIS.2 at same resolution) | + +### 3.2 Front-fidelity score (the axis Pixal3D claims to win) + +Reuse the R7 scorer (`pipeline/object_candidate_score.py`): front-silhouette +proportion match vs the observed crop + mesh sanity. This is exactly the +"pixel-aligned" claim made measurable with code we already trust in the +best-of-N ladder. Run it on both generators' outputs per crop and compare +`total` / `proportion` components. + +### 3.3 Human review (necessary — the numeric gate is not sufficient) + +8-view turntables per object (harness renders them via headless Blender +automatically). Side-by-side review sheet per crop, three judgments: + +1. **Observed-side fidelity** — does the front match the crop (geometry + *and* texture)? This is Pixal3D's claimed win; judge it first. +2. **Inferred-side plausibility** — is the hallucinated back at least as + plausible as TRELLIS.2's? (Backbone is shared, so the completion prior + should be similar; a regression here is disqualifying.) +3. **PBR material quality** — roughness/metallic separation, bake artifacts, + seam visibility at UV islands. + +## 4. Crop set + +The R9 dreamlab set. Committed baseline today is **3 objects** +(`eval/objects/references.json`, live-run 2026-07-09): + +| Crop | TRELLIS.2 reference | +|---|---| +| `0001_metal_container` | 492k faces, 441 s, PBRMaterial, not watertight | +| `0002_bottle` | 483k faces, 158 s, PBRMaterial | +| `0003_wooden_block` | 466k faces, 130 s, PBRMaterial | + +PRD v4 R9 targets a 10-crop set; extend to 10 before the eval verdict (the +audit's "5–10 dreamlab crops"). Selection criteria for the additional 7: +cover the failure axes — one heavily occluded object, one reflective, one +thin-structure, one textureless, one organic shape — so the verdict is not +"wins on three easy solids". + +## 5. Staging steps (before any run) + +1. **Pin**: resolve the current `TencentARC/Pixal3D` HF revision; record the + commit hash. Verify the LICENSE file at that revision is still MIT. +2. **Stage weights** into the unified tree `data/comfyui/models/` (directive + §3: pre-staged, never re-downloaded at runtime). Expect TRELLIS.2-class + VRAM (≥24 GB official; the backbone is shared) — VERIFY-ON-ENV-BUILD. +3. **Register** in `pipeline/sota_registry.py` (licence=MIT, VRAM, pinned + revision) so `python -m pipeline.sota_registry check` gates the run. +4. **Executor** — in preference order (audit rec. 5: native pipelines for the + 3D half): + a. Stand up `scripts/pixal3d_native_service.py` mirroring the + `trellis2_native_service.py` HTTP contract (`/generate` multipart → + `{glb_high_b64, glb_low_b64?, lineage}`); set `pixal3d.native_url`. + The client already speaks this contract. + b. Only if a runtime-verified ComfyUI node pack materialises: author + `src/pipeline/workflows/pixal3d_single_image_pbr.json` and re-pin + `Pixal3DClient._build_prompt` to explicit node ids. As of 2026-07-10 + **no verified node pack exists**; the client fails fast on this path + by design. +5. **Smoke test**: `Pixal3DClient(native_url=...).health_check()` then one + crop end-to-end; confirm the GLB opens with PBRMaterial + UVs before + burning the full sweep. + +## 6. Running the eval + +With the integration diff applied (adds `--generator pixal3d`): + +```bash +# Incumbent (regenerate at eval time — same machine, same day, honest timings) +python3 eval/objects/run_eval.py --crops \ + --out /data/output/eval_pixal3d/trellis2 --generator trellis2 --seed 42 + +# Challenger +python3 eval/objects/run_eval.py --crops \ + --out /data/output/eval_pixal3d/pixal3d --generator pixal3d --seed 42 +``` + +Compare the two `metrics.json` side by side plus turntables. Do **not** +`--write-references` from a Pixal3D run — `references.json` stays the +committed TRELLIS.2 baseline until an adoption decision is made. The +reference "regression" report for the pixal3d run is read as a *diff vs +incumbent*, not a failure (face-count deltas beyond ±30% are expected to +show up there; judge them, don't auto-fail them). + +Seeds: 42 for the primary sweep; then a 3-seed re-roll (42/43/44) on the two +hardest crops to sample variance — a generator that wins on mean but has +higher variance is worse under the R7 best-of-N ladder economics only if its +per-run cost is also higher. + +## 7. Decision criteria + +Adopt **Pixal3D as primary** (TRELLIS.2 demoted to fallback slot ahead of +Hunyuan3D) iff ALL of: + +1. **Front fidelity wins**: R7 proportion/sanity score ≥ TRELLIS.2 on ≥7/10 + crops AND human review prefers Pixal3D's observed side on a clear + majority (no crop where it is *badly* worse). +2. **No PBR regression**: PBRMaterial + UVs on 10/10; material quality judged + at-least-equal in review. +3. **No back-side regression**: inferred surfaces at least as plausible. +4. **Cost sane**: ≤1.5× TRELLIS.2 wall time per object at equivalent + resolution; fits the 24–48 GB envelope without new infrastructure. +5. **Ops sane**: native service stands up under ADR-021 pinning discipline + without dependency conflicts against the TRELLIS.2 env (shared backbone + should make this cheap; if the envs conflict, that cost enters the + verdict). + +**Keep TRELLIS.2 as primary** if any hard gate (2, 4-VRAM) fails or the +fidelity win is not decisive. Intermediate outcome worth naming: adopt +Pixal3D as an **R7 escalation rung** (hero assets only) if it wins on +fidelity but loses on cost — the ladder already selects best-of-N, and a +slow-but-better generator slots naturally as rung (c). + +Whatever the verdict: amend ADR-025 in place (dated block, per the +living-catalogue convention), update `sota_registry`, and record the sweep +under `docs/renders/` like the R9 baseline. + +## 8. Honest status + +- `pixal3d_client.py` is a **contract-validated scaffold**: 13 hermetic tests + pin drop-in compatibility with `Trellis2Client` (result fields, signature, + verbatim-GLB, lineage incl. MIT licence + backbone note, fail-fast when no + executor exists). It has **never generated a real object** — weights are + not staged, no service or node pack exists in this environment. +- The `resolution`/steps knob names assume the TRELLIS.2 backbone surfaces + the same ladder — marked VERIFY-ON-ENV-BUILD in the client; the native + service is the mapping layer if Pixal3D's actual API differs. +- Integration into `config.py` / `stages.py` / `run_eval.py` is prepared as + an unapplied diff (see the R8 task notes) — nothing in the live pipeline + changes until the eval is scheduled. diff --git a/scripts/trellis2_native_service.py b/scripts/trellis2_native_service.py index 81d6fe1c..eb16d302 100644 --- a/scripts/trellis2_native_service.py +++ b/scripts/trellis2_native_service.py @@ -13,15 +13,25 @@ Contract -------- -GET /health - -> 200 {"status": "ok", "pipeline_loaded": bool, "device": str} +GET /health — liveness: is the process up? Always 200 while serving. + -> 200 {"status": "ok", "pipeline_loaded": bool, "model_path": str, ...} + +GET /ready — readiness: is the pipeline loaded and able to generate? + -> 200 {"ready": true, ...} + -> 503 {"ready": false, "native_package_available": bool, "error": str} POST /generate (multipart/form-data) image: PNG/JPEG file — RGBA preferred (alpha = object matte) seed, resolution, texture_size, ss_steps, shape_steps, tex_steps, - face_count_high, face_count_low, label: form fields (all optional) + face_count_high, face_count_low, label: form fields (all optional; + unparseable values fall back to defaults, out-of-range values clamp) -> 200 {"glb_high_b64": str, "glb_low_b64": str, "lineage": {...}} - -> 4xx/5xx {"error": str} + +Error taxonomy (all errors are JSON ``{"error": str}``): + 400 bad input — missing / empty / undecodable ``image`` + 413 oversized upload — > MAX_UPLOAD_MB + 503 env not ready — native package absent or pipeline failed to load + 500 generation fail — the loaded pipeline raised mid-generation Environment ----------- @@ -33,6 +43,10 @@ python3 scripts/trellis2_native_service.py --host 127.0.0.1 --port 8402 +The service starts fine WITHOUT the native package (health stays green so +orchestration can distinguish "down" from "not ready"); /generate and +/ready return 503 until the pinned env is stood up. + NOTE (scaffold status): the HTTP surface and client contract are final; the two functions marked VERIFY-ON-ENV-BUILD wrap the upstream API exactly as documented (``pipeline.run(image)`` -> ``to_glb`` twice) and must be smoke- @@ -44,9 +58,12 @@ import argparse import base64 +import hashlib +import importlib.util import io import logging import os +import threading import time from flask import Flask, jsonify, request @@ -55,14 +72,56 @@ format="%(asctime)s %(levelname)-8s %(name)s: %(message)s") logger = logging.getLogger("trellis2-native") +SERVICE_NAME = "trellis2-native" +SERVICE_VERSION = "1.0.0" +CONTRACT = "trellis2-native/v1" +MAX_UPLOAD_MB = 64 + MODEL_PATH = os.environ.get("TRELLIS2_MODEL_PATH", "microsoft/TRELLIS.2-4B") +# LoadTrellis2Models-compatible resolutions (runtime-verified node pack set). +ALLOWED_RESOLUTIONS = ("512", "1024", "1024_cascade", "1536_cascade") +DEFAULT_RESOLUTION = "1536_cascade" + +# name -> (default, lower bound, upper bound); unparseable -> default, +# out-of-range -> clamped. Defaults mirror pipeline.trellis2_client. +INT_FIELDS = { + "seed": (42, 0, 2**31 - 1), + "texture_size": (4096, 256, 8192), + "ss_steps": (12, 1, 100), + "shape_steps": (12, 1, 100), + "tex_steps": (12, 1, 100), + "face_count_high": (500_000, 1_000, 4_000_000), + "face_count_low": (20_000, 100, 4_000_000), +} + app = Flask(__name__) -_pipeline = None # loaded lazily on first /generate +app.config["MAX_CONTENT_LENGTH"] = MAX_UPLOAD_MB * 1024 * 1024 + +_pipeline = None # loaded lazily on first /generate (or --preload) +_pipeline_lock = threading.Lock() + + +class EnvNotReadyError(RuntimeError): + """The native TRELLIS.2 environment is absent or failed to load (-> 503).""" + + +def _native_available() -> bool: + """Cheap importability probe — never actually imports the package.""" + if _pipeline is not None: + return True + try: + return importlib.util.find_spec("trellis2") is not None + except (ImportError, ValueError): # broken parent package on sys.path + return False def _load_pipeline(): - """Load the upstream TRELLIS.2 image-to-3D pipeline once. + """Load the upstream TRELLIS.2 image-to-3D pipeline once (thread-safe). + + Raises ``EnvNotReadyError`` for anything that goes wrong at load time — + an absent package and a failed weight load are both deployment problems, + not request problems, and map to HTTP 503. VERIFY-ON-ENV-BUILD: import path + class name per the pinned upstream checkout (microsoft/TRELLIS.2). The documented API is @@ -71,12 +130,28 @@ def _load_pipeline(): global _pipeline if _pipeline is not None: return _pipeline - t0 = time.monotonic() - from trellis2.pipelines import Trellis2ImageTo3DPipeline # upstream repo - _pipeline = Trellis2ImageTo3DPipeline.from_pretrained(MODEL_PATH) - _pipeline.cuda() - logger.info("TRELLIS.2 pipeline loaded from %s in %.1fs", - MODEL_PATH, time.monotonic() - t0) + with _pipeline_lock: + if _pipeline is not None: + return _pipeline + t0 = time.monotonic() + try: + from trellis2.pipelines import Trellis2ImageTo3DPipeline # upstream repo + except ImportError as exc: + raise EnvNotReadyError( + "native TRELLIS.2 env not available in this interpreter: " + f"{exc}. Stand up the pinned env per PRD v4 R5 / ADR-021, or " + "leave trellis2.native_url empty to use the ComfyUI executor." + ) from exc + try: + pipe = Trellis2ImageTo3DPipeline.from_pretrained(MODEL_PATH) + pipe.cuda() + except Exception as exc: # noqa: BLE001 — weights/CUDA failures are env problems + raise EnvNotReadyError( + f"TRELLIS.2 pipeline failed to load from {MODEL_PATH}: {exc}" + ) from exc + _pipeline = pipe + logger.info("TRELLIS.2 pipeline loaded from %s in %.1fs", + MODEL_PATH, time.monotonic() - t0) return _pipeline @@ -85,7 +160,9 @@ def _generate(image, params: dict): VERIFY-ON-ENV-BUILD: ``run()`` kwargs and the o-voxel ``to_glb`` signature (decimation target, texture_size, remesh/UV options) per the - pinned checkout. Returns (glb_high_bytes, glb_low_bytes, timings). + pinned checkout — including where ``resolution`` lands (model-load-time + in the ComfyUI port) and the texture-stage steps kwarg for + ``tex_steps``. Returns (glb_high_bytes, glb_low_bytes, timings). """ pipeline = _load_pipeline() t0 = time.monotonic() @@ -115,82 +192,200 @@ def _to_glb(face_count: int) -> bytes: return glb_high, glb_low, {"run_s": round(t_run, 1), "to_glb_s": round(t_glb, 1)} +def _parse_params(form) -> dict: + """Parse + validate form fields: default on garbage, clamp on range. + + Bad parameters never 400 — a crop with a typo'd seed should still + generate (the client sends well-formed fields anyway; this guards + hand-rolled curl calls). + """ + params: dict = {} + for name, (default, lo, hi) in INT_FIELDS.items(): + raw = form.get(name) + if raw is None or str(raw).strip() == "": + params[name] = default + continue + try: + value = int(str(raw).strip()) + except (TypeError, ValueError): + logger.warning("unparseable %s=%r — defaulting to %d", name, raw, default) + params[name] = default + continue + clamped = max(lo, min(hi, value)) + if clamped != value: + logger.warning("%s=%d out of range [%d, %d] — clamped to %d", + name, value, lo, hi, clamped) + params[name] = clamped + + # The low-poly pair can never exceed the high-poly hero. + if params["face_count_low"] > params["face_count_high"]: + logger.warning("face_count_low %d > face_count_high %d — clamping", + params["face_count_low"], params["face_count_high"]) + params["face_count_low"] = params["face_count_high"] + + resolution = form.get("resolution", DEFAULT_RESOLUTION) + if resolution not in ALLOWED_RESOLUTIONS: + logger.warning("unknown resolution %r — defaulting to %s", + resolution, DEFAULT_RESOLUTION) + resolution = DEFAULT_RESOLUTION + params["resolution"] = resolution + + label = str(form.get("label", "object")) + params["label"] = "".join(c if c.isalnum() else "_" for c in label)[:40] or "object" + return params + + +# --------------------------------------------------------------------------- +# HTTP surface +# --------------------------------------------------------------------------- + @app.get("/health") def health(): + """Liveness only: the process is up. Never 503s — use /ready for that.""" return jsonify({ "status": "ok", + "service": SERVICE_NAME, + "version": SERVICE_VERSION, + "contract": CONTRACT, "pipeline_loaded": _pipeline is not None, "model_path": MODEL_PATH, + "device": str(getattr(_pipeline, "device", "cuda")) if _pipeline is not None else None, }) +@app.get("/ready") +def ready(): + """Readiness: 200 only when the pipeline is loaded and can generate.""" + if _pipeline is not None: + return jsonify({"ready": True, "model_path": MODEL_PATH}) + available = _native_available() + return jsonify({ + "ready": False, + "native_package_available": available, + "error": ( + "pipeline not loaded yet (first /generate or --preload loads it)" + if available else + "native 'trellis2' package not importable in this interpreter — " + "stand up the pinned env per PRD v4 R5 / ADR-021" + ), + }), 503 + + @app.post("/generate") def generate(): + # -- input validation (400) -------------------------------------------- if "image" not in request.files: return jsonify({"error": "multipart field 'image' is required"}), 400 - - def _int(name: str, default: int) -> int: - try: - return int(request.form.get(name, default)) - except (TypeError, ValueError): - return default - - params = { - "seed": _int("seed", 42), - "resolution": request.form.get("resolution", "1536_cascade"), - "texture_size": _int("texture_size", 4096), - "ss_steps": _int("ss_steps", 12), - "shape_steps": _int("shape_steps", 12), - "tex_steps": _int("tex_steps", 12), - "face_count_high": _int("face_count_high", 500_000), - "face_count_low": _int("face_count_low", 20_000), - "label": request.form.get("label", "object"), - } - + upload = request.files["image"] + data = upload.read() + if not data: + return jsonify({"error": "empty 'image' upload"}), 400 try: from PIL import Image - image = Image.open(request.files["image"].stream).convert("RGBA") - except Exception as exc: # noqa: BLE001 - return jsonify({"error": f"could not decode image: {exc}"}), 400 + image = Image.open(io.BytesIO(data)).convert("RGBA") + except Exception as exc: # noqa: BLE001 — PIL raises a zoo of decode errors + return jsonify({"error": f"could not decode image {upload.filename!r}: {exc}"}), 400 + + params = _parse_params(request.form) + logger.info("generate '%s': %dx%d seed=%d res=%s tex=%d faces=%d/%d", + params["label"], image.width, image.height, params["seed"], + params["resolution"], params["texture_size"], + params["face_count_high"], params["face_count_low"]) - logger.info("generate '%s': %dx%d seed=%d res=%s", - params["label"], image.width, image.height, - params["seed"], params["resolution"]) + # -- generation (503 env / 500 failure) -------------------------------- + t0 = time.monotonic() try: glb_high, glb_low, timings = _generate(image, params) + except EnvNotReadyError as exc: + return jsonify({"error": str(exc)}), 503 except ImportError as exc: + # Lazy CUDA-extension imports inside the pipeline (nvdiffrast, CuMesh, + # o_voxel) surface here — still an environment problem, not a bug. return jsonify({"error": ( - "native TRELLIS.2 env not available in this interpreter: " - f"{exc}. Stand up the pinned env per PRD v4 R5 / ADR-021, or " - "leave trellis2.native_url empty to use the ComfyUI executor." + f"native TRELLIS.2 env incomplete: {exc}. Stand up the pinned " + "env per PRD v4 R5 / ADR-021." )}), 503 except Exception as exc: # noqa: BLE001 — surface, don't crash the service - logger.exception("generation failed") - return jsonify({"error": str(exc)}), 500 - + logger.exception("generation failed for '%s'", params["label"]) + return jsonify({"error": f"generation failed: {exc}"}), 500 + total_s = round(time.monotonic() - t0, 1) + logger.info("generate '%s' done in %.1fs (high=%dB low=%dB)", + params["label"], total_s, len(glb_high), len(glb_low)) + + # -- response ------------------------------------------------------------ + # Lineage is complete on its own: the client folds it into the asset + # record, so everything needed to reproduce this artifact rides along. return jsonify({ "glb_high_b64": base64.b64encode(glb_high).decode("ascii"), "glb_low_b64": base64.b64encode(glb_low).decode("ascii"), "lineage": { "generator": "TRELLIS.2-4B", "executor": "native-service", + "service": SERVICE_NAME, + "service_version": SERVICE_VERSION, + "contract": CONTRACT, "model_path": MODEL_PATH, + "conditioning": "single-image", **params, + "input": { + "filename": upload.filename or "", + "sha256": hashlib.sha256(data).hexdigest(), + "width": image.width, + "height": image.height, + }, + "artifact_bytes": {"glb_high": len(glb_high), "glb_low": len(glb_low)}, "timings": timings, + "service_total_s": total_s, }, }) +# JSON on every error path — the client always parses the body (ADR-025 D2). + +@app.errorhandler(404) +def _not_found(_e): + return jsonify({"error": "not found — endpoints: /health /ready /generate"}), 404 + + +@app.errorhandler(405) +def _method_not_allowed(_e): + return jsonify({"error": "method not allowed"}), 405 + + +@app.errorhandler(413) +def _too_large(_e): + return jsonify({"error": f"upload too large (max {MAX_UPLOAD_MB} MB)"}), 413 + + def main() -> None: - parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser = argparse.ArgumentParser( + description="Thin HTTP service wrapping the native TRELLIS.2 pipeline (PRD v4 R5)") parser.add_argument("--host", default="127.0.0.1", help="Bind address (loopback by default, ADR-022)") parser.add_argument("--port", type=int, default=8402) parser.add_argument("--preload", action="store_true", help="Load the pipeline at startup instead of first request") args = parser.parse_args() + + if args.host not in ("127.0.0.1", "localhost", "::1"): + # Inside a container 0.0.0.0 is legitimate (the compose publish is the + # boundary, pinned to host loopback) — but flag it either way. + logger.warning("binding non-loopback %s — ensure the host publish is " + "loopback-pinned (ADR-022)", args.host) + + if not _native_available(): + logger.warning("native 'trellis2' package NOT importable — serving " + "anyway; /generate and /ready return 503 until the " + "pinned env is stood up (PRD v4 R5 / ADR-021)") + if args.preload: - _load_pipeline() + try: + _load_pipeline() + except EnvNotReadyError as exc: + logger.error("--preload failed: %s", exc) + raise SystemExit(1) from exc + + # threaded=False: one generation at a time — serial GPU lifecycle (ADR-013). app.run(host=args.host, port=args.port, threaded=False) diff --git a/src/pipeline/image_edit_view.py b/src/pipeline/image_edit_view.py new file mode 100644 index 00000000..541eddda --- /dev/null +++ b/src/pipeline/image_edit_view.py @@ -0,0 +1,512 @@ +# SPDX-FileCopyrightText: 2026 LichtFeld Studio Authors +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Image-edit alternate-view synthesis (ADR-025 D4 / PRD v4 R7, rung b). + +Backsides of single-image-generated objects are model-hallucinated. The second +rung of the quality-escalation ladder synthesizes an ALTERNATE viewpoint of the +object crop ("show the back / rotate 180°") with an instruction-driven image +edit model, and feeds that edited image to the 3D generator as one more +best-of-N candidate. Each attempt remains ONE clean single image — this is +explicitly NOT the retired panel-stitching anti-pattern (ADR-025: multiview +panel conditioning is superseded; escalation is "image-edit second view fed as +an *alternative single-image attempt*, best-of-N selected"). + +Edit model: **Qwen-Image-Edit-2509** (Apache-2.0, commercial-safe — the only +edit model in the SOTA registry benchmarked for instruction-driven view +rotation). Executor: ComfyUI (a 2D stage, in-scope per ADR-014 as narrowed by +ADR-025). FLUX.2-Kontext-style editing is a licence-gated alternative; swap +the workflow template + model filenames via config to use it. + +Staging honesty (sota_registry ground truth 2026-06-19): the Qwen text-encode +nodes are installed and ``qwen_image_vae`` is staged, but the Qwen-Image-Edit +DIFFUSION UNET is **not yet staged**. ``probe_edit_model()`` checks the live +``/object_info/UNETLoader`` list so callers can skip this rung gracefully +until the weights are pulled; nothing is faked. + +The client mirrors the Trellis2Client plumbing (health_check, _upload_image, +_submit_prompt, _poll_completion, _download_file, _free_vram) so the two feel +identical to operate. Lineage records the edit model, the instruction, and +``surface: image-edit-inferred`` (ADR-025's observed-vs-inferred reporting +concept) so downstream consumers know these pixels were synthesized. + +Usage:: + + from pipeline.image_edit_view import ImageEditView + editor = ImageEditView(comfyui_url="http://vitrine-comfyui:8188") + if editor.probe_edit_model(): + res = editor.edit_view("object_crops/0001_vase.png", label="vase") + # res.image_path is a single clean image -> another crop path for + # Trellis2Client.reconstruct_from_image(...) +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Optional + +import requests + +logger = logging.getLogger(__name__) + +WORKFLOW_DIR = Path(__file__).parent / "workflows" +QWEN_EDIT_VIEW_WORKFLOW = WORKFLOW_DIR / "qwen_image_edit_view.json" + +_CROP_PLACEHOLDER = "CROP_IMAGE_PLACEHOLDER" +_INSTRUCTION_PLACEHOLDER = "EDIT_INSTRUCTION_PLACEHOLDER" + +_IMAGE_SUFFIXES = (".png", ".jpg", ".jpeg", ".webp") + +# The default alternate-view instruction. Identity-preserving and +# single-object by construction: the edit must show the SAME object, not +# restyle it, and must stay a clean isolated photograph (the exact contract +# the single-image generator was trained on). +DEFAULT_BACK_INSTRUCTION = ( + "Rotate the camera 180 degrees to directly behind this object and show " + "its back side. Keep the exact same object: identical shape, materials, " + "colors, proportions and lighting. The whole object centered and fully " + "visible on a plain uniform light background. Do not add, remove or " + "restyle anything." +) + + +@dataclass +class ImageEditResult: + """Result of one instruction-driven alternate-view edit. + + ``image_data`` is the edited image byte-for-byte as ComfyUI produced it + (matting, when applied, yields a *separate* RGBA byte string in + ``image_data``; the raw download is kept in ``raw_image_data``). + """ + image_data: Optional[bytes] = None # the candidate image (RGBA if matted) + raw_image_data: Optional[bytes] = None # verbatim ComfyUI output (RGB) + image_path: Optional[Path] = None # where the candidate was written + backend: str = "qwen-image-edit-comfyui" + duration_seconds: float = 0.0 + prompt_id: str = "" + lineage: dict[str, Any] = field(default_factory=dict) + error: Optional[str] = None + + @property + def image_sha256(self) -> str: + return hashlib.sha256(self.image_data).hexdigest() if self.image_data else "" + + +class ImageEditView: + """Instruction-driven alternate-view client (Qwen-Image-Edit-2509 primary). + + Parameters + ---------- + comfyui_url : str + ComfyUI URL (e.g. ``http://vitrine-comfyui:8188``). + timeout : int + Maximum seconds per edit. + diffusion_model / clip_model / vae_model : str + Loader filenames injected into the workflow. The diffusion UNET is the + canonical Comfy-Org release name but is NOT yet staged (registry ground + truth 2026-06-19) — ``probe_edit_model()`` resolves/falls back against + the live server before any submit. + instruction : str + Default edit instruction (``DEFAULT_BACK_INSTRUCTION``). + steps / cfg / shift / denoise : sampling parameters + Defaults follow the Comfy-Org Qwen-Image-Edit-2509 template + (20 steps, cfg 2.5, AuraFlow shift 3.1, full denoise). + matte_output : bool + Re-matte the edited RGB output to RGBA via rembg when available. The + edited view has a *different* silhouette from the source crop, so the + source alpha can never be reused; without rembg the image is emitted + opaque and flagged ``matted: false`` in lineage (the TRELLIS.2 native + path mattes internally; the ComfyUI path prefers a real alpha). + """ + + def __init__( + self, + comfyui_url: str = "http://vitrine-comfyui:8188", + timeout: int = 600, + poll_interval: float = 2.0, + diffusion_model: str = "qwen_image_edit_2509_fp8_e4m3fn.safetensors", + clip_model: str = "qwen_2.5_vl_7b_fp8_scaled.safetensors", + vae_model: str = "qwen_image_vae.safetensors", + instruction: str = DEFAULT_BACK_INSTRUCTION, + negative: str = "", + steps: int = 20, + cfg: float = 2.5, + shift: float = 3.1, + denoise: float = 1.0, + seed: int = 42, + matte_output: bool = True, + workflow_path: str | Path | None = None, + ): + self.comfyui_url = comfyui_url.rstrip("/") + self.timeout = timeout + self.poll_interval = poll_interval + self.diffusion_model = diffusion_model + self.clip_model = clip_model + self.vae_model = vae_model + self.instruction = instruction + self.negative = negative + self.steps = steps + self.cfg = cfg + self.shift = shift + self.denoise = denoise + self.seed = seed + self.matte_output = matte_output + self.workflow_path = Path(workflow_path) if workflow_path else QWEN_EDIT_VIEW_WORKFLOW + self.session = requests.Session() + + @classmethod + def from_config(cls, cfg: Any) -> "ImageEditView": + """Construct from an ImageEditConfig, reading every field defensively.""" + return cls( + comfyui_url=getattr(cfg, "comfyui_url", "http://vitrine-comfyui:8188"), + timeout=getattr(cfg, "timeout", 600), + diffusion_model=getattr( + cfg, "diffusion_model", "qwen_image_edit_2509_fp8_e4m3fn.safetensors"), + clip_model=getattr(cfg, "clip_model", "qwen_2.5_vl_7b_fp8_scaled.safetensors"), + vae_model=getattr(cfg, "vae_model", "qwen_image_vae.safetensors"), + # Empty string in config means "use the built-in back-view + # instruction" (keeps config.py free of an import on this module). + instruction=getattr(cfg, "instruction", "") or DEFAULT_BACK_INSTRUCTION, + negative=getattr(cfg, "negative", ""), + steps=getattr(cfg, "steps", 20), + cfg=getattr(cfg, "cfg", 2.5), + shift=getattr(cfg, "shift", 3.1), + denoise=getattr(cfg, "denoise", 1.0), + seed=getattr(cfg, "seed", 42), + matte_output=getattr(cfg, "matte_output", True), + ) + + # ------------------------------------------------------------------ + # ComfyUI interaction (native API — mirrors Trellis2Client) + # ------------------------------------------------------------------ + + def health_check(self) -> bool: + try: + r = self.session.get(f"{self.comfyui_url}/system_stats", timeout=10) + return r.status_code == 200 + except requests.RequestException: + return False + + def probe_edit_model(self) -> Optional[str]: + """Resolve the edit UNET against the live server, or None if absent. + + The registry ground truth (2026-06-19) is that the Qwen-Image-Edit + diffusion UNET is NOT staged. Rather than submit a workflow that will + fail node validation, probe ``/object_info/UNETLoader``: exact config + name first, then any staged name containing both "qwen" and "edit" + (fuzzy match tolerates fp8/bf16 variant renames). On a fuzzy hit the + client adopts the staged filename. None = skip this escalation rung. + """ + try: + r = self.session.get( + f"{self.comfyui_url}/object_info/UNETLoader", timeout=10) + data = r.json() + except (requests.RequestException, ValueError) as exc: + logger.warning("probe_edit_model: cannot query UNETLoader: %s", exc) + return None + names = ( + data.get("UNETLoader", {}).get("input", {}).get("required", {}) + .get("unet_name", [[]])[0] + ) + if not isinstance(names, list): + return None + if self.diffusion_model in names: + return self.diffusion_model + for name in names: + low = str(name).lower() + if "qwen" in low and "edit" in low: + logger.info("probe_edit_model: adopting staged edit UNET %r " + "(config asked for %r)", name, self.diffusion_model) + self.diffusion_model = str(name) + return self.diffusion_model + logger.info("probe_edit_model: no Qwen-Image-Edit UNET staged " + "(%d UNETs listed) — image-edit escalation unavailable", + len(names)) + return None + + def _upload_image(self, image_path: Path) -> str: + with open(image_path, "rb") as f: + files = {"image": (image_path.name, f, "image/png")} + r = self.session.post( + f"{self.comfyui_url}/upload/image", files=files, timeout=30, + ) + r.raise_for_status() + return r.json().get("name", image_path.name) + + def _submit_prompt(self, prompt: dict) -> str: + r = self.session.post( + f"{self.comfyui_url}/prompt", json={"prompt": prompt}, timeout=30, + ) + data = r.json() + if data.get("error") or data.get("node_errors"): + node_errors = data.get("node_errors", {}) + details = "; ".join( + f"node {nid}: {e.get('errors', e)}" for nid, e in node_errors.items() + ) if node_errors else str(data.get("error")) + raise RuntimeError(f"ComfyUI validation error: {details}") + prompt_id = data.get("prompt_id") + if not prompt_id: + raise RuntimeError(f"No prompt_id in response: {data}") + return prompt_id + + def _poll_completion(self, prompt_id: str) -> dict: + deadline = time.monotonic() + self.timeout + last_log = 0.0 + while time.monotonic() < deadline: + time.sleep(self.poll_interval) + try: + r = self.session.get( + f"{self.comfyui_url}/history/{prompt_id}", timeout=60, + ) + hist = r.json() + except (requests.ReadTimeout, requests.ConnectionError) as e: + logger.debug("History poll transient error: %s", e) + continue + if prompt_id not in hist: + if time.monotonic() - last_log > 30: + logger.info("Waiting for image-edit prompt %s...", prompt_id[:8]) + last_log = time.monotonic() + continue + entry = hist[prompt_id] + status = entry.get("status", {}).get("status_str", "unknown") + if status == "success": + return entry + if status == "error": + messages = entry.get("status", {}).get("messages", []) + raise RuntimeError(f"Image-edit execution error: {messages}") + if time.monotonic() - last_log > 30: + logger.info("Image-edit %s status: %s", prompt_id[:8], status) + last_log = time.monotonic() + raise TimeoutError(f"Image-edit prompt {prompt_id} timed out after {self.timeout}s") + + def _free_vram(self) -> None: + """POST /free to unload models + free VRAM (serial lifecycle, ADR-013). + Best-effort, never raises — the ~40 GB edit model must not stay + co-resident with TRELLIS.2.""" + try: + self.session.post( + f"{self.comfyui_url}/free", + json={"unload_models": True, "free_memory": True}, timeout=30, + ) + logger.info("freed ComfyUI VRAM after image-edit generation") + except requests.RequestException as e: # noqa: BLE001 + logger.warning("free_vram failed: %s", e) + + def _download_file(self, filename: str, subfolder: str = "") -> bytes: + for file_type in ("output", "temp"): + r = self.session.get( + f"{self.comfyui_url}/view", + params={"filename": filename, "subfolder": subfolder, "type": file_type}, + timeout=120, + ) + if r.status_code == 200 and len(r.content) > 0: + return r.content + raise FileNotFoundError(f"Cannot download {subfolder}/{filename} from ComfyUI") + + def _extract_image_refs(self, history: dict) -> list[tuple[str, str]]: + """Scan ComfyUI history outputs for downloadable image references. + + SaveImage registers ``outputs[node]["images"] = [{filename, subfolder, + type}]``; this scan is robust to the exact ui key and returns + (filename, subfolder) pairs for anything with an image suffix. + """ + refs: list[tuple[str, str]] = [] + outputs = history.get("outputs", {}) + + def add(fname: str, sub: str = "") -> None: + if fname and fname.lower().endswith(_IMAGE_SUFFIXES): + refs.append((Path(fname).name, sub or ( + str(Path(fname).parent) if "/" in fname else sub))) + + for _node_id, node_output in outputs.items(): + for _key, items in node_output.items(): + if not isinstance(items, list): + if isinstance(items, str): + add(items) + continue + for item in items: + if isinstance(item, str): + add(item) + elif isinstance(item, dict): + add(item.get("filename", ""), item.get("subfolder", "")) + return refs + + # ------------------------------------------------------------------ + # Image pre/post-processing (best-effort, never fatal) + # ------------------------------------------------------------------ + + def _prepare_input(self, crop_path: Path) -> tuple[Path, bool]: + """Flatten an RGBA crop onto white before upload, when possible. + + ComfyUI ``LoadImage`` drops alpha (the premultiplied-ghost failure mode + the 2026-07-09 audit documented on the retired panel path), so an RGBA + crop must be composited onto a clean background *before* the edit model + sees it. Best-effort: if Pillow is unavailable or the file cannot be + decoded, the original path is used and the fact recorded in lineage. + Returns (path_to_upload, flattened). + """ + try: + from PIL import Image # noqa: PLC0415 — optional dependency + with Image.open(crop_path) as im: + if im.mode != "RGBA": + return crop_path, False + bg = Image.new("RGB", im.size, (255, 255, 255)) + bg.paste(im, mask=im.split()[3]) + flat_path = crop_path.with_name(crop_path.stem + "__flat_rgb.png") + bg.save(flat_path, format="PNG") + return flat_path, True + except Exception as exc: # noqa: BLE001 — non-fatal preprocessing + logger.debug("RGBA flatten skipped for %s: %s", crop_path.name, exc) + return crop_path, False + + def _matte_output(self, image_data: bytes) -> Optional[bytes]: + """RGBA matte of the edited view via rembg, or None when unavailable. + + The edited view has a NEW silhouette (the object was rotated), so the + source crop's alpha can never be reused. rembg is the same fallback + matting backend the object_crops stage uses (ObjectCropsConfig.matting). + """ + try: + from rembg import remove # noqa: PLC0415 — optional dependency + return remove(image_data) + except Exception as exc: # noqa: BLE001 — non-fatal postprocessing + logger.warning("rembg matte of edited view unavailable/failed: %s", exc) + return None + + # ------------------------------------------------------------------ + # Workflow construction + # ------------------------------------------------------------------ + + def _build_prompt(self, uploaded_name: str, instruction: str, + seed: int, label: str) -> dict: + with open(self.workflow_path) as f: + prompt = json.load(f) + prompt = {k: v for k, v in prompt.items() if not k.startswith("_")} + + for node in prompt.values(): + ins = node.get("inputs", {}) + if ins.get("image") == _CROP_PLACEHOLDER: + ins["image"] = uploaded_name + if ins.get("prompt") == _INSTRUCTION_PLACEHOLDER: + ins["prompt"] = instruction + + # Runtime parameters (node ids per qwen_image_edit_view.json). + if "1" in prompt: + prompt["1"]["inputs"]["unet_name"] = self.diffusion_model + if "2" in prompt: + prompt["2"]["inputs"]["clip_name"] = self.clip_model + if "3" in prompt: + prompt["3"]["inputs"]["vae_name"] = self.vae_model + if "21" in prompt: + prompt["21"]["inputs"]["prompt"] = self.negative + if "40" in prompt: + prompt["40"]["inputs"]["shift"] = self.shift + if "50" in prompt: + prompt["50"]["inputs"].update({ + "seed": seed, "steps": self.steps, "cfg": self.cfg, + "denoise": self.denoise, + }) + if "70" in prompt: + prompt["70"]["inputs"]["filename_prefix"] = f"vitrine_editview_{label}" + return prompt + + # ------------------------------------------------------------------ + # Public entry point + # ------------------------------------------------------------------ + + def edit_view( + self, + crop_path: str | Path, + instruction: str | None = None, + seed: int | None = None, + label: str = "object", + provenance: dict | None = None, + output_path: str | Path | None = None, + ) -> ImageEditResult: + """Synthesize ONE alternate-view image of the object crop. + + The result is a single clean image (never a panel/grid) intended to be + fed to ``Trellis2Client.reconstruct_from_image`` as one additional + best-of-N candidate. ``provenance`` (the object_crops manifest entry) + is folded into the lineage so the synthesized view still traces back + to the source observation. Every synthesized pixel is flagged + ``surface: image-edit-inferred``. + """ + crop_path = Path(crop_path) + if not crop_path.exists(): + raise FileNotFoundError(f"Crop not found: {crop_path}") + seed = self.seed if seed is None else seed + instruction = instruction or self.instruction + safe = "".join(c if c.isalnum() else "_" for c in label)[:40] or "object" + + logger.info("Image-edit alternate view: %s (seed=%d) — %r", + crop_path.name, seed, instruction[:60]) + + t0 = time.monotonic() + # Serial lifecycle: begin from a clean GPU (a prior stage that crashed + # before its own /free leaves stale models resident). + self._free_vram() + + upload_path, flattened = self._prepare_input(crop_path) + uploaded = self._upload_image(upload_path) + prompt = self._build_prompt(uploaded, instruction, seed=seed, label=safe) + prompt_id = self._submit_prompt(prompt) + logger.info("Submitted image-edit prompt %s", prompt_id) + + history = self._poll_completion(prompt_id) + elapsed = time.monotonic() - t0 + + result = ImageEditResult( + backend="qwen-image-edit-comfyui", + duration_seconds=elapsed, + prompt_id=prompt_id, + ) + for fname, sub in self._extract_image_refs(history): + try: + result.raw_image_data = self._download_file(fname, sub) + break + except (FileNotFoundError, requests.RequestException) as e: + logger.warning("Could not download %s/%s: %s", sub, fname, e) + + matted = False + if result.raw_image_data is not None: + result.image_data = result.raw_image_data + if self.matte_output: + rgba = self._matte_output(result.raw_image_data) + if rgba is not None: + result.image_data = rgba + matted = True + out = Path(output_path) if output_path else crop_path.with_name( + crop_path.stem + "__editview.png") + out.parent.mkdir(parents=True, exist_ok=True) + out.write_bytes(result.image_data) + result.image_path = out + logger.info("Image-edit view complete in %.1fs -> %s (matted=%s)", + elapsed, out.name, matted) + else: + result.error = "No retrievable image in edit outputs" + logger.warning("Image-edit: %s (history outputs scanned)", result.error) + + # Serial lifecycle: free the edit model so the 3D generator has VRAM. + self._free_vram() + + result.lineage = { + "edit_model": "Qwen-Image-Edit-2509", + "diffusion_checkpoint": self.diffusion_model, + "instruction": instruction, + "surface": "image-edit-inferred", + "source_crop": str(crop_path), + "executor": result.backend, + "seed": seed, + "steps": self.steps, + "cfg": self.cfg, + "input_flattened": flattened, + "matted": matted, + **({"source": provenance} if provenance else {}), + } + return result diff --git a/src/pipeline/object_orientation.py b/src/pipeline/object_orientation.py new file mode 100644 index 00000000..c0c1922a --- /dev/null +++ b/src/pipeline/object_orientation.py @@ -0,0 +1,313 @@ +# SPDX-FileCopyrightText: 2026 LichtFeld Studio Authors +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Upright-object yaw solve from the crop camera pose (ADR-025 D3 / PRD v4 R10). + +``object_placement`` solves WHERE a generated object goes (position) and HOW +BIG it is (uniform scale). This module solves WHICH WAY IT FACES — the piece +``object_placement`` honestly flags ``orientation: "unsolved"``. + +The key observation +------------------- +TRELLIS.2 (and Hunyuan3D) canonicalize their output relative to the +conditioning image: the surface visible in the crop becomes the mesh's +canonical FRONT, which in the exporter's glTF convention is **+Z with +Y up** +(``FRONT_AXIS_USD`` / ``UP_AXIS_USD`` below). The crop was taken from ONE +known COLMAP camera (``object_crops`` records ``quaternion_wxyz`` + +``translation`` in its provenance). Therefore the mesh's +Z axis should +point *from the object toward that camera* in the assembled scene — that +single correspondence pins the yaw. + +A single view cannot pin all three rotational degrees of freedom (the crop +says nothing about the object's roll, and its pitch is confounded with the +camera's elevation). Real captured objects overwhelmingly rest upright, so we +adopt the upright prior: keep the mesh's +Y aligned with scene up and solve +**yaw only** — rotate about +Y until canonical front points at the horizontal +projection of the object→camera ray. + +Frame conventions (all verified against the pipeline sources) +-------------------------------------------------------------- +COLMAP (``colmap_parser.ColmapImage``) stores WORLD→CAMERA: + + p_cam = R(q) @ p_world + t # camera frame: +X right, +Y down, + # +Z forward (RDF) + +hence + + camera center (world): C = -R^T @ t + optical axis (world): fwd = R^T @ (0,0,1) = third ROW of R + +``scripts/assemble_usd_scene.py`` maps COLMAP points into the Y-up USD stage +with ``colmap_to_usd_position``: ``(x, y, z) -> (x*s, -y*s, -z*s)`` where +``s = SCENE_SCALE``. For DIRECTIONS only the linear part applies (a 180° +rotation about X — the same map, sans scale): + + d_usd = (d_x, -d_y, -d_z) + +Scene "up" is therefore +Y in USD, equivalently -Y in COLMAP world. The +generated GLB is authored Y-up (glTF), inlined into the Y-up stage verbatim, +so mesh-local up already matches stage up: the upright prior costs nothing. + +The solve +--------- +1. front ray (COLMAP world): f = normalize(C - object_centroid) when the + Gaussian-subset centroid is available (exact), else f = -fwd (optical + axis — exact only when the object sat dead-centre in the frame, which + the crop selector's centrality score biases toward). +2. map to USD: f_usd = (f_x, -f_y, -f_z). +3. upright projection: f_h = (f_x, 0, f_z) in USD. Degenerate when the + camera looked straight down/up at the object (‖f_h‖ ≈ 0) — then yaw is + unobservable and we return identity, flagged. +4. yaw about +Y taking FRONT_AXIS_USD = +Z onto f_h: + R_y(yaw) @ (0,0,1) = (sin yaw, 0, cos yaw) => yaw = atan2(f_x, f_z). +5. quaternion (wxyz): (cos(yaw/2), 0, sin(yaw/2), 0). + +All outputs are in the USD stage frame, ready for an ``AddOrientOp`` between +the assembler's translate and scale ops. Everything here is pure math — no +numpy, no I/O — mirroring ``object_placement``'s testability contract. + +See ``research/2026-07-orientation-solve.md`` for the full derivation, +limitations (roll ambiguity, upright assumption, canonical-front assumption) +and the integration plan. +""" + +from __future__ import annotations + +import math +from typing import Any, Optional, Sequence + +# The generated mesh's canonical axes in its own (glTF Y-up) frame, which is +# also the USD stage frame once inlined. If empirical validation ever shows +# the generator fronts -Z instead of +Z, flip this ONE constant (equivalent +# to a 180° yaw offset) — nothing else changes. +FRONT_AXIS_USD: tuple[float, float, float] = (0.0, 0.0, 1.0) +UP_AXIS_USD: tuple[float, float, float] = (0.0, 1.0, 0.0) + +# Below this horizontal-component norm the viewing ray is straight up/down +# and yaw is unobservable (elevation ~> 89.9997°). +DEFAULT_DEGENERATE_EPS: float = 1e-8 + +_EPS = 1e-12 + +IDENTITY_QUAT_WXYZ: tuple[float, float, float, float] = (1.0, 0.0, 0.0, 0.0) + + +# --------------------------------------------------------------------------- +# Small pure helpers (quaternion / vector) +# --------------------------------------------------------------------------- + +def _as_vec3(v: Sequence[float] | None, name: str) -> tuple[float, float, float]: + if v is None or len(v) != 3: + raise ValueError(f"{name} must be a 3-sequence, got {v!r}") + return (float(v[0]), float(v[1]), float(v[2])) + + +def _normalize3(v: tuple[float, float, float]) -> Optional[tuple[float, float, float]]: + n = math.sqrt(v[0] * v[0] + v[1] * v[1] + v[2] * v[2]) + if n < _EPS: + return None + return (v[0] / n, v[1] / n, v[2] / n) + + +def normalize_quat_wxyz(q: Sequence[float]) -> tuple[float, float, float, float]: + """Validate + unit-normalize a (w, x, y, z) quaternion. + + Raises ValueError on wrong arity or (near-)zero norm — a zero quaternion + is corrupt input, not a solvable pose. + """ + if q is None or len(q) != 4: + raise ValueError(f"quaternion must be a 4-sequence (w,x,y,z), got {q!r}") + w, x, y, z = (float(q[0]), float(q[1]), float(q[2]), float(q[3])) + n = math.sqrt(w * w + x * x + y * y + z * z) + if n < _EPS: + raise ValueError("zero-norm quaternion") + return (w / n, x / n, y / n, z / n) + + +def quat_to_rotation_matrix(q: Sequence[float]) -> list[list[float]]: + """Rotation matrix for a (w, x, y, z) quaternion (COLMAP/USD convention). + + Same formula as ``assemble_usd_scene._quat_to_rotation_matrix`` so the + two stay numerically consistent. + """ + w, x, y, z = normalize_quat_wxyz(q) + return [ + [1 - 2 * (y * y + z * z), 2 * (x * y - z * w), 2 * (x * z + y * w)], + [2 * (x * y + z * w), 1 - 2 * (x * x + z * z), 2 * (y * z - x * w)], + [2 * (x * z - y * w), 2 * (y * z + x * w), 1 - 2 * (x * x + y * y)], + ] + + +def quat_multiply(a: Sequence[float], b: Sequence[float]) -> tuple[float, float, float, float]: + """Hamilton product a ⊗ b, both (w, x, y, z). R(a⊗b) = R(a) @ R(b).""" + aw, ax, ay, az = (float(a[0]), float(a[1]), float(a[2]), float(a[3])) + bw, bx, by, bz = (float(b[0]), float(b[1]), float(b[2]), float(b[3])) + return ( + aw * bw - ax * bx - ay * by - az * bz, + aw * bx + ax * bw + ay * bz - az * by, + aw * by - ax * bz + ay * bw + az * bx, + aw * bz + ax * by - ay * bx + az * bw, + ) + + +def rotate_vec_by_quat(q: Sequence[float], v: Sequence[float]) -> tuple[float, float, float]: + """Rotate a 3-vector by a (w, x, y, z) quaternion: R(q) @ v.""" + R = quat_to_rotation_matrix(q) + x, y, z = _as_vec3(v, "v") + return ( + R[0][0] * x + R[0][1] * y + R[0][2] * z, + R[1][0] * x + R[1][1] * y + R[1][2] * z, + R[2][0] * x + R[2][1] * y + R[2][2] * z, + ) + + +# --------------------------------------------------------------------------- +# COLMAP camera geometry +# --------------------------------------------------------------------------- + +def camera_center_colmap( + quat_wxyz: Sequence[float], translation: Sequence[float] +) -> tuple[float, float, float]: + """Camera center in COLMAP world coordinates: C = -R^T @ t. + + COLMAP stores world→camera (p_cam = R p_world + t); the center is the + world point mapping to the camera origin. + """ + R = quat_to_rotation_matrix(quat_wxyz) + tx, ty, tz = _as_vec3(translation, "translation") + return ( + -(R[0][0] * tx + R[1][0] * ty + R[2][0] * tz), + -(R[0][1] * tx + R[1][1] * ty + R[2][1] * tz), + -(R[0][2] * tx + R[1][2] * ty + R[2][2] * tz), + ) + + +def camera_forward_colmap(quat_wxyz: Sequence[float]) -> tuple[float, float, float]: + """Camera optical axis (+Z_cam, the viewing direction) in COLMAP world. + + A world direction d maps to camera coords as R @ d, so the world vector + mapping onto +Z_cam is R^T @ (0,0,1) — the third ROW of R. + """ + R = quat_to_rotation_matrix(quat_wxyz) + return (R[2][0], R[2][1], R[2][2]) + + +def colmap_dir_to_usd(v: Sequence[float]) -> tuple[float, float, float]: + """Map a DIRECTION from COLMAP world into the USD stage frame. + + The linear part of ``assemble_usd_scene.colmap_to_usd_position`` — + (x, y, z) → (x, -y, -z), a 180° rotation about X. SCENE_SCALE is a + point-only concern and never applies to directions. + """ + x, y, z = _as_vec3(v, "direction") + return (x, -y, -z) + + +# --------------------------------------------------------------------------- +# The upright yaw solve +# --------------------------------------------------------------------------- + +def yaw_from_front_usd( + front_usd: Sequence[float], + degenerate_eps: float = DEFAULT_DEGENERATE_EPS, +) -> Optional[float]: + """Yaw (radians, about +Y_usd) taking FRONT_AXIS_USD (+Z) onto ``front``. + + Only the horizontal (XZ) component of ``front`` is used — the upright + constraint. Returns None when that component is degenerate (near-vertical + viewing ray: yaw unobservable). + + Derivation: R_y(θ) @ (0,0,1) = (sin θ, 0, cos θ), so matching the + horizontal front direction (f_x, f_z) gives θ = atan2(f_x, f_z). + """ + fx, _fy, fz = _as_vec3(front_usd, "front_usd") + if math.hypot(fx, fz) <= degenerate_eps: + return None + return math.atan2(fx, fz) + + +def quat_wxyz_from_yaw(yaw_rad: float) -> tuple[float, float, float, float]: + """(w, x, y, z) quaternion for a rotation of ``yaw_rad`` about +Y.""" + half = 0.5 * float(yaw_rad) + return (math.cos(half), 0.0, math.sin(half), 0.0) + + +def solve_yaw( + camera_quaternion_wxyz: Sequence[float], + camera_translation: Sequence[float], + object_centroid: Sequence[float] | None = None, + degenerate_eps: float = DEFAULT_DEGENERATE_EPS, +) -> dict[str, Any]: + """Solve the upright-object yaw from the crop camera's COLMAP pose. + + Parameters + ---------- + camera_quaternion_wxyz, camera_translation : + The crop frame's WORLD→CAMERA pose exactly as recorded by + ``object_crops`` provenance (``camera_pose.quaternion_wxyz`` / + ``camera_pose.translation``, i.e. COLMAP images.txt fields). + object_centroid : + Optional Gaussian-subset centroid in COLMAP world coordinates (the + ``placement.centroid`` the position solve already uses). When given, + the front direction is the exact object→camera ray (method + ``"camera-ray"``); otherwise the camera's optical axis is used as a + proxy (method ``"optical-axis"`` — exact only for a centred object). + degenerate_eps : + Horizontal-norm threshold below which yaw is unobservable. + + Returns + ------- + dict with plain-JSON values (the placements.json fragment): + quat_wxyz [w, x, y, z] rotation in the USD stage frame — a pure + rotation about +Y (identity when degenerate). + yaw_deg solved yaw in degrees (0 when degenerate). + method "camera-ray" | "optical-axis" | "degenerate". + elevation_deg camera elevation above the object's horizontal plane — + a confidence signal (high |elevation| → the visible + surface was mostly top/bottom; treat yaw as weak). + front_usd the (unit) desired front direction, USD frame, + pre-projection — diagnostic. + + Deterministic, pure, raises ValueError only on malformed input. + """ + q = normalize_quat_wxyz(camera_quaternion_wxyz) + t = _as_vec3(camera_translation, "camera_translation") + + front_colmap: Optional[tuple[float, float, float]] = None + method = "optical-axis" + if object_centroid is not None: + cx, cy, cz = _as_vec3(object_centroid, "object_centroid") + cam = camera_center_colmap(q, t) + front_colmap = _normalize3((cam[0] - cx, cam[1] - cy, cam[2] - cz)) + if front_colmap is not None: + method = "camera-ray" + # else: centroid coincides with the camera center (corrupt hint) — + # fall through to the optical-axis proxy rather than failing. + + if front_colmap is None: + fwd = camera_forward_colmap(q) + # The object is IN FRONT of the camera; its front points BACK at it. + front_colmap = (-fwd[0], -fwd[1], -fwd[2]) # unit: row of a rotation + method = "optical-axis" + + front_usd = colmap_dir_to_usd(front_colmap) + horiz = math.hypot(front_usd[0], front_usd[2]) + elevation_deg = math.degrees(math.atan2(front_usd[1], horiz)) + + yaw = yaw_from_front_usd(front_usd, degenerate_eps) + if yaw is None: + return { + "quat_wxyz": list(IDENTITY_QUAT_WXYZ), + "yaw_deg": 0.0, + "method": "degenerate", + "elevation_deg": elevation_deg, + "front_usd": list(front_usd), + } + + return { + "quat_wxyz": list(quat_wxyz_from_yaw(yaw)), + "yaw_deg": math.degrees(yaw), + "method": method, + "elevation_deg": elevation_deg, + "front_usd": list(front_usd), + } diff --git a/src/pipeline/pixal3d_client.py b/src/pipeline/pixal3d_client.py new file mode 100644 index 00000000..0c4a062a --- /dev/null +++ b/src/pipeline/pixal3d_client.py @@ -0,0 +1,527 @@ +# SPDX-FileCopyrightText: 2026 LichtFeld Studio Authors +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Pixal3D client — SINGLE-image object generation (PRD v4 R8 gated eval). + +Pixal3D (TencentARC, May 2026, SIGGRAPH 2026, **MIT** — ADR-025 amendment +2026-07-09 corrected the original "non-MIT" claim) is pixel-aligned generation +on the TRELLIS.2 backbone: it back-projects pixel features from the +conditioning image into 3D for near-reconstruction input fidelity. It is the +likely TRELLIS.2 successor and is evaluated head-to-head against TRELLIS.2 on +the R9 crop set (see ``research/2026-07-pixal3d-eval.md``) before any +adoption decision. + +STATUS — HONEST SCAFFOLD, NOT A LIVE-VERIFIED GENERATOR + Pixal3D weights are NOT staged in ``data/comfyui/models/`` and no service + or node pack for it exists in this environment yet. This client is + validated against the *mirrored contract* (``Trellis2Client``'s public + surface + the PRD v4 R5 native-service HTTP contract), so it drops into + the ``stages._generate_object_from_crop`` generator chain unchanged the + day an executor exists. Every point where the real Pixal3D API could not + be confirmed is marked ``VERIFY-ON-ENV-BUILD``. + +Two executors, selected by config (mirroring ``trellis2_client``): + +* ``native_url`` set — a thin HTTP service wrapping the native Pixal3D + pipeline (a future ``scripts/pixal3d_native_service.py``), speaking the + SAME contract as ``scripts/trellis2_native_service.py``: multipart + ``image`` + form params → ``{glb_high_b64, glb_low_b64?, lineage}``. + Preferred (PRD v4 R5 discipline: native pipelines for the 3D half). + VERIFY-ON-ENV-BUILD: the underlying Pixal3D python API (assumed + ``pipeline.run(image)``-shaped since it sits on the TRELLIS.2 backbone). +* ``native_url`` empty — a ComfyUI workflow executor. As of 2026-07-10 there + is NO runtime-verified Pixal3D node pack; this path fails fast with a + clear error unless a workflow JSON (``pixal3d_single_image_pbr.json``) + has been authored and verified at env build. VERIFY-ON-ENV-BUILD: node + pack existence, node class names, and node ids. + +The returned GLB bytes are the artifact: callers persist them verbatim +(hash-recorded) and must not re-export through trimesh (PRD v4 R6). + +Usage:: + + from pipeline.pixal3d_client import Pixal3DClient + client = Pixal3DClient(native_url="http://gaussian-toolkit:8403") + result = client.reconstruct_from_image("object_crops/0001_vase.png") + Path("vase.glb").write_bytes(result.glb_data) +""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import logging +import tempfile +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Optional + +import requests +import trimesh + +logger = logging.getLogger(__name__) + +WORKFLOW_DIR = Path(__file__).parent / "workflows" +# VERIFY-ON-ENV-BUILD: this workflow does not exist yet. It must be authored +# against a runtime-verified Pixal3D node pack (none confirmed as of +# 2026-07-10) before the ComfyUI executor can run. +PIXAL3D_SI_WORKFLOW = WORKFLOW_DIR / "pixal3d_single_image_pbr.json" + +_CROP_PLACEHOLDER = "CROP_IMAGE_PLACEHOLDER" + +# Provenance pins (docs/audit/object-pipeline-audit-2026-07-09.md §B.1). +PIXAL3D_HF_REPO = "TencentARC/Pixal3D" +PIXAL3D_PAPER = "arXiv:2605.10922" +PIXAL3D_LICENCE = "MIT" # ADR-025 amendment 2026-07-09 (GitHub LICENSE + HF card) +PIXAL3D_BACKBONE = "TRELLIS.2 (pixel-aligned conditioning)" + + +@dataclass +class Pixal3DResult: + """Result of a Pixal3D single-image object generation. + + Field-for-field compatible with ``trellis2_client.Trellis2Result`` so the + two generators are interchangeable in ``stages._generate_object_from_crop`` + and ``eval/objects/run_eval.py``. ``glb_data`` is the generator's PBR GLB + byte-for-byte; ``mesh`` is a trimesh view for stats/placement only — + never re-export it (PRD v4 R6). + """ + mesh: Optional[trimesh.Trimesh] = None + glb_data: Optional[bytes] = None + glb_low_data: Optional[bytes] = None # decimated game-res pair (native service) + backend: str = "pixal3d-single-image" + duration_seconds: float = 0.0 + prompt_id: str = "" + output_paths: dict[str, str] = field(default_factory=dict) + lineage: dict[str, Any] = field(default_factory=dict) + error: Optional[str] = None + + @property + def glb_sha256(self) -> str: + return hashlib.sha256(self.glb_data).hexdigest() if self.glb_data else "" + + @property + def vertex_count(self) -> int: + return 0 if self.mesh is None else len(self.mesh.vertices) + + @property + def face_count(self) -> int: + return 0 if self.mesh is None else len(self.mesh.faces) + + @property + def has_texture(self) -> bool: + return ( + self.mesh is not None + and getattr(self.mesh.visual, "kind", None) in ("texture", "vertex") + ) + + +class Pixal3DClient: + """Client for Pixal3D single-image object generation (contract scaffold). + + Mirrors ``Trellis2Client``'s public surface exactly: + ``reconstruct_from_image(image_path, seed, label, provenance)`` returning + a result with ``glb_data`` / ``mesh`` / ``face_count`` / ``lineage`` / + ``error`` / ``glb_sha256``. + + Parameters + ---------- + comfyui_url : str + ComfyUI URL for the (currently unavailable) node-pack executor. + native_url : str + Native-pipeline service URL (future ``scripts/pixal3d_native_service.py``, + same HTTP contract as the TRELLIS.2 native service). Empty selects the + ComfyUI workflow — which fails fast until a verified workflow exists. + timeout : int + Maximum seconds per generation. + resolution : str + Structure resolution ladder. VERIFY-ON-ENV-BUILD: assumed to share + the TRELLIS.2 backbone ladder (``512`` … ``1536_cascade``). + texture_size : int + PBR texture resolution (default 4096). + seed : int + Generation seed (re-rolls are the first escalation rung, PRD v4 R7). + ss_steps / shape_steps / tex_steps : int + Sampling steps for the sparse-structure / shape / texture stages + (TRELLIS.2-backbone naming; VERIFY-ON-ENV-BUILD for Pixal3D's actual + knob names — the native service maps them). + face_count_high / face_count_low : int + Remesh targets for the high-poly artifact and (native service only) + the decimated low-poly pair. + """ + + def __init__( + self, + comfyui_url: str = "http://vitrine-comfyui:8188", + native_url: str = "", + timeout: int = 1800, + poll_interval: float = 2.0, + resolution: str = "1536_cascade", + texture_size: int = 4096, + seed: int = 42, + ss_steps: int = 12, + shape_steps: int = 12, + tex_steps: int = 12, + face_count_high: int = 500_000, + face_count_low: int = 20_000, + workflow_path: str | Path | None = None, + ): + self.comfyui_url = comfyui_url.rstrip("/") + self.native_url = native_url.rstrip("/") if native_url else "" + self.timeout = timeout + self.poll_interval = poll_interval + self.resolution = resolution + self.texture_size = texture_size + self.seed = seed + self.ss_steps = ss_steps + self.shape_steps = shape_steps + self.tex_steps = tex_steps + self.face_count_high = face_count_high + self.face_count_low = face_count_low + self.workflow_path = Path(workflow_path) if workflow_path else PIXAL3D_SI_WORKFLOW + self.session = requests.Session() + + @classmethod + def from_config(cls, cfg: Any) -> "Pixal3DClient": + """Construct from a Pixal3DConfig, reading every field defensively.""" + return cls( + comfyui_url=getattr(cfg, "comfyui_url", "http://vitrine-comfyui:8188"), + native_url=getattr(cfg, "native_url", ""), + timeout=getattr(cfg, "timeout", 1800), + resolution=getattr(cfg, "resolution", "1536_cascade"), + texture_size=getattr(cfg, "texture_size", 4096), + seed=getattr(cfg, "seed", 42), + ss_steps=getattr(cfg, "ss_steps", 12), + shape_steps=getattr(cfg, "shape_steps", 12), + tex_steps=getattr(cfg, "tex_steps", 12), + face_count_high=getattr(cfg, "face_count_high", 500_000), + face_count_low=getattr(cfg, "face_count_low", 20_000), + workflow_path=getattr(cfg, "workflow_path", None) or None, + ) + + # ------------------------------------------------------------------ + # Shared HTTP plumbing (mirrors Trellis2Client; ComfyUI native API is + # executor-generic and stable — these calls are NOT Pixal3D-specific) + # ------------------------------------------------------------------ + + def health_check(self) -> bool: + try: + if self.native_url: + r = self.session.get(f"{self.native_url}/health", timeout=10) + else: + r = self.session.get(f"{self.comfyui_url}/system_stats", timeout=10) + return r.status_code == 200 + except requests.RequestException: + return False + + def _upload_image(self, image_path: Path) -> str: + with open(image_path, "rb") as f: + files = {"image": (image_path.name, f, "image/png")} + r = self.session.post( + f"{self.comfyui_url}/upload/image", files=files, timeout=30, + ) + r.raise_for_status() + return r.json().get("name", image_path.name) + + def _submit_prompt(self, prompt: dict) -> str: + r = self.session.post( + f"{self.comfyui_url}/prompt", json={"prompt": prompt}, timeout=30, + ) + data = r.json() + if data.get("error") or data.get("node_errors"): + node_errors = data.get("node_errors", {}) + details = "; ".join( + f"node {nid}: {e.get('errors', e)}" for nid, e in node_errors.items() + ) if node_errors else str(data.get("error")) + raise RuntimeError(f"ComfyUI validation error: {details}") + prompt_id = data.get("prompt_id") + if not prompt_id: + raise RuntimeError(f"No prompt_id in response: {data}") + return prompt_id + + def _poll_completion(self, prompt_id: str) -> dict: + deadline = time.monotonic() + self.timeout + last_log = 0.0 + while time.monotonic() < deadline: + time.sleep(self.poll_interval) + try: + r = self.session.get( + f"{self.comfyui_url}/history/{prompt_id}", timeout=60, + ) + hist = r.json() + except (requests.ReadTimeout, requests.ConnectionError) as e: + logger.debug("History poll transient error: %s", e) + continue + if prompt_id not in hist: + if time.monotonic() - last_log > 30: + logger.info("Waiting for Pixal3D prompt %s...", prompt_id[:8]) + last_log = time.monotonic() + continue + entry = hist[prompt_id] + status = entry.get("status", {}).get("status_str", "unknown") + if status == "success": + return entry + if status == "error": + messages = entry.get("status", {}).get("messages", []) + raise RuntimeError(f"Pixal3D execution error: {messages}") + if time.monotonic() - last_log > 30: + logger.info("Pixal3D %s status: %s", prompt_id[:8], status) + last_log = time.monotonic() + raise TimeoutError(f"Pixal3D prompt {prompt_id} timed out after {self.timeout}s") + + def _free_vram(self) -> None: + """POST /free to unload models + free VRAM (serial lifecycle, ADR-013). + Best-effort, never raises. No-op for the native service.""" + if self.native_url: + return + try: + self.session.post( + f"{self.comfyui_url}/free", + json={"unload_models": True, "free_memory": True}, timeout=30, + ) + logger.info("freed ComfyUI VRAM after Pixal3D generation") + except requests.RequestException as e: # noqa: BLE001 + logger.warning("free_vram failed: %s", e) + + def _download_file(self, filename: str, subfolder: str = "") -> bytes: + for file_type in ("output", "temp"): + r = self.session.get( + f"{self.comfyui_url}/view", + params={"filename": filename, "subfolder": subfolder, "type": file_type}, + timeout=120, + ) + if r.status_code == 200 and len(r.content) > 0: + return r.content + raise FileNotFoundError(f"Cannot download {subfolder}/{filename} from ComfyUI") + + def _extract_glb_refs(self, history: dict) -> list[tuple[str, str]]: + """Scan ComfyUI history outputs for downloadable GLB references. + + Robust to the exact ui key (``3d``, ``gltf``, ``result``, ``meshes``, + ``text``): returns (filename, subfolder) pairs for anything ending in + .glb — same tolerant scan as the TRELLIS.2 client, since export nodes + frequently terminate in a Preview3D whose ui output carries the file. + """ + refs: list[tuple[str, str]] = [] + outputs = history.get("outputs", {}) + + def add(fname: str, sub: str = "") -> None: + if fname and fname.lower().endswith(".glb"): + refs.append((Path(fname).name, sub or str(Path(fname).parent) if "/" in fname else sub)) + + for _node_id, node_output in outputs.items(): + for _key, items in node_output.items(): + if not isinstance(items, list): + if isinstance(items, str): + add(items) + continue + for item in items: + if isinstance(item, str): + add(item) + elif isinstance(item, dict): + fn = item.get("filename") or item.get("model_file") or "" + add(fn, item.get("subfolder", "")) + return refs + + def _load_glb(self, data: bytes) -> Optional[trimesh.Trimesh]: + """Trimesh view of a GLB for stats/placement — NOT for re-export.""" + with tempfile.NamedTemporaryFile(suffix=".glb", delete=False) as tmp: + tmp.write(data) + tmp.flush() + scene = trimesh.load(tmp.name, file_type="glb", force="scene") + if isinstance(scene, trimesh.Scene): + meshes = [g for g in scene.geometry.values() if isinstance(g, trimesh.Trimesh)] + if not meshes: + return None + return meshes[0] if len(meshes) == 1 else trimesh.util.concatenate(meshes) + if isinstance(scene, trimesh.Trimesh): + return scene + return None + + # ------------------------------------------------------------------ + # Workflow construction (ComfyUI executor — SCAFFOLD) + # ------------------------------------------------------------------ + + def _build_prompt(self, uploaded_name: str, seed: int, label: str) -> dict: + """Load + parameterize the Pixal3D workflow. + + VERIFY-ON-ENV-BUILD: no runtime-verified Pixal3D node pack exists as + of 2026-07-10, so node ids/class names cannot be pinned the way + ``trellis2_client._build_prompt`` pins them. Parameter injection is + therefore *input-name driven* (any node input literally named + ``seed`` / ``texture_size`` / ``target_face_count`` / + ``filename_prefix`` is set) — re-pin to explicit node ids once the + workflow JSON is authored against verified nodes. + """ + with open(self.workflow_path) as f: + prompt = json.load(f) + prompt = {k: v for k, v in prompt.items() if not k.startswith("_")} + + for node in prompt.values(): + ins = node.get("inputs", {}) + if ins.get("image") == _CROP_PLACEHOLDER: + ins["image"] = uploaded_name + if "seed" in ins: + ins["seed"] = seed + if "resolution" in ins: + ins["resolution"] = self.resolution + if "texture_size" in ins: + ins["texture_size"] = self.texture_size + if "target_face_count" in ins: + ins["target_face_count"] = self.face_count_high + if "filename_prefix" in ins: + ins["filename_prefix"] = f"vitrine_object_{label}" + return prompt + + # ------------------------------------------------------------------ + # Executors + # ------------------------------------------------------------------ + + def _generate_native(self, image_path: Path, seed: int, label: str) -> Pixal3DResult: + """POST the crop to the native-pipeline service. + + Contract (mirrors ``scripts/trellis2_native_service.py``, PRD v4 R5): + multipart ``image`` + form params; JSON response + ``{glb_high_b64, glb_low_b64?, lineage}``. A future + ``scripts/pixal3d_native_service.py`` must honour this contract. + VERIFY-ON-ENV-BUILD: the wrapped Pixal3D python API itself. + """ + t0 = time.monotonic() + with open(image_path, "rb") as f: + r = self.session.post( + f"{self.native_url}/generate", + files={"image": (image_path.name, f, "image/png")}, + data={ + "seed": str(seed), + "resolution": self.resolution, + "texture_size": str(self.texture_size), + "ss_steps": str(self.ss_steps), + "shape_steps": str(self.shape_steps), + "tex_steps": str(self.tex_steps), + "face_count_high": str(self.face_count_high), + "face_count_low": str(self.face_count_low), + "label": label, + }, + timeout=self.timeout, + ) + r.raise_for_status() + payload = r.json() + result = Pixal3DResult( + backend="pixal3d-native-single-image", + duration_seconds=time.monotonic() - t0, + lineage=payload.get("lineage", {}), + ) + if payload.get("glb_high_b64"): + result.glb_data = base64.b64decode(payload["glb_high_b64"]) + result.mesh = self._load_glb(result.glb_data) + if payload.get("glb_low_b64"): + result.glb_low_data = base64.b64decode(payload["glb_low_b64"]) + if result.glb_data is None: + result.error = payload.get("error", "native service returned no GLB") + return result + + def _generate_comfyui(self, image_path: Path, seed: int, label: str) -> Pixal3DResult: + """Run the single-image workflow on ComfyUI (SCAFFOLD executor).""" + t0 = time.monotonic() + # Begin from a clean GPU (serial lifecycle, ADR-013). + self._free_vram() + + uploaded = self._upload_image(image_path) + prompt = self._build_prompt(uploaded, seed=seed, label=label) + prompt_id = self._submit_prompt(prompt) + logger.info("Submitted Pixal3D single-image prompt %s", prompt_id) + + history = self._poll_completion(prompt_id) + elapsed = time.monotonic() - t0 + logger.info("Pixal3D completed in %.1fs", elapsed) + + result = Pixal3DResult( + backend="pixal3d-comfyui-single-image", + duration_seconds=elapsed, + prompt_id=prompt_id, + ) + for fname, sub in self._extract_glb_refs(history): + try: + data = self._download_file(fname, sub) + result.glb_data = data + result.output_paths[f"{sub}/{fname}" if sub else fname] = fname + result.mesh = self._load_glb(data) + if result.mesh is not None: + logger.info("Loaded Pixal3D object: %d verts, %d faces", + result.vertex_count, result.face_count) + break + except (FileNotFoundError, requests.RequestException) as e: + logger.warning("Could not download %s/%s: %s", sub, fname, e) + + if result.glb_data is None: + result.error = "No retrievable GLB in Pixal3D outputs" + logger.warning("Pixal3D: %s (history outputs scanned)", result.error) + + # Serial lifecycle: free the models so the next object has full VRAM. + self._free_vram() + return result + + # ------------------------------------------------------------------ + # Public entry point + # ------------------------------------------------------------------ + + def reconstruct_from_image( + self, + image_path: str | Path, + seed: int | None = None, + label: str = "object", + provenance: dict | None = None, + ) -> Pixal3DResult: + """Generate a PBR-textured object GLB from ONE matted crop. + + Drop-in for the ADR-025 generator chain: same signature and result + shape as ``Trellis2Client.reconstruct_from_image``. ``provenance`` + (the object_crops manifest entry) is folded into the result lineage. + Backsides are model-completed (``surface: inferred``); Pixal3D's + pixel-aligned conditioning is expected to improve *front* fidelity + specifically — the head-to-head eval measures exactly that. + + Raises loudly (never fakes a result) when no executor is available: + the generator chain in ``stages._generate_object_from_crop`` catches + and falls through to TRELLIS.2 / Hunyuan3D. + """ + image_path = Path(image_path) + if not image_path.exists(): + raise FileNotFoundError(f"Crop not found: {image_path}") + if not self.native_url and not self.workflow_path.exists(): + # Fail fast BEFORE any HTTP: there is nothing to execute against. + raise RuntimeError( + "Pixal3D has no available executor: native_url is unset and " + f"no ComfyUI workflow exists at {self.workflow_path} " + "(no runtime-verified Pixal3D node pack as of 2026-07-10 — " + "VERIFY-ON-ENV-BUILD; stand up scripts/pixal3d_native_service.py " + "or author + verify pixal3d_single_image_pbr.json)") + seed = self.seed if seed is None else seed + safe = "".join(c if c.isalnum() else "_" for c in label)[:40] or "object" + + logger.info("Pixal3D single-image generation: %s (%s/%d, seed=%d)", + image_path.name, self.resolution, self.texture_size, seed) + + if self.native_url: + result = self._generate_native(image_path, seed, safe) + else: + result = self._generate_comfyui(image_path, seed, safe) + + result.lineage = { + "conditioning": "single-image", + "crop": str(image_path), + "generator": "Pixal3D", + "generator_backbone": PIXAL3D_BACKBONE, + "generator_repo": PIXAL3D_HF_REPO, + "generator_paper": PIXAL3D_PAPER, + "licence": PIXAL3D_LICENCE, + "executor": result.backend, + "resolution": self.resolution, + "seed": seed, + "surface": "observed-front/inferred-back", + **({"source": provenance} if provenance else {}), + **result.lineage, + } + return result diff --git a/src/pipeline/workflows/qwen_image_edit_view.json b/src/pipeline/workflows/qwen_image_edit_view.json new file mode 100644 index 00000000..27e01581 --- /dev/null +++ b/src/pipeline/workflows/qwen_image_edit_view.json @@ -0,0 +1,107 @@ +{ + "_comment": "Qwen-Image-Edit-2509 alternate-view synthesis (ADR-025 D4 / PRD v4 R7 rung b): one object crop + an instruction ('show the back') -> ONE edited single image, fed to the 3D generator as an additional best-of-N candidate. NOT panel stitching. Graph mirrors the Comfy-Org Qwen-Image-Edit-2509 template: TextEncodeQwenImageEditPlus (runtime-verified installed, sota_registry 2026-06-19) conditions on clip+vae+image1; ModelSamplingAuraFlow(shift=3.1)+CFGNorm wrap the UNET; KSampler 20 steps cfg 2.5 denoise 1.0; input scaled to ~1MP. Placeholders: CROP_IMAGE_PLACEHOLDER, EDIT_INSTRUCTION_PLACEHOLDER; the client injects model filenames/seed/steps/cfg/shift at submit time.", + "_scaffold": "SCAFFOLD — NOT yet runtime-verified end-to-end: the Qwen-Image-Edit diffusion UNET is NOT staged in data/comfyui/models (registry ground truth 2026-06-19; UNETLoader lists no qwen). unet_name/clip_name below are the canonical Comfy-Org release filenames and MUST be validated against /object_info once the weights are pulled (ImageEditView.probe_edit_model() does the UNET half at runtime). Node input names for TextEncodeQwenImageEditPlus (clip, prompt, vae, image1) match the installed node per the registry note; sampler wiring follows the official template but has not been executed against the live server.", + "1": { + "class_type": "UNETLoader", + "inputs": { + "unet_name": "qwen_image_edit_2509_fp8_e4m3fn.safetensors", + "weight_dtype": "default" + } + }, + "2": { + "class_type": "CLIPLoader", + "inputs": { + "clip_name": "qwen_2.5_vl_7b_fp8_scaled.safetensors", + "type": "qwen_image", + "device": "default" + } + }, + "3": { + "class_type": "VAELoader", + "inputs": { + "vae_name": "qwen_image_vae.safetensors" + } + }, + "10": { + "class_type": "LoadImage", + "inputs": { + "image": "CROP_IMAGE_PLACEHOLDER" + } + }, + "12": { + "class_type": "ImageScaleToTotalPixels", + "inputs": { + "image": ["10", 0], + "upscale_method": "lanczos", + "megapixels": 1.0 + } + }, + "20": { + "class_type": "TextEncodeQwenImageEditPlus", + "inputs": { + "clip": ["2", 0], + "prompt": "EDIT_INSTRUCTION_PLACEHOLDER", + "vae": ["3", 0], + "image1": ["12", 0] + } + }, + "21": { + "class_type": "TextEncodeQwenImageEditPlus", + "inputs": { + "clip": ["2", 0], + "prompt": "", + "vae": ["3", 0], + "image1": ["12", 0] + } + }, + "30": { + "class_type": "VAEEncode", + "inputs": { + "pixels": ["12", 0], + "vae": ["3", 0] + } + }, + "40": { + "class_type": "ModelSamplingAuraFlow", + "inputs": { + "model": ["1", 0], + "shift": 3.1 + } + }, + "41": { + "class_type": "CFGNorm", + "inputs": { + "model": ["40", 0], + "strength": 1.0 + } + }, + "50": { + "class_type": "KSampler", + "inputs": { + "seed": 42, + "steps": 20, + "cfg": 2.5, + "sampler_name": "euler", + "scheduler": "simple", + "denoise": 1.0, + "model": ["41", 0], + "positive": ["20", 0], + "negative": ["21", 0], + "latent_image": ["30", 0] + } + }, + "60": { + "class_type": "VAEDecode", + "inputs": { + "samples": ["50", 0], + "vae": ["3", 0] + } + }, + "70": { + "class_type": "SaveImage", + "inputs": { + "images": ["60", 0], + "filename_prefix": "vitrine_editview" + } + } +} diff --git a/tests/python/test_image_edit_view.py b/tests/python/test_image_edit_view.py new file mode 100644 index 00000000..e7eb6c03 --- /dev/null +++ b/tests/python/test_image_edit_view.py @@ -0,0 +1,331 @@ +# SPDX-FileCopyrightText: 2026 LichtFeld Studio Authors +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Unit tests for pipeline.image_edit_view (ADR-025 D4 / PRD v4 R7 rung b). + +All HTTP and GPU calls are mocked: no live ComfyUI, no Qwen weights. The tests +pin the escalation-rung contract: ONE crop + an instruction in, ONE edited +single image out (never a panel), lineage flags every synthesized pixel as +``surface: image-edit-inferred``, and the client degrades gracefully when the +edit UNET is not staged (the current registry ground truth). +""" + +from __future__ import annotations + +import json +import sys +import types +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +_SRC = Path(__file__).resolve().parents[2] / "src" +if str(_SRC) not in sys.path: + sys.path.insert(0, str(_SRC)) + +from pipeline.image_edit_view import ( # noqa: E402 + DEFAULT_BACK_INSTRUCTION, + ImageEditResult, + ImageEditView, + QWEN_EDIT_VIEW_WORKFLOW, +) + +EDITED_PNG = b"\x89PNG-EDITED-FAKE-BYTES" * 8 +RGBA_PNG = b"\x89PNG-RGBA-MATTED-FAKE" * 8 + + +@pytest.fixture() +def crop(tmp_path) -> Path: + p = tmp_path / "0001_vase.png" + p.write_bytes(b"\x89PNG-fake") # undecodable -> flatten falls back + return p + + +def _client(**kw) -> ImageEditView: + c = ImageEditView(comfyui_url="http://comfy:8188", poll_interval=0.001, **kw) + return c + + +def _mock_comfy_session(history_outputs: dict) -> MagicMock: + session = MagicMock() + + def post(url, **kw): + resp = MagicMock(status_code=200) + if url.endswith("/upload/image"): + resp.json.return_value = {"name": "uploaded_vase.png"} + elif url.endswith("/prompt"): + resp.json.return_value = {"prompt_id": "e123"} + else: # /free + resp.json.return_value = {} + return resp + + def get(url, **kw): + resp = MagicMock(status_code=200) + if "/history/" in url: + resp.json.return_value = { + "e123": {"status": {"status_str": "success"}, + "outputs": history_outputs}, + } + elif "/view" in url: + resp.content = EDITED_PNG + return resp + + session.post.side_effect = post + session.get.side_effect = get + return session + + +# --------------------------------------------------------------------------- +# Workflow template + prompt construction +# --------------------------------------------------------------------------- + +def test_shipped_workflow_is_single_image_single_output(): + graph = json.loads(QWEN_EDIT_VIEW_WORKFLOW.read_text()) + nodes = {n["class_type"] for k, n in graph.items() if not k.startswith("_")} + # One conditioning image in, one edited image out — never a panel set. + loads = [n for k, n in graph.items() + if not k.startswith("_") and n["class_type"] == "LoadImage"] + assert len(loads) == 1 + saves = [n for k, n in graph.items() + if not k.startswith("_") and n["class_type"] == "SaveImage"] + assert len(saves) == 1 + # Instruction-driven Qwen edit conditioning (runtime-verified node name). + assert "TextEncodeQwenImageEditPlus" in nodes + # The retired panel-path nodes must never reappear here. + assert "ImageBatch" not in nodes + assert "Trellis2MultiViewImageToShape" not in nodes + # The scaffold is honestly marked until the UNET is staged + e2e-verified. + assert "_scaffold" in graph + + +def test_build_prompt_substitutes_image_instruction_and_parameters(): + client = _client(steps=8, cfg=3.0, shift=2.5, denoise=0.9, negative="blurry", + diffusion_model="qwen_custom.safetensors", + clip_model="clip_custom.safetensors", + vae_model="vae_custom.safetensors") + prompt = client._build_prompt("uploaded_vase.png", + "show the back of this object", + seed=7, label="vase") + + assert prompt["10"]["inputs"]["image"] == "uploaded_vase.png" + assert prompt["20"]["inputs"]["prompt"] == "show the back of this object" + assert prompt["21"]["inputs"]["prompt"] == "blurry" # negative + assert prompt["1"]["inputs"]["unet_name"] == "qwen_custom.safetensors" + assert prompt["2"]["inputs"]["clip_name"] == "clip_custom.safetensors" + assert prompt["3"]["inputs"]["vae_name"] == "vae_custom.safetensors" + assert prompt["40"]["inputs"]["shift"] == 2.5 + assert prompt["50"]["inputs"]["seed"] == 7 + assert prompt["50"]["inputs"]["steps"] == 8 + assert prompt["50"]["inputs"]["cfg"] == 3.0 + assert prompt["50"]["inputs"]["denoise"] == 0.9 + assert prompt["70"]["inputs"]["filename_prefix"] == "vitrine_editview_vase" + # Comment/scaffold keys never reach the ComfyUI API. + assert not any(k.startswith("_") for k in prompt) + + +# --------------------------------------------------------------------------- +# edit_view happy path +# --------------------------------------------------------------------------- + +def test_edit_view_returns_single_edited_image(crop, tmp_path): + client = _client() + client.session = _mock_comfy_session({ + "70": {"images": [{"filename": "vitrine_editview_vase_00001_.png", + "subfolder": "", "type": "output"}]}, + }) + client._matte_output = MagicMock(return_value=None) # rembg absent path + out = tmp_path / "alt" / "vase_back.png" + + result = client.edit_view(crop, label="vase", seed=99, + provenance={"source_frame": "f1.jpg"}, + output_path=out) + + assert result.error is None + assert result.raw_image_data == EDITED_PNG # verbatim download + assert result.image_data == EDITED_PNG # unmatted fallback + assert result.image_path == out + assert out.read_bytes() == EDITED_PNG + assert result.image_sha256 + # Lineage: edit model + instruction + inferred-surface flag + provenance. + assert result.lineage["edit_model"] == "Qwen-Image-Edit-2509" + assert result.lineage["instruction"] == DEFAULT_BACK_INSTRUCTION + assert result.lineage["surface"] == "image-edit-inferred" + assert result.lineage["source_crop"] == str(crop) + assert result.lineage["seed"] == 99 + assert result.lineage["matted"] is False + assert result.lineage["source"]["source_frame"] == "f1.jpg" + # Exactly ONE image uploaded — single-image contract, no panels. + uploads = [c for c in client.session.post.call_args_list + if c.args and c.args[0].endswith("/upload/image")] + assert len(uploads) == 1 + # Serial VRAM lifecycle: /free before AND after the edit. + frees = [c for c in client.session.post.call_args_list + if c.args and c.args[0].endswith("/free")] + assert len(frees) == 2 + + +def test_edit_view_mattes_output_when_rembg_available(crop): + client = _client() + client.session = _mock_comfy_session({ + "70": {"images": [{"filename": "v.png", "subfolder": ""}]}, + }) + client._matte_output = MagicMock(return_value=RGBA_PNG) + + result = client.edit_view(crop, label="vase") + assert result.raw_image_data == EDITED_PNG # kept verbatim + assert result.image_data == RGBA_PNG # matted candidate + assert result.lineage["matted"] is True + client._matte_output.assert_called_once_with(EDITED_PNG) + + +def test_edit_view_custom_instruction_lands_in_lineage_and_prompt(crop): + client = _client() + client.session = _mock_comfy_session({ + "70": {"images": [{"filename": "v.png", "subfolder": ""}]}, + }) + client._matte_output = MagicMock(return_value=None) + + result = client.edit_view(crop, instruction="show the left side", label="vase") + assert result.lineage["instruction"] == "show the left side" + submits = [c for c in client.session.post.call_args_list + if c.args and c.args[0].endswith("/prompt")] + graph = submits[0].kwargs["json"]["prompt"] + assert graph["20"]["inputs"]["prompt"] == "show the left side" + + +# --------------------------------------------------------------------------- +# Error paths +# --------------------------------------------------------------------------- + +def test_edit_view_missing_crop_raises(tmp_path): + with pytest.raises(FileNotFoundError): + _client().edit_view(tmp_path / "nope.png") + + +def test_edit_view_reports_missing_output(crop): + client = _client() + client.session = _mock_comfy_session({}) # no outputs at all + result = client.edit_view(crop, label="vase") + assert result.image_data is None + assert result.image_path is None + assert result.error + + +def test_submit_prompt_surfaces_node_errors(crop): + client = _client() + session = MagicMock() + resp = MagicMock(status_code=200) + resp.json.return_value = {"error": "invalid prompt", + "node_errors": {"1": {"errors": ["unet not found"]}}} + session.post.return_value = resp + client.session = session + with pytest.raises(RuntimeError, match="unet not found"): + client._submit_prompt({"1": {}}) + + +# --------------------------------------------------------------------------- +# Staging probe (the UNET is NOT staged yet — registry ground truth) +# --------------------------------------------------------------------------- + +def _object_info_session(unet_names: list[str]) -> MagicMock: + session = MagicMock() + resp = MagicMock(status_code=200) + resp.json.return_value = { + "UNETLoader": {"input": {"required": {"unet_name": [unet_names]}}}, + } + session.get.return_value = resp + return session + + +def test_probe_edit_model_exact_hit(): + client = _client() + client.session = _object_info_session( + ["flux2_dev_fp8mixed.safetensors", "qwen_image_edit_2509_fp8_e4m3fn.safetensors"]) + assert client.probe_edit_model() == "qwen_image_edit_2509_fp8_e4m3fn.safetensors" + + +def test_probe_edit_model_fuzzy_hit_adopts_staged_name(): + client = _client() + client.session = _object_info_session(["Qwen-Image-Edit-2509_bf16.safetensors"]) + assert client.probe_edit_model() == "Qwen-Image-Edit-2509_bf16.safetensors" + assert client.diffusion_model == "Qwen-Image-Edit-2509_bf16.safetensors" + + +def test_probe_edit_model_absent_returns_none(): + client = _client() + client.session = _object_info_session(["flux2_dev_fp8mixed.safetensors"]) + assert client.probe_edit_model() is None + + +def test_probe_edit_model_connection_error_returns_none(): + import requests + client = _client() + session = MagicMock() + session.get.side_effect = requests.ConnectionError("down") + client.session = session + assert client.probe_edit_model() is None + + +# --------------------------------------------------------------------------- +# Input preparation (RGBA flatten) — best-effort, never fatal +# --------------------------------------------------------------------------- + +def test_prepare_input_flattens_rgba_onto_white(tmp_path): + Image = pytest.importorskip("PIL.Image") + p = tmp_path / "crop.png" + im = Image.new("RGBA", (8, 8), (0, 0, 0, 0)) # fully transparent + im.putpixel((4, 4), (200, 10, 10, 255)) # one opaque pixel + im.save(p) + + path, flattened = _client()._prepare_input(p) + assert flattened is True + assert path != p + with Image.open(path) as flat: + assert flat.mode == "RGB" + assert flat.getpixel((0, 0)) == (255, 255, 255) # transparent -> white + assert flat.getpixel((4, 4)) == (200, 10, 10) # object preserved + + +def test_prepare_input_falls_back_on_undecodable_bytes(crop): + path, flattened = _client()._prepare_input(crop) + assert path == crop + assert flattened is False + + +def test_matte_output_returns_none_without_rembg(monkeypatch): + # Force the guarded import to fail even if rembg happens to be installed. + monkeypatch.setitem(sys.modules, "rembg", None) + assert _client()._matte_output(EDITED_PNG) is None + + +# --------------------------------------------------------------------------- +# Config plumbing +# --------------------------------------------------------------------------- + +def test_from_config_reads_fields(): + cfg = types.SimpleNamespace( + comfyui_url="http://c:8188", timeout=99, + diffusion_model="d.safetensors", clip_model="c.safetensors", + vae_model="v.safetensors", instruction="show the back", + negative="ugly", steps=4, cfg=1.0, shift=3.0, denoise=0.8, + seed=5, matte_output=False, + ) + client = ImageEditView.from_config(cfg) + assert client.comfyui_url == "http://c:8188" + assert client.timeout == 99 + assert client.diffusion_model == "d.safetensors" + assert client.instruction == "show the back" + assert (client.steps, client.cfg, client.shift, client.denoise) == (4, 1.0, 3.0, 0.8) + assert client.matte_output is False + + +def test_from_config_defaults_when_fields_absent(): + client = ImageEditView.from_config(types.SimpleNamespace()) + assert client.instruction == DEFAULT_BACK_INSTRUCTION + assert client.diffusion_model == "qwen_image_edit_2509_fp8_e4m3fn.safetensors" + assert client.matte_output is True + + +def test_result_sha_empty_without_image(): + assert ImageEditResult().image_sha256 == "" diff --git a/tests/python/test_object_orientation.py b/tests/python/test_object_orientation.py new file mode 100644 index 00000000..a357829a --- /dev/null +++ b/tests/python/test_object_orientation.py @@ -0,0 +1,254 @@ +# SPDX-FileCopyrightText: 2026 LichtFeld Studio Authors +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Unit tests for pipeline.object_orientation (ADR-025 D3 / PRD v4 R10). + +Pure math — no deps, no I/O. Every fixture below is derived by hand from the +COLMAP world→camera convention (p_cam = R p + t; camera RDF: +X right, +Y +down, +Z forward) and the assembler's colmap→USD map (x, y, z) → (x, -y, -z). + +Fixture derivation (level cameras, object at the COLMAP/USD origin): + * "camera on +X_usd looking at origin": desired USD viewing dir (-1,0,0) + → COLMAP forward (-1,0,0) = row3(R). Level ⇒ camera-down = world-down + = (0,+1,0) (COLMAP up is -Y) = row2. row1 = down × forward = (0,0,1). + R = Ry(90°) ⇒ q = (cos45°, 0, sin45°, 0). Center C = (2,0,0) ⇒ + t = -R C = (0,0,2). Expected: object front faces +X_usd ⇒ yaw = 90°, + quat = (cos45°, 0, sin45°, 0). + * "+Z_usd": R = I, C = (0,0,-2), t = (0,0,2) ⇒ yaw 0 (identity). + * "-X_usd": R = Ry(-90°) ⇒ q = (cos45°, 0, -sin45°, 0), t = (0,0,2) + ⇒ yaw -90°. + * "-Z_usd": R = Ry(180°) ⇒ q = (0,0,1,0), t = (0,0,2) ⇒ |yaw| = 180°. +""" + +from __future__ import annotations + +import math +import sys +from pathlib import Path + +import pytest + +_SRC = Path(__file__).resolve().parents[2] / "src" +if str(_SRC) not in sys.path: + sys.path.insert(0, str(_SRC)) + +from pipeline.object_orientation import ( # noqa: E402 + FRONT_AXIS_USD, + IDENTITY_QUAT_WXYZ, + camera_center_colmap, + camera_forward_colmap, + colmap_dir_to_usd, + normalize_quat_wxyz, + quat_multiply, + quat_to_rotation_matrix, + quat_wxyz_from_yaw, + rotate_vec_by_quat, + solve_yaw, + yaw_from_front_usd, +) + +S = math.sqrt(0.5) # cos45° = sin45° + +# Level cameras looking at the origin, hand-derived above. +CAM_PLUS_X = ((S, 0.0, S, 0.0), (0.0, 0.0, 2.0)) # on +X_usd +CAM_PLUS_Z = ((1.0, 0.0, 0.0, 0.0), (0.0, 0.0, 2.0)) # on +Z_usd +CAM_MINUS_X = ((S, 0.0, -S, 0.0), (0.0, 0.0, 2.0)) # on -X_usd +CAM_MINUS_Z = ((0.0, 0.0, 1.0, 0.0), (0.0, 0.0, 2.0)) # on -Z_usd + + +def _assert_unit_pure_yaw(quat): + w, x, y, z = quat + assert math.sqrt(w * w + x * x + y * y + z * z) == pytest.approx(1.0) + assert x == pytest.approx(0.0, abs=1e-12) # never tips the object + assert z == pytest.approx(0.0, abs=1e-12) # never rolls the object + + +def _rotated_front(quat): + return rotate_vec_by_quat(quat, FRONT_AXIS_USD) + + +# --------------------------------------------------------------------------- +# Geometry helpers pin the conventions +# --------------------------------------------------------------------------- + +def test_camera_center_is_minus_R_transpose_t(): + # R = Ry(90°) as world→cam, t = (0,0,2): C = -R^T t = (2, 0, 0). + c = camera_center_colmap(*CAM_PLUS_X) + assert c == pytest.approx((2.0, 0.0, 0.0)) + + +def test_camera_forward_is_third_row_of_R(): + # Identity pose looks along +Z_colmap. + assert camera_forward_colmap((1, 0, 0, 0)) == pytest.approx((0.0, 0.0, 1.0)) + # The +X_usd camera looks along -X_colmap (toward the origin). + assert camera_forward_colmap(CAM_PLUS_X[0]) == pytest.approx((-1.0, 0.0, 0.0)) + + +def test_colmap_dir_to_usd_flips_y_and_z_without_scale(): + assert colmap_dir_to_usd((1.0, 2.0, 3.0)) == pytest.approx((1.0, -2.0, -3.0)) + + +def test_quat_multiply_composes_rotations(): + qa = quat_wxyz_from_yaw(math.radians(30)) + qb = quat_wxyz_from_yaw(math.radians(60)) + Rab = quat_to_rotation_matrix(quat_multiply(qa, qb)) + R90 = quat_to_rotation_matrix(quat_wxyz_from_yaw(math.radians(90))) + for i in range(3): + assert Rab[i] == pytest.approx(R90[i]) + + +def test_yaw_round_trip_through_front_vector(): + for deg in (-179.0, -90.0, -45.0, 0.0, 30.0, 90.0, 135.0, 179.0): + yaw = math.radians(deg) + front = (math.sin(yaw), 0.0, math.cos(yaw)) + assert math.degrees(yaw_from_front_usd(front)) == pytest.approx(deg) + + +# --------------------------------------------------------------------------- +# The headline solve: known camera pose → expected yaw +# --------------------------------------------------------------------------- + +def test_camera_on_plus_x_makes_object_face_plus_x(): + out = solve_yaw(*CAM_PLUS_X) + assert out["method"] == "optical-axis" + assert out["yaw_deg"] == pytest.approx(90.0) + assert out["quat_wxyz"] == pytest.approx([S, 0.0, S, 0.0]) + _assert_unit_pure_yaw(out["quat_wxyz"]) + assert _rotated_front(out["quat_wxyz"]) == pytest.approx((1.0, 0.0, 0.0)) + assert out["elevation_deg"] == pytest.approx(0.0) + + +def test_camera_on_plus_z_is_identity(): + out = solve_yaw(*CAM_PLUS_Z) + assert out["yaw_deg"] == pytest.approx(0.0) + assert out["quat_wxyz"] == pytest.approx(list(IDENTITY_QUAT_WXYZ)) + assert _rotated_front(out["quat_wxyz"]) == pytest.approx((0.0, 0.0, 1.0)) + + +def test_camera_on_minus_x_makes_object_face_minus_x(): + out = solve_yaw(*CAM_MINUS_X) + assert out["yaw_deg"] == pytest.approx(-90.0) + assert _rotated_front(out["quat_wxyz"]) == pytest.approx((-1.0, 0.0, 0.0)) + + +def test_camera_behind_turns_object_around(): + out = solve_yaw(*CAM_MINUS_Z) + assert abs(out["yaw_deg"]) == pytest.approx(180.0) + assert _rotated_front(out["quat_wxyz"]) == pytest.approx((0.0, 0.0, -1.0)) + _assert_unit_pure_yaw(out["quat_wxyz"]) + + +# --------------------------------------------------------------------------- +# The upright constraint: elevation is discarded, never tips the object +# --------------------------------------------------------------------------- + +def test_elevated_camera_keeps_object_upright_optical_axis(): + # The +X_usd camera pitched 45° down toward the origin: world→cam gains + # a premultiplied Rx(45°) (verified: R = Rx(45°) @ Ry(90°) has + # row3 = forward = normalize((-1, 1, 0)) in COLMAP = toward the origin + # from the raised center C = (2, -2, 0); t = -R C = (0, 0, 2√2)). + q_pitch = (math.cos(math.radians(22.5)), math.sin(math.radians(22.5)), 0.0, 0.0) + q = quat_multiply(q_pitch, CAM_PLUS_X[0]) + t = (0.0, 0.0, 2.0 * math.sqrt(2.0)) + # Self-check the composed fixture before trusting the assertion. + assert camera_forward_colmap(q) == pytest.approx((-S, S, 0.0)) + assert camera_center_colmap(q, t) == pytest.approx((2.0, -2.0, 0.0)) + + out = solve_yaw(q, t) + assert out["method"] == "optical-axis" + assert out["yaw_deg"] == pytest.approx(90.0) # pitch discarded + assert out["elevation_deg"] == pytest.approx(45.0) # ...but reported + _assert_unit_pure_yaw(out["quat_wxyz"]) + # Rotated front stays horizontal: the object is never tipped. + assert _rotated_front(out["quat_wxyz"]) == pytest.approx((1.0, 0.0, 0.0)) + + +def test_elevated_camera_keeps_object_upright_camera_ray(): + # Same level +X-camera ROTATION but center raised to C = (2, -2, 0) + # (t = -R C = (0, 2, 2)); the object centroid pins the exact ray. + q = CAM_PLUS_X[0] + t = (0.0, 2.0, 2.0) + assert camera_center_colmap(q, t) == pytest.approx((2.0, -2.0, 0.0)) + + out = solve_yaw(q, t, object_centroid=(0.0, 0.0, 0.0)) + assert out["method"] == "camera-ray" + assert out["yaw_deg"] == pytest.approx(90.0) + assert out["elevation_deg"] == pytest.approx(45.0) + _assert_unit_pure_yaw(out["quat_wxyz"]) + + +def test_camera_ray_beats_optical_axis_for_off_centre_objects(): + # Identity camera at C = (0,0,-2) looking along +Z_colmap; the object + # sits off-axis at (2, 0, 0). Optical axis says yaw 0; the true + # object→camera ray normalize((-2,0,-2)) → USD (-S, 0, S) says -45°. + q, t = CAM_PLUS_Z + assert solve_yaw(q, t)["yaw_deg"] == pytest.approx(0.0) + out = solve_yaw(q, t, object_centroid=(2.0, 0.0, 0.0)) + assert out["method"] == "camera-ray" + assert out["yaw_deg"] == pytest.approx(-45.0) + + +# --------------------------------------------------------------------------- +# Degenerate + malformed inputs +# --------------------------------------------------------------------------- + +def test_top_down_camera_is_degenerate_identity(): + # Straight-down camera: forward = world-down = (0, +1, 0) in COLMAP + # (COLMAP up is -Y). R = Rx(90°) ⇒ q = (cos45°, sin45°, 0, 0); center + # C = (0, -3, 0) above the object ⇒ t = -R C = (0, 0, 3). + q = (S, S, 0.0, 0.0) + t = (0.0, 0.0, 3.0) + assert camera_forward_colmap(q) == pytest.approx((0.0, 1.0, 0.0)) + + out = solve_yaw(q, t) + assert out["method"] == "degenerate" + assert out["quat_wxyz"] == pytest.approx(list(IDENTITY_QUAT_WXYZ)) + assert out["yaw_deg"] == 0.0 + assert out["elevation_deg"] == pytest.approx(90.0) + + # Same verdict when the ray comes from a centroid directly underneath. + out = solve_yaw(q, t, object_centroid=(0.0, 0.0, 0.0)) + assert out["method"] == "degenerate" + + +def test_centroid_coincident_with_camera_falls_back_to_optical_axis(): + q, t = CAM_PLUS_Z + out = solve_yaw(q, t, object_centroid=(0.0, 0.0, -2.0)) # == camera center + assert out["method"] == "optical-axis" + assert out["yaw_deg"] == pytest.approx(0.0) + + +def test_malformed_inputs_raise_value_error(): + with pytest.raises(ValueError): + solve_yaw((0.0, 0.0, 0.0, 0.0), (0.0, 0.0, 1.0)) # zero quat + with pytest.raises(ValueError): + solve_yaw((1.0, 0.0, 0.0), (0.0, 0.0, 1.0)) # 3-quat + with pytest.raises(ValueError): + solve_yaw((1.0, 0.0, 0.0, 0.0), (0.0, 1.0)) # 2-translation + with pytest.raises(ValueError): + solve_yaw((1.0, 0.0, 0.0, 0.0), None) # missing t + with pytest.raises(ValueError): + normalize_quat_wxyz(None) + + +def test_unnormalized_quaternion_is_accepted(): + # COLMAP text quats can drift off unit norm; scaling must not change yaw. + q, t = CAM_PLUS_X + scaled = tuple(3.7 * c for c in q) + assert solve_yaw(scaled, t)["yaw_deg"] == pytest.approx(90.0) + + +# --------------------------------------------------------------------------- +# Output schema (the placements.json fragment) +# --------------------------------------------------------------------------- + +def test_result_schema_is_json_ready(): + out = solve_yaw(*CAM_PLUS_X, object_centroid=(0.0, 0.0, 0.0)) + assert set(out) == {"quat_wxyz", "yaw_deg", "method", "elevation_deg", + "front_usd"} + assert isinstance(out["quat_wxyz"], list) and len(out["quat_wxyz"]) == 4 + assert isinstance(out["front_usd"], list) and len(out["front_usd"]) == 3 + assert all(isinstance(v, float) for v in out["quat_wxyz"] + out["front_usd"]) + assert isinstance(out["yaw_deg"], float) + assert isinstance(out["elevation_deg"], float) + assert isinstance(out["method"], str) diff --git a/tests/python/test_pixal3d_client.py b/tests/python/test_pixal3d_client.py new file mode 100644 index 00000000..a2fda09c --- /dev/null +++ b/tests/python/test_pixal3d_client.py @@ -0,0 +1,287 @@ +# SPDX-FileCopyrightText: 2026 LichtFeld Studio Authors +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Unit tests for pipeline.pixal3d_client (single-image, PRD v4 R8). + +All HTTP and GPU calls are mocked: no live ComfyUI/native service, no GPU, +and — critically — no Pixal3D weights (they are NOT staged; the client is a +contract scaffold). The tests pin the drop-in contract instead: + +* Pixal3DResult is field-compatible with Trellis2Result, so the generator + chain and eval harness can swap generators without code changes. +* GLB bytes are returned verbatim (PRD v4 R6). +* Lineage records generator=Pixal3D, the TRELLIS.2-backbone note, and the + MIT licence (ADR-025 amendment 2026-07-09). +* With no executor available the client raises loudly (never fakes). +""" + +from __future__ import annotations + +import base64 +import dataclasses +import json +import sys +import types +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +_SRC = Path(__file__).resolve().parents[2] / "src" +if str(_SRC) not in sys.path: + sys.path.insert(0, str(_SRC)) + +# Minimal trimesh stub when the real one is absent (keeps CI slim). +try: + import trimesh as _trimesh_real # noqa: F401 +except ImportError: + _stub = types.ModuleType("trimesh") + _stub.Trimesh = MagicMock + _stub.Scene = MagicMock + _stub.load = MagicMock(return_value=MagicMock()) + _util = types.ModuleType("trimesh.util") + _util.concatenate = MagicMock(return_value=MagicMock()) + _stub.util = _util + sys.modules["trimesh"] = _stub + sys.modules["trimesh.util"] = _util + +from pipeline.pixal3d_client import ( # noqa: E402 + Pixal3DClient, + Pixal3DResult, + PIXAL3D_SI_WORKFLOW, +) +from pipeline.trellis2_client import Trellis2Result # noqa: E402 + +GLB_BYTES = b"glTF-FAKE-PIXAL-PAYLOAD" * 10 + + +@pytest.fixture() +def crop(tmp_path) -> Path: + p = tmp_path / "0001_vase.png" + p.write_bytes(b"\x89PNG-fake") + return p + + +@pytest.fixture() +def workflow(tmp_path) -> Path: + """A placeholder single-image workflow (stands in for the env-build-time + pixal3d_single_image_pbr.json, which does not exist yet).""" + wf = { + "_meta": {"note": "comment keys must be stripped"}, + "10": {"class_type": "LoadImage", + "inputs": {"image": "CROP_IMAGE_PLACEHOLDER"}}, + "40": {"class_type": "Pixal3DImageToShape", + "inputs": {"seed": 0, "resolution": "512"}}, + "60": {"class_type": "Pixal3DBakePBR", + "inputs": {"texture_size": 512, "target_face_count": 1}}, + "70": {"class_type": "ExportGLB", + "inputs": {"filename_prefix": "x"}}, + } + p = tmp_path / "pixal3d_single_image_pbr.json" + p.write_text(json.dumps(wf)) + return p + + +def _client(**kw) -> Pixal3DClient: + c = Pixal3DClient(comfyui_url="http://comfy:8188", **kw) + c._load_glb = MagicMock(return_value=None) # no real GLB parsing in unit tests + return c + + +# --------------------------------------------------------------------------- +# Drop-in contract: Pixal3DResult must be Trellis2Result-compatible +# --------------------------------------------------------------------------- + +def test_result_is_field_compatible_with_trellis2result(): + t2_fields = {f.name for f in dataclasses.fields(Trellis2Result)} + px_fields = {f.name for f in dataclasses.fields(Pixal3DResult)} + assert t2_fields <= px_fields, f"missing fields: {t2_fields - px_fields}" + # The derived properties the generator chain + eval harness consume. + for prop in ("glb_sha256", "vertex_count", "face_count", "has_texture"): + assert isinstance(getattr(Pixal3DResult, prop), property) + + +def test_public_surface_mirrors_trellis2client(): + from pipeline.trellis2_client import Trellis2Client + for name in ("reconstruct_from_image", "from_config", "health_check"): + assert callable(getattr(Pixal3DClient, name)) + assert callable(getattr(Trellis2Client, name)) + import inspect + px_sig = inspect.signature(Pixal3DClient.reconstruct_from_image) + t2_sig = inspect.signature(Trellis2Client.reconstruct_from_image) + assert list(px_sig.parameters) == list(t2_sig.parameters) + + +def test_result_sha_empty_without_glb(): + assert Pixal3DResult().glb_sha256 == "" + + +# --------------------------------------------------------------------------- +# No-executor fail-fast (weights/nodes not staged — never fake a result) +# --------------------------------------------------------------------------- + +def test_shipped_workflow_does_not_exist_yet(): + # Honesty pin: the ComfyUI workflow is VERIFY-ON-ENV-BUILD. If this file + # appears, the fail-fast test below and the scaffold docstrings must be + # revisited (and this test updated) as part of the env build. + assert not PIXAL3D_SI_WORKFLOW.exists() + + +def test_no_executor_raises_loudly(crop): + client = _client() # no native_url, no workflow file + with pytest.raises(RuntimeError, match="VERIFY-ON-ENV-BUILD"): + client.reconstruct_from_image(crop) + # Fail-fast means NO HTTP was attempted. + client.session = MagicMock() + with pytest.raises(RuntimeError): + client.reconstruct_from_image(crop) + client.session.post.assert_not_called() + client.session.get.assert_not_called() + + +def test_missing_crop_raises(tmp_path): + with pytest.raises(FileNotFoundError): + _client().reconstruct_from_image(tmp_path / "nope.png") + + +# --------------------------------------------------------------------------- +# Workflow prompt construction (ComfyUI scaffold executor) +# --------------------------------------------------------------------------- + +def test_build_prompt_substitutes_crop_and_parameters(workflow): + client = _client(workflow_path=workflow, resolution="1024_cascade", + texture_size=2048, face_count_high=123_456) + prompt = client._build_prompt("uploaded_vase.png", seed=7, label="vase") + + assert prompt["10"]["inputs"]["image"] == "uploaded_vase.png" + assert prompt["40"]["inputs"]["seed"] == 7 + assert prompt["40"]["inputs"]["resolution"] == "1024_cascade" + assert prompt["60"]["inputs"]["texture_size"] == 2048 + assert prompt["60"]["inputs"]["target_face_count"] == 123_456 + assert prompt["70"]["inputs"]["filename_prefix"] == "vitrine_object_vase" + # Comment keys never reach the ComfyUI API. + assert not any(k.startswith("_") for k in prompt) + + +# --------------------------------------------------------------------------- +# ComfyUI executor (mocked session) +# --------------------------------------------------------------------------- + +def _mock_comfy_session(history_outputs: dict) -> MagicMock: + session = MagicMock() + + def post(url, **kw): + resp = MagicMock(status_code=200) + if url.endswith("/upload/image"): + resp.json.return_value = {"name": "uploaded_vase.png"} + elif url.endswith("/prompt"): + resp.json.return_value = {"prompt_id": "p123"} + else: # /free + resp.json.return_value = {} + return resp + + def get(url, **kw): + resp = MagicMock(status_code=200) + if "/history/" in url: + resp.json.return_value = { + "p123": {"status": {"status_str": "success"}, + "outputs": history_outputs}, + } + elif "/view" in url: + resp.content = GLB_BYTES + return resp + + session.post.side_effect = post + session.get.side_effect = get + return session + + +def test_comfyui_executor_returns_glb_bytes_verbatim(crop, workflow): + client = _client(workflow_path=workflow, poll_interval=0.001) + client.session = _mock_comfy_session({ + "80": {"result": [{"filename": "vitrine_object_vase.glb", "subfolder": ""}]}, + }) + result = client.reconstruct_from_image(crop, label="vase", + provenance={"source_frame": "f1.jpg"}) + assert result.glb_data == GLB_BYTES # byte-identical (R6) + assert result.glb_sha256 # hash recorded + assert result.backend == "pixal3d-comfyui-single-image" + # Lineage: generator + backbone + licence (ADR-025 amendment). + assert result.lineage["conditioning"] == "single-image" + assert result.lineage["generator"] == "Pixal3D" + assert "TRELLIS.2" in result.lineage["generator_backbone"] + assert result.lineage["licence"] == "MIT" + assert result.lineage["generator_repo"] == "TencentARC/Pixal3D" + assert result.lineage["source"]["source_frame"] == "f1.jpg" + # Only ONE image was uploaded — single-crop conditioning (ADR-025). + uploads = [c for c in client.session.post.call_args_list + if c.args and c.args[0].endswith("/upload/image")] + assert len(uploads) == 1 + + +def test_comfyui_executor_reports_missing_glb(crop, workflow): + client = _client(workflow_path=workflow, poll_interval=0.001) + client.session = _mock_comfy_session({}) # no outputs at all + result = client.reconstruct_from_image(crop, label="vase") + assert result.glb_data is None + assert result.error + + +# --------------------------------------------------------------------------- +# Native-service executor (mirrored PRD v4 R5 contract) +# --------------------------------------------------------------------------- + +def test_native_executor_decodes_high_low_pair(crop): + client = _client(native_url="http://native:8403") + resp = MagicMock(status_code=200) + resp.json.return_value = { + "glb_high_b64": base64.b64encode(GLB_BYTES).decode(), + "glb_low_b64": base64.b64encode(b"low-poly").decode(), + "lineage": {"timings": {"run_s": 30.0}}, + } + client.session.post = MagicMock(return_value=resp) + + result = client.reconstruct_from_image(crop, label="vase") + assert result.glb_data == GLB_BYTES + assert result.glb_low_data == b"low-poly" + assert result.backend == "pixal3d-native-single-image" + assert result.lineage["timings"] == {"run_s": 30.0} + assert result.lineage["generator"] == "Pixal3D" + url = client.session.post.call_args.args[0] + assert url == "http://native:8403/generate" + + +def test_native_executor_error_path(crop): + client = _client(native_url="http://native:8403") + resp = MagicMock(status_code=200) + resp.json.return_value = {"error": "OOM at 1536"} + client.session.post = MagicMock(return_value=resp) + + result = client.reconstruct_from_image(crop, label="vase") + assert result.glb_data is None + assert result.error == "OOM at 1536" + + +# --------------------------------------------------------------------------- +# from_config +# --------------------------------------------------------------------------- + +def test_from_config_reads_all_fields(): + cfg = types.SimpleNamespace( + comfyui_url="http://c:8188", native_url="http://n:8403", timeout=99, + resolution="512", texture_size=1024, seed=1, ss_steps=2, shape_steps=3, + tex_steps=4, face_count_high=5, face_count_low=6, + ) + client = Pixal3DClient.from_config(cfg) + assert client.native_url == "http://n:8403" + assert client.timeout == 99 + assert (client.ss_steps, client.shape_steps, client.tex_steps) == (2, 3, 4) + assert (client.face_count_high, client.face_count_low) == (5, 6) + + +def test_from_config_defensive_defaults(): + client = Pixal3DClient.from_config(types.SimpleNamespace()) + assert client.comfyui_url == "http://vitrine-comfyui:8188" + assert client.native_url == "" + assert client.resolution == "1536_cascade" + assert client.workflow_path == PIXAL3D_SI_WORKFLOW diff --git a/tests/python/test_trellis2_native_service.py b/tests/python/test_trellis2_native_service.py new file mode 100644 index 00000000..3be1ac32 --- /dev/null +++ b/tests/python/test_trellis2_native_service.py @@ -0,0 +1,248 @@ +# SPDX-FileCopyrightText: 2026 LichtFeld Studio Authors +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Unit tests for scripts/trellis2_native_service (PRD v4 R5). + +Hermetic: ``_load_pipeline``/``_generate`` are mocked, so no model, no GPU, +no native trellis2 package needed — just Flask's test client. The tests pin +the HTTP contract the pipeline client (pipeline.trellis2_client, +``_generate_native``) already speaks: multipart ``image`` + form params in; +``{glb_high_b64, glb_low_b64, lineage}`` out; the 400/503/500 taxonomy. +""" + +from __future__ import annotations + +import base64 +import hashlib +import importlib.util +import io +import sys +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +pytest.importorskip("flask", reason="native-service tests need flask") +pytest.importorskip("PIL", reason="native-service tests need pillow") + + +def _load_service(): + """Import the service by path — scripts/ is not a package.""" + path = Path(__file__).resolve().parents[2] / "scripts" / "trellis2_native_service.py" + spec = importlib.util.spec_from_file_location("trellis2_native_service", path) + module = importlib.util.module_from_spec(spec) + sys.modules["trellis2_native_service"] = module + spec.loader.exec_module(module) + return module + + +service = _load_service() + +TIMINGS = {"run_s": 1.2, "to_glb_s": 3.4} + + +@pytest.fixture() +def client(): + service.app.config["TESTING"] = True + return service.app.test_client() + + +@pytest.fixture() +def png_bytes() -> bytes: + from PIL import Image + buf = io.BytesIO() + Image.new("RGBA", (8, 8), (255, 0, 0, 255)).save(buf, "PNG") + return buf.getvalue() + + +def _post(client, png_bytes, **form): + data = {"image": (io.BytesIO(png_bytes), "crop.png"), **form} + return client.post("/generate", data=data, content_type="multipart/form-data") + + +# --------------------------------------------------------------------------- +# /health vs /ready +# --------------------------------------------------------------------------- + +def test_health_is_liveness_only(client): + resp = client.get("/health") + assert resp.status_code == 200 # up even with no env + body = resp.get_json() + assert body["status"] == "ok" + assert body["pipeline_loaded"] is False + assert body["service"] == "trellis2-native" + + +def test_ready_503_until_pipeline_loaded(client, monkeypatch): + monkeypatch.setattr(service, "_pipeline", None) + monkeypatch.setattr(service, "_native_available", lambda: False) + resp = client.get("/ready") + assert resp.status_code == 503 + body = resp.get_json() + assert body["ready"] is False + assert body["native_package_available"] is False + assert body["error"] + + +def test_ready_200_once_loaded(client, monkeypatch): + monkeypatch.setattr(service, "_pipeline", MagicMock()) + resp = client.get("/ready") + assert resp.status_code == 200 + assert resp.get_json()["ready"] is True + + +# --------------------------------------------------------------------------- +# /generate happy path (the client contract) +# --------------------------------------------------------------------------- + +def test_generate_happy_path_returns_glb_pair_and_lineage(client, png_bytes, monkeypatch): + gen = MagicMock(return_value=(b"HIGH-GLB", b"LOW-GLB", TIMINGS)) + monkeypatch.setattr(service, "_generate", gen) + + # Exactly the form fields pipeline.trellis2_client._generate_native sends. + resp = _post(client, png_bytes, + seed="7", resolution="1024_cascade", texture_size="2048", + ss_steps="9", shape_steps="10", tex_steps="11", + face_count_high="100000", face_count_low="5000", label="vase") + assert resp.status_code == 200 + payload = resp.get_json() + assert base64.b64decode(payload["glb_high_b64"]) == b"HIGH-GLB" # verbatim (R6) + assert base64.b64decode(payload["glb_low_b64"]) == b"LOW-GLB" + + lin = payload["lineage"] + assert lin["generator"] == "TRELLIS.2-4B" + assert lin["executor"] == "native-service" + assert lin["conditioning"] == "single-image" + assert lin["seed"] == 7 + assert lin["resolution"] == "1024_cascade" + assert lin["label"] == "vase" + assert lin["timings"] == TIMINGS + # Reproducibility: the input crop is hash-recorded with its dimensions. + assert lin["input"]["sha256"] == hashlib.sha256(png_bytes).hexdigest() + assert (lin["input"]["width"], lin["input"]["height"]) == (8, 8) + assert lin["artifact_bytes"] == {"glb_high": 8, "glb_low": 7} + + # Params landed on the generator parsed + typed; image is matted RGBA. + image, params = gen.call_args.args + assert image.mode == "RGBA" + assert params["texture_size"] == 2048 + assert (params["ss_steps"], params["shape_steps"], params["tex_steps"]) == (9, 10, 11) + assert (params["face_count_high"], params["face_count_low"]) == (100_000, 5_000) + + +def test_generate_no_params_uses_client_defaults(client, png_bytes, monkeypatch): + gen = MagicMock(return_value=(b"H", b"L", {})) + monkeypatch.setattr(service, "_generate", gen) + assert _post(client, png_bytes).status_code == 200 + params = gen.call_args.args[1] + assert params["seed"] == 42 + assert params["resolution"] == "1536_cascade" + assert params["texture_size"] == 4096 + assert (params["face_count_high"], params["face_count_low"]) == (500_000, 20_000) + assert params["label"] == "object" + + +# --------------------------------------------------------------------------- +# Bad input -> 400 +# --------------------------------------------------------------------------- + +def test_generate_missing_image_400(client): + resp = client.post("/generate", data={"seed": "7"}, + content_type="multipart/form-data") + assert resp.status_code == 400 + assert "image" in resp.get_json()["error"] + + +def test_generate_undecodable_image_400(client): + resp = client.post("/generate", + data={"image": (io.BytesIO(b"not-a-png"), "crop.png")}, + content_type="multipart/form-data") + assert resp.status_code == 400 + assert "decode" in resp.get_json()["error"] + + +def test_generate_empty_image_400(client): + resp = client.post("/generate", + data={"image": (io.BytesIO(b""), "crop.png")}, + content_type="multipart/form-data") + assert resp.status_code == 400 + + +# --------------------------------------------------------------------------- +# Bad params default/clamp (never 400 — the crop still generates) +# --------------------------------------------------------------------------- + +def test_generate_bad_params_default_and_clamp(client, png_bytes, monkeypatch): + gen = MagicMock(return_value=(b"H", b"L", {})) + monkeypatch.setattr(service, "_generate", gen) + resp = _post(client, png_bytes, + seed="banana", # unparseable -> default + texture_size="999999", # out of range -> clamped + ss_steps="0", # below floor -> clamped + resolution="4096_mega", # unknown -> default + face_count_high="1000", + face_count_low="999999999", # -> clamped, then <= high + label="weird label!!") # sanitized + assert resp.status_code == 200 + params = gen.call_args.args[1] + assert params["seed"] == 42 + assert params["texture_size"] == 8192 + assert params["ss_steps"] == 1 + assert params["resolution"] == "1536_cascade" + assert params["face_count_high"] == 1000 + assert params["face_count_low"] == 1000 # low can never exceed high + assert params["label"] == "weird_label__" + + +# --------------------------------------------------------------------------- +# Error taxonomy: 503 env-not-ready vs 500 generation failure +# --------------------------------------------------------------------------- + +def test_generate_env_not_ready_503(client, png_bytes, monkeypatch): + monkeypatch.setattr(service, "_generate", MagicMock( + side_effect=service.EnvNotReadyError("trellis2 package not importable"))) + resp = _post(client, png_bytes) + assert resp.status_code == 503 + assert "trellis2" in resp.get_json()["error"] + + +def test_generate_lazy_import_error_maps_to_503(client, png_bytes, monkeypatch): + # CUDA extensions import lazily inside pipeline.run — env, not a bug. + monkeypatch.setattr(service, "_generate", MagicMock( + side_effect=ImportError("No module named 'nvdiffrast'"))) + resp = _post(client, png_bytes) + assert resp.status_code == 503 + assert "nvdiffrast" in resp.get_json()["error"] + + +def test_generate_failure_500(client, png_bytes, monkeypatch): + monkeypatch.setattr(service, "_generate", MagicMock( + side_effect=RuntimeError("CUDA out of memory"))) + resp = _post(client, png_bytes) + assert resp.status_code == 500 + assert "CUDA out of memory" in resp.get_json()["error"] + + +def test_load_pipeline_absent_package_raises_env_not_ready(monkeypatch): + monkeypatch.setattr(service, "_pipeline", None) + # Deterministic even on a machine that HAS trellis2 installed. + monkeypatch.setitem(sys.modules, "trellis2", None) + monkeypatch.setitem(sys.modules, "trellis2.pipelines", None) + with pytest.raises(service.EnvNotReadyError, match="native TRELLIS.2 env"): + service._load_pipeline() + + +# --------------------------------------------------------------------------- +# Every error path is JSON (the client always parses the body) +# --------------------------------------------------------------------------- + +def test_unknown_route_is_json_404(client): + resp = client.get("/nope") + assert resp.status_code == 404 + assert "error" in resp.get_json() + + +def test_wrong_method_is_json_405(client): + resp = client.get("/generate") + assert resp.status_code == 405 + assert "error" in resp.get_json() From 79e8954efb946654859fe26cd36df5b9dc911779 Mon Sep 17 00:00:00 2001 From: jjohare Date: Sat, 18 Jul 2026 17:11:12 +0100 Subject: [PATCH 2/2] feat(objects): integrate the R&D frontier into the pipeline + CI + docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires the four R&D modules into the shared pipeline (all backward compatible — the default single-shot object path is byte-identical; new tiers are opt-in): - config.py: Pixal3DConfig + ImageEditConfig + Trellis2Config escalation fields (image_edit_escalation/escalation_threshold); PipelineConfig + _from_dict. - stages._generate_object_from_crop: Pixal3D as an opt-in primary tier (R8, falls through to TRELLIS.2 on any failure); the image-edit escalation rung in the best-of-N loop (R7 rung b — best tuple widened to carry edit lineage; a winning edit rewrites surface -> image-edit-inferred); _persist records the crop camera_pose for R10. NOTE: camera_pose reads from the provenance dict (the crops manifest), correcting the teammate diff which assumed crop_entry. - object_placement.build_placements + assemble_usd_scene: R10 orientation — solve_yaw from the camera pose -> placements.json orientation_quat -> assembler AddOrientOp (TRS order). Live-verified: yaw-90 -> xformOp:orient=(0.7071,0,0.7071,0). +2 integration tests. - eval/objects/run_eval.py: --generator pixal3d (constructs the client directly, no pipeline flag needed). - CI: runs the 4 new hermetic suites (+flask dep). Container pipeline suite: 209 passed (+47). Clean-venv CI simulation green. Docs: engineering-log R&D entry; PRD R7 done / R8 partial / R10 done. Co-Authored-By: claude-flow --- .github/workflows/vitrine-ci.yml | 13 ++- docs/engineering-log.md | 49 ++++++++++ eval/objects/run_eval.py | 10 +- .../prd-v4-object-pipeline-convergence.md | 20 ++-- scripts/assemble_usd_scene.py | 10 ++ src/pipeline/config.py | 63 +++++++++++++ src/pipeline/object_placement.py | 23 ++++- src/pipeline/stages.py | 93 ++++++++++++++++++- tests/python/test_object_placement.py | 26 ++++++ 9 files changed, 289 insertions(+), 18 deletions(-) diff --git a/.github/workflows/vitrine-ci.yml b/.github/workflows/vitrine-ci.yml index 79ad08b7..19b19bd6 100644 --- a/.github/workflows/vitrine-ci.yml +++ b/.github/workflows/vitrine-ci.yml @@ -44,16 +44,21 @@ jobs: with: python-version: '3.12' - name: Install light CPU test deps - # Covers the hermetic object-arc suite: numpy (object_crops), requests + - # trimesh (trellis2_client), opencv-headless (object_crops, cv2-gated). - # No torch / sam3 / gsplat — those tests defer or stub GPU deps. - run: pip install pytest pillow numpy requests trimesh opencv-python-headless + # Covers the hermetic object-arc + R&D suite: numpy (object_crops), + # requests + trimesh (generator clients), opencv-headless (object_crops, + # cv2-gated), flask (native service test). No torch/sam3/gsplat — those + # tests defer or stub GPU deps. + run: pip install pytest pillow numpy requests trimesh opencv-python-headless flask - name: Hermetic pipeline unit tests (no GPU/container) run: | pytest -q \ tests/python/test_image_decoder.py \ tests/python/test_object_placement.py \ tests/python/test_object_candidate_score.py \ + tests/python/test_object_orientation.py \ + tests/python/test_trellis2_native_service.py \ + tests/python/test_pixal3d_client.py \ + tests/python/test_image_edit_view.py \ tests/python/test_config_hull.py \ tests/python/test_object_crops.py \ tests/python/test_trellis2_client.py \ diff --git a/docs/engineering-log.md b/docs/engineering-log.md index eb6f100b..51334e7c 100644 --- a/docs/engineering-log.md +++ b/docs/engineering-log.md @@ -2,6 +2,55 @@ Development history for Vitrine (a standalone project that vendors LichtFeld Studio as a pinned tool; formerly a fork — see ADR-021). +## 2026-07-10 — R&D frontier: 4 Fable teammates, branch `feat/object-rd-frontier` + +Ran a 4-agent Fable R&D fan-out on the open object-arc frontier; each teammate +owned distinct new files and returned integration diffs, consolidated serially +onto the branch. All integrations compile, the container pipeline suite is +**209 passed** (+47), and both new geometry paths are live-verified in the USD +assembler. + +- **Native TRELLIS.2 service (R5)** — `scripts/trellis2_native_service.py` + fleshed out: `/health` (liveness) vs `/ready` (pipeline loaded), a full error + taxonomy (400/413/503/500, all JSON), param hardening (unparseable→default, + out-of-range→clamp, `face_count_low ≤ high`), thread-safe lazy load, 64 MB + upload cap, complete lineage. This is the durable answer to the ComfyUI drtk + fragility (the native pipeline uses nvdiffrast, not the ABI-brittle drtk + path). 15 hermetic tests. VERIFY-ON-ENV-BUILD kept on the two upstream-API + wrappers (the ComfyUI-ported tree lacks the `pipelines/` module). +- **R10 orientation solve** — `src/pipeline/object_orientation.py`: the mesh's + canonical front (+Z) corresponds to the observing crop camera's ray, so an + upright-object **yaw** is solved from the COLMAP camera pose (frame math + verified against `colmap_parser` + the assembler; COLMAP world→cam vs USD + Y/Z-flip handled). Wired through `object_placement.build_placements` → + `usd/placements.json` → the assembler's `AddOrientOp` (TRS order). Live USD + check: yaw-90 → `xformOp:orient=(0.7071,0,0.7071,0)`. Roll/pitch stay pinned + upright (single-view can't observe them); near-vertical views flagged + `degenerate`→unsolved. 17 solver tests + 2 integration tests. +- **R7 rung (b) image-edit view** — `src/pipeline/image_edit_view.py` + + `qwen_image_edit_view.json`: for objects whose best seed scores below + `escalation_threshold`, synthesize ONE "show the back" alternate view + (Qwen-Image-Edit-2509, Apache-2.0) and add it as another single-image + best-of-N candidate (NOT panel-stitching). Self-gates via + `probe_edit_model()` (the edit UNET isn't staged), flattens input alpha + (avoids the LoadImage premultiplied-ghost trap), re-mattes the output. A + winning edit-view rewrites the asset's `surface` to `image-edit-inferred`. + 18 tests. +- **R8 Pixal3D** — `src/pipeline/pixal3d_client.py` mirrors the single-image + contract as a drop-in generator (opt-in, default-off, eval-gated), + the + `research/2026-07-pixal3d-eval.md` head-to-head plan. Pixal3D is **MIT** (the + ADR-025 correction), so its licence gate passes; weights are NOT staged, so + the client is a contract-validated scaffold (VERIFY-ON-ENV-BUILD), never + live-run. `eval/objects/run_eval.py --generator pixal3d` added. 13 tests. + +Integration hygiene: config.py gained `Pixal3DConfig` + `ImageEditConfig` + +Trellis2Config escalation fields; stages `_generate_object_from_crop` gained a +Pixal3D primary tier + the image-edit rung (best-of-N tuple widened, backward +compatible — the default single-shot path is byte-identical). CI now runs all +four new hermetic suites (+flask dep). One correction applied during +integration: the orientation camera-pose reads from `provenance` (the crops +manifest), not `crop_entry`, which the teammate's diff had assumed. + ## 2026-07-10 — R7 best-of-N quality ladder + drtk self-heal hardened ### R7 — best-of-N seed re-rolls with a silhouette-consistency scorer diff --git a/eval/objects/run_eval.py b/eval/objects/run_eval.py index 55814a42..7d59da75 100644 --- a/eval/objects/run_eval.py +++ b/eval/objects/run_eval.py @@ -142,6 +142,13 @@ def run_generation(crops_dir: Path, out_dir: Path, generator: str, seed: int) -> client = Trellis2Client.from_config(cfg.trellis2) res = client.reconstruct_from_image(crop, seed=seed, label=name) glb = res.glb_data + elif generator == "pixal3d": + # R8 head-to-head: constructs the client directly, so the eval + # never needs the pipeline flag flipped (cfg.pixal3d.enabled). + from pipeline.pixal3d_client import Pixal3DClient + client = Pixal3DClient.from_config(cfg.pixal3d) + res = client.reconstruct_from_image(crop, seed=seed, label=name) + glb = res.glb_data else: from pipeline.hunyuan3d_client import Hunyuan3DClient client = Hunyuan3DClient.from_config(cfg.hunyuan3d) @@ -171,7 +178,8 @@ def main() -> int: ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument("--crops", type=Path, help="Directory of object crops (RGBA PNGs)") ap.add_argument("--out", type=Path, help="Output directory for GLBs + metrics") - ap.add_argument("--generator", choices=["trellis2", "hy3d21"], default="trellis2") + ap.add_argument("--generator", choices=["trellis2", "hy3d21", "pixal3d"], + default="trellis2") ap.add_argument("--seed", type=int, default=42) ap.add_argument("--stats-only", type=Path, metavar="GLB", help="Grade one existing GLB and exit (no generation)") diff --git a/research/decisions/prd-v4-object-pipeline-convergence.md b/research/decisions/prd-v4-object-pipeline-convergence.md index 248ed806..995a0c71 100644 --- a/research/decisions/prd-v4-object-pipeline-convergence.md +++ b/research/decisions/prd-v4-object-pipeline-convergence.md @@ -10,16 +10,18 @@ > R5 ◐ (single-image client live on the ComfyUI executor; native service > scaffolded, env stand-up pending); R6 ✅ (verbatim GLB + lineage, > texture_bake skips generators); R6a ✅ (Hy3D21 graph in code, broken JSONs -> deleted); R7 ◐ (2026-07-10: best-of-N seed re-rolls + a front-silhouette -> proportion/sanity scorer landed — `object_candidate_score.py`, wired into the -> TRELLIS.2 branch, all candidate scores recorded in lineage; the image-edit -> second-view rung remains); R8 ⏳ (note: Pixal3D is MIT — see ADR-025 -> amendment); R9 ✅ +> deleted); R7 ✅ (best-of-N seed re-rolls + front-silhouette scorer + +> 2026-07-10 the image-edit "show the back" rung (b) — `image_edit_view.py`, +> Qwen-Image-Edit, self-gated + eval-scored; a winning edit rewrites `surface` +> to `image-edit-inferred`); R8 ◐ (2026-07-10: `pixal3d_client.py` drop-in +> scaffold + `--generator pixal3d` eval + head-to-head plan; MIT gate passes; +> weights not staged so VERIFY-ON-ENV-BUILD, not live-run); R9 ✅ > (harness + 3-object baseline committed, live run: vessel 492k faces/441s, -> bottle 483k/158s, block 466k/130s, 0 regressions); R10 ◐ (2026-07-10: -> position + uniform scale SOLVED and applied at assembly via -> `object_placement.py` → `usd/placements.json`; orientation flagged -> `unsolved` — a crop-pose orientation solve remains). Proof assets: +> bottle 483k/158s, block 466k/130s, 0 regressions); R10 ✅ (2026-07-10: +> position + uniform scale + **upright yaw orientation** all solved and applied +> at assembly — `object_placement.py` + `object_orientation.py` → +> `usd/placements.json` → assembler `AddOrientOp`, live-verified; roll/pitch +> pinned upright per the single-view constraint). Proof assets: > `docs/renders/object-pipeline-2026-07-09/`. **Supersedes scope**: the object-generation half of [prd-v3-e2e-closure.md](prd-v3-e2e-closure.md) (selection + recovery); does **not** reopen ingest/provenance/annotation (v3 scope) or diff --git a/scripts/assemble_usd_scene.py b/scripts/assemble_usd_scene.py index b52fffec..8095e9c4 100755 --- a/scripts/assemble_usd_scene.py +++ b/scripts/assemble_usd_scene.py @@ -649,6 +649,16 @@ def assemble_scene( wc = placement.get("world_centroid", [0.0, 0.0, 0.0]) ux, uy, uz = colmap_to_usd_position(wc[0], wc[1], wc[2]) xform.AddTranslateOp().Set(Gf.Vec3d(ux, uy, uz)) + # R10 orientation: apply the solved upright yaw (TRS order; a pure-Y + # rotation commutes with uniform scale anyway). Absent/unsolved -> + # identity (no op), matching legacy placements.json. + oq = placement.get("orientation_quat") + if placement.get("orientation") == "yaw-solved" and oq and len(oq) == 4: + xform.AddOrientOp().Set(Gf.Quatf(oq[0], oq[1], oq[2], oq[3])) + prim.SetCustomDataByKey("v2g:orientation_yaw_deg", + float(placement.get("orientation_yaw_deg", 0.0))) + prim.SetCustomDataByKey("v2g:orientation_method", + placement.get("orientation_method", "")) # scale_ratio is in raw world units; USD positions carry SCENE_SCALE, # so the object's size must be scaled by the same factor to match. s = float(placement.get("scale_ratio", 1.0)) * SCENE_SCALE diff --git a/src/pipeline/config.py b/src/pipeline/config.py index def1c767..b486f6a4 100644 --- a/src/pipeline/config.py +++ b/src/pipeline/config.py @@ -315,6 +315,14 @@ class Trellis2Config: # recorded in the winner's lineage. Default 1 = single-shot (unchanged). best_of_n: int = 1 seeds: list[int] = field(default_factory=list) # explicit seeds; else derived + # Rung (b) of the escalation ladder (ADR-025 D4 / PRD v4 R7): when the best + # seed re-roll scores below ``escalation_threshold``, synthesize ONE + # image-edit alternate view ("show the back", Qwen-Image-Edit-2509) and add + # it as an additional single-image best-of-N candidate — never panel + # synthesis. Off by default (the Qwen edit UNET is not staged yet; the rung + # also self-gates at runtime via ImageEditView.probe_edit_model()). + image_edit_escalation: bool = False + escalation_threshold: float = 0.55 ss_steps: int = 12 # sparse-structure sampling steps shape_steps: int = 12 # shape diffusion steps tex_steps: int = 12 # texture diffusion steps @@ -323,6 +331,57 @@ class Trellis2Config: face_count_low: int = 20_000 +@dataclass +class Pixal3DConfig: + """Pixal3D single-image object generation (PRD v4 R8 — gated eval). + + Pixel-aligned generation on the TRELLIS.2 backbone (TencentARC, MIT — + ADR-025 amendment). DISABLED by default: weights not staged, no executor + stood up; the R8 head-to-head eval (research/2026-07-pixal3d-eval.md) gates + any promotion. When enabled, stages tries Pixal3D FIRST, falling through to + TRELLIS.2 on failure. ``native_url`` selects the thin HTTP service (mirrors + the TRELLIS.2 R5 contract); empty selects the ComfyUI workflow, which fails + fast until a verified node pack + workflow JSON exist (VERIFY-ON-ENV-BUILD). + """ + enabled: bool = False + comfyui_url: str = "http://vitrine-comfyui:8188" + native_url: str = "" + resolution: str = "1536_cascade" # VERIFY-ON-ENV-BUILD: assumed backbone ladder + texture_size: int = 4096 + timeout: int = 1800 + seed: int = 42 + ss_steps: int = 12 + shape_steps: int = 12 + tex_steps: int = 12 + face_count_high: int = 500_000 + face_count_low: int = 20_000 + workflow_path: str = "" # "" = default (not authored yet) + + +@dataclass +class ImageEditConfig: + """Instruction-driven alternate-view edit (ADR-025 D4 / PRD v4 R7 rung b). + + Qwen-Image-Edit-2509 (Apache-2.0, commercial-safe) on ComfyUI — a 2D stage, + in-scope per ADR-014 as narrowed by ADR-025. Loader filenames are the + canonical Comfy-Org release names; the diffusion UNET is NOT staged yet, so + ImageEditView.probe_edit_model() resolves/gates at runtime. + """ + comfyui_url: str = "http://vitrine-comfyui:8188" + diffusion_model: str = "qwen_image_edit_2509_fp8_e4m3fn.safetensors" + clip_model: str = "qwen_2.5_vl_7b_fp8_scaled.safetensors" + vae_model: str = "qwen_image_vae.safetensors" + instruction: str = "" # empty -> DEFAULT_BACK_INSTRUCTION (no import cycle) + negative: str = "" + steps: int = 20 + cfg: float = 2.5 + shift: float = 3.1 + denoise: float = 1.0 + timeout: int = 600 + seed: int = 42 + matte_output: bool = True # rembg RGBA matte of the edited view + + @dataclass class ViewCompletionConfig: """RETIRED from the object arc (ADR-025 supersedes ADR-017). @@ -444,6 +503,8 @@ class PipelineConfig: object_crops: ObjectCropsConfig = field(default_factory=ObjectCropsConfig) mesh: MeshConfig = field(default_factory=MeshConfig) trellis2: Trellis2Config = field(default_factory=Trellis2Config) + pixal3d: Pixal3DConfig = field(default_factory=Pixal3DConfig) + image_edit: ImageEditConfig = field(default_factory=ImageEditConfig) view_completion: ViewCompletionConfig = field(default_factory=ViewCompletionConfig) hunyuan3d: Hunyuan3DConfig = field(default_factory=Hunyuan3DConfig) inpaint: InpaintConfig = field(default_factory=InpaintConfig) @@ -490,6 +551,8 @@ def _from_dict(cls, data: dict[str, Any]) -> PipelineConfig: "object_crops": ObjectCropsConfig, "mesh": MeshConfig, "trellis2": Trellis2Config, + "pixal3d": Pixal3DConfig, + "image_edit": ImageEditConfig, "view_completion": ViewCompletionConfig, "hunyuan3d": Hunyuan3DConfig, "inpaint": InpaintConfig, diff --git a/src/pipeline/object_placement.py b/src/pipeline/object_placement.py index b8507ed4..30d288f5 100644 --- a/src/pipeline/object_placement.py +++ b/src/pipeline/object_placement.py @@ -143,5 +143,26 @@ def build_placements(meshes: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: placement.get("extent", [0, 0, 0]), m.get("glb_extent"), ) - out[label] = p.to_dict() + d = p.to_dict() + + # R10 orientation solve (PRD v4): when the crop's COLMAP camera pose is + # recorded, solve the upright yaw so the object faces the direction it + # was observed from. Additive + best-effort — a bad/absent pose leaves + # orientation "unsolved" (identity), which the assembler already handles. + pose = m.get("camera_pose") + if pose and pose.get("quaternion_wxyz") and pose.get("translation"): + try: + from pipeline.object_orientation import solve_yaw + o = solve_yaw(pose["quaternion_wxyz"], pose["translation"], + object_centroid=placement.get("centroid")) + if o.get("method") != "degenerate": + d["orientation"] = "yaw-solved" + d["orientation_quat"] = o["quat_wxyz"] + d["orientation_yaw_deg"] = o["yaw_deg"] + d["orientation_method"] = o["method"] + d["orientation_elevation_deg"] = o.get("elevation_deg") + except (ValueError, KeyError, ImportError): + pass # stay "unsolved" + + out[label] = d return out diff --git a/src/pipeline/stages.py b/src/pipeline/stages.py index dd6666f5..4d28b4ec 100644 --- a/src/pipeline/stages.py +++ b/src/pipeline/stages.py @@ -2636,6 +2636,9 @@ def _persist(glb_data: bytes, method: str, lineage: dict[str, Any], "crop": str(crop_path), "placement": placement, "glb_extent": glb_extent, + # R10 orientation solve reads the crop's COLMAP camera pose + # (recorded in the object_crops manifest -> provenance here). + "camera_pose": provenance.get("camera_pose") if provenance else None, } if glb_low is not None: low_path = mesh_glb_path.with_name(mesh_glb_path.stem + "_low.glb") @@ -2650,6 +2653,27 @@ def _persist(glb_data: bytes, method: str, lineage: dict[str, Any], errors: list[str] = [] + # PRD v4 R8: Pixal3D (MIT, TRELLIS.2 backbone) as an OPT-IN primary. + # Default-off, eval-gated; on any failure the chain falls through to + # TRELLIS.2 unchanged, so enabling only adds a candidate tier. + pixal3d_cfg = getattr(self.config, "pixal3d", None) + if pixal3d_cfg is not None and getattr(pixal3d_cfg, "enabled", False): + try: + from pipeline.pixal3d_client import Pixal3DClient + px = Pixal3DClient.from_config(pixal3d_cfg) + res = px.reconstruct_from_image( + crop_path, seed=getattr(pixal3d_cfg, "seed", 42), + label=label, provenance=provenance) + if res.glb_data: + logger.info("Pixal3D object '%s': %d verts (%.0fs)", + label, res.vertex_count, res.duration_seconds) + return _persist(res.glb_data, "pixal3d", dict(res.lineage), + glb_low=res.glb_low_data, mesh=res.mesh) + errors.append(f"pixal3d: {res.error or 'no GLB'}") + except Exception as exc: # noqa: BLE001 — fall through to TRELLIS.2 + logger.warning("Pixal3D failed for '%s': %s", label, exc) + errors.append(f"pixal3d: {exc}") + trellis2_cfg = getattr(self.config, "trellis2", None) if trellis2_cfg is not None and getattr(trellis2_cfg, "enabled", False): try: @@ -2665,7 +2689,7 @@ def _persist(glb_data: bytes, method: str, lineage: dict[str, Any], # Generate each seed; score by front-silhouette proportion + # mesh sanity (R7). best_of_n==1 -> single shot, unchanged. - best = None + best = None # (total, result, edit_lineage | None) scoreboard: list[dict[str, Any]] = [] for seed in seeds: res = t2.reconstruct_from_image( @@ -2685,10 +2709,67 @@ def _persist(glb_data: bytes, method: str, lineage: dict[str, Any], "(prop=%.2f sanity=%.2f)", label, seed, res.vertex_count, cs.total, cs.proportion, cs.sanity) if best is None or cs.total > best[0]: - best = (cs.total, res) + best = (cs.total, res, None) + + # Escalation rung (b), ADR-025 D4 / PRD v4 R7: the best seed + # re-roll still scores poorly -> synthesize ONE image-edit + # alternate view and add it as an additional single-image + # candidate. Self-gating: a missing Qwen UNET degrades to + # rung (a) silently-but-logged; a rung-(b) failure never kills + # the seed-re-roll winner. + threshold = float(getattr(trellis2_cfg, "escalation_threshold", 0.55)) + if (getattr(trellis2_cfg, "image_edit_escalation", False) + and (best is None or best[0] < threshold)): + try: + from pipeline.image_edit_view import ImageEditView + iev = ImageEditView.from_config( + getattr(self.config, "image_edit", None)) + if iev.probe_edit_model() is None: + logger.info("image-edit escalation skipped for '%s': " + "edit UNET not staged", label) + else: + edited = iev.edit_view( + crop_path, label=label, provenance=provenance, + output_path=mesh_glb_path.with_name( + mesh_glb_path.stem + "__editview.png")) + if edited.image_path is None: + errors.append(f"image-edit: {edited.error}") + else: + res2 = t2.reconstruct_from_image( + edited.image_path, + seed=getattr(trellis2_cfg, "seed", 42), + label=f"{label}_editview", provenance=provenance) + if not res2.glb_data: + errors.append(f"trellis2(edit-view): " + f"{res2.error or 'no GLB'}") + else: + extent2 = list(res2.mesh.extents) if ( + res2.mesh is not None and getattr( + res2.mesh, "extents", None) is not None + ) else None + wt2 = bool(getattr( + res2.mesh, "is_watertight", False)) \ + if res2.mesh is not None else False + # A 180deg back view preserves the front + # silhouette's width/height for most objects, + # so proportion-vs-crop stays a fair score. + cs2 = ocs.score_candidate( + getattr(trellis2_cfg, "seed", 42), extent2, + crop_aspect, res2.face_count, wt2) + scoreboard.append( + {"view": "image-edit-back", **cs2.__dict__}) + logger.info("TRELLIS.2 '%s' edit-view candidate:" + " %d verts, score=%.3f", label, + res2.vertex_count, cs2.total) + if best is None or cs2.total > best[0]: + best = (cs2.total, res2, edited.lineage) + except Exception as exc: # noqa: BLE001 — rung (b) optional + logger.warning("image-edit escalation failed for '%s': %s", + label, exc) + errors.append(f"image-edit: {exc}") if best is not None: - _, res = best + _, res, edit_lineage = best lineage = dict(res.lineage) if n > 1: lineage["best_of_n"] = n @@ -2697,6 +2778,12 @@ def _persist(glb_data: bytes, method: str, lineage: dict[str, Any], (c["seed"] for c in scoreboard if c["total"] == max(s["total"] for s in scoreboard)), res.lineage.get("seed")) + if edit_lineage is not None: + # Winner came from the synthesized view — flag it: the + # entire conditioning image was model-inferred. + lineage["escalation"] = {"rung": "image-edit-view", + **edit_lineage} + lineage["surface"] = "image-edit-inferred" logger.info("TRELLIS.2 object '%s': kept best of %d (%.0fs)", label, len(scoreboard), res.duration_seconds) return _persist(res.glb_data, "trellis2", lineage, diff --git a/tests/python/test_object_placement.py b/tests/python/test_object_placement.py index a70723f5..93c455a8 100644 --- a/tests/python/test_object_placement.py +++ b/tests/python/test_object_placement.py @@ -84,3 +84,29 @@ def test_build_placements_tolerates_missing_glb_extent(): "extent": [0.5, 0.5, 0.5]}}] out = build_placements(meshes) assert out["vase"]["scale_ratio"] == 1.0 # unknown normalized size + + +def test_build_placements_solves_orientation_from_camera_pose(): + # R10 orientation integration: a mesh carrying a crop camera pose gets a + # yaw-solved orientation quaternion in its placement (camera on +X_usd + # looking at origin -> object front faces +X, yaw 90deg). + meshes = [{ + "label": "vase", + "placement": {"centroid": [0, 0, 0], "extent": [0.5, 0.5, 0.5]}, + "glb_extent": [1.0, 1.0, 1.0], + "camera_pose": {"quaternion_wxyz": [0.7071, 0.0, 0.7071, 0.0], + "translation": [0.0, 0.0, 2.0]}, + }] + out = build_placements(meshes)["vase"] + assert out["orientation"] == "yaw-solved" + assert len(out["orientation_quat"]) == 4 + assert out["orientation_yaw_deg"] == pytest.approx(90.0, abs=1.0) + + +def test_build_placements_without_pose_stays_unsolved(): + meshes = [{"label": "vase", + "placement": {"centroid": [0, 0, 0], "extent": [0.5, 0.5, 0.5]}, + "glb_extent": [1.0, 1.0, 1.0]}] + out = build_placements(meshes)["vase"] + assert out["orientation"] == "unsolved" + assert "orientation_quat" not in out