add embedding for all image nfts, show similar nfts in nft page, expo…

microservices/semantic/requirements.txt

@@ -0,0 +1,7 @@
+fastapi==0.115.0
+uvicorn==0.30.0
+sentence-transformers==3.0.1
+Pillow>=11.0.0
+numpy==1.26.4
+requests==2.32.3
+python-multipart==0.0.12

microservices/semantic/server.py

@@ -0,0 +1,102 @@
+import requests
+from contextlib import asynccontextmanager
+from io import BytesIO
+from PIL import Image
+from fastapi import FastAPI, File, HTTPException, UploadFile
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel, Field
+from sentence_transformers import SentenceTransformer
+
+ARWEAVE_GATEWAY = "https://arweave.net"
+ARWEAVE_ID_PATTERN = r"^[A-Za-z0-9_-]{43}$"
+MAX_UPLOAD_BYTES = 10 * 1024 * 1024
+
+model = None
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    global model
+    print("Loading CLIP model...")
+    model = SentenceTransformer("clip-ViT-B-32")
+    print("Model loaded.")
+    yield
+
+
+app = FastAPI(title="NFT Embedding Server", lifespan=lifespan)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+class EmbedTextRequest(BaseModel):
+    query: str
+
+
+class EmbedArweaveRequest(BaseModel):
+    arweave_id: str = Field(pattern=ARWEAVE_ID_PATTERN)
+
+
+def _decode_image(data: bytes) -> Image.Image:
+    try:
+        return Image.open(BytesIO(data)).convert("RGB")
+    except Exception:
+        raise HTTPException(400, "content is not a valid image")
+
+
+@app.post("/embed/text")
+def embed_text(req: EmbedTextRequest):
+    """Embed a text query into CLIP vector space."""
+    embedding = model.encode(req.query)
+    return {"embedding": embedding.tolist()}
+
+
+@app.post("/embed/image")
+async def embed_image(file: UploadFile = File(...)):
+    """Embed a user-uploaded image. PIL decode is the real image check —
+    the client-supplied content-type header is not trusted."""
+    data = await file.read()
+    if len(data) > MAX_UPLOAD_BYTES:
+        raise HTTPException(413, f"image exceeds {MAX_UPLOAD_BYTES} bytes")
+
+    img = _decode_image(data)
+    embedding = model.encode(img)
+    return {"embedding": embedding.tolist()}
+
+
+@app.post("/embed/arweave")
+def embed_arweave(req: EmbedArweaveRequest):
+    """Verify an Arweave transaction is an image, then CLIP-encode it.
+    Two checks because the gateway is untrusted: HEAD content-type filters
+    cheaply, then PIL decode rejects bytes that don't actually parse."""
+    url = f"{ARWEAVE_GATEWAY}/{req.arweave_id}"
+
+    try:
+        head = requests.head(url, timeout=10, allow_redirects=True)
+        head.raise_for_status()
+    except requests.RequestException as e:
+        raise HTTPException(502, f"Arweave gateway unreachable: {e}")
+
+    content_type = head.headers.get("content-type", "")
+    if not content_type.startswith("image/"):
+        raise HTTPException(
+            400,
+            f"{req.arweave_id} is not an image (content-type: {content_type or 'unknown'})",
+        )
+
+    try:
+        resp = requests.get(url, timeout=30)
+        resp.raise_for_status()
+    except requests.RequestException as e:
+        raise HTTPException(502, f"Failed to download image: {e}")
+
+    img = _decode_image(resp.content)
+    embedding = model.encode(img)
+    return {
+        "arweave_id": req.arweave_id,
+        "embedding": embedding.tolist(),
+    }

src/alex_frontend/core/features/nft/components/SimilarNfts.tsx

@@ -0,0 +1,25 @@
+import React from "react";
+import { useSimilarNfts } from "../hooks/useSimilarNfts";
+import { useAppSelector } from "@/store/hooks/useAppSelector";
+import NftProvider from "@/components/NftProvider";
+import { NFTCard } from "@/features/nft";
+import type { AlexandrianToken } from "@/features/alexandrian/types";
+
+export default function SimilarNfts({ arweaveId }: { arweaveId: string }) {
+	const { data: tokens, isLoading, error } = useSimilarNfts(arweaveId);
+	const { safe } = useAppSelector((state) => state.alexandrian);
+
+	if (error) return null;
+	if (!isLoading && (!tokens || tokens.length === 0)) return null;
+
+	return (
+		<section className="space-y-3">
+			<h2 className="text-lg font-semibold">Similar NFTs</h2>
+			<NftProvider loading={isLoading} items={tokens ?? []} safe={safe}>
+				{(token: AlexandrianToken) => (
+					<NFTCard id={token.arweaveId} token={token} />
+				)}
+			</NftProvider>
+		</section>
+	);
+}

src/alex_frontend/core/features/nft/hooks/useSimilarNfts.ts

@@ -0,0 +1,77 @@
+import { useQuery } from "@tanstack/react-query";
+import { useAlexBackend } from "@/hooks/actors";
+import { arweaveIdToNat } from "@/utils/id_convert";
+import { createTokenAdapter } from "@/features/alexandrian/adapters/TokenAdapter";
+import type { AlexandrianToken } from "@/features/alexandrian/types";
+
+const EMBEDDING_SERVER =
+	process.env.REACT_APP_EMBEDDING_SERVER || "https://lbry.youthumber.com";
+
+const TOP_K = 20;
+
+interface SimilarityHit {
+	arweave_id: string;
+	score: number;
+}
+
+export function useSimilarNfts(arweaveId: string | null) {
+	const { actor } = useAlexBackend();
+
+	return useQuery({
+		queryKey: ["similar-nfts", arweaveId],
+		queryFn: async (): Promise<AlexandrianToken[]> => {
+			const id = arweaveId!;
+
+			// Fast path: canister already has an embedding for this ID.
+			const indexed = await actor!.search_similar(id, TOP_K);
+			let hits: SimilarityHit[];
+
+			if ("Ok" in indexed) {
+				hits = indexed.Ok;
+			} else {
+				// Fallback: ask the server to verify+embed the Arweave image,
+				// then cosine-search the canister by the returned vector.
+				const res = await fetch(`${EMBEDDING_SERVER}/embed/arweave`, {
+					method: "POST",
+					headers: { "Content-Type": "application/json" },
+					body: JSON.stringify({ arweave_id: id }),
+				});
+				if (!res.ok) {
+					const body = await res.json().catch(() => ({}));
+					throw new Error(body.detail ?? `embed/arweave ${res.status}`);
+				}
+				const { embedding } = await res.json();
+				const vecRes = await actor!.search_by_vector(embedding, TOP_K);
+				if (!("Ok" in vecRes)) throw new Error(vecRes.Err);
+				hits = vecRes.Ok;
+			}
+
+			const adapter = createTokenAdapter("NFT");
+			return Promise.all(
+				hits.map(async (r) => {
+					const tokenId = arweaveIdToNat(r.arweave_id);
+					const ownerRes = await adapter.getOwnerOf([tokenId]);
+					const owner = ownerRes?.[0]?.[0]?.owner?.toString() || "";
+					const icpInfo = await adapter.tokenToIcpInfo(tokenId);
+					return {
+						id: tokenId.toString(),
+						arweaveId: r.arweave_id,
+						owner,
+						collection: "NFT" as const,
+						...icpInfo,
+					} as AlexandrianToken;
+				}),
+			);
+		},
+		enabled: !!actor && !!arweaveId,
+		staleTime: Infinity,
+		gcTime: 60 * 60 * 1000,
+		refetchOnWindowFocus: false,
+		refetchOnReconnect: false,
+		retry: (n, err) => {
+			const msg = (err as Error).message?.toLowerCase() ?? "";
+			if (msg.includes("not an image") || msg.includes("not a valid image")) return false;
+			return n < 1;
+		},
+	});
+}

src/alex_frontend/core/features/perpetua/components/ShelfDetail.tsx

@@ -14,7 +14,7 @@ import {
 	useSortable,
 } from "@dnd-kit/sortable";
 import { CSS } from "@dnd-kit/utilities";
-import { ArrowLeft, BookOpen, Copy, Check, GripVertical, Info, Layers, LayoutGrid, List, Loader2, Package } from "lucide-react";
+import { ArrowLeft, BookOpen, Copy, Check, Download, GripVertical, Info, Layers, LayoutGrid, List, Loader2, Package } from "lucide-react";
 import { Button } from "@/lib/components/button";
 import { Badge } from "@/lib/components/badge";
 import { Skeleton } from "@/lib/components/skeleton";
@@ -29,6 +29,7 @@ import { useSetItemOrder } from "../hooks/useMutations";
 import { useUsername } from "@/hooks/useUsername";
 import { convertTimestamp } from "@/utils/general";
 import { shortenPrincipal, getItemContentValue } from "../utils";
+import { natToArweaveId } from "@/utils/id_convert";
 import type { Item } from "../types";
 import MarkdownRenderer from "@/components/MarkdownRenderer";
 import ItemCard from "./ItemCard";
@@ -321,6 +322,33 @@ export default function ShelfDetail({ shelfId, userId }: ShelfDetailProps) {
 					<>
 						<ContentTypeFilter value={contentFilter} onChange={setContentFilter} />
 						<ViewSwitch viewMode={viewMode} onChange={setViewMode} />
+						{contentFilter === "Nft" && filteredItems.length > 0 && (
+							<button
+								type="button"
+								onClick={() => {
+									const ids = filteredItems
+										.filter(([, item]) => "Nft" in item.content)
+										.map(([, item]) =>
+											natToArweaveId(BigInt(getItemContentValue(item.content))),
+										);
+									const blob = new Blob([JSON.stringify(ids, null, 2)], {
+										type: "application/json",
+									});
+									const url = URL.createObjectURL(blob);
+									const a = document.createElement("a");
+									a.href = url;
+									a.download = `${shelfId}-arweave-ids.json`;
+									document.body.appendChild(a);
+									a.click();
+									document.body.removeChild(a);
+									URL.revokeObjectURL(url);
+								}}
+								className="inline-flex items-center h-[22px] gap-1 rounded-full border border-gray-300 dark:border-gray-600 bg-gray-50 dark:bg-gray-800 px-2.5 text-xs text-muted-foreground dark:text-gray-400 hover:border-foreground transition-colors"
+							>
+								<Download className="h-3 w-3" />
+								Export
+							</button>
+						)}
 					</>
 				)}
 

src/alex_frontend/lbry/src/pages/NftPage.tsx

@@ -8,6 +8,7 @@ import { NftContext } from "@/components/NftProvider";
 import IcpInfo from "@/features/nft/components/Info/Icp";
 import Tags from "@/features/nft/components/Info/Tags";
 import Comment from "@/features/nft/components/Comment";
+import SimilarNfts from "@/features/nft/components/SimilarNfts";
 import useTransactionMetadata from "@/features/nft/hooks/useTransactionMetadata";
 import { AlexandrianToken } from "@/features/alexandrian/types";
 import { natToArweaveId, arweaveIdToNat } from "@/utils/id_convert";
@@ -523,6 +524,9 @@ function NftPage() {
 					</Tabs>
 				</div>
 			</div>
+
+			{/* Row 2: Similar NFTs */}
+			<SimilarNfts arweaveId={arweaveId} />
 		</div>
 		</>
 	);