From 1d6c534513c59dde4b04be711633bc89a8472261 Mon Sep 17 00:00:00 2001
From: recrudesce <recrudesce@gmail.com>
Date: Sun, 30 Aug 2026 19:20:59 +0100
Subject: [PATCH 1/2] add: Add Dockerfile, docker-compose, and docs/docker.md
 to allow containerisation of FreeToken.

---
 Dockerfile         |  34 ++++++++++
 docker-compose.yml |  27 ++++++++
 docs/docker.md     | 151 +++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 212 insertions(+)
 create mode 100644 Dockerfile
 create mode 100644 docker-compose.yml
 create mode 100644 docs/docker.md

diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 000000000..e6f9d4386
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,34 @@
+# Use CUDA devel base image so nvcc and headers are available for kernel compilation
+FROM nvidia/cuda:13.3.1-devel-ubuntu26.04
+
+ENV DEBIAN_FRONTEND=noninteractive \
+    PYTHONUNBUFFERED=1 \
+    CUDA_HOME=/usr/local/cuda \
+    PATH="/opt/venv/bin:/usr/local/cuda/bin:$PATH" \
+    LD_LIBRARY_PATH="/usr/local/cuda/lib64:$LD_LIBRARY_PATH" \
+    CPATH="/usr/local/cuda/include:$CPATH" \
+    LIBRARY_PATH="/usr/local/cuda/lib64:$LIBRARY_PATH"
+
+# Install build prerequisites and git
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    ninja-build \
+    git \
+    curl \
+    ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install uv binary
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+
+WORKDIR /app
+
+# Provision Python 3.11/3.12 virtualenv via uv (avoids Python 3.14 ABI incompatibilities)
+RUN uv venv /opt/venv --python 3.11 && \
+    uv pip install --no-cache "setuptools<70.0.0" wheel torch ninja && \
+    uv pip install --no-cache "freetoken[accel]"
+
+EXPOSE 1919
+
+ENTRYPOINT ["ft", "serve"]
+CMD ["--host", "0.0.0.0", "--port", "1919"]
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 000000000..3c1c73235
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,27 @@
+services:
+  freetoken:
+    image: freetoken:latest
+    container_name: freetoken-server
+    ipc: host
+    ports:
+      - "1919:1919"
+    environment:
+      - HF_TOKEN=${HF_TOKEN:-}
+      - HF_HOME=/root/.cache/huggingface
+    volumes:
+      - ~/.cache/huggingface:/root/.cache/huggingface
+    command:
+      - "--model"
+      - "${MODEL_NAME:-Qwen/Qwen3.6-35B-A3B}"
+      - "--host"
+      - "0.0.0.0"
+      - "--port"
+      - "1919"
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: all
+              capabilities: [gpu]
+    restart: unless-stopped
diff --git a/docs/docker.md b/docs/docker.md
new file mode 100644
index 000000000..736f22fb2
--- /dev/null
+++ b/docs/docker.md
@@ -0,0 +1,151 @@
+# Running FreeToken with Docker & Docker Compose
+
+This guide explains how to build, configure, and serve LLMs using FreeToken in a GPU-accelerated Docker container with automatic Hugging Face model downloading.
+
+---
+
+## Prerequisites
+
+- **NVIDIA GPU** with Pascal architecture or newer (Hopper, Ada Lovelace, Ampere, Turing supported).
+- **NVIDIA Driver**: Recommended `r580+` (or latest CUDA-compatible driver).
+- **Docker Engine** (24.0+) & **Docker Compose** (v2+).
+- **NVIDIA Container Toolkit** installed and configured as the default Docker runtime.
+
+Verify GPU passthrough is working before proceeding:
+```bash
+docker run --rm --gpus all nvidia/cuda:13.3.1-base-ubuntu22.04 nvidia-smi
+```
+
+---
+
+## Project Structure
+
+Ensure the following files are in your project directory:
+
+```text
+.
+├── Dockerfile
+├── docker-compose.yml
+├── .env                  # Optional: override default models & tokens
+└── DOCKER_README.md
+```
+
+---
+
+## 1. Build the Docker Image
+
+Build the image locally and tag it as `freetoken:latest`:
+
+```bash
+docker build -t freetoken:latest .
+```
+
+---
+
+## 2. Configuration (`.env`)
+
+Create a `.env` file in the same directory to configure your model and credentials:
+
+```env
+# Required only for gated/private models (e.g., meta-llama/Llama-3.1-8B-Instruct)
+HF_TOKEN=hf_your_token_here
+
+# Hugging Face model repository ID
+MODEL_NAME=Qwen/Qwen3.6-35B-A3B
+```
+
+---
+
+## 3. Running with Docker Compose
+
+### Start the Service
+```bash
+docker compose up -d
+```
+
+### View Logs & Download Progress
+FreeToken downloads weights directly to the mounted Hugging Face cache on the host (`~/.cache/huggingface`):
+```bash
+docker compose logs -f freetoken
+```
+
+Wait until you see:
+```text
+API server is ready to serve on 0.0.0.0:1919
+```
+
+### Stop the Service
+```bash
+docker compose down
+```
+
+---
+
+## 4. Alternative: Running via `docker run` (CLI)
+
+If you prefer running a standalone container without Compose:
+
+```bash
+docker run -d \
+  --name freetoken-server \
+  --gpus all \
+  --ipc=host \
+  -p 1919:1919 \
+  -e HF_TOKEN="${HF_TOKEN}" \
+  -e HF_HOME="/root/.cache/huggingface" \
+  -v ~/.cache/huggingface:/root/.cache/huggingface \
+  freetoken:latest \
+  --model "Qwen/Qwen3.6-35B-A3B" \
+  --host 0.0.0.0 \
+  --port 1919
+```
+
+---
+
+## 5. Testing the API
+
+FreeToken exposes an OpenAI-compatible HTTP API on port `1919`.
+
+### OpenAI-Compatible Chat Completion (`curl`)
+
+```bash
+curl [http://127.0.0.1:1919/v1/chat/completions](http://127.0.0.1:1919/v1/chat/completions) \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "Qwen/Qwen2.5-Coder-32B-Instruct",
+    "messages": [
+      {"role": "system", "content": "You are a helpful coding assistant."},
+      {"role": "user", "content": "Write a quick Python script to calculate Fibonacci numbers."}
+    ],
+    "temperature": 0.7,
+    "max_tokens": 512
+  }'
+```
+
+### Python Client Example (`openai` SDK)
+
+```python
+from openai import OpenAI
+
+client = OpenAI(
+    base_url="http://localhost:1919/v1",
+    api_key="none"
+)
+
+response = client.chat.completions.create(
+    model="Qwen/Qwen2.5-Coder-32B-Instruct",
+    messages=[
+        {"role": "user", "content": "Explain KV caching in two sentences."}
+    ]
+)
+
+print(response.choices[0].message.content)
+```
+
+---
+
+## 6. Performance & Troubleshooting Notes
+
+- **`ipc: host`**: Required. High-performance LLM engines use shared memory (`/dev/shm`) for multi-worker communication and KV cache transfers. Omitting this can cause immediate `Bus error` or `CUDA OOM` crashes.
+- **Persistent Cache**: Model weights are stored in `~/.cache/huggingface` on the host machine. Subsequent runs will load from disk instantly without re-downloading.
+- **Custom Model Paths**: To load local safetensors or GGUF files instead of downloading from Hugging Face, mount your local directory into the container (e.g., `-v /path/to/models:/models`) and set `--model /models/your-model-folder`.

From 86d6424e12f7ee0ae2f6b3fc885c3176c26c0759 Mon Sep 17 00:00:00 2001
From: recrudesce <recrudesce@gmail.com>
Date: Sun, 30 Aug 2026 19:26:04 +0100
Subject: [PATCH 2/2] fix: correct the base image in Dockerfile

---
 docs/docker.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/docker.md b/docs/docker.md
index 736f22fb2..495275da1 100644
--- a/docs/docker.md
+++ b/docs/docker.md
@@ -13,7 +13,7 @@ This guide explains how to build, configure, and serve LLMs using FreeToken in a
 
 Verify GPU passthrough is working before proceeding:
 ```bash
-docker run --rm --gpus all nvidia/cuda:13.3.1-base-ubuntu22.04 nvidia-smi
+docker run --rm --gpus all nvidia/cuda:13.3.1-base-ubuntu26.04 nvidia-smi
 ```
 
 ---