switch to tanstack router

.github/dependabot.yml

@@ -0,0 +1,11 @@
+version: 2
+
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+    groups:
+      actions:
+        patterns:
+          - "*"

.github/workflows/ci.yml

@@ -7,14 +7,31 @@ on:
     branches: [master]
 
 jobs:
+  python-format:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - name: Check backend formatting
+        uses: astral-sh/ruff-action@v4.1.0
+        with:
+          version: "0.16.0"
+          args: format --check --diff
+          src: backend
+      - name: Check icegraph-client formatting
+        uses: astral-sh/ruff-action@v4.1.0
+        with:
+          version: "0.16.0"
+          args: format --check --diff
+          src: icegraph-client
+
   frontend:
     runs-on: ubuntu-latest
     defaults:
       run:
         working-directory: frontend
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-node@v4
+      - uses: actions/checkout@v7
+      - uses: actions/setup-node@v7
         with:
           node-version: 22
           cache: npm

.github/workflows/deploy.yml

@@ -1,9 +1,7 @@
 name: Deploy to GitHub Pages
 
 on:
-  push:
-    tags:
-      - "v*"
+  workflow_call:
 
 permissions:
   contents: write
@@ -15,10 +13,10 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
 
       - name: Setup Node.js
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v7
         with:
           node-version: "24"
           cache: "npm"

.github/workflows/docker-publish.yml

@@ -1,9 +1,12 @@
 name: Publish Docker image
 
 on:
-  push:
-    tags:
-      - 'v*'
+  workflow_call:
+    secrets:
+      DOCKERHUB_USERNAME:
+        required: true
+      DOCKERHUB_TOKEN:
+        required: true
 
 jobs:
   push_to_registry:
@@ -13,28 +16,28 @@ jobs:
       contents: read
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       - name: Log in to Docker Hub
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@v6
         with:
           images: ${{ secrets.DOCKERHUB_USERNAME }}/icegraph
           tags: |
             type=ref,event=tag
             type=raw,value=latest
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         with:
           context: .
           push: true

.github/workflows/publish-icegraph-client.yml

@@ -1,22 +1,25 @@
 name: Publish icegraph-client to PyPI
 
 on:
-  push:
-    tags:
-      - "v*"
+  workflow_call:
+    secrets:
+      PYPI_API_TOKEN:
+        required: true
 
 jobs:
   publish:
     name: Build and publish to PyPI
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
         with:
           fetch-depth: 0
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@v7
 
       - name: Build
         working-directory: icegraph-client

.github/workflows/release.yml

@@ -0,0 +1,79 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - "v*"
+
+permissions: {}
+
+jobs:
+  docker:
+    name: Docker image
+    uses: ./.github/workflows/docker-publish.yml
+    permissions:
+      contents: read
+    secrets:
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+
+  pypi:
+    name: Python client
+    uses: ./.github/workflows/publish-icegraph-client.yml
+    permissions:
+      contents: read
+    secrets:
+      PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
+
+  pages:
+    name: Frontend demo
+    uses: ./.github/workflows/deploy.yml
+    permissions:
+      contents: write
+
+  release-notes:
+    name: Write release notes
+    runs-on: ubuntu-latest
+    needs: [docker, pypi, pages]
+    permissions:
+      contents: write
+    steps:
+      - name: Build notes and publish release
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+          TAG: ${{ github.ref_name }}
+          DOCKER_IMAGE: yanivzalach/icegraph
+          DEMO_URL: https://yanivzalach.github.io/IceGraph/
+        run: |
+          set -euo pipefail
+
+          VERSION="${TAG#v}"
+          NOTES_FILE="$RUNNER_TEMP/release-notes.md"
+
+          GENERATED="$(gh api "repos/$REPO/releases/generate-notes" -f tag_name="$TAG" --jq .body)"
+          CONTRIBUTORS="$(printf '%s\n' "$GENERATED" | grep -oE '@[A-Za-z0-9-]+' | sort -u | paste -sd , - | sed 's/,/, /g' || true)"
+
+          cat > "$NOTES_FILE" <<EOF
+          ## Artifacts
+
+          | Artifact | Get it |
+          | --- | --- |
+          | 🐳 Docker image | \`docker pull $DOCKER_IMAGE:$TAG\`<br>[Docker Hub](https://hub.docker.com/r/$DOCKER_IMAGE/tags?name=$TAG) |
+          | 🐍 Python client | \`pip install icegraph-client==$VERSION\`<br>[PyPI](https://pypi.org/project/icegraph-client/$VERSION/) |
+          | 🌐 Live demo | [$DEMO_URL]($DEMO_URL) |
+          | 📖 Docs | [${DEMO_URL}docs](${DEMO_URL}docs) |
+
+          EOF
+
+          printf '%s\n' "$GENERATED" >> "$NOTES_FILE"
+
+          if [ -n "$CONTRIBUTORS" ]; then
+            printf '\n## Contributors\n\nThanks to %s 🧊\n' "$CONTRIBUTORS" >> "$NOTES_FILE"
+          fi
+
+          if gh release view "$TAG" --repo "$REPO" >/dev/null 2>&1; then
+            gh release edit "$TAG" --repo "$REPO" --notes-file "$NOTES_FILE" --latest
+          else
+            gh release create "$TAG" --repo "$REPO" --title "IceGraph $TAG" --notes-file "$NOTES_FILE" --latest
+          fi

ARCHITECTURE_PHILOSOPHY.md

@@ -12,7 +12,7 @@ Introduce no service or infrastructure component unless the feature is impossibl
 
 ## 3. Simple deployment
 
-`SPARK_REMOTE` is the only mandatory configuration. All other tunables default in `backend/constants.py`, overridable via environment variable. One Docker image serves both frontend and backend; no second service to provision or wire up.
+`SPARK_REMOTE` is the only mandatory configuration. All other tunables are defined in `backend/env.py` with defaults and can be overridden via environment variables. One Docker image serves both frontend and backend; no second service to provision or wire up.
 
 ## 4. The process is disposable
 

CLAUDE.md

@@ -24,7 +24,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
 - Python files must not exceed 400 lines.
 - Never run git commands — only the user does.
 - When `icegraph-client`'s public API or CLI commands change, update the CLI section in `frontend/src/pages/DocsPage.jsx` and the Python Client & CLI bullet in `README.md` to match.
-- Whenever creating a constant that is settable via an environment variable, define its default value in `backend/constants.py` and add its description to `README.md`.
+- Whenever creating a setting that is configurable via an environment variable, define it in `backend/env.py`.
 - Frontend work has its own rules, philosophy, and styling conventions in [frontend/CLAUDE.md](frontend/CLAUDE.md) and [frontend/PHILOSOPHY.md](frontend/PHILOSOPHY.md).
 
 ## Project Overview
@@ -116,7 +116,7 @@ Backend environment variables (set in `backend/.env`):
 | `MAX_SNAPSHOTS_TO_COMPUTE` | 50 | Max snapshots processed per job |
 | `MAX_DATA_FILES_TO_COLLECT` | 5000 | Data file iteration limit |
 | `MAX_NUMBER_OF_GRAPHS_TO_COMPUTE` | 15 | Concurrent job limit |
-| `MAX_SNAPSHOTS_TO_SHOW` | 2000 | Snapshot selection UI limit |
+| `MAX_SNAPSHOTS_TO_SHOW` | 20 | Snapshot selection UI limit |
 | `INCLUDE_NONE_ICEBERG_CATALOGS` | `true` | Include non-Iceberg catalogs (e.g. Spark session catalog) in `/api/v1/tables` |
 | `TABLE_LIST_CACHE_TTL_SECONDS` | 60 | Cache TTL for table list endpoint |
 

README.md

@@ -93,6 +93,11 @@ cd backend
 uv sync
 ```
 
+```bash
+cd icegraph-client
+uv sync
+```
+
 ```bash
 cd frontend
 npm i
@@ -106,18 +111,7 @@ We will create an `.env` file in the root of the backend directory:
 SPARK_REMOTE=sc://localhost:15002 # Our local testing spark, If you use docker, change it to your ip.
 ```
 
-If you want to change the default values of the application, you can set the following environment variables:
-
-- `MAX_NUMBER_OF_GRAPHS_TO_COMPUTE`: The maximum number of graphs to compute in parallel. Default is 15.
-- `MAX_SNAPSHOTS_TO_SHOW`: The maximum number of snapshots to show in the snapshot selection page. Default is 2000.
-- `COMPUTE_CLEANUP_TIME_SECONDS`: The time to wait before cleaning up the computed graphs. Default is 12.
-- `MAX_DATA_FILES_TO_COLLECT`: The maximum number of data files to collect. Default is 5000.
-- `MAX_SNAPSHOTS_TO_COMPUTE`: The maximum number of snapshots to compute. Default is 50.
-- `MAX_GRACEFUL_SHUTDOWN_TIME_SECONDS`: The time to wait before forcing an exit of the application when shutting it down. Default is 10.
-- `INCLUDE_NONE_ICEBERG_CATALOGS`: Whether to include catalogs that are not exclusively Iceberg (such as the Spark session catalog) in the table selection autocompletion. Default is true.
-- `TABLE_LIST_CACHE_TTL_SECONDS`: The cache time-to-live for table selection autocompletion endpoint before refresh. Default is 60.
-- `PRODUCTION_MODE`: Whether to serve the app with the Waitress production WSGI server instead of Flask's development server. Always single-process (multi-threaded) so the in-memory job-polling state stays consistent. Default is false; the Docker image sets this to true.
-- `WSGI_THREADS`: The number of threads Waitress uses to handle concurrent requests when `PRODUCTION_MODE` is enabled. Default is 20.
+The supported environment variables, their defaults, and their descriptions are defined in the [`Env` class](backend/env.py).
 
 ### 3. Run
 
@@ -136,9 +130,29 @@ Go to `http://localhost:3000` and explore your tables.
 
 ### 4. Before Every Commit
 
-CI fails the build on any violation, so run all three from the `frontend` directory before you commit:
+CI checks Python formatting and the frontend toolchain. Format each Python project from its own directory.
+
+Backend:
 
 ```bash
+cd backend
+
+uv run ruff format .
+```
+
+Python client:
+
+```bash
+cd icegraph-client
+
+uv run ruff format .
+```
+
+Run all three frontend checks from the `frontend` directory:
+
+```bash
+cd frontend
+
 npm run format
 npm run lint
 npm run typecheck

backend/base_classes/base_file.py

@@ -1,5 +1,5 @@
-from dataclasses import dataclass, fields
-from typing import List
+from dataclasses import dataclass, field, fields
+from typing import List, Optional
 
 from constants import FileType
 
@@ -14,6 +14,8 @@ class BaseFile:
     type: FileType
     file_path: str
     child_files: List[str]
+    error: Optional[str] = field(default=None, kw_only=True)
+    warning: Optional[str] = field(default=None, kw_only=True)
 
     def to_dict(self):
         result_dict = {field.name: getattr(self, field.name) for field in fields(self) if not isinstance(getattr(self, field.name), HiddenFile)}

backend/collectors/collect_data_files.py

@@ -38,19 +38,16 @@ def __init__(
     ):
         super().__init__(full_table_name)
         self._manifests = manifests
-        self._errors: Dict[str, str] = {}
 
         self._data_files: List[DataFileRecord] = []
 
     @timed
     def collect(self) -> FilesCollection:
-        data_files_extraction_result = DataFilesExtractor(self._table_name, self._manifests).extract_dataframe()
-        self._errors = data_files_extraction_result.errors
+        data_files_rows = DataFilesExtractor(self._table_name, self._manifests).extract_dataframe().collect()
 
-        data_files_rows = data_files_extraction_result.dataframe.collect()
         self._data_files = [self._process_data_file_row(data_file_row) for data_file_row in data_files_rows]
 
-        return FilesCollection(files=self._data_files, errors=self._errors)
+        return FilesCollection(files=self._data_files)
 
     def _process_data_file_row(self, data_file_row) -> DataFileRecord:
         data_file_dict = data_file_row.asDict(recursive=True)
@@ -59,7 +56,7 @@ def _process_data_file_row(self, data_file_row) -> DataFileRecord:
             type=self._detect_file_type(data_file_dict["content"]),
             file_path=data_file_dict["file_path"],
             format=data_file_dict["file_format"],
-            size_gb=f"{(data_file_dict['file_size_in_bytes'] / 1024 ** 3):.10f}",
+            size_gb=f"{(data_file_dict['file_size_in_bytes'] / 1024**3):.10f}",
             row_count=data_file_dict["record_count"],
             partition=format_partition(data_file_dict["partition"]),
             earliest_appearing_snapshot_id=data_file_dict["earliest_snapshot_id"],

backend/collectors/collect_manifests.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import Dict, List, Optional
+from typing import List, Optional
 
 import pyspark
 
@@ -37,19 +37,16 @@ def __init__(
         super().__init__(full_table_name)
         self._snapshots = snapshots
         self._manifests_to_ignore_df = manifests_to_ignore_df
-        self._errors: Dict[str, str] = {}
 
         self._manifests: List[ManifestRecord] = []
 
     @timed
     def collect(self) -> FilesCollection:
-        manifest_extraction_result = ManifestsExtractor(self._table_name, self._snapshots, self._manifests_to_ignore_df).extract_dataframe()
-        self._errors = manifest_extraction_result.errors
+        manifests_rows = ManifestsExtractor(self._table_name, self._snapshots, self._manifests_to_ignore_df).extract_dataframe().collect()
 
-        manifests_rows = manifest_extraction_result.dataframe.collect()
         self._manifests = [self._process_manifest_row(manifest_row) for manifest_row in manifests_rows]
 
-        return FilesCollection(files=self._manifests, errors=self._errors)
+        return FilesCollection(files=self._manifests)
 
     @staticmethod
     def _process_manifest_row(manifest_row) -> ManifestRecord: