From cb7959c5b11f055707c3d5ce2ee256d3b3070f6c Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Thu, 11 Jun 2026 13:34:13 +0200 Subject: [PATCH 01/57] automate notebook examples end-to-end MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - scripts/get_or_create_api_key.py: create/retrieve AIGatewayKey (importable + CLI) - scripts/setup_mlflow_credentials.py: one-time MLflow PAT → k8s secret (importable + CLI, optional --uri/--username/--password args) - notebooks/dask/dask_example.ipynb: auto-detect namespace from SA token file - mlflow/{quickstart,image}: read MLflow creds from mlflow-credentials k8s secret instead of hardcoded placeholders - mlflow/{kfp,mobile-price}: update markdown to reference setup_mlflow_credentials.py - serving/minimal-s3-model: auto-create API key via get_or_create_api_key - serving/mlflow-kserve-inference-protocols: add setup cell that applies YAML templates with substituted namespace/username, waits for ISVC readiness, and auto-fetches URIs and API key - serving/mlflow-kserve-minimal/apply.py: deploy InferenceService with substituted values, wait for ready, print URI - pipelines/lightweight-python-package/submit-cluster.py: print KFP_RUN_ID for CI polling - cleanup.py per example: idempotent kubectl delete for k8s resources only - ci/run_all.py: papermill-based orchestrator with parallel phases, KFP run polling, and cleanup in finally block --- ci/run_all.py | 454 ++++++++++++++++++ hparam-tuning/minimal-mnist/cleanup.py | 54 +++ mlflow/mlflow-image-example.ipynb | 25 +- mlflow/mlflow-kfp-example.ipynb | 11 +- mlflow/mlflow-quickstart-example.ipynb | 72 +-- .../mlflow-mobile-price-classification.ipynb | 8 +- notebooks/dask/cleanup.py | 53 ++ notebooks/dask/dask_example.ipynb | 6 +- .../submit-cluster.py | 5 +- scripts/get_or_create_api_key.py | 116 +++++ scripts/setup_mlflow_credentials.py | 125 +++++ serving/minimal-s3-model/cleanup.py | 63 +++ .../minimal-s3-model/minimal-s3-model.ipynb | 9 +- .../cleanup.py | 56 +++ .../inference_protocol_version_example.ipynb | 107 ++++- serving/mlflow-kserve-minimal/apply.py | 150 ++++++ serving/mlflow-kserve-minimal/cleanup.py | 49 ++ 17 files changed, 1271 insertions(+), 92 deletions(-) create mode 100644 ci/run_all.py create mode 100644 hparam-tuning/minimal-mnist/cleanup.py create mode 100644 notebooks/dask/cleanup.py create mode 100644 scripts/get_or_create_api_key.py create mode 100644 scripts/setup_mlflow_credentials.py create mode 100644 serving/minimal-s3-model/cleanup.py create mode 100644 serving/mlflow-kserve-inference-protocols/cleanup.py create mode 100644 serving/mlflow-kserve-minimal/apply.py create mode 100644 serving/mlflow-kserve-minimal/cleanup.py diff --git a/ci/run_all.py b/ci/run_all.py new file mode 100644 index 0000000..17d61ea --- /dev/null +++ b/ci/run_all.py @@ -0,0 +1,454 @@ +""" +CI orchestrator — runs all automatable examples and reports results. + +Execution model +--------------- +Phase 1 (parallel, self-contained): + notebooks/dask/dask_example.ipynb + mlflow/mlflow-quickstart-example.ipynb + mlflow/mlflow-image-example.ipynb + mlflow/mlflow-kfp-example.ipynb + serving/minimal-s3-model/minimal-s3-model.ipynb + +Phase 2 (parallel, pipeline submissions — return fast): + mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb + pipelines/lightweight-components/mobile-price-classifications.ipynb + pipelines/lightweight-python-package/submit-cluster.py (Python script) + +Phase 3 (depends on Phase 2 mlflow-mobile-price notebook completing): + serving/mlflow-kserve-minimal/apply.py + test_inference_service.py + serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb + +Phase 4 (KFP run polling — runs alongside Phase 3): + Poll all KFP run IDs extracted from Phase 2 output notebooks until + Succeeded or timeout. + +Phase 5 — cleanup (always, in finally block): + All cleanup.py scripts in parallel. + +Usage from a Jupyter notebook +------------------------------ + import subprocess + subprocess.run(["python", "ci/run_all.py"], check=True) + +CLI usage +--------- + python ci/run_all.py [--timeout-notebook 1800] [--timeout-pipeline 3600] + [--include-keda] [--dry-run] + +Prerequisites +------------- + pip install papermill kfp + python scripts/setup_mlflow_credentials.py +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import os +import re +import subprocess +import sys +import time +from concurrent.futures import Future, ThreadPoolExecutor, as_completed +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + +# ── Repo root ───────────────────────────────────────────────────────────────── +_REPO_ROOT = Path( + subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip() +) + +# ── KFP run ID pattern (UUID v4) ───────────────────────────────────────────── +_RUN_ID_RE = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") + + +# ── Result dataclass ────────────────────────────────────────────────────────── + + +@dataclass +class Result: + name: str + status: str = "PENDING" # PASS | FAIL | SKIP + duration: float = 0.0 + error: str = "" + kfp_run_ids: list[str] = field(default_factory=list) + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _run_notebook(nb_path: Path, output_dir: Path, timeout: int) -> tuple[Path, str]: + """Execute a notebook with papermill. Returns (output_path, stderr).""" + try: + import papermill as pm + except ImportError: + raise RuntimeError("papermill is not installed. Run: pip install papermill") + + output_path = output_dir / nb_path.name + output_dir.mkdir(parents=True, exist_ok=True) + pm.execute_notebook( + str(nb_path), + str(output_path), + kernel_name="python3", + execution_timeout=timeout, + cwd=str(nb_path.parent), + ) + return output_path, "" + + +def _run_script(script_path: Path, timeout: int) -> tuple[str, str]: + """Run a plain Python script. Returns (stdout, stderr).""" + result = subprocess.run( + [sys.executable, str(script_path)], + capture_output=True, + text=True, + timeout=timeout, + cwd=str(script_path.parent), + ) + if result.returncode != 0: + raise RuntimeError(result.stderr) + return result.stdout, result.stderr + + +def _extract_run_ids_from_notebook(output_nb: Path) -> list[str]: + """Parse a papermill output notebook for KFP run IDs.""" + with open(output_nb) as fh: + nb = json.load(fh) + ids: list[str] = [] + for cell in nb.get("cells", []): + for output in cell.get("outputs", []): + text = "".join( + output.get("text", []) + output.get("data", {}).get("text/plain", []) + ) + ids.extend(_RUN_ID_RE.findall(text)) + return list(dict.fromkeys(ids)) # deduplicate, preserve order + + +def _extract_run_ids_from_stdout(stdout: str) -> list[str]: + """Parse stdout from a script for KFP_RUN_ID= lines.""" + ids = [] + for line in stdout.splitlines(): + if line.startswith("KFP_RUN_ID="): + ids.append(line.split("=", 1)[1].strip()) + return ids + + +def _poll_kfp_run(run_id: str, timeout: int, interval: int = 30) -> str: + """Poll a KFP run until terminal state. Returns final status string.""" + try: + from kfp.client import Client + except ImportError: + return "SKIP (kfp not installed)" + client = Client() + deadline = time.time() + timeout + while time.time() < deadline: + run = client.get_run(run_id) + state = run.run.status + if state in ("Succeeded", "Failed", "Error", "Skipped"): + return state + time.sleep(interval) + return f"TIMEOUT after {timeout}s" + + +def _run_cleanup(cleanup_path: Path) -> None: + """Run a cleanup.py; log but don't raise on failure.""" + try: + subprocess.run( + [sys.executable, str(cleanup_path)], + capture_output=True, + text=True, + cwd=str(cleanup_path.parent), + timeout=120, + ) + except Exception as exc: # noqa: BLE001 + print(f" WARNING: cleanup {cleanup_path.name} failed: {exc}", file=sys.stderr) + + +# ── Executor ────────────────────────────────────────────────────────────────── + + +def _submit_notebook( + executor: ThreadPoolExecutor, + results: dict[str, Result], + name: str, + nb_path: Path, + output_dir: Path, + timeout: int, +) -> Future: + result = Result(name=name) + results[name] = result + + def _run(): + t0 = time.time() + try: + out, _ = _run_notebook(nb_path, output_dir, timeout) + result.status = "PASS" + result.kfp_run_ids = _extract_run_ids_from_notebook(out) + except Exception as exc: # noqa: BLE001 + result.status = "FAIL" + result.error = str(exc)[:300] + finally: + result.duration = time.time() - t0 + + return executor.submit(_run) + + +def _submit_script( + executor: ThreadPoolExecutor, + results: dict[str, Result], + name: str, + script_path: Path, + timeout: int, +) -> Future: + result = Result(name=name) + results[name] = result + + def _run(): + t0 = time.time() + try: + stdout, _ = _run_script(script_path, timeout) + result.status = "PASS" + result.kfp_run_ids = _extract_run_ids_from_stdout(stdout) + except Exception as exc: # noqa: BLE001 + result.status = "FAIL" + result.error = str(exc)[:300] + finally: + result.duration = time.time() - t0 + + return executor.submit(_run) + + +# ── Report ──────────────────────────────────────────────────────────────────── + + +def _print_report(results: dict[str, Result], poll_results: dict[str, str]) -> None: + col = 50 + print("\n" + "=" * 70) + print(f"{'EXAMPLE':<{col}} {'STATUS':<10} {'DURATION'}") + print("-" * 70) + passed = failed = 0 + for r in results.values(): + dur = f"{r.duration:.0f}s" if r.duration else "-" + print(f"{r.name:<{col}} {r.status:<10} {dur}") + if r.status == "FAIL": + print(f" {'':>{col}} {r.error}") + failed += 1 + else: + passed += 1 + if poll_results: + print() + print(f"{'KFP PIPELINE POLL':<{col}} {'STATUS':<10}") + print("-" * 70) + for run_id, status in poll_results.items(): + short = run_id[:8] + "..." + ok = status == "Succeeded" + if not ok: + failed += 1 + else: + passed += 1 + print(f"{short:<{col}} {status:<10}") + print("=" * 70) + print(f"PASSED: {passed} FAILED: {failed} TOTAL: {passed + failed}") + print() + + +# ── Main ────────────────────────────────────────────────────────────────────── + + +def run_all( + timeout_notebook: int = 1800, + timeout_pipeline: int = 3600, + include_keda: bool = False, + dry_run: bool = False, +) -> dict[str, Result]: + root = _REPO_ROOT + output_dir = root / "ci" / "output" + results: dict[str, Result] = {} + poll_results: dict[str, str] = {} + + if dry_run: + print("[dry-run] Would execute the following phases:") + for label in [ + "Phase 1: dask, mlflow-quickstart, mlflow-image, mlflow-kfp, minimal-s3-model", + "Phase 2: mlflow-mobile-price, lightweight-components, lightweight-python-package", + "Phase 3: mlflow-kserve-minimal, mlflow-kserve-inference-protocols", + "Phase 4: KFP run polling", + "Phase 5: cleanup", + ]: + print(f" {label}") + return results + + cleanup_scripts = [ + root / "notebooks" / "dask" / "cleanup.py", + root / "serving" / "minimal-s3-model" / "cleanup.py", + root / "serving" / "mlflow-kserve-minimal" / "cleanup.py", + root / "serving" / "mlflow-kserve-inference-protocols" / "cleanup.py", + root / "hparam-tuning" / "minimal-mnist" / "cleanup.py", + ] + + try: + with ThreadPoolExecutor(max_workers=8) as executor: + # ── Phase 1: independent notebooks ─────────────────────────────── + print("Phase 1: running independent notebooks in parallel...") + phase1_futures = {} + for name, rel in [ + ("notebooks/dask", "notebooks/dask/dask_example.ipynb"), + ("mlflow/mlflow-quickstart", "mlflow/mlflow-quickstart-example.ipynb"), + ("mlflow/mlflow-image-example", "mlflow/mlflow-image-example.ipynb"), + ("mlflow/mlflow-kfp-example", "mlflow/mlflow-kfp-example.ipynb"), + ( + "serving/minimal-s3-model", + "serving/minimal-s3-model/minimal-s3-model.ipynb", + ), + ]: + f = _submit_notebook( + executor, results, name, root / rel, output_dir, timeout_notebook + ) + phase1_futures[name] = f + + # wait for all phase 1 + for name, f in phase1_futures.items(): + f.result() + print(f" [{results[name].status}] {name}") + + # ── Phase 2: pipeline submissions (return fast) ─────────────────── + print("\nPhase 2: submitting pipelines...") + phase2_futures = {} + for name, rel in [ + ( + "mlflow/mobile-price-classification", + "mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb", + ), + ( + "pipelines/lightweight-components", + "pipelines/lightweight-components/mobile-price-classifications.ipynb", + ), + ]: + f = _submit_notebook( + executor, results, name, root / rel, output_dir, timeout_notebook + ) + phase2_futures[name] = f + + lpp_future = _submit_script( + executor, + results, + "pipelines/lightweight-python-package", + root / "pipelines" / "lightweight-python-package" / "submit-cluster.py", + timeout_notebook, + ) + phase2_futures["pipelines/lightweight-python-package"] = lpp_future + + # Wait only for mlflow-mobile-price before launching phase 3 + mlflow_mobile_future = phase2_futures["mlflow/mobile-price-classification"] + mlflow_mobile_future.result() + print( + f" [{results['mlflow/mobile-price-classification'].status}] mlflow/mobile-price-classification" + ) + + # ── Phase 3: MLflow KServe examples (need registered model) ─────── + print("\nPhase 3: deploying MLflow KServe InferenceServices...") + phase3_futures = {} + for name, rel in [ + ( + "serving/mlflow-kserve-minimal", + "serving/mlflow-kserve-minimal/apply.py", + ), + ( + "serving/mlflow-kserve-inference-protocols", + "serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb", + ), + ]: + path = root / rel + if path.suffix == ".ipynb": + f = _submit_notebook( + executor, results, name, path, output_dir, timeout_notebook + ) + else: + f = _submit_script(executor, results, name, path, timeout_notebook) + phase3_futures[name] = f + + # Wait for remaining phase 2 futures while phase 3 runs + for name, f in phase2_futures.items(): + if name != "mlflow/mobile-price-classification": + f.result() + print(f" [{results[name].status}] {name}") + + # Wait for phase 3 + for name, f in phase3_futures.items(): + f.result() + print(f" [{results[name].status}] {name}") + + # ── Phase 4: KFP run polling ────────────────────────────────────── + all_run_ids: list[str] = [] + for r in results.values(): + all_run_ids.extend(r.kfp_run_ids) + + if all_run_ids: + print(f"\nPhase 4: polling {len(all_run_ids)} KFP run(s)...") + poll_futures = { + run_id: executor.submit(_poll_kfp_run, run_id, timeout_pipeline) + for run_id in all_run_ids + } + for run_id, f in poll_futures.items(): + poll_results[run_id] = f.result() + print(f" [{poll_results[run_id]}] {run_id[:8]}...") + else: + print("\nPhase 4: no KFP run IDs found, skipping poll.") + + finally: + # ── Phase 5: cleanup ───────────────────────────────────────────────── + print("\nPhase 5: running cleanup scripts...") + with ThreadPoolExecutor(max_workers=4) as ex: + futs = [ex.submit(_run_cleanup, p) for p in cleanup_scripts if p.exists()] + for f in as_completed(futs): + f.result() # already swallows exceptions inside _run_cleanup + + _print_report(results, poll_results) + return results + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--timeout-notebook", + type=int, + default=1800, + help="Per-notebook execution timeout in seconds (default: 1800)", + ) + parser.add_argument( + "--timeout-pipeline", + type=int, + default=3600, + help="KFP run poll timeout in seconds (default: 3600)", + ) + parser.add_argument( + "--include-keda", + action="store_true", + help="Include GPU-dependent KEDA autoscaling example (opt-in)", + ) + parser.add_argument( + "--dry-run", action="store_true", help="Print plan without executing anything" + ) + args = parser.parse_args() + + results = run_all( + timeout_notebook=args.timeout_notebook, + timeout_pipeline=args.timeout_pipeline, + include_keda=args.include_keda, + dry_run=args.dry_run, + ) + + failed = sum(1 for r in results.values() if r.status == "FAIL") + sys.exit(1 if failed else 0) diff --git a/hparam-tuning/minimal-mnist/cleanup.py b/hparam-tuning/minimal-mnist/cleanup.py new file mode 100644 index 0000000..32d6475 --- /dev/null +++ b/hparam-tuning/minimal-mnist/cleanup.py @@ -0,0 +1,54 @@ +""" +Cleanup script for the hparam-tuning/minimal-mnist example. + +Deletes (idempotently): + - Katib Experiment ``minimal-mnist`` + - Any Trial pods created by the Experiment (Katib garbage-collects these + automatically, but this speeds things up if you need to clean up quickly) + +Usage +----- + python cleanup.py [--dry-run] +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys + +_EXPERIMENT_NAME = "minimal-mnist" + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _kubectl_delete(*args: str, dry_run: bool = False) -> None: + cmd = ["kubectl", "delete", *args, "--ignore-not-found"] + if dry_run: + print(f"[dry-run] {' '.join(cmd)}") + return + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"WARNING: {result.stderr.strip()}", file=sys.stderr) + else: + print(result.stdout.strip() or f"deleted (or not found): {' '.join(args)}") + + +def cleanup(dry_run: bool = False) -> None: + ns = _namespace() + # Deleting the Experiment cascades to Trials via owner references + _kubectl_delete("experiment", _EXPERIMENT_NAME, "-n", ns, dry_run=dry_run) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--dry-run", action="store_true", help="Print commands without executing them" + ) + args = parser.parse_args() + cleanup(dry_run=args.dry_run) diff --git a/mlflow/mlflow-image-example.ipynb b/mlflow/mlflow-image-example.ipynb index 186cdf6..0a5b36d 100644 --- a/mlflow/mlflow-image-example.ipynb +++ b/mlflow/mlflow-image-example.ipynb @@ -42,12 +42,27 @@ "metadata": {}, "outputs": [], "source": [ - "import os\n", + "import base64, json, os, subprocess\n", "\n", - "# Set your MLflow credentials\n", - "os.environ['MLFLOW_TRACKING_URI'] = 'https:///mlflow'\n", - "os.environ['MLFLOW_TRACKING_USERNAME'] = '' \n", - "os.environ['MLFLOW_TRACKING_PASSWORD'] = ''" + "# Load MLflow credentials from the mlflow-credentials Kubernetes secret.\n", + "# Run scripts/setup_mlflow_credentials.py once to create this secret.\n", + "with open(\"/var/run/secrets/kubernetes.io/serviceaccount/namespace\") as _f:\n", + " _ns = _f.read().strip()\n", + "\n", + "_result = subprocess.run(\n", + " [\"kubectl\", \"get\", \"secret\", \"mlflow-credentials\", \"-n\", _ns, \"-o\", \"json\"],\n", + " capture_output=True, text=True,\n", + ")\n", + "if _result.returncode != 0:\n", + " raise RuntimeError(\n", + " \"mlflow-credentials secret not found.\\n\"\n", + " \"Run scripts/setup_mlflow_credentials.py to create it.\"\n", + " )\n", + "_data = json.loads(_result.stdout)[\"data\"]\n", + "os.environ[\"MLFLOW_TRACKING_URI\"] = base64.b64decode(_data[\"MLFLOW_TRACKING_URI\"]).decode()\n", + "os.environ[\"MLFLOW_TRACKING_USERNAME\"] = base64.b64decode(_data[\"MLFLOW_TRACKING_USERNAME\"]).decode()\n", + "os.environ[\"MLFLOW_TRACKING_PASSWORD\"] = base64.b64decode(_data[\"MLFLOW_TRACKING_PASSWORD\"]).decode()\n", + "os.environ[\"MLFLOW_ENABLE_PROXY_MULTIPART_UPLOAD\"] = \"true\"\n" ] }, { diff --git a/mlflow/mlflow-kfp-example.ipynb b/mlflow/mlflow-kfp-example.ipynb index 7ef127f..4477f41 100644 --- a/mlflow/mlflow-kfp-example.ipynb +++ b/mlflow/mlflow-kfp-example.ipynb @@ -23,15 +23,12 @@ " - Copy the generated token and store it securely\n", " - Note: You won't be able to see the token again!\n", "\n", - "Now change copy below command and change it to your values:\n", + "Run `scripts/setup_mlflow_credentials.py` once to create the required secret:\n", "```bash\n", - "kubectl create secret generic mlflow-credentials \\\n", - "--from-literal=MLFLOW_TRACKING_URI='https:///mlflow' \\\n", - "--from-literal=MLFLOW_TRACKING_USERNAME='your-email@example.com' \\\n", - "--from-literal=MLFLOW_TRACKING_PASSWORD='your-personal-access-token'\n", + "# From a JupyterLab terminal:\n", + "python examples/scripts/setup_mlflow_credentials.py\n", "```\n", - "\n", - "You can then paste the above command in the terminal built into jupyter-lab and press enter to run it.\n", + "You will be prompted for your MLflow tracking URI, username (e-mail), and Personal Access Token.\n", "\n", "## How it works\n", "\n", diff --git a/mlflow/mlflow-quickstart-example.ipynb b/mlflow/mlflow-quickstart-example.ipynb index f1927bc..06c796a 100644 --- a/mlflow/mlflow-quickstart-example.ipynb +++ b/mlflow/mlflow-quickstart-example.ipynb @@ -52,61 +52,27 @@ }, "outputs": [], "source": [ - "import os\n", + "import base64, json, os, subprocess\n", "\n", - "# Set your MLflow credentials\n", - "os.environ['MLFLOW_TRACKING_URI'] = 'https:///mlflow'\n", - "os.environ['MLFLOW_TRACKING_USERNAME'] = '' \n", - "os.environ['MLFLOW_TRACKING_PASSWORD'] = ''\n", + "# Load MLflow credentials from the mlflow-credentials Kubernetes secret.\n", + "# Run scripts/setup_mlflow_credentials.py once to create this secret.\n", + "with open(\"/var/run/secrets/kubernetes.io/serviceaccount/namespace\") as _f:\n", + " _ns = _f.read().strip()\n", "\n", - "# Enable presigned URL uploads to avoid RAM buffering in MLflow server\n", - "os.environ['MLFLOW_ENABLE_PROXY_MULTIPART_UPLOAD'] = 'true'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "02a920aa-df99-4902-b9ca-b534542882c7", - "metadata": {}, - "outputs": [], - "source": [ - "# Validate MLflow configuration\n", - "import os\n", - "\n", - "REQUIRED_ENV_VARS = [\"MLFLOW_TRACKING_URI\", \"MLFLOW_TRACKING_USERNAME\", \"MLFLOW_TRACKING_PASSWORD\"]\n", - "missing_vars = [var for var in REQUIRED_ENV_VARS if not os.environ.get(var)]\n", - "placeholder_vars = [\n", - " var for var in REQUIRED_ENV_VARS \n", - " if os.environ.get(var, \"\").startswith((\"https://<\", \"/mlflow' \\\n", - " --from-literal=MLFLOW_TRACKING_USERNAME='your-email@example.com' \\\n", - " --from-literal=MLFLOW_TRACKING_PASSWORD='your-personal-access-token'\n", + "# From a JupyterLab terminal:\n", + "python examples/scripts/setup_mlflow_credentials.py\n", "```\n", + "You will be prompted for your MLflow tracking URI, username (e-mail), and Personal Access Token.\n", "\n", "To generate a Personal Access Token, open the [MLflow UI](/mlflow) and navigate to [Permissions](/mlflow/oidc/ui/#)." ] diff --git a/notebooks/dask/cleanup.py b/notebooks/dask/cleanup.py new file mode 100644 index 0000000..0269852 --- /dev/null +++ b/notebooks/dask/cleanup.py @@ -0,0 +1,53 @@ +""" +Cleanup script for the dask example. + +Deletes (idempotently): + - Any DaskCluster CRs left running in the current namespace + (dask-kubernetes names them after the KubeCluster object; the example + uses the default name ``dask-cluster``) + +Usage +----- + python cleanup.py [--dry-run] +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys + +_CLUSTER_NAME = "dask-cluster" + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _kubectl_delete(*args: str, dry_run: bool = False) -> None: + cmd = ["kubectl", "delete", *args, "--ignore-not-found"] + if dry_run: + print(f"[dry-run] {' '.join(cmd)}") + return + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"WARNING: {result.stderr.strip()}", file=sys.stderr) + else: + print(result.stdout.strip() or f"deleted (or not found): {' '.join(args)}") + + +def cleanup(dry_run: bool = False) -> None: + ns = _namespace() + _kubectl_delete("daskcluster", _CLUSTER_NAME, "-n", ns, dry_run=dry_run) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--dry-run", action="store_true", help="Print commands without executing them" + ) + args = parser.parse_args() + cleanup(dry_run=args.dry_run) diff --git a/notebooks/dask/dask_example.ipynb b/notebooks/dask/dask_example.ipynb index a0686ce..77fc563 100644 --- a/notebooks/dask/dask_example.ipynb +++ b/notebooks/dask/dask_example.ipynb @@ -17,7 +17,8 @@ "metadata": {}, "outputs": [], "source": [ - "namespace = 'your-namespace-name' # put the name of your namespace here" + "with open(\"/var/run/secrets/kubernetes.io/serviceaccount/namespace\") as _f:\n", + " namespace = _f.read().strip()\n" ] }, { @@ -27,8 +28,7 @@ "metadata": {}, "outputs": [], "source": [ - "if namespace == 'your-namespace-name':\n", - " raise ValueError('You need to set the `namespace` variable to your personal namespace')" + "print(f\"Using namespace: {namespace}\")\n" ] }, { diff --git a/pipelines/lightweight-python-package/submit-cluster.py b/pipelines/lightweight-python-package/submit-cluster.py index d9499ae..fc4485b 100644 --- a/pipelines/lightweight-python-package/submit-cluster.py +++ b/pipelines/lightweight-python-package/submit-cluster.py @@ -4,7 +4,9 @@ if __name__ == "__main__": - with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace", "r", encoding="utf-8") as namespace_file: + with open( + "/var/run/secrets/kubernetes.io/serviceaccount/namespace", "r", encoding="utf-8" + ) as namespace_file: namespace = namespace_file.read().strip() s3_bucket = f"{namespace}-data" @@ -30,3 +32,4 @@ experiment_name="mobile-price-classification-containerized", run_name=f"Containerized pipeline {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", ) + print(f"KFP_RUN_ID={run.run_id}") diff --git a/scripts/get_or_create_api_key.py b/scripts/get_or_create_api_key.py new file mode 100644 index 0000000..534aa12 --- /dev/null +++ b/scripts/get_or_create_api_key.py @@ -0,0 +1,116 @@ +""" +Utility to create or retrieve a prokube AIGatewayKey for use in examples. + +Usage from a notebook +--------------------- + import sys, subprocess, os + _root = subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip() + sys.path.insert(0, os.path.join(_root, "scripts")) + from get_or_create_api_key import get_or_create_api_key + + API_KEY = get_or_create_api_key() + +CLI usage +--------- + python get_or_create_api_key.py + # prints the key to stdout + +The key is stored as an AIGatewayKey CR named ``examples-key`` backed by a +Secret named ``examples-key-secret`` in the notebook's namespace. Both +resources are created if they do not already exist. Re-running is safe. +""" + +from __future__ import annotations + +import base64 +import json +import secrets +import subprocess +import sys + +_KEY_NAME = "examples-key" +_SECRET_NAME = "examples-key-secret" + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _kubectl(*args: str, input: str | None = None) -> subprocess.CompletedProcess: + return subprocess.run( + ["kubectl", *args], + input=input, + capture_output=True, + text=True, + ) + + +def _key_exists(namespace: str) -> bool: + result = _kubectl( + "get", "aigatewaykey", _KEY_NAME, "-n", namespace, "--ignore-not-found" + ) + return bool(result.stdout.strip()) + + +def _read_key_from_secret(namespace: str) -> str: + result = _kubectl("get", "secret", _SECRET_NAME, "-n", namespace, "-o", "json") + if result.returncode != 0: + raise RuntimeError(f"Could not read secret {_SECRET_NAME}:\n{result.stderr}") + data = json.loads(result.stdout)["data"] + return base64.b64decode(data["token"]).decode() + + +def _apply(manifest: str, namespace: str) -> None: + result = _kubectl("apply", "-n", namespace, "-f", "-", input=manifest) + if result.returncode != 0: + raise RuntimeError(f"kubectl apply failed:\n{result.stderr}") + + +def _create_key(namespace: str) -> str: + key_value = f"pk_live_{secrets.token_hex(20)}" + + _apply( + f"apiVersion: v1\n" + f"kind: Secret\n" + f"metadata:\n" + f" name: {_SECRET_NAME}\n" + f" namespace: {namespace}\n" + f"stringData:\n" + f" token: {key_value}\n", + namespace, + ) + + _apply( + f"apiVersion: prokube.ai/v1alpha1\n" + f"kind: AIGatewayKey\n" + f"metadata:\n" + f" name: {_KEY_NAME}\n" + f" namespace: {namespace}\n" + f"spec:\n" + f" displayName: Examples API key\n" + f" secretRef:\n" + f" name: {_SECRET_NAME}\n" + f" key: token\n" + f" scopes:\n" + f" - type: workspace\n", + namespace, + ) + + return key_value + + +def get_or_create_api_key() -> str: + """Return the API key for the current namespace, creating it if needed.""" + ns = _namespace() + if _key_exists(ns): + return _read_key_from_secret(ns) + return _create_key(ns) + + +if __name__ == "__main__": + try: + print(get_or_create_api_key()) + except Exception as exc: # noqa: BLE001 + print(f"ERROR: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/setup_mlflow_credentials.py b/scripts/setup_mlflow_credentials.py new file mode 100644 index 0000000..b1b3011 --- /dev/null +++ b/scripts/setup_mlflow_credentials.py @@ -0,0 +1,125 @@ +""" +One-time setup: store MLflow credentials in a Kubernetes secret so that +example notebooks can read them without hardcoded values. + +This is the only step in the automation plan that requires human input — +the MLflow Personal Access Token (PAT) must be created via the MLflow UI +(Permissions → Create access key) and cannot be obtained programmatically. + +Usage from a notebook +--------------------- + import sys, subprocess, os + _root = subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip() + sys.path.insert(0, os.path.join(_root, "scripts")) + from setup_mlflow_credentials import setup_mlflow_credentials + + setup_mlflow_credentials( + uri="https://my-cluster.example.com/mlflow/", + username="alice@example.com", + password="", + ) + +CLI usage (any argument can be omitted; you will be prompted for it) +--------------------------------------------------------------------- + python setup_mlflow_credentials.py \\ + --uri https://my-cluster.example.com/mlflow/ \\ + --username alice@example.com \\ + --password + +The credentials are stored in a Secret named ``mlflow-credentials`` in the +notebook's namespace. Re-running updates the secret in place. +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys + +_SECRET_NAME = "mlflow-credentials" + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _prompt(label: str, secret: bool = False) -> str: + if secret: + import getpass + + return getpass.getpass(f"{label}: ") + return input(f"{label}: ").strip() + + +def setup_mlflow_credentials( + uri: str | None = None, + username: str | None = None, + password: str | None = None, +) -> None: + """Create or update the ``mlflow-credentials`` secret. + + Any parameter left as ``None`` will be requested interactively. + """ + if uri is None: + print("MLflow tracking URI — typically https:///mlflow/") + uri = _prompt("MLFLOW_TRACKING_URI") + if username is None: + username = _prompt("MLFLOW_TRACKING_USERNAME (your login e-mail)") + if password is None: + password = _prompt( + "MLFLOW_TRACKING_PASSWORD (Personal Access Token)", secret=True + ) + + ns = _namespace() + + result = subprocess.run( + [ + "kubectl", + "create", + "secret", + "generic", + _SECRET_NAME, + "-n", + ns, + f"--from-literal=MLFLOW_TRACKING_URI={uri}", + f"--from-literal=MLFLOW_TRACKING_USERNAME={username}", + f"--from-literal=MLFLOW_TRACKING_PASSWORD={password}", + "--save-config", + "--dry-run=client", + "-o", + "yaml", + ], + capture_output=True, + text=True, + check=True, + ) + + apply = subprocess.run( + ["kubectl", "apply", "-n", ns, "-f", "-"], + input=result.stdout, + capture_output=True, + text=True, + ) + if apply.returncode != 0: + raise RuntimeError(f"kubectl apply failed:\n{apply.stderr}") + + print(f"Secret '{_SECRET_NAME}' created/updated in namespace '{ns}'.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--uri", default=None, help="MLFLOW_TRACKING_URI") + parser.add_argument("--username", default=None, help="MLFLOW_TRACKING_USERNAME") + parser.add_argument( + "--password", default=None, help="MLFLOW_TRACKING_PASSWORD (PAT)" + ) + args = parser.parse_args() + + try: + setup_mlflow_credentials(args.uri, args.username, args.password) + except Exception as exc: # noqa: BLE001 + print(f"ERROR: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/serving/minimal-s3-model/cleanup.py b/serving/minimal-s3-model/cleanup.py new file mode 100644 index 0000000..9059dd3 --- /dev/null +++ b/serving/minimal-s3-model/cleanup.py @@ -0,0 +1,63 @@ +""" +Cleanup script for the minimal-s3-model example. + +Deletes (idempotently): + - InferenceService ``kserve-object-storage-test`` + - AIGatewayKey ``examples-key`` and its backing Secret ``examples-key-secret`` + (only when --include-api-key is passed, since the key is shared across examples) + +Usage +----- + python cleanup.py [--include-api-key] [--dry-run] +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _kubectl_delete(*args: str, dry_run: bool = False) -> None: + cmd = ["kubectl", "delete", *args, "--ignore-not-found"] + if dry_run: + print(f"[dry-run] {' '.join(cmd)}") + return + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"WARNING: {result.stderr.strip()}", file=sys.stderr) + else: + print(result.stdout.strip() or f"deleted (or not found): {' '.join(args)}") + + +def cleanup(include_api_key: bool = False, dry_run: bool = False) -> None: + ns = _namespace() + + _kubectl_delete( + "inferenceservice", "kserve-object-storage-test", "-n", ns, dry_run=dry_run + ) + + if include_api_key: + _kubectl_delete("aigatewaykey", "examples-key", "-n", ns, dry_run=dry_run) + _kubectl_delete("secret", "examples-key-secret", "-n", ns, dry_run=dry_run) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--include-api-key", + action="store_true", + help="Also delete the shared AIGatewayKey", + ) + parser.add_argument( + "--dry-run", action="store_true", help="Print commands without executing them" + ) + args = parser.parse_args() + cleanup(include_api_key=args.include_api_key, dry_run=args.dry_run) diff --git a/serving/minimal-s3-model/minimal-s3-model.ipynb b/serving/minimal-s3-model/minimal-s3-model.ipynb index 6d1b063..f1b5e91 100644 --- a/serving/minimal-s3-model/minimal-s3-model.ipynb +++ b/serving/minimal-s3-model/minimal-s3-model.ipynb @@ -307,9 +307,12 @@ "metadata": {}, "outputs": [], "source": [ - "INFERENCE_SERVICE_API_KEY = \"\"\n", - "if not INFERENCE_SERVICE_API_KEY:\n", - " raise RuntimeError(\"Please provide the API Key that will be used to test the deployed InferenceService\")" + "import sys, subprocess, os\n", + "_root = subprocess.check_output([\"git\", \"rev-parse\", \"--show-toplevel\"], text=True).strip()\n", + "sys.path.insert(0, os.path.join(_root, \"scripts\"))\n", + "from get_or_create_api_key import get_or_create_api_key\n", + "\n", + "INFERENCE_SERVICE_API_KEY = get_or_create_api_key()\n" ] }, { diff --git a/serving/mlflow-kserve-inference-protocols/cleanup.py b/serving/mlflow-kserve-inference-protocols/cleanup.py new file mode 100644 index 0000000..f4fb07d --- /dev/null +++ b/serving/mlflow-kserve-inference-protocols/cleanup.py @@ -0,0 +1,56 @@ +""" +Cleanup script for the mlflow-kserve-inference-protocols example. + +Deletes (idempotently): + - InferenceService ``v1-mobile-price-classification-inference`` + - InferenceService ``v2-mobile-price-classification-inference`` + +Usage +----- + python cleanup.py [--dry-run] +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys + +_ISVC_NAMES = [ + "v1-mobile-price-classification-inference", + "v2-mobile-price-classification-inference", +] + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _kubectl_delete(*args: str, dry_run: bool = False) -> None: + cmd = ["kubectl", "delete", *args, "--ignore-not-found"] + if dry_run: + print(f"[dry-run] {' '.join(cmd)}") + return + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"WARNING: {result.stderr.strip()}", file=sys.stderr) + else: + print(result.stdout.strip() or f"deleted (or not found): {' '.join(args)}") + + +def cleanup(dry_run: bool = False) -> None: + ns = _namespace() + for name in _ISVC_NAMES: + _kubectl_delete("inferenceservice", name, "-n", ns, dry_run=dry_run) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--dry-run", action="store_true", help="Print commands without executing them" + ) + args = parser.parse_args() + cleanup(dry_run=args.dry_run) diff --git a/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb b/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb index d6971c5..46e1d2c 100644 --- a/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb +++ b/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb @@ -340,10 +340,88 @@ "source": [ "---\n", "\n", - "## 6. Live Testing (Optional)\n", + "## 6. Live Testing\n", "\n", - "If you have deployed both InferenceServices, you can test them here. \n", - "Fill in your connection details and run the cells below." + "The cells below deploy both InferenceServices, wait for them to become ready,\n", + "and run inference requests automatically.\n", + "\n", + "**Prerequisite:** the `mlflow-credentials` secret must exist in your namespace\n", + "(run `scripts/setup_mlflow_credentials.py` once if you have not done so).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "auto-setup-isvc", + "metadata": {}, + "outputs": [], + "source": [ + "import base64, json, os, subprocess, sys\n", + "\n", + "# ── 1. Namespace & MLflow username ───────────────────────────────────────────\n", + "with open(\"/var/run/secrets/kubernetes.io/serviceaccount/namespace\") as _f:\n", + " _ns = _f.read().strip()\n", + "\n", + "_secret = subprocess.run(\n", + " [\"kubectl\", \"get\", \"secret\", \"mlflow-credentials\", \"-n\", _ns, \"-o\", \"json\"],\n", + " capture_output=True, text=True,\n", + ")\n", + "if _secret.returncode != 0:\n", + " raise RuntimeError(\n", + " \"mlflow-credentials secret not found.\\n\"\n", + " \"Run scripts/setup_mlflow_credentials.py to create it.\"\n", + " )\n", + "_creds = json.loads(_secret.stdout)[\"data\"]\n", + "_username = base64.b64decode(_creds[\"MLFLOW_TRACKING_USERNAME\"]).decode()\n", + "\n", + "# ── 2. Apply InferenceService YAMLs with substituted namespace & username ────\n", + "_isvc_names = [\n", + " (\"v1-InferenceService.yaml\", \"v1-mobile-price-classification-inference\"),\n", + " (\"v2-InferenceService.yaml\", \"v2-mobile-price-classification-inference\"),\n", + "]\n", + "for _yaml_file, _ in _isvc_names:\n", + " _manifest = open(_yaml_file).read()\n", + " _manifest = _manifest.replace(\"\", _ns).replace(\"\", _username)\n", + " _r = subprocess.run(\n", + " [\"kubectl\", \"apply\", \"-n\", _ns, \"-f\", \"-\"],\n", + " input=_manifest, capture_output=True, text=True,\n", + " )\n", + " if _r.returncode != 0:\n", + " raise RuntimeError(f\"Failed to apply {_yaml_file}:\\n{_r.stderr}\")\n", + " print(f\"Applied {_yaml_file}\")\n", + "\n", + "# ── 3. Wait for both ISVCs to be ready (up to 10 min each) ───────────────────\n", + "print(\"Waiting for InferenceServices to become ready...\")\n", + "for _, _name in _isvc_names:\n", + " _r = subprocess.run(\n", + " [\"kubectl\", \"wait\", \"inferenceservice\", _name, \"-n\", _ns,\n", + " \"--for=condition=Ready\", \"--timeout=600s\"],\n", + " capture_output=True, text=True,\n", + " )\n", + " if _r.returncode != 0:\n", + " raise RuntimeError(f\"InferenceService {_name} did not become ready:\\n{_r.stderr}\")\n", + " print(f\" {_name}: Ready\")\n", + "\n", + "# ── 4. Fetch external URLs from ISVC status ───────────────────────────────────\n", + "def _isvc_url(name):\n", + " return subprocess.check_output(\n", + " [\"kubectl\", \"get\", \"inferenceservice\", name, \"-n\", _ns,\n", + " \"-o\", \"jsonpath={.status.url}\"],\n", + " text=True,\n", + " ).strip()\n", + "\n", + "_uri_v1 = _isvc_url(\"v1-mobile-price-classification-inference\")\n", + "_uri_v2 = _isvc_url(\"v2-mobile-price-classification-inference\")\n", + "\n", + "# ── 5. Get / create inference API key ────────────────────────────────────────\n", + "_root = subprocess.check_output([\"git\", \"rev-parse\", \"--show-toplevel\"], text=True).strip()\n", + "sys.path.insert(0, os.path.join(_root, \"scripts\"))\n", + "from get_or_create_api_key import get_or_create_api_key\n", + "_api_key = get_or_create_api_key()\n", + "\n", + "print(f\"\\nv1 URI : {_uri_v1}\")\n", + "print(f\"v2 URI : {_uri_v2}\")\n", + "print(\"API key: [set]\")\n" ] }, { @@ -356,27 +434,24 @@ "import requests\n", "\n", "# ---------------------------------------------------------------------------\n", - "# Connection settings (fill these in)\n", + "# Connection settings (auto-populated by the setup cell above)\n", "# ---------------------------------------------------------------------------\n", - "INFERENCE_SERVICE_URI_V1 = \"\" # external URL, no trailing slash (copied from KServe InferenceService status.url)\n", - "INFERENCE_SERVICE_URI_V2 = \"\" # external URL, no trailing slash (copied from KServe InferenceService status.url)\n", - "API_KEY = \"\" # API key, provided by your administrator \n", + "INFERENCE_SERVICE_URI_V1 = _uri_v1\n", + "INFERENCE_SERVICE_URI_V2 = _uri_v2\n", + "API_KEY = _api_key\n", "\n", "# ---------------------------------------------------------------------------\n", "# Model names (must match metadata.name in the InferenceService YAMLs)\n", "# ---------------------------------------------------------------------------\n", - "MODEL_NAME_V1 = \"v1-mobile-price-classification-inference\" # from v1_InferenceService.yaml\n", - "MODEL_NAME_V2 = \"v2-mobile-price-classification-inference\" # from v2_InferenceService.yaml\n", + "MODEL_NAME_V1 = \"v1-mobile-price-classification-inference\"\n", + "MODEL_NAME_V2 = \"v2-mobile-price-classification-inference\"\n", "\n", "HEADERS = {\"X-Api-Key\": API_KEY, \"Content-Type\": \"application/json\"}\n", "\n", - "if not INFERENCE_SERVICE_URI_V1 or not INFERENCE_SERVICE_URI_V2 or not API_KEY:\n", - " raise ValueError(\"Please fill in the connection settings (URIs and API key) before running the tests.\")\n", - "\n", - "print(f\"v1 URI : {INFERENCE_SERVICE_URI_V1}\")\n", - "print(f\"v2 URI : {INFERENCE_SERVICE_URI_V2}\")\n", - "print(f\"v1 model : {MODEL_NAME_V1}\")\n", - "print(f\"v2 model : {MODEL_NAME_V2}\")" + "print(f\"v1 URI : {INFERENCE_SERVICE_URI_V1}\")\n", + "print(f\"v2 URI : {INFERENCE_SERVICE_URI_V2}\")\n", + "print(f\"v1 model: {MODEL_NAME_V1}\")\n", + "print(f\"v2 model: {MODEL_NAME_V2}\")\n" ] }, { diff --git a/serving/mlflow-kserve-minimal/apply.py b/serving/mlflow-kserve-minimal/apply.py new file mode 100644 index 0000000..b3896bd --- /dev/null +++ b/serving/mlflow-kserve-minimal/apply.py @@ -0,0 +1,150 @@ +""" +Deploy the mlflow-kserve-minimal InferenceService. + +Substitutes ```` and ```` in InferenceService.yaml +from the cluster environment, applies it, waits for readiness, and prints the +external URL so it can be used by ``test_inference_service.py``. + +Usage from a notebook +--------------------- + import sys, subprocess, os + _root = subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip() + sys.path.insert(0, os.path.join(_root, "serving", "mlflow-kserve-minimal")) + from apply import deploy + + isvc_uri = deploy() + +CLI usage +--------- + python apply.py + # prints ISVC_URI= to stdout + +Prerequisites +------------- +- ``mlflow-credentials`` secret must exist (run scripts/setup_mlflow_credentials.py). +- The MLflow model ``mobile-price-svm-`` version 1 must be registered + (run mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb first). +""" + +from __future__ import annotations + +import base64 +import json +import os +import subprocess +import sys + +_ISVC_NAME = "mobile-price-svm" +_YAML_TEMPLATE = os.path.join(os.path.dirname(__file__), "InferenceService.yaml") + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _mlflow_username(namespace: str) -> str: + result = subprocess.run( + [ + "kubectl", + "get", + "secret", + "mlflow-credentials", + "-n", + namespace, + "-o", + "json", + ], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError( + "mlflow-credentials secret not found.\n" + "Run scripts/setup_mlflow_credentials.py to create it." + ) + data = json.loads(result.stdout)["data"] + return base64.b64decode(data["MLFLOW_TRACKING_USERNAME"]).decode() + + +def _kubectl_apply(manifest: str, namespace: str) -> None: + result = subprocess.run( + ["kubectl", "apply", "-n", namespace, "-f", "-"], + input=manifest, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError(f"kubectl apply failed:\n{result.stderr}") + + +def _wait_ready(name: str, namespace: str, timeout: int = 600) -> None: + print( + f"Waiting for InferenceService '{name}' to become ready (timeout {timeout}s)..." + ) + result = subprocess.run( + [ + "kubectl", + "wait", + "inferenceservice", + name, + "-n", + namespace, + "--for=condition=Ready", + f"--timeout={timeout}s", + ], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError( + f"InferenceService '{name}' did not become ready:\n{result.stderr}" + ) + print(f" '{name}' is Ready.") + + +def _isvc_url(name: str, namespace: str) -> str: + return subprocess.check_output( + [ + "kubectl", + "get", + "inferenceservice", + name, + "-n", + namespace, + "-o", + "jsonpath={.status.url}", + ], + text=True, + ).strip() + + +def deploy(timeout: int = 600) -> str: + """Apply InferenceService.yaml, wait for readiness, return the external URL.""" + ns = _namespace() + username = _mlflow_username(ns) + + with open(_YAML_TEMPLATE) as fh: + manifest = fh.read() + + manifest = ( + manifest.replace("", _ISVC_NAME) + .replace("", ns) + .replace("", username) + ) + + _kubectl_apply(manifest, ns) + print(f"Applied InferenceService '{_ISVC_NAME}' in namespace '{ns}'.") + + _wait_ready(_ISVC_NAME, ns, timeout=timeout) + url = _isvc_url(_ISVC_NAME, ns) + print(f"ISVC_URI={url}") + return url + + +if __name__ == "__main__": + try: + deploy() + except Exception as exc: # noqa: BLE001 + print(f"ERROR: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/serving/mlflow-kserve-minimal/cleanup.py b/serving/mlflow-kserve-minimal/cleanup.py new file mode 100644 index 0000000..96c341c --- /dev/null +++ b/serving/mlflow-kserve-minimal/cleanup.py @@ -0,0 +1,49 @@ +""" +Cleanup script for the mlflow-kserve-minimal example. + +Deletes (idempotently): + - InferenceService ``mobile-price-svm`` + +Usage +----- + python cleanup.py [--dry-run] +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _kubectl_delete(*args: str, dry_run: bool = False) -> None: + cmd = ["kubectl", "delete", *args, "--ignore-not-found"] + if dry_run: + print(f"[dry-run] {' '.join(cmd)}") + return + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"WARNING: {result.stderr.strip()}", file=sys.stderr) + else: + print(result.stdout.strip() or f"deleted (or not found): {' '.join(args)}") + + +def cleanup(dry_run: bool = False) -> None: + ns = _namespace() + _kubectl_delete("inferenceservice", "mobile-price-svm", "-n", ns, dry_run=dry_run) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--dry-run", action="store_true", help="Print commands without executing them" + ) + args = parser.parse_args() + cleanup(dry_run=args.dry_run) From ca88b4e828892eaddd5ee5de4dc4b3a194fb4ec4 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Thu, 11 Jun 2026 14:26:08 +0200 Subject: [PATCH 02/57] Better error handling --- ci/run_all.py | 108 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 86 insertions(+), 22 deletions(-) diff --git a/ci/run_all.py b/ci/run_all.py index 17d61ea..1cf3229 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -45,9 +45,7 @@ from __future__ import annotations import argparse -import importlib.util import json -import os import re import subprocess import sys @@ -55,13 +53,30 @@ from concurrent.futures import Future, ThreadPoolExecutor, as_completed from dataclasses import dataclass, field from pathlib import Path -from typing import Optional + # ── Repo root ───────────────────────────────────────────────────────────────── _REPO_ROOT = Path( subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip() ) + +# ── Prerequisite: papermill ─────────────────────────────────────────────────── + + +def _ensure_papermill() -> None: + """Install papermill if it is not already available.""" + try: + import papermill # noqa: F401 + except ImportError: + print("papermill not found — installing...") + subprocess.run( + [sys.executable, "-m", "pip", "install", "papermill"], + check=True, + ) + print("papermill installed.") + + # ── KFP run ID pattern (UUID v4) ───────────────────────────────────────────── _RUN_ID_RE = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") @@ -86,23 +101,53 @@ def _namespace() -> str: return fh.read().strip() -def _run_notebook(nb_path: Path, output_dir: Path, timeout: int) -> tuple[Path, str]: - """Execute a notebook with papermill. Returns (output_path, stderr).""" +def _format_papermill_error(exc: Exception) -> str: + """Extract a human-readable summary from a PapermillExecutionError.""" try: - import papermill as pm + from papermill.exceptions import PapermillExecutionError + + if not isinstance(exc, PapermillExecutionError): + return str(exc) except ImportError: - raise RuntimeError("papermill is not installed. Run: pip install papermill") + return str(exc) + + lines = [ + f"Cell {exc.exec_count} raised {exc.ename}: {exc.evalue}", + ] + # Show the failing cell source (first 5 lines) + if exc.source: + src_lines = exc.source.strip().splitlines()[:5] + lines.append(" Cell source:") + for src_line in src_lines: + lines.append(f" {src_line}") + if len(exc.source.strip().splitlines()) > 5: + lines.append(" ...") + # Show the innermost traceback frame (last non-empty line) + if exc.traceback: + tb_lines = [l for l in exc.traceback if l.strip()] + if tb_lines: + lines.append(f" Traceback (last): {tb_lines[-1].strip()}") + return "\n".join(lines) + + +def _run_notebook(nb_path: Path, output_dir: Path, timeout: int) -> Path: + """Execute a notebook with papermill. Returns the output notebook path.""" + import papermill as pm + from papermill.exceptions import PapermillExecutionError output_path = output_dir / nb_path.name output_dir.mkdir(parents=True, exist_ok=True) - pm.execute_notebook( - str(nb_path), - str(output_path), - kernel_name="python3", - execution_timeout=timeout, - cwd=str(nb_path.parent), - ) - return output_path, "" + try: + pm.execute_notebook( + str(nb_path), + str(output_path), + kernel_name="python3", + execution_timeout=timeout, + cwd=str(nb_path.parent), + ) + except PapermillExecutionError as exc: + raise RuntimeError(_format_papermill_error(exc)) from exc + return output_path def _run_script(script_path: Path, timeout: int) -> tuple[str, str]: @@ -115,7 +160,9 @@ def _run_script(script_path: Path, timeout: int) -> tuple[str, str]: cwd=str(script_path.parent), ) if result.returncode != 0: - raise RuntimeError(result.stderr) + # Include both stderr and stdout for full context + detail = (result.stderr or result.stdout or "(no output)").strip() + raise RuntimeError(detail) return result.stdout, result.stderr @@ -190,12 +237,12 @@ def _submit_notebook( def _run(): t0 = time.time() try: - out, _ = _run_notebook(nb_path, output_dir, timeout) + out = _run_notebook(nb_path, output_dir, timeout) result.status = "PASS" result.kfp_run_ids = _extract_run_ids_from_notebook(out) except Exception as exc: # noqa: BLE001 result.status = "FAIL" - result.error = str(exc)[:300] + result.error = str(exc) finally: result.duration = time.time() - t0 @@ -220,7 +267,7 @@ def _run(): result.kfp_run_ids = _extract_run_ids_from_stdout(stdout) except Exception as exc: # noqa: BLE001 result.status = "FAIL" - result.error = str(exc)[:300] + result.error = str(exc) finally: result.duration = time.time() - t0 @@ -230,6 +277,15 @@ def _run(): # ── Report ──────────────────────────────────────────────────────────────────── +def _print_result(r: Result) -> None: + """Print a single result line, with error detail if failed.""" + dur = f"{r.duration:.0f}s" if r.duration else "-" + print(f" [{r.status}] {r.name} ({dur})") + if r.status == "FAIL" and r.error: + for line in r.error.splitlines(): + print(f" {line}") + + def _print_report(results: dict[str, Result], poll_results: dict[str, str]) -> None: col = 50 print("\n" + "=" * 70) @@ -240,7 +296,6 @@ def _print_report(results: dict[str, Result], poll_results: dict[str, str]) -> N dur = f"{r.duration:.0f}s" if r.duration else "-" print(f"{r.name:<{col}} {r.status:<10} {dur}") if r.status == "FAIL": - print(f" {'':>{col}} {r.error}") failed += 1 else: passed += 1 @@ -250,14 +305,21 @@ def _print_report(results: dict[str, Result], poll_results: dict[str, str]) -> N print("-" * 70) for run_id, status in poll_results.items(): short = run_id[:8] + "..." - ok = status == "Succeeded" - if not ok: + if status != "Succeeded": failed += 1 else: passed += 1 print(f"{short:<{col}} {status:<10}") print("=" * 70) print(f"PASSED: {passed} FAILED: {failed} TOTAL: {passed + failed}") + if failed: + print("\nFailed details:") + print("-" * 70) + for r in results.values(): + if r.status == "FAIL" and r.error: + print(f"\n{r.name}:") + for line in r.error.splitlines(): + print(f" {line}") print() @@ -275,6 +337,8 @@ def run_all( results: dict[str, Result] = {} poll_results: dict[str, str] = {} + _ensure_papermill() + if dry_run: print("[dry-run] Would execute the following phases:") for label in [ From ca176040ab1308179e5965d507be2533e270e4be Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Thu, 11 Jun 2026 14:28:45 +0200 Subject: [PATCH 03/57] ci: fix output readability and error reporting - Auto-install papermill if missing (pip install at startup) - Disable papermill progress bars (progress_bar=False) to prevent interleaved tqdm output from parallel notebook executions - Print [START] when each notebook/script is submitted so names are visible immediately, not just after completion - Switch from sequential f.result() to as_completed() so results print as each finishes rather than in submission order - Extract structured PapermillExecutionError details: cell number, cell source (first 5 lines), innermost traceback frame - Include full stderr/stdout in script failure messages (was truncated) - Print error details inline after each [FAIL] line during execution - Print full 'Failed details' section at the bottom of the report - Fix phase 2 remaining results printing under Phase 3 header --- ci/run_all.py | 41 ++++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/ci/run_all.py b/ci/run_all.py index 1cf3229..4c38585 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -144,6 +144,7 @@ def _run_notebook(nb_path: Path, output_dir: Path, timeout: int) -> Path: kernel_name="python3", execution_timeout=timeout, cwd=str(nb_path.parent), + progress_bar=False, # avoid interleaved tqdm bars from parallel threads ) except PapermillExecutionError as exc: raise RuntimeError(_format_papermill_error(exc)) from exc @@ -233,6 +234,7 @@ def _submit_notebook( ) -> Future: result = Result(name=name) results[name] = result + print(f" [START ] {name}") def _run(): t0 = time.time() @@ -258,6 +260,7 @@ def _submit_script( ) -> Future: result = Result(name=name) results[name] = result + print(f" [START ] {name}") def _run(): t0 = time.time() @@ -379,10 +382,10 @@ def run_all( ) phase1_futures[name] = f - # wait for all phase 1 - for name, f in phase1_futures.items(): - f.result() - print(f" [{results[name].status}] {name}") + # wait for all phase 1, printing results as each finishes + _future_to_name = {v: k for k, v in phase1_futures.items()} + for f in as_completed(phase1_futures.values()): + _print_result(results[_future_to_name[f]]) # ── Phase 2: pipeline submissions (return fast) ─────────────────── print("\nPhase 2: submitting pipelines...") @@ -412,11 +415,8 @@ def run_all( phase2_futures["pipelines/lightweight-python-package"] = lpp_future # Wait only for mlflow-mobile-price before launching phase 3 - mlflow_mobile_future = phase2_futures["mlflow/mobile-price-classification"] - mlflow_mobile_future.result() - print( - f" [{results['mlflow/mobile-price-classification'].status}] mlflow/mobile-price-classification" - ) + phase2_futures["mlflow/mobile-price-classification"].result() + _print_result(results["mlflow/mobile-price-classification"]) # ── Phase 3: MLflow KServe examples (need registered model) ─────── print("\nPhase 3: deploying MLflow KServe InferenceServices...") @@ -440,16 +440,19 @@ def run_all( f = _submit_script(executor, results, name, path, timeout_notebook) phase3_futures[name] = f - # Wait for remaining phase 2 futures while phase 3 runs - for name, f in phase2_futures.items(): - if name != "mlflow/mobile-price-classification": - f.result() - print(f" [{results[name].status}] {name}") - - # Wait for phase 3 - for name, f in phase3_futures.items(): - f.result() - print(f" [{results[name].status}] {name}") + # Phase 2 remaining + phase 3 run concurrently; print as each finishes + print("\nPhase 2 (remaining) + Phase 3 running concurrently...") + remaining = { + **{ + k: v + for k, v in phase2_futures.items() + if k != "mlflow/mobile-price-classification" + }, + **phase3_futures, + } + _future_to_name2 = {v: k for k, v in remaining.items()} + for f in as_completed(remaining.values()): + _print_result(results[_future_to_name2[f]]) # ── Phase 4: KFP run polling ────────────────────────────────────── all_run_ids: list[str] = [] From caf78e114c16a6f41f099baf0665999f5d4ddf08 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Thu, 11 Jun 2026 16:05:22 +0200 Subject: [PATCH 04/57] ci: fix KFP v2 polling and prevent Phase 4 crash from aborting run - _poll_kfp_run: use run.state (KFP v2 V2beta1Run) with fallback to run.run.status (KFP v1) for compatibility; normalise to uppercase - Terminal states updated to KFP v2 set: SUCCEEDED/FAILED/ERROR/CANCELED/SKIPPED - Phase 4: wrap f.result() in try/except so a polling error records POLL_ERROR in the report instead of crashing the whole CI run - _print_report: compare status.upper() == 'SUCCEEDED' (case-insensitive) --- ci/run_all.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/ci/run_all.py b/ci/run_all.py index 4c38585..932ffaa 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -190,6 +190,9 @@ def _extract_run_ids_from_stdout(stdout: str) -> list[str]: return ids +_KFP_TERMINAL_STATES = {"SUCCEEDED", "FAILED", "ERROR", "CANCELED", "SKIPPED"} + + def _poll_kfp_run(run_id: str, timeout: int, interval: int = 30) -> str: """Poll a KFP run until terminal state. Returns final status string.""" try: @@ -200,8 +203,14 @@ def _poll_kfp_run(run_id: str, timeout: int, interval: int = 30) -> str: deadline = time.time() + timeout while time.time() < deadline: run = client.get_run(run_id) - state = run.run.status - if state in ("Succeeded", "Failed", "Error", "Skipped"): + # KFP v2 SDK: state is on the run object directly as a string + # KFP v1 SDK: state is on run.run.status + state = str( + getattr(run, "state", None) + or getattr(getattr(run, "run", None), "status", None) + or "UNKNOWN" + ).upper() + if state in _KFP_TERMINAL_STATES: return state time.sleep(interval) return f"TIMEOUT after {timeout}s" @@ -308,10 +317,10 @@ def _print_report(results: dict[str, Result], poll_results: dict[str, str]) -> N print("-" * 70) for run_id, status in poll_results.items(): short = run_id[:8] + "..." - if status != "Succeeded": - failed += 1 - else: + if status.upper() == "SUCCEEDED": passed += 1 + else: + failed += 1 print(f"{short:<{col}} {status:<10}") print("=" * 70) print(f"PASSED: {passed} FAILED: {failed} TOTAL: {passed + failed}") @@ -466,8 +475,12 @@ def run_all( for run_id in all_run_ids } for run_id, f in poll_futures.items(): - poll_results[run_id] = f.result() - print(f" [{poll_results[run_id]}] {run_id[:8]}...") + try: + poll_results[run_id] = f.result() + except Exception as exc: # noqa: BLE001 + poll_results[run_id] = f"POLL_ERROR: {exc}" + state = poll_results[run_id] + print(f" [{state}] {run_id[:8]}...") else: print("\nPhase 4: no KFP run IDs found, skipping poll.") From d629a2bc6acd2b4fb2f26600ca8895d71b268535 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Thu, 11 Jun 2026 16:41:18 +0200 Subject: [PATCH 05/57] fix: YAML filename typo and CI report/skip improvements - inference_protocol_version_example.ipynb: fix pre-existing bug where cell read v1_InferenceService.yaml (underscore) but file is v1-InferenceService.yaml (dash); same for v2 - ci/run_all.py: move summary table to the very end (after Failed details) so final verdict is the last thing visible in the terminal - ci/run_all.py: skip Phase 3 ISVC tests when mlflow/mobile-price-classification fails, avoiding a 10-minute ISVC timeout on a model that was never registered --- ci/run_all.py | 94 ++++++++++++------- .../inference_protocol_version_example.ipynb | 4 +- 2 files changed, 62 insertions(+), 36 deletions(-) diff --git a/ci/run_all.py b/ci/run_all.py index 932ffaa..afb0895 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -300,30 +300,19 @@ def _print_result(r: Result) -> None: def _print_report(results: dict[str, Result], poll_results: dict[str, str]) -> None: col = 50 - print("\n" + "=" * 70) - print(f"{'EXAMPLE':<{col}} {'STATUS':<10} {'DURATION'}") - print("-" * 70) passed = failed = 0 for r in results.values(): - dur = f"{r.duration:.0f}s" if r.duration else "-" - print(f"{r.name:<{col}} {r.status:<10} {dur}") if r.status == "FAIL": failed += 1 else: passed += 1 - if poll_results: - print() - print(f"{'KFP PIPELINE POLL':<{col}} {'STATUS':<10}") - print("-" * 70) - for run_id, status in poll_results.items(): - short = run_id[:8] + "..." - if status.upper() == "SUCCEEDED": - passed += 1 - else: - failed += 1 - print(f"{short:<{col}} {status:<10}") - print("=" * 70) - print(f"PASSED: {passed} FAILED: {failed} TOTAL: {passed + failed}") + for run_id, status in poll_results.items(): + if status.upper() == "SUCCEEDED": + passed += 1 + else: + failed += 1 + + # Failed details first so they're easy to scroll back to if failed: print("\nFailed details:") print("-" * 70) @@ -332,6 +321,25 @@ def _print_report(results: dict[str, Result], poll_results: dict[str, str]) -> N print(f"\n{r.name}:") for line in r.error.splitlines(): print(f" {line}") + for run_id, status in poll_results.items(): + if status.upper() != "SUCCEEDED": + print(f"\nKFP run {run_id[:8]}...: {status}") + + # Summary table last — easy to see final verdict at a glance + print("\n" + "=" * 70) + print(f"{'EXAMPLE':<{col}} {'STATUS':<10} {'DURATION'}") + print("-" * 70) + for r in results.values(): + dur = f"{r.duration:.0f}s" if r.duration else "-" + print(f"{r.name:<{col}} {r.status:<10} {dur}") + if poll_results: + print() + print(f"{'KFP PIPELINE POLL':<{col}} {'STATUS':<10}") + print("-" * 70) + for run_id, status in poll_results.items(): + print(f"{run_id[:8] + '...':<{col}} {status:<10}") + print("=" * 70) + print(f"PASSED: {passed} FAILED: {failed} TOTAL: {passed + failed}") print() @@ -429,25 +437,43 @@ def run_all( # ── Phase 3: MLflow KServe examples (need registered model) ─────── print("\nPhase 3: deploying MLflow KServe InferenceServices...") - phase3_futures = {} - for name, rel in [ - ( + if results["mlflow/mobile-price-classification"].status == "FAIL": + print( + " [SKIP] mlflow-mobile-price-classification failed — " + "skipping ISVC tests that depend on the registered model" + ) + for name in ( "serving/mlflow-kserve-minimal", - "serving/mlflow-kserve-minimal/apply.py", - ), - ( "serving/mlflow-kserve-inference-protocols", - "serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb", - ), - ]: - path = root / rel - if path.suffix == ".ipynb": - f = _submit_notebook( - executor, results, name, path, output_dir, timeout_notebook + ): + results[name] = Result( + name=name, + status="SKIP", + error="prerequisite mlflow/mobile-price-classification failed", ) - else: - f = _submit_script(executor, results, name, path, timeout_notebook) - phase3_futures[name] = f + phase3_futures = {} + else: + phase3_futures = {} + for name, rel in [ + ( + "serving/mlflow-kserve-minimal", + "serving/mlflow-kserve-minimal/apply.py", + ), + ( + "serving/mlflow-kserve-inference-protocols", + "serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb", + ), + ]: + path = root / rel + if path.suffix == ".ipynb": + f = _submit_notebook( + executor, results, name, path, output_dir, timeout_notebook + ) + else: + f = _submit_script( + executor, results, name, path, timeout_notebook + ) + phase3_futures[name] = f # Phase 2 remaining + phase 3 run concurrently; print as each finishes print("\nPhase 2 (remaining) + Phase 3 running concurrently...") diff --git a/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb b/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb index 46e1d2c..83f7898 100644 --- a/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb +++ b/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb @@ -53,8 +53,8 @@ "metadata": {}, "outputs": [], "source": [ - "v1_yaml = Path(\"v1_InferenceService.yaml\").read_text()\n", - "v2_yaml = Path(\"v2_InferenceService.yaml\").read_text()\n", + "v1_yaml = Path(\"v1-InferenceService.yaml\").read_text()\n", + "v2_yaml = Path(\"v2-InferenceService.yaml\").read_text()\n", "\n", "display(Markdown(\n", " \"| v1 InferenceService | v2 InferenceService |\\n\"\n", From d2bee85416d2cc4601fc80fa1e9489d17cc0986e Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Thu, 11 Jun 2026 17:40:39 +0200 Subject: [PATCH 06/57] ci: label KFP poll rows with source notebook name --- ci/run_all.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/ci/run_all.py b/ci/run_all.py index afb0895..d2b477d 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -298,7 +298,11 @@ def _print_result(r: Result) -> None: print(f" {line}") -def _print_report(results: dict[str, Result], poll_results: dict[str, str]) -> None: +def _print_report( + results: dict[str, Result], + poll_results: dict[str, str], + run_id_to_name: dict[str, str], +) -> None: col = 50 passed = failed = 0 for r in results.values(): @@ -323,7 +327,8 @@ def _print_report(results: dict[str, Result], poll_results: dict[str, str]) -> N print(f" {line}") for run_id, status in poll_results.items(): if status.upper() != "SUCCEEDED": - print(f"\nKFP run {run_id[:8]}...: {status}") + source = run_id_to_name.get(run_id, "unknown") + print(f"\nKFP run {run_id[:8]}... (from {source}): {status}") # Summary table last — easy to see final verdict at a glance print("\n" + "=" * 70) @@ -334,10 +339,12 @@ def _print_report(results: dict[str, Result], poll_results: dict[str, str]) -> N print(f"{r.name:<{col}} {r.status:<10} {dur}") if poll_results: print() - print(f"{'KFP PIPELINE POLL':<{col}} {'STATUS':<10}") + print(f"{'KFP RUN (source notebook)':<{col}} {'STATUS':<10}") print("-" * 70) for run_id, status in poll_results.items(): - print(f"{run_id[:8] + '...':<{col}} {status:<10}") + source = run_id_to_name.get(run_id, "unknown") + label = f"{run_id[:8]}... ({source})" + print(f"{label:<{col}} {status:<10}") print("=" * 70) print(f"PASSED: {passed} FAILED: {failed} TOTAL: {passed + failed}") print() @@ -518,7 +525,13 @@ def run_all( for f in as_completed(futs): f.result() # already swallows exceptions inside _run_cleanup - _print_report(results, poll_results) + # Build run_id → source notebook name map from all results + run_id_to_name: dict[str, str] = {} + for r in results.values(): + for run_id in r.kfp_run_ids: + run_id_to_name[run_id] = r.name + + _print_report(results, poll_results, run_id_to_name) return results From 58f575b7e5cf51c34df51d85bfd4f8e70cf9a4a2 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Fri, 12 Jun 2026 15:43:34 +0200 Subject: [PATCH 07/57] fix: AIGatewayKey owner field, MLflow pre-flight check, ISVC timing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scripts/get_or_create_api_key.py: - Add required 'owner' field to AIGatewayKey manifest (CRD validation was rejecting the resource without it); reads MLFLOW_TRACKING_USERNAME from the mlflow-credentials secret when available, falls back to notebook-examples@ ci/run_all.py: - Add _check_mlflow_credentials() pre-flight check: validates secret exists AND credentials work (quick MLflow REST API call); skips all MLflow-dependent tests with a clear message if either check fails - Phase 1 and Phase 2 skip notebooks already resolved by pre-flight - Fix Phase 3 timing: after mlflow-mobile-price notebook finishes, poll its KFP run inline (blocking) before deploying ISVCs — previously Phase 3 deployed immediately while the pipeline (and model registration) was still running, causing ISVC timeouts - Phase 4 skips run IDs already polled inline before Phase 3 --- ci/run_all.py | 141 +++++++++++++++++++++++++++---- scripts/get_or_create_api_key.py | 23 +++++ 2 files changed, 149 insertions(+), 15 deletions(-) diff --git a/ci/run_all.py b/ci/run_all.py index d2b477d..265508a 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -350,6 +350,72 @@ def _print_report( print() +# ── MLflow pre-flight check ─────────────────────────────────────────────────── + +# Notebooks that require working MLflow credentials +_MLFLOW_DEPENDENT = frozenset( + { + "mlflow/mlflow-quickstart", + "mlflow/mlflow-image-example", + "mlflow/mlflow-kfp-example", + "mlflow/mobile-price-classification", + } +) + + +def _check_mlflow_credentials() -> tuple[bool, str]: + """Return (ok, reason). Checks secret existence then validates credentials + with a quick call to the MLflow REST API.""" + import base64 as _b64 + import json as _json + import urllib.error + import urllib.request + + ns = _namespace() + r = subprocess.run( + ["kubectl", "get", "secret", "mlflow-credentials", "-n", ns, "-o", "json"], + capture_output=True, + text=True, + ) + if r.returncode != 0: + return False, ( + "mlflow-credentials secret not found — " + "run scripts/setup_mlflow_credentials.py first" + ) + + try: + data = _json.loads(r.stdout)["data"] + uri = _b64.b64decode(data["MLFLOW_TRACKING_URI"]).decode().rstrip("/") + username = _b64.b64decode(data["MLFLOW_TRACKING_USERNAME"]).decode() + password = _b64.b64decode(data["MLFLOW_TRACKING_PASSWORD"]).decode() + except KeyError as exc: + return ( + False, + f"mlflow-credentials secret is missing key {exc} — re-run setup_mlflow_credentials.py", + ) + + creds = _b64.b64encode(f"{username}:{password}".encode()).decode() + req = urllib.request.Request( + f"{uri}/api/2.0/mlflow/experiments/search?max_results=1", + headers={"Authorization": f"Basic {creds}"}, + ) + try: + urllib.request.urlopen(req, timeout=8) + return True, "OK" + except urllib.error.HTTPError as exc: + if exc.code in (401, 403): + return False, ( + "MLflow credentials are invalid or the PAT has expired — " + "re-run scripts/setup_mlflow_credentials.py" + ) + return ( + False, + f"MLflow API returned HTTP {exc.code} — check MLFLOW_TRACKING_URI in the secret", + ) + except Exception as exc: # noqa: BLE001 + return False, f"Could not reach MLflow at {uri}: {exc}" + + # ── Main ────────────────────────────────────────────────────────────────────── @@ -366,6 +432,16 @@ def run_all( _ensure_papermill() + # Pre-flight: validate MLflow credentials; skip dependent tests if unavailable + print("Pre-flight: checking MLflow credentials...") + mlflow_ok, mlflow_reason = _check_mlflow_credentials() + if mlflow_ok: + print(" [OK] MLflow credentials valid") + else: + print(f" [SKIP] {mlflow_reason}") + for name in _MLFLOW_DEPENDENT: + results[name] = Result(name=name, status="SKIP", error=mlflow_reason) + if dry_run: print("[dry-run] Would execute the following phases:") for label in [ @@ -401,6 +477,9 @@ def run_all( "serving/minimal-s3-model/minimal-s3-model.ipynb", ), ]: + if name in results: # already SKIP from pre-flight + print(f" [SKIP ] {name}") + continue f = _submit_notebook( executor, results, name, root / rel, output_dir, timeout_notebook ) @@ -424,6 +503,9 @@ def run_all( "pipelines/lightweight-components/mobile-price-classifications.ipynb", ), ]: + if name in results: # already SKIP from pre-flight + print(f" [SKIP ] {name}") + continue f = _submit_notebook( executor, results, name, root / rel, output_dir, timeout_notebook ) @@ -438,26 +520,51 @@ def run_all( ) phase2_futures["pipelines/lightweight-python-package"] = lpp_future - # Wait only for mlflow-mobile-price before launching phase 3 - phase2_futures["mlflow/mobile-price-classification"].result() - _print_result(results["mlflow/mobile-price-classification"]) + # Wait for mlflow-mobile-price notebook; then poll its KFP run + # inline before Phase 3 — the ISVC needs the registered model. + _mobile_price_skipped = ( + "mlflow/mobile-price-classification" not in phase2_futures + ) + if not _mobile_price_skipped: + phase2_futures["mlflow/mobile-price-classification"].result() + _print_result(results["mlflow/mobile-price-classification"]) + + _mobile_price_ok = ( + not _mobile_price_skipped + and results["mlflow/mobile-price-classification"].status == "PASS" + ) + + if _mobile_price_ok: + _mobile_run_ids = results[ + "mlflow/mobile-price-classification" + ].kfp_run_ids + if _mobile_run_ids: + print( + " Polling mlflow-mobile-price KFP pipeline (model must be registered before ISVCs)..." + ) + for _run_id in _mobile_run_ids: + _state = _poll_kfp_run(_run_id, timeout_pipeline) + poll_results[_run_id] = ( + _state # recorded here; Phase 4 will skip it + ) + print(f" [{_state}] {_run_id[:8]}...") + if _state.upper() != "SUCCEEDED": + _mobile_price_ok = False # ── Phase 3: MLflow KServe examples (need registered model) ─────── print("\nPhase 3: deploying MLflow KServe InferenceServices...") - if results["mlflow/mobile-price-classification"].status == "FAIL": - print( - " [SKIP] mlflow-mobile-price-classification failed — " - "skipping ISVC tests that depend on the registered model" + if not _mobile_price_ok: + _reason = ( + "prerequisite mlflow/mobile-price-classification skipped (MLflow credentials)" + if _mobile_price_skipped + else "prerequisite mlflow/mobile-price-classification notebook or KFP pipeline did not succeed" ) + print(f" [SKIP] {_reason}") for name in ( "serving/mlflow-kserve-minimal", "serving/mlflow-kserve-inference-protocols", ): - results[name] = Result( - name=name, - status="SKIP", - error="prerequisite mlflow/mobile-price-classification failed", - ) + results[name] = Result(name=name, status="SKIP", error=_reason) phase3_futures = {} else: phase3_futures = {} @@ -497,9 +604,13 @@ def run_all( _print_result(results[_future_to_name2[f]]) # ── Phase 4: KFP run polling ────────────────────────────────────── - all_run_ids: list[str] = [] - for r in results.values(): - all_run_ids.extend(r.kfp_run_ids) + # Exclude run IDs already polled inline before Phase 3 + all_run_ids: list[str] = [ + rid + for r in results.values() + for rid in r.kfp_run_ids + if rid not in poll_results + ] if all_run_ids: print(f"\nPhase 4: polling {len(all_run_ids)} KFP run(s)...") diff --git a/scripts/get_or_create_api_key.py b/scripts/get_or_create_api_key.py index 534aa12..a9a9528 100644 --- a/scripts/get_or_create_api_key.py +++ b/scripts/get_or_create_api_key.py @@ -67,8 +67,30 @@ def _apply(manifest: str, namespace: str) -> None: raise RuntimeError(f"kubectl apply failed:\n{result.stderr}") +def _owner(namespace: str) -> str: + """Determine the key owner. + + Tries to read MLFLOW_TRACKING_USERNAME from the mlflow-credentials secret + (the canonical identity in the prokube platform). Falls back to a + namespace-scoped placeholder if the secret is absent. + """ + result = _kubectl( + "get", + "secret", + "mlflow-credentials", + "-n", + namespace, + "-o", + "jsonpath={.data.MLFLOW_TRACKING_USERNAME}", + ) + if result.returncode == 0 and result.stdout.strip(): + return base64.b64decode(result.stdout.strip()).decode() + return f"notebook-examples@{namespace}" + + def _create_key(namespace: str) -> str: key_value = f"pk_live_{secrets.token_hex(20)}" + owner = _owner(namespace) _apply( f"apiVersion: v1\n" @@ -89,6 +111,7 @@ def _create_key(namespace: str) -> str: f" namespace: {namespace}\n" f"spec:\n" f" displayName: Examples API key\n" + f" owner: {owner}\n" f" secretRef:\n" f" name: {_SECRET_NAME}\n" f" key: token\n" From 9c04ad47218dcfb2e83aa2ac52dbf387be5289a8 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Fri, 12 Jun 2026 16:24:26 +0200 Subject: [PATCH 08/57] ci: pipeline error details, ci-skip cell tag mechanism, tag debug sections MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ci/run_all.py: - _poll_kfp_run now returns (status, error_detail); on FAILED state fetches run.error.message from the KFP v2 API so the user can see what went wrong without opening the KFP UI - Add poll_errors dict; Failed details section in the report shows the KFP error message under each failed run - Add _strip_ci_skip_cells(): preprocess notebooks before papermill runs by replacing cells tagged 'ci-skip' with a comment placeholder - _run_notebook() calls _strip_ci_skip_cells() transparently (cwd stays the original notebook directory so relative imports still work) pipelines/lightweight-components/mobile-price-classifications.ipynb: - Tag cells 26-30 (the 'Debugging with Lightweight Components' section) with 'ci-skip' — these cells hardcode specific S3 artifact UUIDs from past manual runs and cannot execute in CI --- ci/run_all.py | 56 +++++++++++++++---- .../mobile-price-classifications.ipynb | 50 +++++++++++++---- 2 files changed, 85 insertions(+), 21 deletions(-) diff --git a/ci/run_all.py b/ci/run_all.py index 265508a..8228cdc 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -130,20 +130,41 @@ def _format_papermill_error(exc: Exception) -> str: return "\n".join(lines) +def _strip_ci_skip_cells(nb_path: Path, output_dir: Path) -> Path: + """Return a copy of the notebook with 'ci-skip' tagged cells replaced by a comment. + Returns the original path unchanged if no cells are tagged.""" + with open(nb_path) as fh: + nb = json.load(fh) + skipped = 0 + for cell in nb["cells"]: + if "ci-skip" in cell.get("metadata", {}).get("tags", []): + cell["source"] = ["# [CI] cell skipped (ci-skip tag)\n"] + skipped += 1 + if skipped == 0: + return nb_path + stripped = output_dir / f"_stripped_{nb_path.name}" + stripped.parent.mkdir(parents=True, exist_ok=True) + with open(stripped, "w") as fh: + json.dump(nb, fh, indent=1) + print(f" ({skipped} ci-skip cell(s) stripped from {nb_path.name})") + return stripped + + def _run_notebook(nb_path: Path, output_dir: Path, timeout: int) -> Path: """Execute a notebook with papermill. Returns the output notebook path.""" import papermill as pm from papermill.exceptions import PapermillExecutionError - output_path = output_dir / nb_path.name output_dir.mkdir(parents=True, exist_ok=True) + nb_to_run = _strip_ci_skip_cells(nb_path, output_dir) + output_path = output_dir / nb_path.name try: pm.execute_notebook( - str(nb_path), + str(nb_to_run), str(output_path), kernel_name="python3", execution_timeout=timeout, - cwd=str(nb_path.parent), + cwd=str(nb_path.parent), # always cwd to original notebook directory progress_bar=False, # avoid interleaved tqdm bars from parallel threads ) except PapermillExecutionError as exc: @@ -193,12 +214,12 @@ def _extract_run_ids_from_stdout(stdout: str) -> list[str]: _KFP_TERMINAL_STATES = {"SUCCEEDED", "FAILED", "ERROR", "CANCELED", "SKIPPED"} -def _poll_kfp_run(run_id: str, timeout: int, interval: int = 30) -> str: - """Poll a KFP run until terminal state. Returns final status string.""" +def _poll_kfp_run(run_id: str, timeout: int, interval: int = 30) -> tuple[str, str]: + """Poll a KFP run until terminal state. Returns (status, error_detail).""" try: from kfp.client import Client except ImportError: - return "SKIP (kfp not installed)" + return "SKIP (kfp not installed)", "" client = Client() deadline = time.time() + timeout while time.time() < deadline: @@ -211,9 +232,14 @@ def _poll_kfp_run(run_id: str, timeout: int, interval: int = 30) -> str: or "UNKNOWN" ).upper() if state in _KFP_TERMINAL_STATES: - return state + error_detail = "" + if state not in ("SUCCEEDED", "SKIPPED"): + err = getattr(run, "error", None) + if err: + error_detail = getattr(err, "message", "") or "" + return state, error_detail time.sleep(interval) - return f"TIMEOUT after {timeout}s" + return f"TIMEOUT after {timeout}s", "" def _run_cleanup(cleanup_path: Path) -> None: @@ -301,6 +327,7 @@ def _print_result(r: Result) -> None: def _print_report( results: dict[str, Result], poll_results: dict[str, str], + poll_errors: dict[str, str], run_id_to_name: dict[str, str], ) -> None: col = 50 @@ -329,6 +356,10 @@ def _print_report( if status.upper() != "SUCCEEDED": source = run_id_to_name.get(run_id, "unknown") print(f"\nKFP run {run_id[:8]}... (from {source}): {status}") + err = poll_errors.get(run_id, "") + if err: + for line in err.splitlines(): + print(f" {line}") # Summary table last — easy to see final verdict at a glance print("\n" + "=" * 70) @@ -429,6 +460,7 @@ def run_all( output_dir = root / "ci" / "output" results: dict[str, Result] = {} poll_results: dict[str, str] = {} + poll_errors: dict[str, str] = {} _ensure_papermill() @@ -543,10 +575,11 @@ def run_all( " Polling mlflow-mobile-price KFP pipeline (model must be registered before ISVCs)..." ) for _run_id in _mobile_run_ids: - _state = _poll_kfp_run(_run_id, timeout_pipeline) + _state, _err = _poll_kfp_run(_run_id, timeout_pipeline) poll_results[_run_id] = ( _state # recorded here; Phase 4 will skip it ) + poll_errors[_run_id] = _err print(f" [{_state}] {_run_id[:8]}...") if _state.upper() != "SUCCEEDED": _mobile_price_ok = False @@ -620,9 +653,10 @@ def run_all( } for run_id, f in poll_futures.items(): try: - poll_results[run_id] = f.result() + poll_results[run_id], poll_errors[run_id] = f.result() except Exception as exc: # noqa: BLE001 poll_results[run_id] = f"POLL_ERROR: {exc}" + poll_errors[run_id] = "" state = poll_results[run_id] print(f" [{state}] {run_id[:8]}...") else: @@ -642,7 +676,7 @@ def run_all( for run_id in r.kfp_run_ids: run_id_to_name[run_id] = r.name - _print_report(results, poll_results, run_id_to_name) + _print_report(results, poll_results, poll_errors, run_id_to_name) return results diff --git a/pipelines/lightweight-components/mobile-price-classifications.ipynb b/pipelines/lightweight-components/mobile-price-classifications.ipynb index 5179316..fd6af05 100644 --- a/pipelines/lightweight-components/mobile-price-classifications.ipynb +++ b/pipelines/lightweight-components/mobile-price-classifications.ipynb @@ -357,7 +357,9 @@ { "cell_type": "markdown", "id": "40b9ef30-3352-44ec-a0e4-9e52f032194e", - "metadata": {}, + "metadata": { + "tags": [] + }, "source": [ "### Evaluate model on validation dataset" ] @@ -366,7 +368,9 @@ "cell_type": "code", "execution_count": null, "id": "3f9453f2-0c56-4c25-af05-7dfe42eaf3b1", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "@dsl.component(\n", @@ -416,7 +420,9 @@ { "cell_type": "markdown", "id": "836412b8-fe89-48e9-8383-53a5db9761bf", - "metadata": {}, + "metadata": { + "tags": [] + }, "source": [ "### Run predictions on test dataset" ] @@ -425,7 +431,9 @@ "cell_type": "code", "execution_count": null, "id": "12e40c2d-889e-4cc6-bb9d-a722653acb30", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "@dsl.component(\n", @@ -478,7 +486,9 @@ { "cell_type": "markdown", "id": "604fab6f-5dc8-47e0-b15f-868032f076b2", - "metadata": {}, + "metadata": { + "tags": [] + }, "source": [ "## Build pipeline" ] @@ -622,7 +632,11 @@ { "cell_type": "markdown", "id": "2e73f8fc", - "metadata": {}, + "metadata": { + "tags": [ + "ci-skip" + ] + }, "source": [ "# Debugging with Lightweight Components\n", "Debugging can be challenging when using lightweight components in Kubeflow Pipelines. A practical approach is to download the artifacts from the steps preceding the failing one from object storage, and then run the functions used in the components locally. You can easily copy the paths to these artifacts from the Kubeflow Pipelines UI. Once you have these paths, you can use them as shown in the example below to download and read in Pandas DataFrames or perform similar operations. Make sure to adjust the paths according to your specific setup requirements." @@ -632,7 +646,11 @@ "cell_type": "code", "execution_count": null, "id": "315f6c39-4341-45a8-8c24-a81e0a92225f", - "metadata": {}, + "metadata": { + "tags": [ + "ci-skip" + ] + }, "outputs": [], "source": [ "storage_options={\n", @@ -646,7 +664,11 @@ "cell_type": "code", "execution_count": null, "id": "09f1655a-2e79-49c7-aae3-d698169cb7ee", - "metadata": {}, + "metadata": { + "tags": [ + "ci-skip" + ] + }, "outputs": [], "source": [ "# reading pandas dataframes (Modify paths!)\n", @@ -664,7 +686,11 @@ "cell_type": "code", "execution_count": null, "id": "f6fc60c0-714e-4623-bc2c-b3366de6f2b1", - "metadata": {}, + "metadata": { + "tags": [ + "ci-skip" + ] + }, "outputs": [], "source": [ "# use the mc tool to download model artifact\n", @@ -675,7 +701,11 @@ "cell_type": "code", "execution_count": null, "id": "e14a9baa-e110-45f1-90e9-c6eb2df71d9e", - "metadata": {}, + "metadata": { + "tags": [ + "ci-skip" + ] + }, "outputs": [], "source": [ "from joblib import load\n", From 1e5b8ada7083c9bf6510771da2ffabf6f13f8077 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Fri, 12 Jun 2026 16:55:16 +0200 Subject: [PATCH 09/57] fix: dask version pins + KFP task-level error extraction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit notebooks/dask/dask_example.ipynb: - Drop pinned dask-kubernetes==2024.9.0 and dask[complete]==2024.9.1 which require anyio<4. Unpinned install resolves versions compatible with the cluster's anyio 4.12.1 and the updated dask operator. ci/run_all.py: - Add _get_failed_task_logs(): when a KFP run is FAILED and run.error is None (common in KFP v2), walks task_details → child_tasks → pod_name and tails the last 10 lines of the failed task pod's logs via kubectl. Falls back silently if pods are already gone or logs are unavailable. - _poll_kfp_run: reads namespace from SA token file and passes it to _get_failed_task_logs so CI now surfaces the actual pipeline failure message rather than showing nothing. --- ci/run_all.py | 61 ++++++++++++++++++++++++++++++- notebooks/dask/dask_example.ipynb | 2 +- 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/ci/run_all.py b/ci/run_all.py index 8228cdc..a88bebb 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -214,6 +214,55 @@ def _extract_run_ids_from_stdout(stdout: str) -> list[str]: _KFP_TERMINAL_STATES = {"SUCCEEDED", "FAILED", "ERROR", "CANCELED", "SKIPPED"} +def _get_failed_task_logs(run: object, namespace: str) -> str: + """Best-effort: tail logs from the pod(s) of the first failed KFP task. + + Walks run.run_details.task_details → child_tasks → pod_name and tries + kubectl logs on those pods. Silently returns "" on any failure. + """ + try: + task_details = ( + getattr(getattr(run, "run_details", None), "task_details", None) or [] + ) + for task in task_details: + if "FAIL" not in str(getattr(task, "state", "")).upper(): + continue + task_name = getattr(task, "display_name", "unknown-task") + for ct in getattr(task, "child_tasks", None) or []: + pod = ( + ct.get("pod_name") + if isinstance(ct, dict) + else getattr(ct, "pod_name", None) + ) + if not pod: + continue + for container in ("main", "user-main"): + result = subprocess.run( + [ + "kubectl", + "logs", + pod, + "-n", + namespace, + "-c", + container, + "--tail=15", + ], + capture_output=True, + text=True, + timeout=10, + ) + if result.returncode == 0 and result.stdout.strip(): + header = f"[Task '{task_name}' / pod {pod[:30]}...]" + tail = "\n".join( + f" {l}" for l in result.stdout.strip().splitlines()[-10:] + ) + return f"{header}\n{tail}" + except Exception: # noqa: BLE001 + pass + return "" + + def _poll_kfp_run(run_id: str, timeout: int, interval: int = 30) -> tuple[str, str]: """Poll a KFP run until terminal state. Returns (status, error_detail).""" try: @@ -221,6 +270,11 @@ def _poll_kfp_run(run_id: str, timeout: int, interval: int = 30) -> tuple[str, s except ImportError: return "SKIP (kfp not installed)", "" client = Client() + try: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + namespace = fh.read().strip() + except OSError: + namespace = "" deadline = time.time() + timeout while time.time() < deadline: run = client.get_run(run_id) @@ -234,9 +288,12 @@ def _poll_kfp_run(run_id: str, timeout: int, interval: int = 30) -> tuple[str, s if state in _KFP_TERMINAL_STATES: error_detail = "" if state not in ("SUCCEEDED", "SKIPPED"): + # run.error is often None even for failed runs in KFP v2 — + # the real failure is in the task pod logs err = getattr(run, "error", None) - if err: - error_detail = getattr(err, "message", "") or "" + error_detail = (getattr(err, "message", "") or "") if err else "" + if not error_detail and namespace: + error_detail = _get_failed_task_logs(run, namespace) return state, error_detail time.sleep(interval) return f"TIMEOUT after {timeout}s", "" diff --git a/notebooks/dask/dask_example.ipynb b/notebooks/dask/dask_example.ipynb index 77fc563..21f4f86 100644 --- a/notebooks/dask/dask_example.ipynb +++ b/notebooks/dask/dask_example.ipynb @@ -40,7 +40,7 @@ }, "outputs": [], "source": [ - "!pip install --upgrade ipywidgets dask-kubernetes==2024.9.0 pandas \"dask[complete]\"==2024.9.1" + "!pip install --upgrade ipywidgets dask-kubernetes pandas \"dask[complete]\"\n" ] }, { From 857dbeeb9440acd7254adcd934363342c1ff9518 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Fri, 12 Jun 2026 17:07:49 +0200 Subject: [PATCH 10/57] =?UTF-8?q?fix:=20mlflow=3D=3D3.10.0=20(non-existent?= =?UTF-8?q?)=20=E2=86=92=20mlflow>=3D3.0.0,<4=20in=20pipeline=20components?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The train_model and evaluate_model KFP components had mlflow==3.10.0 pinned in packages_to_install. That version does not exist (latest 3.x is 3.1.4), causing every pipeline run to fail at the package install step. Changed to mlflow>=3.0.0,<4 which installs the latest available 3.x release and stays compatible with the 3.x MLflow tracking server. --- .../mlflow-mobile-price-classification.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb b/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb index 13036b0..b2a3023 100644 --- a/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb +++ b/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb @@ -364,7 +364,7 @@ "outputs": [], "source": [ "@dsl.component(\n", - " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\", \"mlflow==3.10.0\", \"s3fs\"],\n", + " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\", \"mlflow>=3.0.0,<4\", \"s3fs\"],\n", " base_image=\"python:3.9\",\n", ")\n", "def train_model(\n", @@ -444,7 +444,7 @@ "outputs": [], "source": [ "@dsl.component(\n", - " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\", \"mlflow==3.10.0\", \"s3fs\", \"matplotlib\"],\n", + " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\", \"mlflow>=3.0.0,<4\", \"s3fs\", \"matplotlib\"],\n", " base_image=\"python:3.9\",\n", ")\n", "def evaluate_model(\n", From aa00d4452fce323c0fa6dd76cbb73ec9bfd145de Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Fri, 12 Jun 2026 17:11:30 +0200 Subject: [PATCH 11/57] revert: restore mlflow==3.10.0 in pipeline components MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pin was correct — 3.10.0 exists on public PyPI and matches the server. The failure is that KFP component pods (python:3.9 base) have restricted network access and can only see an older PyPI snapshot. Root cause: network policy issue for component pods, not a version mismatch. Needs a platform-level fix (open network access or use a base image that already has mlflow pre-installed). --- .../mlflow-mobile-price-classification.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb b/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb index b2a3023..13036b0 100644 --- a/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb +++ b/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb @@ -364,7 +364,7 @@ "outputs": [], "source": [ "@dsl.component(\n", - " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\", \"mlflow>=3.0.0,<4\", \"s3fs\"],\n", + " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\", \"mlflow==3.10.0\", \"s3fs\"],\n", " base_image=\"python:3.9\",\n", ")\n", "def train_model(\n", @@ -444,7 +444,7 @@ "outputs": [], "source": [ "@dsl.component(\n", - " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\", \"mlflow>=3.0.0,<4\", \"s3fs\", \"matplotlib\"],\n", + " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\", \"mlflow==3.10.0\", \"s3fs\", \"matplotlib\"],\n", " base_image=\"python:3.9\",\n", ")\n", "def evaluate_model(\n", From 81d0c5caeea5a148373373157fbd1a36a2c42373 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Fri, 12 Jun 2026 17:13:28 +0200 Subject: [PATCH 12/57] fix: upgrade KFP component base images from python:3.9 to python:3.11 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mlflow>=3.2.0 requires Python >=3.10. With python:3.9 as the base image, pip silently excludes all mlflow versions that have Requires-Python>=3.10, leaving only versions up to 3.1.4 visible — which is why mlflow==3.10.0 appeared 'not found'. Upgrading to python:3.11 makes the full PyPI version range visible and satisfies the requirement. Affects all @dsl.component decorators in: - mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb (7 components) - pipelines/lightweight-components/mobile-price-classifications.ipynb (7 components) --- .../mlflow-mobile-price-classification.ipynb | 14 +++++++------- .../mobile-price-classifications.ipynb | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb b/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb index 13036b0..ef18fd4 100644 --- a/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb +++ b/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb @@ -171,7 +171,7 @@ "source": [ "@dsl.component(\n", " packages_to_install=[\"pandas\", \"pyarrow\", \"s3fs\"],\n", - " base_image=\"python:3.9\",\n", + " base_image=\"python:3.11\",\n", ")\n", "def read_data(\n", " train_data_path: str,\n", @@ -206,7 +206,7 @@ "source": [ "@dsl.component(\n", " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\"],\n", - " base_image=\"python:3.9\",\n", + " base_image=\"python:3.11\",\n", ")\n", "def split_data(\n", " train_df: Input[Dataset],\n", @@ -254,7 +254,7 @@ "source": [ "@dsl.component(\n", " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\"],\n", - " base_image=\"python:3.9\",\n", + " base_image=\"python:3.11\",\n", ")\n", "def fit_scaler(\n", " train_x: Input[Dataset],\n", @@ -295,7 +295,7 @@ "source": [ "@dsl.component(\n", " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\"],\n", - " base_image=\"python:3.9\",\n", + " base_image=\"python:3.11\",\n", ")\n", "def tune_hyperparams(\n", " train_x: Input[Dataset],\n", @@ -365,7 +365,7 @@ "source": [ "@dsl.component(\n", " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\", \"mlflow==3.10.0\", \"s3fs\"],\n", - " base_image=\"python:3.9\",\n", + " base_image=\"python:3.11\",\n", ")\n", "def train_model(\n", " train_x: Input[Dataset],\n", @@ -445,7 +445,7 @@ "source": [ "@dsl.component(\n", " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\", \"mlflow==3.10.0\", \"s3fs\", \"matplotlib\"],\n", - " base_image=\"python:3.9\",\n", + " base_image=\"python:3.11\",\n", ")\n", "def evaluate_model(\n", " val_x: Input[Dataset],\n", @@ -530,7 +530,7 @@ "source": [ "@dsl.component(\n", " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\", \"plotly\"],\n", - " base_image=\"python:3.9\",\n", + " base_image=\"python:3.11\",\n", ")\n", "def test_model(\n", " test_x: Input[Dataset],\n", diff --git a/pipelines/lightweight-components/mobile-price-classifications.ipynb b/pipelines/lightweight-components/mobile-price-classifications.ipynb index fd6af05..d4954c4 100644 --- a/pipelines/lightweight-components/mobile-price-classifications.ipynb +++ b/pipelines/lightweight-components/mobile-price-classifications.ipynb @@ -125,7 +125,7 @@ "source": [ "@dsl.component(\n", " packages_to_install=[\"pandas\", \"pyarrow\", \"s3fs\"],\n", - " base_image=\"python:3.9\",\n", + " base_image=\"python:3.11\",\n", ")\n", "def read_data(\n", " train_data_path: str,\n", @@ -160,7 +160,7 @@ "source": [ "@dsl.component(\n", " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\"],\n", - " base_image=\"python:3.9\",\n", + " base_image=\"python:3.11\",\n", ")\n", "def split_data(\n", " train_df: Input[Dataset],\n", @@ -208,7 +208,7 @@ "source": [ "@dsl.component(\n", " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\"],\n", - " base_image=\"python:3.9\",\n", + " base_image=\"python:3.11\",\n", ")\n", "def fit_scaler(\n", " train_x: Input[Dataset],\n", @@ -249,7 +249,7 @@ "source": [ "@dsl.component(\n", " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\"],\n", - " base_image=\"python:3.9\",\n", + " base_image=\"python:3.11\",\n", ")\n", "def tune_hyperparams(\n", " train_x: Input[Dataset],\n", @@ -319,7 +319,7 @@ "source": [ "@dsl.component(\n", " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\"],\n", - " base_image=\"python:3.9\",\n", + " base_image=\"python:3.11\",\n", ")\n", "def train_model(\n", " train_x: Input[Dataset],\n", @@ -375,7 +375,7 @@ "source": [ "@dsl.component(\n", " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\"],\n", - " base_image=\"python:3.9\",\n", + " base_image=\"python:3.11\",\n", ")\n", "def evaluate_model(\n", " val_x: Input[Dataset],\n", @@ -438,7 +438,7 @@ "source": [ "@dsl.component(\n", " packages_to_install=[\"pandas\", \"scikit-learn\", \"pyarrow\", \"plotly\"],\n", - " base_image=\"python:3.9\",\n", + " base_image=\"python:3.11\",\n", ")\n", "def test_model(\n", " test_x: Input[Dataset],\n", From a66758f6e168ad74523f44b341c6e38a14ac0765 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Fri, 12 Jun 2026 17:17:56 +0200 Subject: [PATCH 13/57] build: add mobile-price-classification components image v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dockerfile: python:3.9-slim → python:3.11-slim (same Python version issue as the KFP lightweight components — newer packages require Python >=3.10) pipeline.py: COMPONENTS_IMAGE default bumped to :v2; falls back to COMPONENTS_IMAGE env var so the image path can be overridden without a code change. .github/workflows/mobile-price-classification-components.yaml: new workflow_dispatch workflow to build and push the image. Accepts a tag input (default: v2) and pushes :v2, :latest, and :commit-. --- ...obile-price-classification-components.yaml | 53 +++++++++++++++++++ .../lightweight-python-package/Dockerfile | 2 +- .../lightweight-python-package/pipeline.py | 30 ++++++++--- 3 files changed, 76 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/mobile-price-classification-components.yaml diff --git a/.github/workflows/mobile-price-classification-components.yaml b/.github/workflows/mobile-price-classification-components.yaml new file mode 100644 index 0000000..612d5fe --- /dev/null +++ b/.github/workflows/mobile-price-classification-components.yaml @@ -0,0 +1,53 @@ +name: Build Mobile Price Classification Components Image + +on: + workflow_dispatch: + inputs: + tag: + description: "Image tag (e.g. v2, v3)" + required: true + default: "v2" + +env: + DOCKERFILE_PATH: "pipelines/lightweight-python-package/Dockerfile" + BUILD_CONTEXT_DIR: "pipelines/lightweight-python-package/" + IMAGE_NAME: "mobile-price-classification" + IMAGE_PUSH_PATH: "${{ secrets.GCP_ARTIFACT_REGISTRY_PATH }}" + IMAGE_PUSH_SECRET: "${{ secrets.GCP_SA_KEY }}" + +jobs: + build-image: + runs-on: ubuntu-latest + steps: + - name: Delete huge unnecessary tools folder + run: rm -rf /opt/hostedtoolcache + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Auth GCloud CLI + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ env.IMAGE_PUSH_SECRET }} + + - name: Docker Login to Google Artifact Registry + uses: docker/login-action@v3 + with: + registry: europe-west3-docker.pkg.dev + username: _json_key + password: ${{ env.IMAGE_PUSH_SECRET }} + + - name: Build and push image + run: | + TAG=${{ github.event.inputs.tag }} + VERSIONED=${{ env.IMAGE_PUSH_PATH }}/${{ env.IMAGE_NAME }}:${TAG} + LATEST=${{ env.IMAGE_PUSH_PATH }}/${{ env.IMAGE_NAME }}:latest + COMMIT=${{ env.IMAGE_PUSH_PATH }}/${{ env.IMAGE_NAME }}:commit-${{ github.sha }} + BUILD_CONTEXT="${GITHUB_WORKSPACE}/${{ env.BUILD_CONTEXT_DIR }}" + docker build -t $VERSIONED -f ${{ env.DOCKERFILE_PATH }} "${BUILD_CONTEXT}" + docker tag $VERSIONED $LATEST + docker tag $VERSIONED $COMMIT + docker push $VERSIONED + docker push $LATEST + docker push $COMMIT + echo "Pushed: $VERSIONED" diff --git a/pipelines/lightweight-python-package/Dockerfile b/pipelines/lightweight-python-package/Dockerfile index 071487b..7a50c53 100644 --- a/pipelines/lightweight-python-package/Dockerfile +++ b/pipelines/lightweight-python-package/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-slim +FROM python:3.11-slim WORKDIR /app diff --git a/pipelines/lightweight-python-package/pipeline.py b/pipelines/lightweight-python-package/pipeline.py index f1d89b2..f770aaf 100644 --- a/pipelines/lightweight-python-package/pipeline.py +++ b/pipelines/lightweight-python-package/pipeline.py @@ -2,10 +2,22 @@ from typing import Dict, List from kfp import dsl -from kfp.dsl import HTML, Input, Output, Dataset, Artifact, Model, ClassificationMetrics, Markdown +from kfp.dsl import ( + HTML, + Input, + Output, + Dataset, + Artifact, + Model, + ClassificationMetrics, + Markdown, +) -COMPONENTS_IMAGE = "europe-west3-docker.pkg.dev/prokube-internal/prokube-customer/mobile-price-classification:v1" +COMPONENTS_IMAGE = os.environ.get( + "COMPONENTS_IMAGE", + "europe-west3-docker.pkg.dev/prokube-internal/prokube-customer/mobile-price-classification:v2", +) @dsl.component(base_image=COMPONENTS_IMAGE) @@ -187,16 +199,18 @@ def mobile_price_classification_pipeline( test_data_path=test_data_path, ) # Use the cluster internal object storage endpoint - read_data_task.set_env_variable('AWS_ENDPOINT_URL', f'http://{os.environ["S3_ENDPOINT"]}') + read_data_task.set_env_variable( + "AWS_ENDPOINT_URL", f"http://{os.environ['S3_ENDPOINT']}" + ) # Use Kubernetes secrets to provide AWS credentials to the read_data component kubernetes.use_secret_as_env( read_data_task, - secret_name='s3creds', + secret_name="s3creds", secret_key_to_env={ - 'AWS_ACCESS_KEY_ID': 'AWS_ACCESS_KEY_ID', - 'AWS_SECRET_ACCESS_KEY': 'AWS_SECRET_ACCESS_KEY', - } - ) + "AWS_ACCESS_KEY_ID": "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY": "AWS_SECRET_ACCESS_KEY", + }, + ) # Step 2: Split the data split_data_task = split_data( From 407b5f335668cf6fd4b84f50d17e62aaad85f7bb Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Fri, 12 Jun 2026 18:22:41 +0200 Subject: [PATCH 14/57] fix: upgrade anyio>=4.12 in dask notebook pip install httpx_ws (a dask-kubernetes dependency via kr8s) uses anyio.AsyncContextManagerMixin which was removed in anyio 4.0 and re-added as a compat alias in anyio 4.12+. The etst-0 CI notebook has anyio 4.6.0 (no alias) while test-0 has 4.12.1 (alias present). Adding anyio>=4.12 to the pip install ensures the compat alias is available regardless of which notebook image version is running. --- notebooks/dask/dask_example.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/dask/dask_example.ipynb b/notebooks/dask/dask_example.ipynb index 21f4f86..4733128 100644 --- a/notebooks/dask/dask_example.ipynb +++ b/notebooks/dask/dask_example.ipynb @@ -40,7 +40,7 @@ }, "outputs": [], "source": [ - "!pip install --upgrade ipywidgets dask-kubernetes pandas \"dask[complete]\"\n" + "!pip install --upgrade ipywidgets dask-kubernetes pandas \"dask[complete]\" \"anyio>=4.12\"\n" ] }, { From 16322db492e793cdf1140e2b6fc968d79dd57a4a Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Fri, 12 Jun 2026 18:30:08 +0200 Subject: [PATCH 15/57] fix: pin dask to 2026.3.0 and update worker image tag Pin dask[complete] and dask-kubernetes to 2026.3.0 to match the latest available daskdev/dask Docker image (no 2026.6.0 image exists yet on Docker Hub). Mismatched scheduler versions between the client (2026.6.0 from an unpinned install) and the worker image caused: TypeError: Scheduler.identity() got an unexpected keyword argument 'n_workers' Also updates the KubeCluster image from daskdev/dask:2024.9.1-py3.11 to daskdev/dask:2026.3.0-py3.11. --- notebooks/dask/dask_example.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/notebooks/dask/dask_example.ipynb b/notebooks/dask/dask_example.ipynb index 4733128..b8ac14b 100644 --- a/notebooks/dask/dask_example.ipynb +++ b/notebooks/dask/dask_example.ipynb @@ -40,7 +40,7 @@ }, "outputs": [], "source": [ - "!pip install --upgrade ipywidgets dask-kubernetes pandas \"dask[complete]\" \"anyio>=4.12\"\n" + "!pip install --upgrade ipywidgets \"dask-kubernetes==2026.3.0\" pandas \"dask[complete]==2026.3.0\" \"anyio>=4.12\"\n" ] }, { @@ -120,7 +120,7 @@ }, "outputs": [], "source": [ - "cluster = KubeCluster(name=\"test-cluster\", namespace=namespace, image=\"daskdev/dask:2024.9.1-py3.11\")\n", + "cluster = KubeCluster(name=\"test-cluster\", namespace=namespace, image=\"daskdev/dask:2026.3.0-py3.11\")\n", "cluster.scale(3)\n", "client = Client(cluster)" ] From bfc6024b96f8e5f92c7d1fa9c2e9b2783d838975 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Mon, 15 Jun 2026 10:35:48 +0200 Subject: [PATCH 16/57] ci: add mobile-price-classification, mnist-vae, minimal-container-components, mlflow-kserve test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pipelines/minimal-container-components/submit-cluster.py: - Print KFP_RUN_ID so CI can poll the pipeline run status notebooks/mnist-vae/visualizations.ipynb: - Tag the ipywidgets.interact cell with ci-skip (GUI-only, blocks headless CI) serving/mlflow-kserve-minimal/apply.py: - Add test() function: gets API key, runs test_inference_service.py with API_KEY and INFERENCE_SERVICE_URI env vars set, asserts success - Add deploy_and_test() which chains deploy() then test() - __main__ now calls deploy_and_test() so CI gets a full smoke test ci/run_all.py: - _run_script: accept extra_args for passing CLI flags (e.g. --max_epochs) - _submit_script: propagate extra_args - _submit_chain: new helper to run a sequential list of notebook/script steps as a single named CI result (used for mnist-vae train→visualize) - Phase 1: add notebooks/mobile-price-classification (standalone notebook) and notebooks/mnist-vae (training 3 epochs → visualizations chain) - Phase 2: add pipelines/minimal-container-components - Phase 3: mlflow-kserve-minimal now calls apply.py which does deploy_and_test; inference-protocols unchanged --- ci/run_all.py | 127 ++++++++++++++---- notebooks/mnist-vae/visualizations.ipynb | 6 +- .../submit-cluster.py | 14 +- serving/mlflow-kserve-minimal/apply.py | 46 ++++++- 4 files changed, 156 insertions(+), 37 deletions(-) diff --git a/ci/run_all.py b/ci/run_all.py index a88bebb..6750e6c 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -172,17 +172,21 @@ def _run_notebook(nb_path: Path, output_dir: Path, timeout: int) -> Path: return output_path -def _run_script(script_path: Path, timeout: int) -> tuple[str, str]: +def _run_script( + script_path: Path, + timeout: int, + extra_args: list[str] | None = None, +) -> tuple[str, str]: """Run a plain Python script. Returns (stdout, stderr).""" + cmd = [sys.executable, str(script_path)] + (extra_args or []) result = subprocess.run( - [sys.executable, str(script_path)], + cmd, capture_output=True, text=True, timeout=timeout, cwd=str(script_path.parent), ) if result.returncode != 0: - # Include both stderr and stdout for full context detail = (result.stderr or result.stdout or "(no output)").strip() raise RuntimeError(detail) return result.stdout, result.stderr @@ -349,6 +353,7 @@ def _submit_script( name: str, script_path: Path, timeout: int, + extra_args: list[str] | None = None, ) -> Future: result = Result(name=name) results[name] = result @@ -357,7 +362,7 @@ def _submit_script( def _run(): t0 = time.time() try: - stdout, _ = _run_script(script_path, timeout) + stdout, _ = _run_script(script_path, timeout, extra_args=extra_args) result.status = "PASS" result.kfp_run_ids = _extract_run_ids_from_stdout(stdout) except Exception as exc: # noqa: BLE001 @@ -369,6 +374,45 @@ def _run(): return executor.submit(_run) +def _submit_chain( + executor: ThreadPoolExecutor, + results: dict[str, Result], + name: str, + steps: list[tuple[str, Path, dict]], + output_dir: Path, + timeout: int, +) -> Future: + """Submit a sequential chain of (kind, path, kwargs) steps as a single named result. + + kind is 'notebook' or 'script'. kwargs may include 'extra_args' for scripts. + All steps run in one thread; the first failure stops the chain. + """ + result = Result(name=name) + results[name] = result + print(f" [START ] {name}") + + def _run(): + t0 = time.time() + try: + for kind, path, kwargs in steps: + if kind == "notebook": + out = _run_notebook(path, output_dir, timeout) + result.kfp_run_ids.extend(_extract_run_ids_from_notebook(out)) + elif kind == "script": + stdout, _ = _run_script( + path, timeout, extra_args=kwargs.get("extra_args") + ) + result.kfp_run_ids.extend(_extract_run_ids_from_stdout(stdout)) + result.status = "PASS" + except Exception as exc: # noqa: BLE001 + result.status = "FAIL" + result.error = str(exc) + finally: + result.duration = time.time() - t0 + + return executor.submit(_run) + + # ── Report ──────────────────────────────────────────────────────────────────── @@ -558,6 +602,10 @@ def run_all( phase1_futures = {} for name, rel in [ ("notebooks/dask", "notebooks/dask/dask_example.ipynb"), + ( + "notebooks/mobile-price-classification", + "notebooks/mobile-price-classification/mobile-price-classifications.ipynb", + ), ("mlflow/mlflow-quickstart", "mlflow/mlflow-quickstart-example.ipynb"), ("mlflow/mlflow-image-example", "mlflow/mlflow-image-example.ipynb"), ("mlflow/mlflow-kfp-example", "mlflow/mlflow-kfp-example.ipynb"), @@ -574,6 +622,24 @@ def run_all( ) phase1_futures[name] = f + # mnist-vae: training script (reduced epochs) → visualization notebook + # run as a sequential chain in a single thread, parallel to other Phase 1 + phase1_futures["notebooks/mnist-vae"] = _submit_chain( + executor, + results, + "notebooks/mnist-vae", + steps=[ + ( + "script", + root / "notebooks/mnist-vae/run_training.py", + {"extra_args": ["--max_epochs", "3"]}, + ), + ("notebook", root / "notebooks/mnist-vae/visualizations.ipynb", {}), + ], + output_dir=output_dir, + timeout=timeout_notebook, + ) + # wait for all phase 1, printing results as each finishes _future_to_name = {v: k for k, v in phase1_futures.items()} for f in as_completed(phase1_futures.values()): @@ -609,6 +675,18 @@ def run_all( ) phase2_futures["pipelines/lightweight-python-package"] = lpp_future + mcc_future = _submit_script( + executor, + results, + "pipelines/minimal-container-components", + root + / "pipelines" + / "minimal-container-components" + / "submit-cluster.py", + timeout_notebook, + ) + phase2_futures["pipelines/minimal-container-components"] = mcc_future + # Wait for mlflow-mobile-price notebook; then poll its KFP run # inline before Phase 3 — the ISVC needs the registered model. _mobile_price_skipped = ( @@ -658,26 +736,29 @@ def run_all( phase3_futures = {} else: phase3_futures = {} - for name, rel in [ - ( - "serving/mlflow-kserve-minimal", - "serving/mlflow-kserve-minimal/apply.py", - ), - ( + + # mlflow-kserve-minimal: deploy then immediately smoke-test + # deploy_and_test() in apply.py handles both steps + phase3_futures["serving/mlflow-kserve-minimal"] = _submit_script( + executor, + results, + "serving/mlflow-kserve-minimal", + root / "serving" / "mlflow-kserve-minimal" / "apply.py", + timeout_notebook, + ) + + # inference-protocols notebook handles its own deploy + test + phase3_futures["serving/mlflow-kserve-inference-protocols"] = ( + _submit_notebook( + executor, + results, "serving/mlflow-kserve-inference-protocols", - "serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb", - ), - ]: - path = root / rel - if path.suffix == ".ipynb": - f = _submit_notebook( - executor, results, name, path, output_dir, timeout_notebook - ) - else: - f = _submit_script( - executor, results, name, path, timeout_notebook - ) - phase3_futures[name] = f + root + / "serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb", + output_dir, + timeout_notebook, + ) + ) # Phase 2 remaining + phase 3 run concurrently; print as each finishes print("\nPhase 2 (remaining) + Phase 3 running concurrently...") diff --git a/notebooks/mnist-vae/visualizations.ipynb b/notebooks/mnist-vae/visualizations.ipynb index 95a78da..354fa49 100644 --- a/notebooks/mnist-vae/visualizations.ipynb +++ b/notebooks/mnist-vae/visualizations.ipynb @@ -101,7 +101,11 @@ "cell_type": "code", "execution_count": null, "id": "d032c5d3-1476-40b6-b32f-68b462c27890", - "metadata": {}, + "metadata": { + "tags": [ + "ci-skip" + ] + }, "outputs": [], "source": [ "def generate_image(z1=0, z2=0):\n", diff --git a/pipelines/minimal-container-components/submit-cluster.py b/pipelines/minimal-container-components/submit-cluster.py index 334d1f8..6b95a77 100644 --- a/pipelines/minimal-container-components/submit-cluster.py +++ b/pipelines/minimal-container-components/submit-cluster.py @@ -7,15 +7,13 @@ if __name__ == "__main__": client = Client() - compiler.Compiler().compile(container_components_pipeline, 'pipeline.yaml') + compiler.Compiler().compile(container_components_pipeline, "pipeline.yaml") run = client.create_run_from_pipeline_package( - 'pipeline.yaml', + "pipeline.yaml", enable_caching=False, - arguments={ - "input1": "Hello", - "input2": "world!" - }, - experiment_name='minimal-container-components-experiment', - run_name=f'Simple pipeline {datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")}' + arguments={"input1": "Hello", "input2": "world!"}, + experiment_name="minimal-container-components-experiment", + run_name=f"Simple pipeline {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", ) + print(f"KFP_RUN_ID={run.run_id}") diff --git a/serving/mlflow-kserve-minimal/apply.py b/serving/mlflow-kserve-minimal/apply.py index b3896bd..8268ef8 100644 --- a/serving/mlflow-kserve-minimal/apply.py +++ b/serving/mlflow-kserve-minimal/apply.py @@ -1,9 +1,9 @@ """ -Deploy the mlflow-kserve-minimal InferenceService. +Deploy the mlflow-kserve-minimal InferenceService and run an inference test. Substitutes ```` and ```` in InferenceService.yaml -from the cluster environment, applies it, waits for readiness, and prints the -external URL so it can be used by ``test_inference_service.py``. +from the cluster environment, applies it, waits for readiness, runs a smoke +test via ``test_inference_service.py``, and prints the external URL. Usage from a notebook --------------------- @@ -17,7 +17,7 @@ CLI usage --------- python apply.py - # prints ISVC_URI= to stdout + # prints ISVC_URI= and runs inference smoke test Prerequisites ------------- @@ -142,9 +142,45 @@ def deploy(timeout: int = 600) -> str: return url +_HERE = os.path.dirname(os.path.abspath(__file__)) +_TEST_SCRIPT = os.path.join(_HERE, "test_inference_service.py") +_TEST_JSON = os.path.join(_HERE, "v2-mlflow-inference-body.json") + + +def test(uri: str) -> None: + """Run the inference smoke test against the deployed ISVC.""" + # Get / create the API key + _root = subprocess.check_output( + ["git", "rev-parse", "--show-toplevel"], text=True + ).strip() + sys.path.insert(0, os.path.join(_root, "scripts")) + from get_or_create_api_key import get_or_create_api_key # noqa: PLC0415 + + api_key = get_or_create_api_key() + + print(f"Running inference smoke test against {uri} ...") + result = subprocess.run( + [sys.executable, _TEST_SCRIPT, "--json", _TEST_JSON, "--model", _ISVC_NAME], + capture_output=True, + text=True, + env={**os.environ, "API_KEY": api_key, "INFERENCE_SERVICE_URI": uri}, + ) + if result.returncode != 0: + raise RuntimeError(f"Inference test failed:\n{result.stdout}\n{result.stderr}") + print("Inference test passed.") + print(result.stdout.strip()) + + +def deploy_and_test(timeout: int = 600) -> str: + """Deploy the ISVC, wait for readiness, run smoke test, return the URL.""" + uri = deploy(timeout=timeout) + test(uri) + return uri + + if __name__ == "__main__": try: - deploy() + deploy_and_test() except Exception as exc: # noqa: BLE001 print(f"ERROR: {exc}", file=sys.stderr) sys.exit(1) From ebd9da27656d0764db0823d603f27cc7ada18bf9 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Mon, 15 Jun 2026 13:45:33 +0200 Subject: [PATCH 17/57] ci: make notebooks/mnist-vae opt-in via --include-pytorch pytorch_lightning is not installed in all notebook images. Running the example without it fails immediately with ModuleNotFoundError. Add --include-pytorch flag (default off); mnist-vae is skipped with a clear message unless the flag is passed. Same pattern as --include-keda. --- ci/run_all.py | 49 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/ci/run_all.py b/ci/run_all.py index 6750e6c..84d269b 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -555,6 +555,7 @@ def run_all( timeout_notebook: int = 1800, timeout_pipeline: int = 3600, include_keda: bool = False, + include_pytorch: bool = False, dry_run: bool = False, ) -> dict[str, Result]: root = _REPO_ROOT @@ -622,23 +623,31 @@ def run_all( ) phase1_futures[name] = f - # mnist-vae: training script (reduced epochs) → visualization notebook - # run as a sequential chain in a single thread, parallel to other Phase 1 - phase1_futures["notebooks/mnist-vae"] = _submit_chain( - executor, - results, - "notebooks/mnist-vae", - steps=[ - ( - "script", - root / "notebooks/mnist-vae/run_training.py", - {"extra_args": ["--max_epochs", "3"]}, - ), - ("notebook", root / "notebooks/mnist-vae/visualizations.ipynb", {}), - ], - output_dir=output_dir, - timeout=timeout_notebook, - ) + # mnist-vae: opt-in (requires pytorch_lightning, not in all images) + if include_pytorch: + phase1_futures["notebooks/mnist-vae"] = _submit_chain( + executor, + results, + "notebooks/mnist-vae", + steps=[ + ( + "script", + root / "notebooks/mnist-vae/run_training.py", + {"extra_args": ["--max_epochs", "3"]}, + ), + ( + "notebook", + root / "notebooks/mnist-vae/visualizations.ipynb", + {}, + ), + ], + output_dir=output_dir, + timeout=timeout_notebook, + ) + else: + print( + " [SKIP ] notebooks/mnist-vae (pass --include-pytorch to enable)" + ) # wait for all phase 1, printing results as each finishes _future_to_name = {v: k for k, v in phase1_futures.items()} @@ -840,6 +849,11 @@ def run_all( action="store_true", help="Include GPU-dependent KEDA autoscaling example (opt-in)", ) + parser.add_argument( + "--include-pytorch", + action="store_true", + help="Include pytorch_lightning examples (opt-in; requires pytorch in the notebook image)", + ) parser.add_argument( "--dry-run", action="store_true", help="Print plan without executing anything" ) @@ -849,6 +863,7 @@ def run_all( timeout_notebook=args.timeout_notebook, timeout_pipeline=args.timeout_pipeline, include_keda=args.include_keda, + include_pytorch=args.include_pytorch, dry_run=args.dry_run, ) From e07aa7f5b0c8b14512b9b4f7e9264ffbd442dff1 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Mon, 15 Jun 2026 15:13:21 +0200 Subject: [PATCH 18/57] ci: install pytorch-lightning before mnist-vae training MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pytorch-full image ships PyTorch but not pytorch-lightning. Add a 'pip' step kind to _submit_chain and use it to install pytorch-lightning before running run_training.py when --include-pytorch is set. Also documents the numpy/sklearn issue: the pytorch-full image pins numpy to 1.26.4 but scikit-learn 1.6.1 requires numpy >=2.0 for the _signature_descriptor ABI. Fix is in the image (upgrade numpy to 2.x — PyTorch 2.5.1 supports it) not in the examples. --- ci/run_all.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ci/run_all.py b/ci/run_all.py index 84d269b..70e41b1 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -378,7 +378,7 @@ def _submit_chain( executor: ThreadPoolExecutor, results: dict[str, Result], name: str, - steps: list[tuple[str, Path, dict]], + steps: list[tuple[str, Path | None, dict]], output_dir: Path, timeout: int, ) -> Future: @@ -403,6 +403,14 @@ def _run(): path, timeout, extra_args=kwargs.get("extra_args") ) result.kfp_run_ids.extend(_extract_run_ids_from_stdout(stdout)) + elif kind == "pip": + r = subprocess.run( + [sys.executable, "-m", "pip", "install", "-q"] + + kwargs.get("packages", []), + capture_output=True, + text=True, + check=True, + ) result.status = "PASS" except Exception as exc: # noqa: BLE001 result.status = "FAIL" @@ -630,6 +638,8 @@ def run_all( results, "notebooks/mnist-vae", steps=[ + # install pytorch-lightning if not already present in the image + ("pip", None, {"packages": ["pytorch-lightning"]}), ( "script", root / "notebooks/mnist-vae/run_training.py", From c03700dc30d18ecb64d31866706aaf8921cce502 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Mon, 15 Jun 2026 15:16:05 +0200 Subject: [PATCH 19/57] fix: self-install pytorch-lightning in run_training.py pytorch-lightning is not bundled in all notebook images. The script now installs it silently on first run if absent, so users and CI don't need to pre-install it. The explicit pip step in the CI chain is removed since the script handles its own dependency. --- ci/run_all.py | 3 +-- notebooks/mnist-vae/run_training.py | 30 +++++++++++++++++++++++------ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/ci/run_all.py b/ci/run_all.py index 70e41b1..f5cef9b 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -638,8 +638,7 @@ def run_all( results, "notebooks/mnist-vae", steps=[ - # install pytorch-lightning if not already present in the image - ("pip", None, {"packages": ["pytorch-lightning"]}), + # run_training.py self-installs pytorch-lightning if absent ( "script", root / "notebooks/mnist-vae/run_training.py", diff --git a/notebooks/mnist-vae/run_training.py b/notebooks/mnist-vae/run_training.py index e52458f..1967d93 100644 --- a/notebooks/mnist-vae/run_training.py +++ b/notebooks/mnist-vae/run_training.py @@ -1,3 +1,15 @@ +import subprocess +import sys + +# pytorch-lightning is not bundled in all notebook images — install if absent +try: + import pytorch_lightning # noqa: F401 +except ImportError: + subprocess.run( + [sys.executable, "-m", "pip", "install", "-q", "pytorch-lightning"], + check=True, + ) + import click import pytorch_lightning as pl from pytorch_lightning.loggers import TensorBoardLogger @@ -5,9 +17,14 @@ from model.datamodule import MNISTDataModule import torch + @click.command() -@click.option('--hidden_dim', default=400, type=int, help='Dimension of the hidden layer.') -@click.option('--latent_dim', default=2, type=int, help='Dimension of the latent space.') +@click.option( + "--hidden_dim", default=400, type=int, help="Dimension of the hidden layer." +) +@click.option( + "--latent_dim", default=2, type=int, help="Dimension of the latent space." +) def run(hidden_dim: int, latent_dim: int) -> None: """ Train a VAE model on the MNIST dataset using PyTorch Lightning. @@ -23,8 +40,8 @@ def run(hidden_dim: int, latent_dim: int) -> None: # Initialize model model = VAE( - input_dim=784, # 28x28 pixels - hidden_dim=hidden_dim, # Dimension of the hidden layer + input_dim=784, # 28x28 pixels + hidden_dim=hidden_dim, # Dimension of the hidden layer latent_dim=latent_dim, # Dimension of the latent space ) @@ -36,11 +53,12 @@ def run(hidden_dim: int, latent_dim: int) -> None: max_epochs=50, accelerator="gpu" if torch.cuda.is_available() else "cpu", devices=1, - logger=logger + logger=logger, ) # Start training trainer.fit(model=model, datamodule=dm) -if __name__ == '__main__': + +if __name__ == "__main__": run() From d710ea4e4ac2a5401d968a18f27bd100504f95b8 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Mon, 15 Jun 2026 16:53:43 +0200 Subject: [PATCH 20/57] fix: remove multi_class param from LogisticRegression (removed in sklearn 1.6) --- mlflow/mlflow-quickstart-example.ipynb | 1 - 1 file changed, 1 deletion(-) diff --git a/mlflow/mlflow-quickstart-example.ipynb b/mlflow/mlflow-quickstart-example.ipynb index 06c796a..b60514d 100644 --- a/mlflow/mlflow-quickstart-example.ipynb +++ b/mlflow/mlflow-quickstart-example.ipynb @@ -112,7 +112,6 @@ "params = {\n", " \"solver\": \"lbfgs\",\n", " \"max_iter\": 1000,\n", - " \"multi_class\": \"auto\",\n", " \"random_state\": 8888,\n", "}\n", "\n", From 63727634d518ac48e88588fd0481924116f7f0e1 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Mon, 15 Jun 2026 17:38:06 +0200 Subject: [PATCH 21/57] fix: strip @domain from MLflow username when constructing ISVC model name The KFP pipeline registers models as 'mobile-price-svm-{local_user}' using MLFLOW_TRACKING_USERNAME.split('@')[0] (cell 19 of the notebook). Both apply.py and the inference-protocols setup cell were passing the full email (e.g. 'developer1@prokube.cloud') as the model name suffix, causing the ISVC storageUri to reference a non-existent model and timing out waiting for readiness. --- .../inference_protocol_version_example.ipynb | 2 +- serving/mlflow-kserve-minimal/apply.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb b/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb index 83f7898..0d07609 100644 --- a/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb +++ b/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb @@ -372,7 +372,7 @@ " \"Run scripts/setup_mlflow_credentials.py to create it.\"\n", " )\n", "_creds = json.loads(_secret.stdout)[\"data\"]\n", - "_username = base64.b64decode(_creds[\"MLFLOW_TRACKING_USERNAME\"]).decode()\n", + "_username = base64.b64decode(_creds[\"MLFLOW_TRACKING_USERNAME\"]).decode().split(\"@\")[0]\n", "\n", "# ── 2. Apply InferenceService YAMLs with substituted namespace & username ────\n", "_isvc_names = [\n", diff --git a/serving/mlflow-kserve-minimal/apply.py b/serving/mlflow-kserve-minimal/apply.py index 8268ef8..ba24be9 100644 --- a/serving/mlflow-kserve-minimal/apply.py +++ b/serving/mlflow-kserve-minimal/apply.py @@ -64,7 +64,11 @@ def _mlflow_username(namespace: str) -> str: "Run scripts/setup_mlflow_credentials.py to create it." ) data = json.loads(result.stdout)["data"] - return base64.b64decode(data["MLFLOW_TRACKING_USERNAME"]).decode() + full = base64.b64decode(data["MLFLOW_TRACKING_USERNAME"]).decode() + # Pipeline registers models as "mobile-price-svm-{local_user}" where + # local_user = MLFLOW_TRACKING_USERNAME.split('@')[0], e.g. + # "developer1@prokube.cloud" → "developer1" + return full.split("@")[0] def _kubectl_apply(manifest: str, namespace: str) -> None: From 023c31f198369f17d3e7c38dff8c0b2f98d7900a Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Mon, 15 Jun 2026 18:43:00 +0200 Subject: [PATCH 22/57] fix: add mlflow-isvc-sa SA to avoid KServe S3 credential injection conflict KServe v0.18.0 injects S3 env vars (S3_ENDPOINT, AWS_ENDPOINT_URL, S3_USE_HTTPS) into the storage-initializer for every ISVC whose SA references s3creds. The MLflow storage-initializer sets the same vars via valueFrom, producing a value+valueFrom conflict Kubernetes rejects at admission. Pods are never created and the ISVC times out. - Add ServiceAccount.yaml: mlflow-isvc-sa with regcred-prokube and regcred-dev imagePullSecrets but no s3creds reference - Set serviceAccountName: mlflow-isvc-sa in InferenceService.yaml - Update apply.py to apply the SA manifest before the ISVC - Document the root cause and fix in README.md --- .../InferenceService.yaml | 1 + serving/mlflow-kserve-minimal/README.md | 57 ++++++++++++++++++- .../mlflow-kserve-minimal/ServiceAccount.yaml | 20 +++++++ serving/mlflow-kserve-minimal/apply.py | 9 ++- 4 files changed, 83 insertions(+), 4 deletions(-) create mode 100644 serving/mlflow-kserve-minimal/ServiceAccount.yaml diff --git a/serving/mlflow-kserve-minimal/InferenceService.yaml b/serving/mlflow-kserve-minimal/InferenceService.yaml index ecbdc2d..6000116 100644 --- a/serving/mlflow-kserve-minimal/InferenceService.yaml +++ b/serving/mlflow-kserve-minimal/InferenceService.yaml @@ -5,6 +5,7 @@ metadata: namespace: "" spec: predictor: + serviceAccountName: mlflow-isvc-sa model: modelFormat: name: mlflow diff --git a/serving/mlflow-kserve-minimal/README.md b/serving/mlflow-kserve-minimal/README.md index b3c7461..cdd0e24 100644 --- a/serving/mlflow-kserve-minimal/README.md +++ b/serving/mlflow-kserve-minimal/README.md @@ -12,9 +12,58 @@ InferenceService using the v2 inference protocol. It uses the built-in - `kubectl` access to your prokube namespace. (already installed in a pk-notebook) - Python with the `requests` package installed (for testing, already installed in a pk-notebook) +## Why a dedicated ServiceAccount is required + +> [!IMPORTANT] +> MLflow ISVCs **must** use the dedicated `mlflow-isvc-sa` ServiceAccount +> defined in `ServiceAccount.yaml`. Do not use the namespace `default` SA. + +### Root cause: KServe S3 credential injection conflict + +KServe v0.18.0 automatically injects S3 credentials into the +`storage-initializer` init container for every ISVC whose ServiceAccount +references a secret annotated as an S3 credential source (e.g. the +`s3creds` secret present in every prokube namespace). + +The injection produces two kinds of env entries for the same variable names +(`S3_ENDPOINT`, `AWS_ENDPOINT_URL`, `S3_USE_HTTPS`, etc.): + +| Source | Env form | +|---|---| +| Secret annotations (`serving.kserve.io/s3-endpoint`, `s3-usehttps`, …) | `value: ` | +| Secret keys (mounted via `valueFrom.secretKeyRef`) | `valueFrom: {secretKeyRef: …}` | + +Kubernetes rejects a container spec that sets **both** `value` and +`valueFrom` on the same env var. The admission webhook blocks the pod, the +storage-initializer never runs, and the ISVC times out with +`timed out waiting for condition`. + +This only affects `mlflow://` ISVCs: S3-backed ISVCs (`s3://`) rely on +exactly that injection and work correctly with the `default` SA. + +### The fix + +`ServiceAccount.yaml` defines `mlflow-isvc-sa`, a dedicated SA that: + +- Holds the `regcred-prokube` and `regcred-dev` imagePullSecrets so that + the private prokube storage-initializer image can be pulled. +- Does **not** reference `s3creds`, so KServe skips S3 credential injection + entirely for any ISVC that uses it. + +Both `InferenceService.yaml` and `apply.py` already reference this SA. + ## Deploy the InferenceService -1. Open `InferenceService.yaml` and replace the placeholder values: +1. Open `ServiceAccount.yaml` and replace `` with your + Kubeflow namespace, then apply it: + + ```sh + kubectl apply -f ServiceAccount.yaml -n + ``` + + You only need to do this once per namespace. + +2. Open `InferenceService.yaml` and replace the placeholder values: | Placeholder | Description | |---|---| @@ -32,17 +81,19 @@ InferenceService using the v2 inference protocol. It uses the built-in > You need to the a MLFlow ClusterStorageContainer in order to use the > `mlflow://` scheme (prokube platform versions >= 1.7.0) -2. Apply the manifest: +3. Apply the manifest: ```sh kubectl apply -f InferenceService.yaml -n ``` -3. Wait for the InferenceService to become ready. You can check the status in +4. Wait for the InferenceService to become ready. You can check the status in the Kubeflow Endpoints UI or via: ```sh kubectl get inferenceservice -n ``` +Alternatively, `apply.py` handles steps 1–4 automatically (see below). + ## Test the Deployment A test script and sample request body are provided to verify the deployment. diff --git a/serving/mlflow-kserve-minimal/ServiceAccount.yaml b/serving/mlflow-kserve-minimal/ServiceAccount.yaml new file mode 100644 index 0000000..ae26f66 --- /dev/null +++ b/serving/mlflow-kserve-minimal/ServiceAccount.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mlflow-isvc-sa + namespace: "" +# imagePullSecrets grant access to the private prokube container registry +# where the custom MLflow storage-initializer image lives. +# These secrets are provisioned automatically by the prokube platform. +imagePullSecrets: + - name: regcred-prokube + - name: regcred-dev +# NOTE: intentionally does not reference the 's3creds' Secret. +# KServe v0.18.0 injects S3 env vars into the storage-initializer init +# container for every ISVC whose ServiceAccount references 's3creds'. When +# the storageUri uses the mlflow:// scheme the injected env contains both a +# plain 'value' field (from the secret's annotations) and a 'valueFrom' +# field on the same variable names (S3_ENDPOINT, AWS_ENDPOINT_URL, +# S3_USE_HTTPS). Kubernetes rejects that combination at admission, so pods +# are never created and the ISVC times out. By using a dedicated SA without +# 's3creds' the injection is skipped entirely. diff --git a/serving/mlflow-kserve-minimal/apply.py b/serving/mlflow-kserve-minimal/apply.py index ba24be9..fd259b8 100644 --- a/serving/mlflow-kserve-minimal/apply.py +++ b/serving/mlflow-kserve-minimal/apply.py @@ -36,6 +36,7 @@ _ISVC_NAME = "mobile-price-svm" _YAML_TEMPLATE = os.path.join(os.path.dirname(__file__), "InferenceService.yaml") +_SA_YAML_TEMPLATE = os.path.join(os.path.dirname(__file__), "ServiceAccount.yaml") def _namespace() -> str: @@ -124,10 +125,16 @@ def _isvc_url(name: str, namespace: str) -> str: def deploy(timeout: int = 600) -> str: - """Apply InferenceService.yaml, wait for readiness, return the external URL.""" + """Apply ServiceAccount.yaml and InferenceService.yaml, wait for readiness, return the external URL.""" ns = _namespace() username = _mlflow_username(ns) + with open(_SA_YAML_TEMPLATE) as fh: + sa_manifest = fh.read().replace("", ns) + + _kubectl_apply(sa_manifest, ns) + print(f"Applied ServiceAccount 'mlflow-isvc-sa' in namespace '{ns}'.") + with open(_YAML_TEMPLATE) as fh: manifest = fh.read() From f2a20159195de209bb99bc977cd86b8830c33b4a Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Mon, 15 Jun 2026 18:43:06 +0200 Subject: [PATCH 23/57] fix: add mlflow-isvc-sa SA to inference-protocols example Same KServe S3 credential injection conflict fix as mlflow-kserve-minimal: the default SA references s3creds, causing a value+valueFrom env conflict for mlflow:// ISVCs that Kubernetes rejects at admission. - Add ServiceAccount.yaml: mlflow-isvc-sa with imagePullSecrets and no s3creds reference - Set serviceAccountName: mlflow-isvc-sa in v1 and v2 InferenceService manifests - Add README.md documenting the root cause, fix, deploy steps, and cleanup --- .../README.md | 110 ++++++++++++++++++ .../ServiceAccount.yaml | 20 ++++ .../v1-InferenceService.yaml | 1 + .../v2-InferenceService.yaml | 1 + 4 files changed, 132 insertions(+) create mode 100644 serving/mlflow-kserve-inference-protocols/README.md create mode 100644 serving/mlflow-kserve-inference-protocols/ServiceAccount.yaml diff --git a/serving/mlflow-kserve-inference-protocols/README.md b/serving/mlflow-kserve-inference-protocols/README.md new file mode 100644 index 0000000..04feceb --- /dev/null +++ b/serving/mlflow-kserve-inference-protocols/README.md @@ -0,0 +1,110 @@ +# MLflow KServe Inference Protocols + +This directory demonstrates both the v1 and v2 inference protocols for +MLflow-tracked models deployed as KServe InferenceServices on prokube. + +It uses two ISVCs side-by-side: + +| ISVC | Model format | Protocol | YAML | +|---|---|---|---| +| `v1-mobile-price-classification-inference` | `sklearn` | v1 | `v1-InferenceService.yaml` | +| `v2-mobile-price-classification-inference` | `mlflow` | v2 | `v2-InferenceService.yaml` | + +The notebook `inference_protocol_version_example.ipynb` walks through deploying +both and comparing request/response shapes for each protocol. + +## Prerequisites + +- A model trained and registered in MLflow. See the + [mobile price classification MLflow example](../../mlflow/mobile-price-classification/) + for how to train and register the SVM model used here. +- `kubectl` access to your prokube namespace. (already installed in a pk-notebook) +- Python with the `requests` package installed (for testing, already installed in a pk-notebook) + +## Why a dedicated ServiceAccount is required + +> [!IMPORTANT] +> MLflow ISVCs **must** use the dedicated `mlflow-isvc-sa` ServiceAccount +> defined in `ServiceAccount.yaml`. Do not use the namespace `default` SA. + +### Root cause: KServe S3 credential injection conflict + +KServe v0.18.0 automatically injects S3 credentials into the +`storage-initializer` init container for every ISVC whose ServiceAccount +references a secret annotated as an S3 credential source (e.g. the +`s3creds` secret present in every prokube namespace). + +The injection produces two kinds of env entries for the same variable names +(`S3_ENDPOINT`, `AWS_ENDPOINT_URL`, `S3_USE_HTTPS`, etc.): + +| Source | Env form | +|---|---| +| Secret annotations (`serving.kserve.io/s3-endpoint`, `s3-usehttps`, …) | `value: ` | +| Secret keys (mounted via `valueFrom.secretKeyRef`) | `valueFrom: {secretKeyRef: …}` | + +Kubernetes rejects a container spec that sets **both** `value` and +`valueFrom` on the same env var. The admission webhook blocks the pod, the +storage-initializer never runs, and the ISVC times out with +`timed out waiting for condition`. + +This only affects `mlflow://` ISVCs: S3-backed ISVCs (`s3://`) rely on +exactly that injection and work correctly with the `default` SA. + +### The fix + +`ServiceAccount.yaml` defines `mlflow-isvc-sa`, a dedicated SA that: + +- Holds the `regcred-prokube` and `regcred-dev` imagePullSecrets so that + the private prokube storage-initializer image can be pulled. +- Does **not** reference `s3creds`, so KServe skips S3 credential injection + entirely for any ISVC that uses it. + +Both `v1-InferenceService.yaml` and `v2-InferenceService.yaml` already +reference this SA. + +## Deploy + +1. Replace `` in `ServiceAccount.yaml` with your namespace + and apply it (once per namespace): + + ```sh + kubectl apply -f ServiceAccount.yaml -n + ``` + +2. Replace the placeholder values in both ISVC YAMLs: + + | Placeholder | Description | + |---|---| + | `` | Your Kubeflow namespace / workspace | + | `` | Your username, matching the model registered in MLflow | + +3. Apply both ISVCs: + + ```sh + kubectl apply -f v1-InferenceService.yaml -n + kubectl apply -f v2-InferenceService.yaml -n + ``` + +4. Wait for both to become ready: + + ```sh + kubectl get inferenceservice -n + ``` + + > [!WARNING] + > You need a MLFlow ClusterStorageContainer in order to use the + > `mlflow://` scheme (prokube platform versions >= 1.7.0) + +## Cleanup + +To delete both ISVCs: + +```sh +python cleanup.py +``` + +Or with `--dry-run` to preview the commands: + +```sh +python cleanup.py --dry-run +``` diff --git a/serving/mlflow-kserve-inference-protocols/ServiceAccount.yaml b/serving/mlflow-kserve-inference-protocols/ServiceAccount.yaml new file mode 100644 index 0000000..ae26f66 --- /dev/null +++ b/serving/mlflow-kserve-inference-protocols/ServiceAccount.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mlflow-isvc-sa + namespace: "" +# imagePullSecrets grant access to the private prokube container registry +# where the custom MLflow storage-initializer image lives. +# These secrets are provisioned automatically by the prokube platform. +imagePullSecrets: + - name: regcred-prokube + - name: regcred-dev +# NOTE: intentionally does not reference the 's3creds' Secret. +# KServe v0.18.0 injects S3 env vars into the storage-initializer init +# container for every ISVC whose ServiceAccount references 's3creds'. When +# the storageUri uses the mlflow:// scheme the injected env contains both a +# plain 'value' field (from the secret's annotations) and a 'valueFrom' +# field on the same variable names (S3_ENDPOINT, AWS_ENDPOINT_URL, +# S3_USE_HTTPS). Kubernetes rejects that combination at admission, so pods +# are never created and the ISVC times out. By using a dedicated SA without +# 's3creds' the injection is skipped entirely. diff --git a/serving/mlflow-kserve-inference-protocols/v1-InferenceService.yaml b/serving/mlflow-kserve-inference-protocols/v1-InferenceService.yaml index c2b2f1f..bb11b3e 100644 --- a/serving/mlflow-kserve-inference-protocols/v1-InferenceService.yaml +++ b/serving/mlflow-kserve-inference-protocols/v1-InferenceService.yaml @@ -5,6 +5,7 @@ metadata: namespace: "" spec: predictor: + serviceAccountName: mlflow-isvc-sa model: modelFormat: name: sklearn diff --git a/serving/mlflow-kserve-inference-protocols/v2-InferenceService.yaml b/serving/mlflow-kserve-inference-protocols/v2-InferenceService.yaml index 1f4db07..c933783 100644 --- a/serving/mlflow-kserve-inference-protocols/v2-InferenceService.yaml +++ b/serving/mlflow-kserve-inference-protocols/v2-InferenceService.yaml @@ -5,6 +5,7 @@ metadata: namespace: "" spec: predictor: + serviceAccountName: mlflow-isvc-sa model: modelFormat: name: mlflow From 411b023e03678c59378d6bf5d420484aee05d832 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Mon, 15 Jun 2026 19:18:45 +0200 Subject: [PATCH 24/57] fix: skip KFP run ID extraction for ISVC-only notebooks in CI The inference-protocols notebook deploys InferenceServices and makes inference calls. Its output contains UUIDs (ISVC UIDs, request IDs) that the broad UUID regex in _extract_run_ids_from_notebook picks up. Phase 4 then polls these as KFP run IDs, which returns 404. Add extract_run_ids flag to _submit_notebook (default True) and pass False for serving/mlflow-kserve-inference-protocols, which never submits KFP pipelines. --- ci/run_all.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ci/run_all.py b/ci/run_all.py index f5cef9b..a170a6b 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -327,6 +327,7 @@ def _submit_notebook( nb_path: Path, output_dir: Path, timeout: int, + extract_run_ids: bool = True, ) -> Future: result = Result(name=name) results[name] = result @@ -337,7 +338,8 @@ def _run(): try: out = _run_notebook(nb_path, output_dir, timeout) result.status = "PASS" - result.kfp_run_ids = _extract_run_ids_from_notebook(out) + if extract_run_ids: + result.kfp_run_ids = _extract_run_ids_from_notebook(out) except Exception as exc: # noqa: BLE001 result.status = "FAIL" result.error = str(exc) @@ -765,7 +767,10 @@ def run_all( timeout_notebook, ) - # inference-protocols notebook handles its own deploy + test + # inference-protocols notebook handles its own deploy + test. + # extract_run_ids=False: this notebook is not a pipeline submission; + # its output contains ISVC/request UUIDs that must not be polled + # as KFP run IDs (they return 404 from the KFP API). phase3_futures["serving/mlflow-kserve-inference-protocols"] = ( _submit_notebook( executor, @@ -775,6 +780,7 @@ def run_all( / "serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb", output_dir, timeout_notebook, + extract_run_ids=False, ) ) From 074f62e7e6a79ccec022686633da356127eb5f67 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Mon, 15 Jun 2026 19:40:10 +0200 Subject: [PATCH 25/57] fix: resolve libstdc++ CXXABI_1.3.15 error and wire --max_epochs in mnist-vae scipy 1.17+ is compiled against CXXABI_1.3.15 which the system libstdc++ lacks (tops out at 1.3.13). The conda-bundled libstdc++ provides it but is not on LD_LIBRARY_PATH by default. Re-exec the process with /opt/conda/lib prepended to LD_LIBRARY_PATH before any imports so the dynamic linker picks up the right library. Also add --max_epochs click option (default 50) so the CI can pass --max_epochs 3 for a fast smoke-test run; previously the option was silently ignored and the trainer always ran 50 epochs. --- notebooks/mnist-vae/run_training.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/notebooks/mnist-vae/run_training.py b/notebooks/mnist-vae/run_training.py index 1967d93..4064976 100644 --- a/notebooks/mnist-vae/run_training.py +++ b/notebooks/mnist-vae/run_training.py @@ -1,6 +1,23 @@ +import os import subprocess import sys +# Re-exec with conda's libstdc++ prepended to LD_LIBRARY_PATH when needed. +# scipy 1.17+ requires CXXABI_1.3.15 which the system libstdc++ may lack; the +# conda-bundled libstdc++ provides it. LD_LIBRARY_PATH must be set before the +# process starts (the dynamic linker reads it once), so we re-exec if the +# conda lib dir is not already in the path. +_conda_lib = os.path.normpath( + os.path.join(os.path.dirname(sys.executable), "..", "lib") +) +_ld = os.environ.get("LD_LIBRARY_PATH", "") +if os.path.isdir(_conda_lib) and _conda_lib not in _ld.split(":"): + os.execvpe( + sys.executable, + [sys.executable] + sys.argv, + {**os.environ, "LD_LIBRARY_PATH": f"{_conda_lib}:{_ld}"}, + ) + # pytorch-lightning is not bundled in all notebook images — install if absent try: import pytorch_lightning # noqa: F401 @@ -25,13 +42,15 @@ @click.option( "--latent_dim", default=2, type=int, help="Dimension of the latent space." ) -def run(hidden_dim: int, latent_dim: int) -> None: +@click.option("--max_epochs", default=50, type=int, help="Number of training epochs.") +def run(hidden_dim: int, latent_dim: int, max_epochs: int) -> None: """ Train a VAE model on the MNIST dataset using PyTorch Lightning. Args: hidden_dim (int): Dimension of the hidden layer. latent_dim (int): Dimension of the latent space. + max_epochs (int): Number of training epochs. """ # Initialize data module @@ -50,7 +69,7 @@ def run(hidden_dim: int, latent_dim: int) -> None: # Initialize trainer trainer = pl.Trainer( - max_epochs=50, + max_epochs=max_epochs, accelerator="gpu" if torch.cuda.is_available() else "cpu", devices=1, logger=logger, From 0e91774759dfc2554e9baa3cc9d37a53b2d035e8 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Mon, 15 Jun 2026 19:55:39 +0200 Subject: [PATCH 26/57] fix: also self-install torchvision when missing in mnist-vae torchvision is absent from the base notebook image independently of pytorch-lightning. Extend the self-install guard to check both and install whichever packages are missing in a single pip call. --- notebooks/mnist-vae/run_training.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/notebooks/mnist-vae/run_training.py b/notebooks/mnist-vae/run_training.py index 4064976..290ac12 100644 --- a/notebooks/mnist-vae/run_training.py +++ b/notebooks/mnist-vae/run_training.py @@ -18,12 +18,19 @@ {**os.environ, "LD_LIBRARY_PATH": f"{_conda_lib}:{_ld}"}, ) -# pytorch-lightning is not bundled in all notebook images — install if absent +# pytorch-lightning and torchvision are not bundled in all notebook images +_missing = [] try: import pytorch_lightning # noqa: F401 except ImportError: + _missing.append("pytorch-lightning") +try: + import torchvision # noqa: F401 +except ImportError: + _missing.append("torchvision") +if _missing: subprocess.run( - [sys.executable, "-m", "pip", "install", "-q", "pytorch-lightning"], + [sys.executable, "-m", "pip", "install", "-q"] + _missing, check=True, ) From 9adf9a898c4d25fe52ad67aae95ac2060f70b49c Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Mon, 15 Jun 2026 19:58:15 +0200 Subject: [PATCH 27/57] fix: self-install tensorboard when missing in mnist-vae --- notebooks/mnist-vae/run_training.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/notebooks/mnist-vae/run_training.py b/notebooks/mnist-vae/run_training.py index 290ac12..afa0cdd 100644 --- a/notebooks/mnist-vae/run_training.py +++ b/notebooks/mnist-vae/run_training.py @@ -18,7 +18,7 @@ {**os.environ, "LD_LIBRARY_PATH": f"{_conda_lib}:{_ld}"}, ) -# pytorch-lightning and torchvision are not bundled in all notebook images +# pytorch-lightning, torchvision, and tensorboard are not bundled in all notebook images _missing = [] try: import pytorch_lightning # noqa: F401 @@ -28,6 +28,10 @@ import torchvision # noqa: F401 except ImportError: _missing.append("torchvision") +try: + import tensorboard # noqa: F401 +except ImportError: + _missing.append("tensorboard") if _missing: subprocess.run( [sys.executable, "-m", "pip", "install", "-q"] + _missing, From d65e2ccdd1c483e84f489db5057161e247e38f6f Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Mon, 15 Jun 2026 20:11:16 +0200 Subject: [PATCH 28/57] feat: add CI coverage for hf-vllm-completion, kserve-keda, shadow-deployment Three previously untested examples now have apply.py + cleanup.py and are wired into ci/run_all.py: serving/hf-vllm-completion (always-on in Phase 1) - Deploys distilbert-inf-serv (HuggingFace CPU backend) - Smoke-tests via cluster-internal URL; no API key required serving/kserve-keda-autoscaling (opt-in: --include-keda) - Checks scaledobjects.keda.sh CRD; exits 0 with SKIP if absent - Deploys OPT-125M ISVC + ScaledObject, waits for Active=True - Smoke-tests via cluster-internal predictor Service serving/minimal-example-shadow-deployment (opt-in: --include-shadow) - Checks postgresclusters CRD; exits 0 with SKIP if absent - Deploys PostgresCluster, waits for primary, creates schema via one-shot psql pod, deploys doubler + tripler ISVCs - Smoke-tests primary via cluster-internal URL (factor=2 check) - Istio VirtualService mirroring is not tested in CI Also fixes the dead --include-keda flag (was parsed but never passed into run_all()) and adds --include-shadow on the same pattern. --- ci/run_all.py | 93 +++++- serving/hf-vllm-completion/apply.py | 126 ++++++++ serving/hf-vllm-completion/cleanup.py | 51 +++ serving/kserve-keda-autoscaling/apply.py | 185 +++++++++++ serving/kserve-keda-autoscaling/cleanup.py | 53 +++ .../apply.py | 303 ++++++++++++++++++ .../cleanup.py | 59 ++++ 7 files changed, 858 insertions(+), 12 deletions(-) create mode 100644 serving/hf-vllm-completion/apply.py create mode 100644 serving/hf-vllm-completion/cleanup.py create mode 100644 serving/kserve-keda-autoscaling/apply.py create mode 100644 serving/kserve-keda-autoscaling/cleanup.py create mode 100644 serving/minimal-example-shadow-deployment/apply.py create mode 100644 serving/minimal-example-shadow-deployment/cleanup.py diff --git a/ci/run_all.py b/ci/run_all.py index a170a6b..204a141 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -566,6 +566,7 @@ def run_all( timeout_pipeline: int = 3600, include_keda: bool = False, include_pytorch: bool = False, + include_shadow: bool = False, dry_run: bool = False, ) -> dict[str, Result]: root = _REPO_ROOT @@ -586,23 +587,27 @@ def run_all( for name in _MLFLOW_DEPENDENT: results[name] = Result(name=name, status="SKIP", error=mlflow_reason) - if dry_run: - print("[dry-run] Would execute the following phases:") - for label in [ - "Phase 1: dask, mlflow-quickstart, mlflow-image, mlflow-kfp, minimal-s3-model", - "Phase 2: mlflow-mobile-price, lightweight-components, lightweight-python-package", - "Phase 3: mlflow-kserve-minimal, mlflow-kserve-inference-protocols", - "Phase 4: KFP run polling", - "Phase 5: cleanup", - ]: - print(f" {label}") - return results + if dry_run: + print("[dry-run] Would execute the following phases:") + for label in [ + "Phase 1: dask, mlflow-quickstart, mlflow-image, mlflow-kfp, minimal-s3-model, hf-vllm-completion", + "Phase 1 (opt-in): kserve-keda-autoscaling (--include-keda), minimal-example-shadow-deployment (--include-shadow)", + "Phase 2: mlflow-mobile-price, lightweight-components, lightweight-python-package", + "Phase 3: mlflow-kserve-minimal, mlflow-kserve-inference-protocols", + "Phase 4: KFP run polling", + "Phase 5: cleanup", + ]: + print(f" {label}") + return results cleanup_scripts = [ root / "notebooks" / "dask" / "cleanup.py", root / "serving" / "minimal-s3-model" / "cleanup.py", root / "serving" / "mlflow-kserve-minimal" / "cleanup.py", root / "serving" / "mlflow-kserve-inference-protocols" / "cleanup.py", + root / "serving" / "hf-vllm-completion" / "cleanup.py", + root / "serving" / "kserve-keda-autoscaling" / "cleanup.py", + root / "serving" / "minimal-example-shadow-deployment" / "cleanup.py", root / "hparam-tuning" / "minimal-mnist" / "cleanup.py", ] @@ -633,6 +638,61 @@ def run_all( ) phase1_futures[name] = f + # hf-vllm-completion (CPU): self-contained, no KFP pipelines + phase1_futures["serving/hf-vllm-completion"] = _submit_script( + executor, + results, + "serving/hf-vllm-completion", + root / "serving" / "hf-vllm-completion" / "apply.py", + timeout_notebook, + ) + + # kserve-keda-autoscaling: opt-in; apply.py self-checks KEDA CRDs + # and exits 0 with a skip message if KEDA is not installed. + if include_keda: + phase1_futures["serving/kserve-keda-autoscaling"] = _submit_script( + executor, + results, + "serving/kserve-keda-autoscaling", + root / "serving" / "kserve-keda-autoscaling" / "apply.py", + timeout_notebook, + ) + else: + print( + " [SKIP ] serving/kserve-keda-autoscaling (pass --include-keda to enable)" + ) + results["serving/kserve-keda-autoscaling"] = Result( + name="serving/kserve-keda-autoscaling", + status="SKIP", + error="opt-in: pass --include-keda to run this example", + ) + + # minimal-example-shadow-deployment: opt-in; apply.py self-checks for + # the postgres-operator CRD and exits 0 if not found. Istio + # VirtualService mirroring is not verified in CI (requires domain config). + if include_shadow: + phase1_futures["serving/minimal-example-shadow-deployment"] = ( + _submit_script( + executor, + results, + "serving/minimal-example-shadow-deployment", + root + / "serving" + / "minimal-example-shadow-deployment" + / "apply.py", + timeout_notebook, + ) + ) + else: + print( + " [SKIP ] serving/minimal-example-shadow-deployment (pass --include-shadow to enable)" + ) + results["serving/minimal-example-shadow-deployment"] = Result( + name="serving/minimal-example-shadow-deployment", + status="SKIP", + error="opt-in: pass --include-shadow to run this example", + ) + # mnist-vae: opt-in (requires pytorch_lightning, not in all images) if include_pytorch: phase1_futures["notebooks/mnist-vae"] = _submit_chain( @@ -862,7 +922,15 @@ def run_all( parser.add_argument( "--include-keda", action="store_true", - help="Include GPU-dependent KEDA autoscaling example (opt-in)", + help="Include KEDA autoscaling example (opt-in; requires KEDA installed in the cluster)", + ) + parser.add_argument( + "--include-shadow", + action="store_true", + help=( + "Include minimal-example-shadow-deployment (opt-in; " + "requires CrunchyData postgres-operator; Istio VirtualService not tested)" + ), ) parser.add_argument( "--include-pytorch", @@ -878,6 +946,7 @@ def run_all( timeout_notebook=args.timeout_notebook, timeout_pipeline=args.timeout_pipeline, include_keda=args.include_keda, + include_shadow=args.include_shadow, include_pytorch=args.include_pytorch, dry_run=args.dry_run, ) diff --git a/serving/hf-vllm-completion/apply.py b/serving/hf-vllm-completion/apply.py new file mode 100644 index 0000000..19b7c7c --- /dev/null +++ b/serving/hf-vllm-completion/apply.py @@ -0,0 +1,126 @@ +""" +Deploy the DistilBERT CPU InferenceService (HuggingFace backend) and smoke-test it. + +The model (distilbert-base-uncased-finetuned-sst-2-english, ~250 MB) is +downloaded from the HuggingFace Hub on first start, so the ISVC may take a +few minutes to become ready. + +Usage +----- + python apply.py +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import time +import urllib.error +import urllib.request + +_ISVC_NAME = "distilbert-inf-serv" +_MODEL_NAME = "distilbert" +_YAML = os.path.join(os.path.dirname(__file__), "inference-service-cpu.yaml") + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _kubectl_apply(manifest: str, namespace: str) -> None: + result = subprocess.run( + ["kubectl", "apply", "-f", "-", "-n", namespace], + input=manifest, + text=True, + capture_output=True, + ) + if result.returncode != 0: + raise RuntimeError(result.stderr or result.stdout) + print(result.stdout.strip()) + + +def _wait_isvc_ready(name: str, namespace: str, timeout: int) -> None: + print( + f"Waiting for InferenceService '{name}' to become ready (timeout {timeout}s)..." + ) + result = subprocess.run( + [ + "kubectl", + "wait", + "inferenceservice", + name, + "--for=condition=Ready", + f"--timeout={timeout}s", + "-n", + namespace, + ], + capture_output=True, + text=True, + ) + if result.returncode != 0: + # Dump ISVC status to help diagnose + subprocess.run( + ["kubectl", "describe", "inferenceservice", name, "-n", namespace], + check=False, + ) + raise RuntimeError( + f"InferenceService '{name}' did not become ready within {timeout}s:\n" + + (result.stderr or result.stdout) + ) + + +def _smoke_test(namespace: str, timeout: int = 120) -> None: + """POST to the internal cluster URL; retries until the model responds.""" + url = ( + f"http://{_ISVC_NAME}.{namespace}.svc.cluster.local" + f"/v1/models/{_MODEL_NAME}:predict" + ) + payload = json.dumps({"instances": ["KServe is wonderful!"]}).encode() + deadline = time.time() + timeout + last_err: Exception | None = None + while time.time() < deadline: + try: + req = urllib.request.Request( + url, + data=payload, + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=15) as resp: + body = json.loads(resp.read()) + if "predictions" not in body: + raise RuntimeError(f"unexpected response body: {body}") + print(f"Smoke test passed: {body}") + return + except Exception as exc: + last_err = exc + time.sleep(5) + raise RuntimeError(f"Smoke test failed after {timeout}s: {last_err}") + + +# ── Entry point ─────────────────────────────────────────────────────────────── + + +def deploy(timeout: int = 900) -> None: + ns = _namespace() + + with open(_YAML) as fh: + manifest = fh.read() + + _kubectl_apply(manifest, ns) + print(f"Applied InferenceService '{_ISVC_NAME}' in namespace '{ns}'.") + + _wait_isvc_ready(_ISVC_NAME, ns, timeout) + print(f"InferenceService '{_ISVC_NAME}' is ready.") + + _smoke_test(ns) + + +if __name__ == "__main__": + deploy() diff --git a/serving/hf-vllm-completion/cleanup.py b/serving/hf-vllm-completion/cleanup.py new file mode 100644 index 0000000..a874d37 --- /dev/null +++ b/serving/hf-vllm-completion/cleanup.py @@ -0,0 +1,51 @@ +""" +Cleanup script for the hf-vllm-completion example. + +Deletes (idempotently): + - InferenceService ``distilbert-inf-serv`` + +Usage +----- + python cleanup.py [--dry-run] +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _kubectl_delete(*args: str, dry_run: bool = False) -> None: + cmd = ["kubectl", "delete", *args, "--ignore-not-found"] + if dry_run: + print(f"[dry-run] {' '.join(cmd)}") + return + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"WARNING: {result.stderr.strip()}", file=sys.stderr) + else: + print(result.stdout.strip() or f"deleted (or not found): {' '.join(args)}") + + +def cleanup(dry_run: bool = False) -> None: + ns = _namespace() + _kubectl_delete( + "inferenceservice", "distilbert-inf-serv", "-n", ns, dry_run=dry_run + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--dry-run", action="store_true", help="Print commands without executing" + ) + args = parser.parse_args() + cleanup(dry_run=args.dry_run) diff --git a/serving/kserve-keda-autoscaling/apply.py b/serving/kserve-keda-autoscaling/apply.py new file mode 100644 index 0000000..8cca44c --- /dev/null +++ b/serving/kserve-keda-autoscaling/apply.py @@ -0,0 +1,185 @@ +""" +Deploy the KEDA autoscaling example (OPT-125M, vLLM backend, CPU) and verify +that both the InferenceService and the ScaledObject become active. + +KEDA CRD check +-------------- +If the ``scaledobjects.keda.sh`` CRD is absent the script exits with 0 and a +SKIP message rather than failing — the example is opt-in because KEDA is not +installed in every prokube cluster. + +Usage +----- + python apply.py +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import time +import urllib.error +import urllib.request + +_ISVC_NAME = "opt-125m" +_SCALED_OBJECT_NAME = "opt-125m-scaledobject" +_ISVC_YAML = os.path.join(os.path.dirname(__file__), "inference-service.yaml") +_SO_YAML = os.path.join(os.path.dirname(__file__), "scaled-object.yaml") + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _kubectl_apply(manifest: str, namespace: str) -> None: + result = subprocess.run( + ["kubectl", "apply", "-f", "-", "-n", namespace], + input=manifest, + text=True, + capture_output=True, + ) + if result.returncode != 0: + raise RuntimeError(result.stderr or result.stdout) + print(result.stdout.strip()) + + +def _keda_available() -> bool: + """Return True if KEDA CRDs are installed in this cluster.""" + result = subprocess.run( + ["kubectl", "get", "crd", "scaledobjects.keda.sh"], + capture_output=True, + text=True, + ) + return result.returncode == 0 + + +def _wait_isvc_ready(name: str, namespace: str, timeout: int) -> None: + print( + f"Waiting for InferenceService '{name}' to become ready (timeout {timeout}s)..." + ) + result = subprocess.run( + [ + "kubectl", + "wait", + "inferenceservice", + name, + "--for=condition=Ready", + f"--timeout={timeout}s", + "-n", + namespace, + ], + capture_output=True, + text=True, + ) + if result.returncode != 0: + subprocess.run( + ["kubectl", "describe", "inferenceservice", name, "-n", namespace], + check=False, + ) + raise RuntimeError( + f"InferenceService '{name}' did not become ready within {timeout}s:\n" + + (result.stderr or result.stdout) + ) + + +def _wait_scaledobject_active(name: str, namespace: str, timeout: int = 120) -> None: + """Poll until KEDA reports the ScaledObject as Active=True.""" + print(f"Waiting for ScaledObject '{name}' to become active...") + deadline = time.time() + timeout + while time.time() < deadline: + r = subprocess.run( + [ + "kubectl", + "get", + "scaledobject", + name, + "-n", + namespace, + "-o", + "jsonpath={.status.conditions[?(@.type=='Active')].status}", + ], + capture_output=True, + text=True, + ) + if r.stdout.strip() == "True": + print(f"ScaledObject '{name}' is active.") + return + time.sleep(10) + raise RuntimeError( + f"ScaledObject '{name}' did not become active within {timeout}s. " + "Check 'kubectl describe scaledobject' for trigger errors." + ) + + +def _smoke_test(namespace: str, timeout: int = 60) -> None: + """Send a single completion request via the cluster-internal predictor Service.""" + # RawDeployment mode exposes the predictor as a plain Kubernetes Service named + # -predictor — no external gateway or API key required. + url = ( + f"http://opt-125m-predictor.{namespace}.svc.cluster.local/openai/v1/completions" + ) + payload = json.dumps( + {"model": "opt-125m", "prompt": "KServe is", "max_tokens": 8} + ).encode() + deadline = time.time() + timeout + last_err: Exception | None = None + while time.time() < deadline: + try: + req = urllib.request.Request( + url, + data=payload, + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=20) as resp: + body = json.loads(resp.read()) + if "choices" not in body: + raise RuntimeError(f"unexpected response body: {body}") + text = body["choices"][0].get("text", "") + print(f"Smoke test passed: {text!r}") + return + except Exception as exc: + last_err = exc + time.sleep(5) + raise RuntimeError(f"Smoke test failed after {timeout}s: {last_err}") + + +# ── Entry point ─────────────────────────────────────────────────────────────── + + +def deploy(timeout: int = 900) -> None: + if not _keda_available(): + print( + "SKIP: KEDA CRDs (scaledobjects.keda.sh) not found in this cluster.\n" + "Ask your cluster admin to install KEDA, or pass --include-keda only\n" + "when targeting a cluster with KEDA installed." + ) + sys.exit(0) + + ns = _namespace() + + with open(_ISVC_YAML) as fh: + isvc_manifest = fh.read() + _kubectl_apply(isvc_manifest, ns) + print(f"Applied InferenceService '{_ISVC_NAME}' in namespace '{ns}'.") + + _wait_isvc_ready(_ISVC_NAME, ns, timeout) + print(f"InferenceService '{_ISVC_NAME}' is ready.") + + with open(_SO_YAML) as fh: + so_manifest = fh.read() + _kubectl_apply(so_manifest, ns) + print(f"Applied ScaledObject '{_SCALED_OBJECT_NAME}' in namespace '{ns}'.") + + _wait_scaledobject_active(_SCALED_OBJECT_NAME, ns) + _smoke_test(ns) + + +if __name__ == "__main__": + deploy() diff --git a/serving/kserve-keda-autoscaling/cleanup.py b/serving/kserve-keda-autoscaling/cleanup.py new file mode 100644 index 0000000..d0c5946 --- /dev/null +++ b/serving/kserve-keda-autoscaling/cleanup.py @@ -0,0 +1,53 @@ +""" +Cleanup script for the kserve-keda-autoscaling example. + +Deletes (idempotently, ScaledObject first to stop KEDA activity): + - ScaledObject ``opt-125m-scaledobject`` + - InferenceService ``opt-125m`` + +Usage +----- + python cleanup.py [--dry-run] +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _kubectl_delete(*args: str, dry_run: bool = False) -> None: + cmd = ["kubectl", "delete", *args, "--ignore-not-found"] + if dry_run: + print(f"[dry-run] {' '.join(cmd)}") + return + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"WARNING: {result.stderr.strip()}", file=sys.stderr) + else: + print(result.stdout.strip() or f"deleted (or not found): {' '.join(args)}") + + +def cleanup(dry_run: bool = False) -> None: + ns = _namespace() + # Delete the ScaledObject before the ISVC so KEDA stops managing the HPA + # before the underlying Deployment is gone. + _kubectl_delete("scaledobject", "opt-125m-scaledobject", "-n", ns, dry_run=dry_run) + _kubectl_delete("inferenceservice", "opt-125m", "-n", ns, dry_run=dry_run) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--dry-run", action="store_true", help="Print commands without executing" + ) + args = parser.parse_args() + cleanup(dry_run=args.dry_run) diff --git a/serving/minimal-example-shadow-deployment/apply.py b/serving/minimal-example-shadow-deployment/apply.py new file mode 100644 index 0000000..272b932 --- /dev/null +++ b/serving/minimal-example-shadow-deployment/apply.py @@ -0,0 +1,303 @@ +""" +Deploy the shadow deployment example and smoke-test the primary InferenceService. + +What this script does +--------------------- +1. Checks for the CrunchyData postgres-operator CRD — exits 0 with a SKIP + message if not found (it is installed by default in prokube clusters). +2. Deploys the PostgresCluster (postgres-cluster.yaml) and waits for the + primary pod to become ready. +3. Creates the two tables the transformer needs (inference_requests and + inference_response) via a short-lived psql pod. +4. Deploys the primary (doubler) and shadow (tripler) InferenceServices and + waits for both to become ready. +5. Sends a smoke-test request to the primary via its cluster-internal URL + and verifies the response contains predictions. + +What this script does NOT do +----------------------------- +- Apply the Istio VirtualService manifests — those embed namespace and domain + name and must be configured manually for each environment. +- Verify the shadow mirroring behaviour (that requires traffic via the + VirtualService, which is out of scope for automated CI). + +Usage +----- + python apply.py +""" + +from __future__ import annotations + +import base64 +import json +import os +import subprocess +import sys +import time +import urllib.error +import urllib.request + +_ROOT = os.path.dirname(__file__) + +_PG_CLUSTER_NAME = "inferencing-postgres" +_PG_SECRET_NAME = "inferencing-postgres-pguser-transformer-admin" +_PG_HOST_TEMPLATE = "inferencing-postgres-primary.{ns}.svc" +_PG_USER = "transformer-admin" +_PG_DB = "scale-inference" + +_DOUBLER_ISVC = "double-minimal-custom-inference" +_TRIPLER_ISVC = "triple-minimal-custom-inference" + +_SCHEMA_SQL = """\ +CREATE TABLE IF NOT EXISTS public.inference_requests ( + request_id uuid NOT NULL, + request_time timestamp with time zone NULL, + request_data json NULL, + predict_url text NULL, + created_at timestamp NULL, + PRIMARY KEY (request_id) +); +CREATE TABLE IF NOT EXISTS public.inference_response ( + request_id uuid NOT NULL, + request_data json NULL, + created_at timestamp NULL, + PRIMARY KEY (request_id) +); +""" + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _kubectl_apply(manifest: str, namespace: str) -> None: + result = subprocess.run( + ["kubectl", "apply", "-f", "-", "-n", namespace], + input=manifest, + text=True, + capture_output=True, + ) + if result.returncode != 0: + raise RuntimeError(result.stderr or result.stdout) + print(result.stdout.strip()) + + +def _crd_available(crd_name: str) -> bool: + result = subprocess.run( + ["kubectl", "get", "crd", crd_name], + capture_output=True, + text=True, + ) + return result.returncode == 0 + + +def _wait_for_secret(name: str, namespace: str, timeout: int = 300) -> None: + """Block until the CrunchyData operator creates the user secret.""" + print( + f"Waiting for secret '{name}' to appear (postgres-operator creates it after cluster init)..." + ) + deadline = time.time() + timeout + while time.time() < deadline: + r = subprocess.run( + ["kubectl", "get", "secret", name, "-n", namespace], + capture_output=True, + text=True, + ) + if r.returncode == 0: + return + time.sleep(10) + raise RuntimeError( + f"Secret '{name}' did not appear within {timeout}s. " + "Check the PostgresCluster status." + ) + + +def _wait_pg_primary_ready(namespace: str, timeout: int = 300) -> None: + """Wait for the postgres primary pod to be ready.""" + print("Waiting for PostgresCluster primary pod to be ready...") + result = subprocess.run( + [ + "kubectl", + "wait", + "pod", + "-l", + f"postgres-operator.crunchydata.com/cluster={_PG_CLUSTER_NAME}," + "postgres-operator.crunchydata.com/role=master", + "--for=condition=Ready", + f"--timeout={timeout}s", + "-n", + namespace, + ], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError( + f"PostgresCluster primary did not become ready within {timeout}s:\n" + + (result.stderr or result.stdout) + ) + print("PostgresCluster primary is ready.") + + +def _get_secret_value(name: str, key: str, namespace: str) -> str: + result = subprocess.run( + [ + "kubectl", + "get", + "secret", + name, + "-n", + namespace, + "-o", + f"jsonpath={{.data.{key}}}", + ], + capture_output=True, + text=True, + ) + if result.returncode != 0 or not result.stdout.strip(): + raise RuntimeError( + f"Could not read key '{key}' from secret '{name}': {result.stderr}" + ) + return base64.b64decode(result.stdout.strip()).decode() + + +def _create_schema(namespace: str, password: str) -> None: + """Run a one-shot psql pod to create the required tables.""" + print("Creating database schema via temporary psql pod...") + host = _PG_HOST_TEMPLATE.format(ns=namespace) + result = subprocess.run( + [ + "kubectl", + "run", + "pg-schema-init", + "--rm", + "-i", + "--restart=Never", + f"-n={namespace}", + "--image=postgres:17", + f"--env=PGPASSWORD={password}", + "--", + "psql", + f"-h={host}", + f"-U={_PG_USER}", + f"-d={_PG_DB}", + ], + input=_SCHEMA_SQL, + text=True, + capture_output=True, + timeout=120, + ) + if result.returncode != 0: + raise RuntimeError(f"Schema creation failed:\n{result.stderr or result.stdout}") + print("Schema created (or already existed).") + + +def _wait_isvc_ready(name: str, namespace: str, timeout: int) -> None: + print(f"Waiting for InferenceService '{name}' (timeout {timeout}s)...") + result = subprocess.run( + [ + "kubectl", + "wait", + "inferenceservice", + name, + "--for=condition=Ready", + f"--timeout={timeout}s", + "-n", + namespace, + ], + capture_output=True, + text=True, + ) + if result.returncode != 0: + subprocess.run( + ["kubectl", "describe", "inferenceservice", name, "-n", namespace], + check=False, + ) + raise RuntimeError( + f"InferenceService '{name}' did not become ready within {timeout}s:\n" + + (result.stderr or result.stdout) + ) + + +def _smoke_test(namespace: str, timeout: int = 120) -> None: + """POST numeric values to the primary (doubler) ISVC and verify predictions.""" + url = ( + f"http://{_DOUBLER_ISVC}.{namespace}.svc.cluster.local/v1/models/model:predict" + ) + payload = json.dumps({"values": [1.0, 2.0, 3.0]}).encode() + deadline = time.time() + timeout + last_err: Exception | None = None + while time.time() < deadline: + try: + req = urllib.request.Request( + url, + data=payload, + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=15) as resp: + body = json.loads(resp.read()) + if "predictions" not in body: + raise RuntimeError(f"unexpected response body: {body}") + print(f"Smoke test passed: {body}") + return + except Exception as exc: + last_err = exc + time.sleep(5) + raise RuntimeError(f"Smoke test failed after {timeout}s: {last_err}") + + +# ── Entry point ─────────────────────────────────────────────────────────────── + + +def deploy(timeout: int = 600) -> None: + if not _crd_available("postgresclusters.postgres-operator.crunchydata.com"): + print( + "SKIP: CrunchyData postgres-operator CRD not found in this cluster.\n" + "The shadow deployment example requires the postgres-operator to be installed." + ) + sys.exit(0) + + ns = _namespace() + + # 1. Postgres cluster + with open(os.path.join(_ROOT, "postgres-cluster.yaml")) as fh: + pg_manifest = fh.read() + _kubectl_apply(pg_manifest, ns) + print(f"Applied PostgresCluster '{_PG_CLUSTER_NAME}' in namespace '{ns}'.") + + _wait_pg_primary_ready(ns, timeout=300) + _wait_for_secret(_PG_SECRET_NAME, ns, timeout=120) + + password = _get_secret_value(_PG_SECRET_NAME, "password", ns) + _create_schema(ns, password) + + # 2. Both InferenceServices + for yaml_file in ( + "doubler-inference-service.yaml", + "tripler-inference-service.yaml", + ): + with open(os.path.join(_ROOT, yaml_file)) as fh: + manifest = fh.read() + _kubectl_apply(manifest, ns) + + print("Applied doubler and tripler InferenceServices.") + + for isvc_name in (_DOUBLER_ISVC, _TRIPLER_ISVC): + _wait_isvc_ready(isvc_name, ns, timeout) + + print("Both InferenceServices are ready.") + print( + "NOTE: Istio VirtualService manifests (istio/) are not applied in CI — " + "they require namespace and domain substitution and must be configured manually." + ) + + _smoke_test(ns) + + +if __name__ == "__main__": + deploy() diff --git a/serving/minimal-example-shadow-deployment/cleanup.py b/serving/minimal-example-shadow-deployment/cleanup.py new file mode 100644 index 0000000..eafd42b --- /dev/null +++ b/serving/minimal-example-shadow-deployment/cleanup.py @@ -0,0 +1,59 @@ +""" +Cleanup script for the minimal-example-shadow-deployment example. + +Deletes (idempotently): + - InferenceService ``double-minimal-custom-inference`` + - InferenceService ``triple-minimal-custom-inference`` + - PostgresCluster ``inferencing-postgres`` + +Usage +----- + python cleanup.py [--dry-run] +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _kubectl_delete(*args: str, dry_run: bool = False) -> None: + cmd = ["kubectl", "delete", *args, "--ignore-not-found"] + if dry_run: + print(f"[dry-run] {' '.join(cmd)}") + return + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"WARNING: {result.stderr.strip()}", file=sys.stderr) + else: + print(result.stdout.strip() or f"deleted (or not found): {' '.join(args)}") + + +def cleanup(dry_run: bool = False) -> None: + ns = _namespace() + _kubectl_delete( + "inferenceservice", "double-minimal-custom-inference", "-n", ns, dry_run=dry_run + ) + _kubectl_delete( + "inferenceservice", "triple-minimal-custom-inference", "-n", ns, dry_run=dry_run + ) + _kubectl_delete( + "postgrescluster", "inferencing-postgres", "-n", ns, dry_run=dry_run + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--dry-run", action="store_true", help="Print commands without executing" + ) + args = parser.parse_args() + cleanup(dry_run=args.dry_run) From d9a27fe832f3c742fc2e4b6c4938f8c80562d98c Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Tue, 16 Jun 2026 10:24:46 +0200 Subject: [PATCH 29/57] fix: detect postgres-operator via apply instead of CRD get The previous approach checked 'kubectl get crd postgresclusters...' which requires cluster-level RBAC that notebook SAs typically don't have, causing the check to fail and the example to silently skip regardless of whether the operator is installed. Instead, attempt to apply postgres-cluster.yaml and treat a 'no matches for kind' / 'server doesn't have a resource type' error as a graceful skip. Any other error (permissions, connectivity) is raised so CI marks it as FAIL rather than a false PASS. Also strengthen the smoke test to verify the doubler's actual output (FACTOR=2: [1,2,3] must produce [2,4,6]) rather than just checking that a predictions key exists. --- .../apply.py | 65 ++++++++++++++----- 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/serving/minimal-example-shadow-deployment/apply.py b/serving/minimal-example-shadow-deployment/apply.py index 272b932..d7089ea 100644 --- a/serving/minimal-example-shadow-deployment/apply.py +++ b/serving/minimal-example-shadow-deployment/apply.py @@ -86,13 +86,30 @@ def _kubectl_apply(manifest: str, namespace: str) -> None: print(result.stdout.strip()) -def _crd_available(crd_name: str) -> bool: +def _try_apply(manifest: str, namespace: str) -> str | None: + """Apply a manifest; return the error string if the resource type is unknown. + + Returns None on success, or a non-empty string if kubectl rejected the + manifest because the CRD does not exist in this cluster. Any other error + (permissions, connectivity, …) is raised as RuntimeError so CI marks the + example as FAIL rather than silently skipping it. + """ result = subprocess.run( - ["kubectl", "get", "crd", crd_name], - capture_output=True, + ["kubectl", "apply", "-f", "-", "-n", namespace], + input=manifest, text=True, + capture_output=True, ) - return result.returncode == 0 + if result.returncode == 0: + print(result.stdout.strip()) + return None + stderr = result.stderr or result.stdout + if ( + "no matches for kind" in stderr + or "the server doesn't have a resource type" in stderr + ): + return stderr.strip() + raise RuntimeError(stderr) def _wait_for_secret(name: str, namespace: str, timeout: int = 300) -> None: @@ -224,11 +241,17 @@ def _wait_isvc_ready(name: str, namespace: str, timeout: int) -> None: def _smoke_test(namespace: str, timeout: int = 120) -> None: - """POST numeric values to the primary (doubler) ISVC and verify predictions.""" + """POST numeric values to the primary (doubler) ISVC and verify predictions. + + The doubler predictor multiplies each input value by FACTOR=2, so + [1.0, 2.0, 3.0] must produce predictions [2.0, 4.0, 6.0]. + """ url = ( f"http://{_DOUBLER_ISVC}.{namespace}.svc.cluster.local/v1/models/model:predict" ) - payload = json.dumps({"values": [1.0, 2.0, 3.0]}).encode() + inputs = [1.0, 2.0, 3.0] + expected = [2.0, 4.0, 6.0] + payload = json.dumps({"values": inputs}).encode() deadline = time.time() + timeout last_err: Exception | None = None while time.time() < deadline: @@ -241,9 +264,15 @@ def _smoke_test(namespace: str, timeout: int = 120) -> None: ) with urllib.request.urlopen(req, timeout=15) as resp: body = json.loads(resp.read()) - if "predictions" not in body: - raise RuntimeError(f"unexpected response body: {body}") - print(f"Smoke test passed: {body}") + predictions = body.get("predictions") + if predictions is None: + raise RuntimeError(f"no 'predictions' key in response: {body}") + if predictions != expected: + raise RuntimeError( + f"doubler (FACTOR=2) returned wrong predictions: " + f"got {predictions}, expected {expected}" + ) + print(f"Smoke test passed: {inputs} → {predictions} (FACTOR=2 verified)") return except Exception as exc: last_err = exc @@ -255,19 +284,19 @@ def _smoke_test(namespace: str, timeout: int = 120) -> None: def deploy(timeout: int = 600) -> None: - if not _crd_available("postgresclusters.postgres-operator.crunchydata.com"): - print( - "SKIP: CrunchyData postgres-operator CRD not found in this cluster.\n" - "The shadow deployment example requires the postgres-operator to be installed." - ) - sys.exit(0) - ns = _namespace() - # 1. Postgres cluster + # 1. Postgres cluster — detect operator availability via the apply itself with open(os.path.join(_ROOT, "postgres-cluster.yaml")) as fh: pg_manifest = fh.read() - _kubectl_apply(pg_manifest, ns) + skip_reason = _try_apply(pg_manifest, ns) + if skip_reason: + print( + f"SKIP: postgres-operator is not installed in this cluster " + f"(kubectl apply returned: {skip_reason}).\n" + "The shadow deployment example requires the CrunchyData postgres-operator." + ) + sys.exit(0) print(f"Applied PostgresCluster '{_PG_CLUSTER_NAME}' in namespace '{ns}'.") _wait_pg_primary_ready(ns, timeout=300) From 6fcfac666aa4bd3d9a4e84f3c743c2b5c5033662 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Tue, 16 Jun 2026 10:25:14 +0200 Subject: [PATCH 30/57] fix: detect KEDA via ScaledObject apply instead of CRD get Same RBAC issue as the shadow-deployment fix: notebook SAs lack cluster-level permission to 'kubectl get crd', so the KEDA check always returned False and the example silently skipped. Deploy the ISVC first (always), then attempt to apply the ScaledObject and treat 'no matches for kind' as a graceful skip. This also gives more useful CI signal: the ISVC smoke-test still runs even when KEDA is absent. --- serving/kserve-keda-autoscaling/apply.py | 44 ++++++++++++++++-------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/serving/kserve-keda-autoscaling/apply.py b/serving/kserve-keda-autoscaling/apply.py index 8cca44c..ee0518a 100644 --- a/serving/kserve-keda-autoscaling/apply.py +++ b/serving/kserve-keda-autoscaling/apply.py @@ -49,14 +49,29 @@ def _kubectl_apply(manifest: str, namespace: str) -> None: print(result.stdout.strip()) -def _keda_available() -> bool: - """Return True if KEDA CRDs are installed in this cluster.""" +def _try_apply_scaledobject(manifest: str, namespace: str) -> str | None: + """Apply the ScaledObject manifest; return an error string if KEDA is absent. + + Returns None on success, or a non-empty string if kubectl rejected the + manifest because the ScaledObject CRD is not installed. Any other error + is raised so CI marks the example as FAIL instead of silently skipping. + """ result = subprocess.run( - ["kubectl", "get", "crd", "scaledobjects.keda.sh"], - capture_output=True, + ["kubectl", "apply", "-f", "-", "-n", namespace], + input=manifest, text=True, + capture_output=True, ) - return result.returncode == 0 + if result.returncode == 0: + print(result.stdout.strip()) + return None + stderr = result.stderr or result.stdout + if ( + "no matches for kind" in stderr + or "the server doesn't have a resource type" in stderr + ): + return stderr.strip() + raise RuntimeError(stderr) def _wait_isvc_ready(name: str, namespace: str, timeout: int) -> None: @@ -154,14 +169,6 @@ def _smoke_test(namespace: str, timeout: int = 60) -> None: def deploy(timeout: int = 900) -> None: - if not _keda_available(): - print( - "SKIP: KEDA CRDs (scaledobjects.keda.sh) not found in this cluster.\n" - "Ask your cluster admin to install KEDA, or pass --include-keda only\n" - "when targeting a cluster with KEDA installed." - ) - sys.exit(0) - ns = _namespace() with open(_ISVC_YAML) as fh: @@ -172,9 +179,18 @@ def deploy(timeout: int = 900) -> None: _wait_isvc_ready(_ISVC_NAME, ns, timeout) print(f"InferenceService '{_ISVC_NAME}' is ready.") + # Apply ScaledObject — detect KEDA availability via the apply itself rather + # than 'kubectl get crd' which requires cluster-level RBAC notebook SAs lack. with open(_SO_YAML) as fh: so_manifest = fh.read() - _kubectl_apply(so_manifest, ns) + skip_reason = _try_apply_scaledobject(so_manifest, ns) + if skip_reason: + print( + f"SKIP: KEDA is not installed in this cluster " + f"(kubectl apply returned: {skip_reason}).\n" + "The ScaledObject was not created; the ISVC is running without autoscaling." + ) + sys.exit(0) print(f"Applied ScaledObject '{_SCALED_OBJECT_NAME}' in namespace '{ns}'.") _wait_scaledobject_active(_SCALED_OBJECT_NAME, ns) From e7665fec17bcb0e90ac2316b7f12089329fac00b Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Tue, 16 Jun 2026 12:34:53 +0200 Subject: [PATCH 31/57] fix: surface actionable error messages for postgres-operator failures Replace the raw kubectl stderr blob with targeted messages: - 'no matches for kind': operator not installed - Forbidden on postgres-operator resources: SA lacks RBAC, with the exact grant needed (get/create/update/patch on postgresclusters) The example still FAILs in CI (correct) but the error line now tells the operator admin exactly what to fix. --- .../apply.py | 47 +++++++------------ 1 file changed, 16 insertions(+), 31 deletions(-) diff --git a/serving/minimal-example-shadow-deployment/apply.py b/serving/minimal-example-shadow-deployment/apply.py index d7089ea..ac70806 100644 --- a/serving/minimal-example-shadow-deployment/apply.py +++ b/serving/minimal-example-shadow-deployment/apply.py @@ -32,7 +32,6 @@ import json import os import subprocess -import sys import time import urllib.error import urllib.request @@ -75,25 +74,6 @@ def _namespace() -> str: def _kubectl_apply(manifest: str, namespace: str) -> None: - result = subprocess.run( - ["kubectl", "apply", "-f", "-", "-n", namespace], - input=manifest, - text=True, - capture_output=True, - ) - if result.returncode != 0: - raise RuntimeError(result.stderr or result.stdout) - print(result.stdout.strip()) - - -def _try_apply(manifest: str, namespace: str) -> str | None: - """Apply a manifest; return the error string if the resource type is unknown. - - Returns None on success, or a non-empty string if kubectl rejected the - manifest because the CRD does not exist in this cluster. Any other error - (permissions, connectivity, …) is raised as RuntimeError so CI marks the - example as FAIL rather than silently skipping it. - """ result = subprocess.run( ["kubectl", "apply", "-f", "-", "-n", namespace], input=manifest, @@ -102,13 +82,24 @@ def _try_apply(manifest: str, namespace: str) -> str | None: ) if result.returncode == 0: print(result.stdout.strip()) - return None + return stderr = result.stderr or result.stdout + # Translate common failure modes into actionable messages if ( "no matches for kind" in stderr or "the server doesn't have a resource type" in stderr ): - return stderr.strip() + raise RuntimeError( + "postgres-operator is not installed in this cluster. " + "The shadow deployment example requires the CrunchyData postgres-operator." + ) + if "Forbidden" in stderr and "postgres-operator.crunchydata.com" in stderr: + raise RuntimeError( + "The notebook ServiceAccount lacks RBAC permission to manage PostgresCluster " + "resources.\n" + "Ask your cluster admin to grant get/create/update/patch on " + "postgresclusters.postgres-operator.crunchydata.com in this namespace." + ) raise RuntimeError(stderr) @@ -286,17 +277,11 @@ def _smoke_test(namespace: str, timeout: int = 120) -> None: def deploy(timeout: int = 600) -> None: ns = _namespace() - # 1. Postgres cluster — detect operator availability via the apply itself + # 1. Postgres cluster — _kubectl_apply raises with an actionable message on + # "no matches for kind" (operator not installed) or Forbidden (RBAC missing). with open(os.path.join(_ROOT, "postgres-cluster.yaml")) as fh: pg_manifest = fh.read() - skip_reason = _try_apply(pg_manifest, ns) - if skip_reason: - print( - f"SKIP: postgres-operator is not installed in this cluster " - f"(kubectl apply returned: {skip_reason}).\n" - "The shadow deployment example requires the CrunchyData postgres-operator." - ) - sys.exit(0) + _kubectl_apply(pg_manifest, ns) print(f"Applied PostgresCluster '{_PG_CLUSTER_NAME}' in namespace '{ns}'.") _wait_pg_primary_ready(ns, timeout=300) From dcdfb3ee52abddd845ecc26dc5c7d8409fb29819 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Tue, 16 Jun 2026 14:03:20 +0200 Subject: [PATCH 32/57] fix: suppress traceback in apply.py scripts, print clean error message Unhandled exceptions print a full Python traceback to stderr which the CI captures verbatim. Catch at __main__ level and print just str(exc) so CI shows the actionable message without the stack noise. --- serving/hf-vllm-completion/apply.py | 6 +++++- serving/kserve-keda-autoscaling/apply.py | 6 +++++- serving/minimal-example-shadow-deployment/apply.py | 7 ++++++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/serving/hf-vllm-completion/apply.py b/serving/hf-vllm-completion/apply.py index 19b7c7c..8b13032 100644 --- a/serving/hf-vllm-completion/apply.py +++ b/serving/hf-vllm-completion/apply.py @@ -123,4 +123,8 @@ def deploy(timeout: int = 900) -> None: if __name__ == "__main__": - deploy() + try: + deploy() + except Exception as exc: + print(str(exc), file=sys.stderr) + sys.exit(1) diff --git a/serving/kserve-keda-autoscaling/apply.py b/serving/kserve-keda-autoscaling/apply.py index ee0518a..5f928f0 100644 --- a/serving/kserve-keda-autoscaling/apply.py +++ b/serving/kserve-keda-autoscaling/apply.py @@ -198,4 +198,8 @@ def deploy(timeout: int = 900) -> None: if __name__ == "__main__": - deploy() + try: + deploy() + except Exception as exc: + print(str(exc), file=sys.stderr) + sys.exit(1) diff --git a/serving/minimal-example-shadow-deployment/apply.py b/serving/minimal-example-shadow-deployment/apply.py index ac70806..1bbe915 100644 --- a/serving/minimal-example-shadow-deployment/apply.py +++ b/serving/minimal-example-shadow-deployment/apply.py @@ -32,6 +32,7 @@ import json import os import subprocess +import sys import time import urllib.error import urllib.request @@ -314,4 +315,8 @@ def deploy(timeout: int = 600) -> None: if __name__ == "__main__": - deploy() + try: + deploy() + except Exception as exc: + print(str(exc), file=sys.stderr) + sys.exit(1) From 69ef93b80eb72c5cb1ab21660ec788c4b9cbbd6a Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Tue, 16 Jun 2026 14:10:02 +0200 Subject: [PATCH 33/57] refactor: break run_all() into phase helpers and add _timed_run wrapper --- ci/run_all.py | 767 +++++++++++++++++++++++++++----------------------- 1 file changed, 413 insertions(+), 354 deletions(-) diff --git a/ci/run_all.py b/ci/run_all.py index 204a141..35cd151 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -53,6 +53,7 @@ from concurrent.futures import Future, ThreadPoolExecutor, as_completed from dataclasses import dataclass, field from pathlib import Path +from typing import Callable # ── Repo root ───────────────────────────────────────────────────────────────── @@ -320,15 +321,17 @@ def _run_cleanup(cleanup_path: Path) -> None: # ── Executor ────────────────────────────────────────────────────────────────── -def _submit_notebook( +def _timed_run( executor: ThreadPoolExecutor, results: dict[str, Result], name: str, - nb_path: Path, - output_dir: Path, - timeout: int, - extract_run_ids: bool = True, + work: Callable[[Result], None], ) -> Future: + """Submit `work(result)` to the executor with shared timing + error handling. + + `work` mutates the Result (e.g. sets kfp_run_ids); status/duration are set + here. The first exception is recorded as FAIL. + """ result = Result(name=name) results[name] = result print(f" [START ] {name}") @@ -336,10 +339,8 @@ def _submit_notebook( def _run(): t0 = time.time() try: - out = _run_notebook(nb_path, output_dir, timeout) + work(result) result.status = "PASS" - if extract_run_ids: - result.kfp_run_ids = _extract_run_ids_from_notebook(out) except Exception as exc: # noqa: BLE001 result.status = "FAIL" result.error = str(exc) @@ -349,6 +350,23 @@ def _run(): return executor.submit(_run) +def _submit_notebook( + executor: ThreadPoolExecutor, + results: dict[str, Result], + name: str, + nb_path: Path, + output_dir: Path, + timeout: int, + extract_run_ids: bool = True, +) -> Future: + def work(result: Result) -> None: + out = _run_notebook(nb_path, output_dir, timeout) + if extract_run_ids: + result.kfp_run_ids = _extract_run_ids_from_notebook(out) + + return _timed_run(executor, results, name, work) + + def _submit_script( executor: ThreadPoolExecutor, results: dict[str, Result], @@ -357,23 +375,11 @@ def _submit_script( timeout: int, extra_args: list[str] | None = None, ) -> Future: - result = Result(name=name) - results[name] = result - print(f" [START ] {name}") + def work(result: Result) -> None: + stdout, _ = _run_script(script_path, timeout, extra_args=extra_args) + result.kfp_run_ids = _extract_run_ids_from_stdout(stdout) - def _run(): - t0 = time.time() - try: - stdout, _ = _run_script(script_path, timeout, extra_args=extra_args) - result.status = "PASS" - result.kfp_run_ids = _extract_run_ids_from_stdout(stdout) - except Exception as exc: # noqa: BLE001 - result.status = "FAIL" - result.error = str(exc) - finally: - result.duration = time.time() - t0 - - return executor.submit(_run) + return _timed_run(executor, results, name, work) def _submit_chain( @@ -389,38 +395,27 @@ def _submit_chain( kind is 'notebook' or 'script'. kwargs may include 'extra_args' for scripts. All steps run in one thread; the first failure stops the chain. """ - result = Result(name=name) - results[name] = result - print(f" [START ] {name}") - def _run(): - t0 = time.time() - try: - for kind, path, kwargs in steps: - if kind == "notebook": - out = _run_notebook(path, output_dir, timeout) - result.kfp_run_ids.extend(_extract_run_ids_from_notebook(out)) - elif kind == "script": - stdout, _ = _run_script( - path, timeout, extra_args=kwargs.get("extra_args") - ) - result.kfp_run_ids.extend(_extract_run_ids_from_stdout(stdout)) - elif kind == "pip": - r = subprocess.run( - [sys.executable, "-m", "pip", "install", "-q"] - + kwargs.get("packages", []), - capture_output=True, - text=True, - check=True, - ) - result.status = "PASS" - except Exception as exc: # noqa: BLE001 - result.status = "FAIL" - result.error = str(exc) - finally: - result.duration = time.time() - t0 + def work(result: Result) -> None: + for kind, path, kwargs in steps: + if kind == "notebook": + out = _run_notebook(path, output_dir, timeout) + result.kfp_run_ids.extend(_extract_run_ids_from_notebook(out)) + elif kind == "script": + stdout, _ = _run_script( + path, timeout, extra_args=kwargs.get("extra_args") + ) + result.kfp_run_ids.extend(_extract_run_ids_from_stdout(stdout)) + elif kind == "pip": + subprocess.run( + [sys.executable, "-m", "pip", "install", "-q"] + + kwargs.get("packages", []), + capture_output=True, + text=True, + check=True, + ) - return executor.submit(_run) + return _timed_run(executor, results, name, work) # ── Report ──────────────────────────────────────────────────────────────────── @@ -558,26 +553,67 @@ def _check_mlflow_credentials() -> tuple[bool, str]: return False, f"Could not reach MLflow at {uri}: {exc}" -# ── Main ────────────────────────────────────────────────────────────────────── +# ── Orchestration context ───────────────────────────────────────────────────── -def run_all( - timeout_notebook: int = 1800, - timeout_pipeline: int = 3600, - include_keda: bool = False, - include_pytorch: bool = False, - include_shadow: bool = False, - dry_run: bool = False, -) -> dict[str, Result]: - root = _REPO_ROOT - output_dir = root / "ci" / "output" - results: dict[str, Result] = {} - poll_results: dict[str, str] = {} - poll_errors: dict[str, str] = {} +@dataclass +class _Context: + """Shared mutable state threaded through every phase.""" + + executor: ThreadPoolExecutor + root: Path + output_dir: Path + timeout_notebook: int + timeout_pipeline: int + results: dict[str, Result] + poll_results: dict[str, str] + poll_errors: dict[str, str] + + +_CLEANUP_RELPATHS = [ + "notebooks/dask/cleanup.py", + "serving/minimal-s3-model/cleanup.py", + "serving/mlflow-kserve-minimal/cleanup.py", + "serving/mlflow-kserve-inference-protocols/cleanup.py", + "serving/hf-vllm-completion/cleanup.py", + "serving/kserve-keda-autoscaling/cleanup.py", + "serving/minimal-example-shadow-deployment/cleanup.py", + "hparam-tuning/minimal-mnist/cleanup.py", +] + +_DRY_RUN_PHASES = [ + "Phase 1: dask, mlflow-quickstart, mlflow-image, mlflow-kfp, minimal-s3-model, hf-vllm-completion", + "Phase 1 (opt-in): kserve-keda-autoscaling (--include-keda), minimal-example-shadow-deployment (--include-shadow)", + "Phase 2: mlflow-mobile-price, lightweight-components, lightweight-python-package", + "Phase 3: mlflow-kserve-minimal, mlflow-kserve-inference-protocols", + "Phase 4: KFP run polling", + "Phase 5: cleanup", +] + + +def _drain(ctx: _Context, futures: dict[str, Future]) -> None: + """Wait on a {name: Future} map, printing each result as it completes.""" + by_future = {v: k for k, v in futures.items()} + for f in as_completed(futures.values()): + _print_result(ctx.results[by_future[f]]) + + +def _skip(ctx: _Context, name: str, reason: str, label: str | None = None) -> None: + """Record a SKIP result and print a one-line notice.""" + print(f" [SKIP ] {label or name}") + ctx.results[name] = Result(name=name, status="SKIP", error=reason) + + +# ── Pre-flight ──────────────────────────────────────────────────────────────── - _ensure_papermill() - # Pre-flight: validate MLflow credentials; skip dependent tests if unavailable +def _preflight(results: dict[str, Result]) -> tuple[bool, str]: + """Ensure papermill, then validate MLflow credentials. + + Returns (mlflow_ok, reason). On failure, MLflow-dependent examples are + recorded as SKIP in `results`. + """ + _ensure_papermill() print("Pre-flight: checking MLflow credentials...") mlflow_ok, mlflow_reason = _check_mlflow_credentials() if mlflow_ok: @@ -586,311 +622,334 @@ def run_all( print(f" [SKIP] {mlflow_reason}") for name in _MLFLOW_DEPENDENT: results[name] = Result(name=name, status="SKIP", error=mlflow_reason) + return mlflow_ok, mlflow_reason - if dry_run: - print("[dry-run] Would execute the following phases:") - for label in [ - "Phase 1: dask, mlflow-quickstart, mlflow-image, mlflow-kfp, minimal-s3-model, hf-vllm-completion", - "Phase 1 (opt-in): kserve-keda-autoscaling (--include-keda), minimal-example-shadow-deployment (--include-shadow)", - "Phase 2: mlflow-mobile-price, lightweight-components, lightweight-python-package", - "Phase 3: mlflow-kserve-minimal, mlflow-kserve-inference-protocols", - "Phase 4: KFP run polling", - "Phase 5: cleanup", - ]: - print(f" {label}") - return results - - cleanup_scripts = [ - root / "notebooks" / "dask" / "cleanup.py", - root / "serving" / "minimal-s3-model" / "cleanup.py", - root / "serving" / "mlflow-kserve-minimal" / "cleanup.py", - root / "serving" / "mlflow-kserve-inference-protocols" / "cleanup.py", - root / "serving" / "hf-vllm-completion" / "cleanup.py", - root / "serving" / "kserve-keda-autoscaling" / "cleanup.py", - root / "serving" / "minimal-example-shadow-deployment" / "cleanup.py", - root / "hparam-tuning" / "minimal-mnist" / "cleanup.py", - ] - try: - with ThreadPoolExecutor(max_workers=8) as executor: - # ── Phase 1: independent notebooks ─────────────────────────────── - print("Phase 1: running independent notebooks in parallel...") - phase1_futures = {} - for name, rel in [ - ("notebooks/dask", "notebooks/dask/dask_example.ipynb"), - ( - "notebooks/mobile-price-classification", - "notebooks/mobile-price-classification/mobile-price-classifications.ipynb", - ), - ("mlflow/mlflow-quickstart", "mlflow/mlflow-quickstart-example.ipynb"), - ("mlflow/mlflow-image-example", "mlflow/mlflow-image-example.ipynb"), - ("mlflow/mlflow-kfp-example", "mlflow/mlflow-kfp-example.ipynb"), - ( - "serving/minimal-s3-model", - "serving/minimal-s3-model/minimal-s3-model.ipynb", - ), - ]: - if name in results: # already SKIP from pre-flight - print(f" [SKIP ] {name}") - continue - f = _submit_notebook( - executor, results, name, root / rel, output_dir, timeout_notebook - ) - phase1_futures[name] = f - - # hf-vllm-completion (CPU): self-contained, no KFP pipelines - phase1_futures["serving/hf-vllm-completion"] = _submit_script( - executor, - results, - "serving/hf-vllm-completion", - root / "serving" / "hf-vllm-completion" / "apply.py", - timeout_notebook, - ) +# ── Phases ────────────────────────────────────────────────────────────────── - # kserve-keda-autoscaling: opt-in; apply.py self-checks KEDA CRDs - # and exits 0 with a skip message if KEDA is not installed. - if include_keda: - phase1_futures["serving/kserve-keda-autoscaling"] = _submit_script( - executor, - results, - "serving/kserve-keda-autoscaling", - root / "serving" / "kserve-keda-autoscaling" / "apply.py", - timeout_notebook, - ) - else: - print( - " [SKIP ] serving/kserve-keda-autoscaling (pass --include-keda to enable)" - ) - results["serving/kserve-keda-autoscaling"] = Result( - name="serving/kserve-keda-autoscaling", - status="SKIP", - error="opt-in: pass --include-keda to run this example", - ) - # minimal-example-shadow-deployment: opt-in; apply.py self-checks for - # the postgres-operator CRD and exits 0 if not found. Istio - # VirtualService mirroring is not verified in CI (requires domain config). - if include_shadow: - phase1_futures["serving/minimal-example-shadow-deployment"] = ( - _submit_script( - executor, - results, - "serving/minimal-example-shadow-deployment", - root - / "serving" - / "minimal-example-shadow-deployment" - / "apply.py", - timeout_notebook, - ) - ) - else: - print( - " [SKIP ] serving/minimal-example-shadow-deployment (pass --include-shadow to enable)" - ) - results["serving/minimal-example-shadow-deployment"] = Result( - name="serving/minimal-example-shadow-deployment", - status="SKIP", - error="opt-in: pass --include-shadow to run this example", - ) +def _phase1( + ctx: _Context, + include_keda: bool, + include_shadow: bool, + include_pytorch: bool, +) -> None: + """Phase 1: independent notebooks + self-contained serving examples.""" + print("Phase 1: running independent notebooks in parallel...") + futures: dict[str, Future] = {} + for name, rel in [ + ("notebooks/dask", "notebooks/dask/dask_example.ipynb"), + ( + "notebooks/mobile-price-classification", + "notebooks/mobile-price-classification/mobile-price-classifications.ipynb", + ), + ("mlflow/mlflow-quickstart", "mlflow/mlflow-quickstart-example.ipynb"), + ("mlflow/mlflow-image-example", "mlflow/mlflow-image-example.ipynb"), + ("mlflow/mlflow-kfp-example", "mlflow/mlflow-kfp-example.ipynb"), + ("serving/minimal-s3-model", "serving/minimal-s3-model/minimal-s3-model.ipynb"), + ]: + if name in ctx.results: # already SKIP from pre-flight + print(f" [SKIP ] {name}") + continue + futures[name] = _submit_notebook( + ctx.executor, + ctx.results, + name, + ctx.root / rel, + ctx.output_dir, + ctx.timeout_notebook, + ) - # mnist-vae: opt-in (requires pytorch_lightning, not in all images) - if include_pytorch: - phase1_futures["notebooks/mnist-vae"] = _submit_chain( - executor, - results, - "notebooks/mnist-vae", - steps=[ - # run_training.py self-installs pytorch-lightning if absent - ( - "script", - root / "notebooks/mnist-vae/run_training.py", - {"extra_args": ["--max_epochs", "3"]}, - ), - ( - "notebook", - root / "notebooks/mnist-vae/visualizations.ipynb", - {}, - ), - ], - output_dir=output_dir, - timeout=timeout_notebook, - ) - else: - print( - " [SKIP ] notebooks/mnist-vae (pass --include-pytorch to enable)" - ) + # hf-vllm-completion (CPU): self-contained, no KFP pipelines + futures["serving/hf-vllm-completion"] = _submit_script( + ctx.executor, + ctx.results, + "serving/hf-vllm-completion", + ctx.root / "serving" / "hf-vllm-completion" / "apply.py", + ctx.timeout_notebook, + ) - # wait for all phase 1, printing results as each finishes - _future_to_name = {v: k for k, v in phase1_futures.items()} - for f in as_completed(phase1_futures.values()): - _print_result(results[_future_to_name[f]]) + # kserve-keda-autoscaling: opt-in; apply.py self-checks KEDA CRDs + # and exits 0 with a skip message if KEDA is not installed. + if include_keda: + futures["serving/kserve-keda-autoscaling"] = _submit_script( + ctx.executor, + ctx.results, + "serving/kserve-keda-autoscaling", + ctx.root / "serving" / "kserve-keda-autoscaling" / "apply.py", + ctx.timeout_notebook, + ) + else: + _skip( + ctx, + "serving/kserve-keda-autoscaling", + "opt-in: pass --include-keda to run this example", + label="serving/kserve-keda-autoscaling (pass --include-keda to enable)", + ) - # ── Phase 2: pipeline submissions (return fast) ─────────────────── - print("\nPhase 2: submitting pipelines...") - phase2_futures = {} - for name, rel in [ - ( - "mlflow/mobile-price-classification", - "mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb", - ), + # minimal-example-shadow-deployment: opt-in; apply.py self-checks for + # the postgres-operator CRD and exits 0 if not found. Istio + # VirtualService mirroring is not verified in CI (requires domain config). + if include_shadow: + futures["serving/minimal-example-shadow-deployment"] = _submit_script( + ctx.executor, + ctx.results, + "serving/minimal-example-shadow-deployment", + ctx.root / "serving" / "minimal-example-shadow-deployment" / "apply.py", + ctx.timeout_notebook, + ) + else: + _skip( + ctx, + "serving/minimal-example-shadow-deployment", + "opt-in: pass --include-shadow to run this example", + label="serving/minimal-example-shadow-deployment (pass --include-shadow to enable)", + ) + + # mnist-vae: opt-in (requires pytorch_lightning, not in all images) + if include_pytorch: + futures["notebooks/mnist-vae"] = _submit_chain( + ctx.executor, + ctx.results, + "notebooks/mnist-vae", + steps=[ + # run_training.py self-installs pytorch-lightning if absent ( - "pipelines/lightweight-components", - "pipelines/lightweight-components/mobile-price-classifications.ipynb", + "script", + ctx.root / "notebooks/mnist-vae/run_training.py", + {"extra_args": ["--max_epochs", "3"]}, ), - ]: - if name in results: # already SKIP from pre-flight - print(f" [SKIP ] {name}") - continue - f = _submit_notebook( - executor, results, name, root / rel, output_dir, timeout_notebook - ) - phase2_futures[name] = f - - lpp_future = _submit_script( - executor, - results, - "pipelines/lightweight-python-package", - root / "pipelines" / "lightweight-python-package" / "submit-cluster.py", - timeout_notebook, - ) - phase2_futures["pipelines/lightweight-python-package"] = lpp_future - - mcc_future = _submit_script( - executor, - results, - "pipelines/minimal-container-components", - root - / "pipelines" - / "minimal-container-components" - / "submit-cluster.py", - timeout_notebook, - ) - phase2_futures["pipelines/minimal-container-components"] = mcc_future + ("notebook", ctx.root / "notebooks/mnist-vae/visualizations.ipynb", {}), + ], + output_dir=ctx.output_dir, + timeout=ctx.timeout_notebook, + ) + else: + print(" [SKIP ] notebooks/mnist-vae (pass --include-pytorch to enable)") + + _drain(ctx, futures) - # Wait for mlflow-mobile-price notebook; then poll its KFP run - # inline before Phase 3 — the ISVC needs the registered model. - _mobile_price_skipped = ( - "mlflow/mobile-price-classification" not in phase2_futures - ) - if not _mobile_price_skipped: - phase2_futures["mlflow/mobile-price-classification"].result() - _print_result(results["mlflow/mobile-price-classification"]) - _mobile_price_ok = ( - not _mobile_price_skipped - and results["mlflow/mobile-price-classification"].status == "PASS" +def _phase2_submit(ctx: _Context) -> dict[str, Future]: + """Phase 2: submit pipeline notebooks + scripts (they return fast).""" + print("\nPhase 2: submitting pipelines...") + futures: dict[str, Future] = {} + for name, rel in [ + ( + "mlflow/mobile-price-classification", + "mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb", + ), + ( + "pipelines/lightweight-components", + "pipelines/lightweight-components/mobile-price-classifications.ipynb", + ), + ]: + if name in ctx.results: # already SKIP from pre-flight + print(f" [SKIP ] {name}") + continue + futures[name] = _submit_notebook( + ctx.executor, + ctx.results, + name, + ctx.root / rel, + ctx.output_dir, + ctx.timeout_notebook, + ) + + futures["pipelines/lightweight-python-package"] = _submit_script( + ctx.executor, + ctx.results, + "pipelines/lightweight-python-package", + ctx.root / "pipelines" / "lightweight-python-package" / "submit-cluster.py", + ctx.timeout_notebook, + ) + futures["pipelines/minimal-container-components"] = _submit_script( + ctx.executor, + ctx.results, + "pipelines/minimal-container-components", + ctx.root / "pipelines" / "minimal-container-components" / "submit-cluster.py", + ctx.timeout_notebook, + ) + return futures + + +def _await_mobile_price(ctx: _Context, phase2_futures: dict[str, Future]) -> bool: + """Wait for the mlflow-mobile-price notebook and poll its KFP run inline. + + Phase 3 ISVCs need the registered model, so this must complete before + Phase 3 starts. Returns True if the model is registered and ready. + """ + name = "mlflow/mobile-price-classification" + skipped = name not in phase2_futures + if skipped: + return False + + phase2_futures[name].result() + _print_result(ctx.results[name]) + if ctx.results[name].status != "PASS": + return False + + run_ids = ctx.results[name].kfp_run_ids + if not run_ids: + return True + + print( + " Polling mlflow-mobile-price KFP pipeline (model must be registered before ISVCs)..." + ) + ok = True + for run_id in run_ids: + state, err = _poll_kfp_run(run_id, ctx.timeout_pipeline) + ctx.poll_results[run_id] = state # recorded here; Phase 4 will skip it + ctx.poll_errors[run_id] = err + print(f" [{state}] {run_id[:8]}...") + if state.upper() != "SUCCEEDED": + ok = False + return ok + + +def _phase3_submit( + ctx: _Context, mobile_price_ok: bool, mobile_price_skipped: bool +) -> dict[str, Future]: + """Phase 3: deploy MLflow KServe InferenceServices (need registered model).""" + print("\nPhase 3: deploying MLflow KServe InferenceServices...") + isvc_names = ( + "serving/mlflow-kserve-minimal", + "serving/mlflow-kserve-inference-protocols", + ) + if not mobile_price_ok: + reason = ( + "prerequisite mlflow/mobile-price-classification skipped (MLflow credentials)" + if mobile_price_skipped + else "prerequisite mlflow/mobile-price-classification notebook or KFP pipeline did not succeed" + ) + print(f" [SKIP] {reason}") + for name in isvc_names: + ctx.results[name] = Result(name=name, status="SKIP", error=reason) + return {} + + futures: dict[str, Future] = {} + # mlflow-kserve-minimal: deploy then immediately smoke-test + # deploy_and_test() in apply.py handles both steps + futures["serving/mlflow-kserve-minimal"] = _submit_script( + ctx.executor, + ctx.results, + "serving/mlflow-kserve-minimal", + ctx.root / "serving" / "mlflow-kserve-minimal" / "apply.py", + ctx.timeout_notebook, + ) + # inference-protocols notebook handles its own deploy + test. + # extract_run_ids=False: this notebook is not a pipeline submission; + # its output contains ISVC/request UUIDs that must not be polled + # as KFP run IDs (they return 404 from the KFP API). + futures["serving/mlflow-kserve-inference-protocols"] = _submit_notebook( + ctx.executor, + ctx.results, + "serving/mlflow-kserve-inference-protocols", + ctx.root + / "serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb", + ctx.output_dir, + ctx.timeout_notebook, + extract_run_ids=False, + ) + return futures + + +def _phase4_poll(ctx: _Context) -> None: + """Phase 4: poll all KFP run IDs not already polled inline before Phase 3.""" + all_run_ids = [ + rid + for r in ctx.results.values() + for rid in r.kfp_run_ids + if rid not in ctx.poll_results + ] + if not all_run_ids: + print("\nPhase 4: no KFP run IDs found, skipping poll.") + return + + print(f"\nPhase 4: polling {len(all_run_ids)} KFP run(s)...") + poll_futures = { + run_id: ctx.executor.submit(_poll_kfp_run, run_id, ctx.timeout_pipeline) + for run_id in all_run_ids + } + for run_id, f in poll_futures.items(): + try: + ctx.poll_results[run_id], ctx.poll_errors[run_id] = f.result() + except Exception as exc: # noqa: BLE001 + ctx.poll_results[run_id] = f"POLL_ERROR: {exc}" + ctx.poll_errors[run_id] = "" + print(f" [{ctx.poll_results[run_id]}] {run_id[:8]}...") + + +def _phase5_cleanup(cleanup_scripts: list[Path]) -> None: + """Phase 5: run all cleanup scripts in parallel (always, even on failure).""" + print("\nPhase 5: running cleanup scripts...") + with ThreadPoolExecutor(max_workers=4) as ex: + futs = [ex.submit(_run_cleanup, p) for p in cleanup_scripts if p.exists()] + for f in as_completed(futs): + f.result() # already swallows exceptions inside _run_cleanup + + +# ── Main ────────────────────────────────────────────────────────────────────── + + +def run_all( + timeout_notebook: int = 1800, + timeout_pipeline: int = 3600, + include_keda: bool = False, + include_pytorch: bool = False, + include_shadow: bool = False, + dry_run: bool = False, +) -> dict[str, Result]: + root = _REPO_ROOT + results: dict[str, Result] = {} + + mlflow_ok, _ = _preflight(results) + if not mlflow_ok and dry_run: + print("[dry-run] Would execute the following phases:") + for label in _DRY_RUN_PHASES: + print(f" {label}") + return results + + cleanup_scripts = [root / rel for rel in _CLEANUP_RELPATHS] + poll_results: dict[str, str] = {} + poll_errors: dict[str, str] = {} + + try: + with ThreadPoolExecutor(max_workers=8) as executor: + ctx = _Context( + executor=executor, + root=root, + output_dir=root / "ci" / "output", + timeout_notebook=timeout_notebook, + timeout_pipeline=timeout_pipeline, + results=results, + poll_results=poll_results, + poll_errors=poll_errors, ) - if _mobile_price_ok: - _mobile_run_ids = results[ - "mlflow/mobile-price-classification" - ].kfp_run_ids - if _mobile_run_ids: - print( - " Polling mlflow-mobile-price KFP pipeline (model must be registered before ISVCs)..." - ) - for _run_id in _mobile_run_ids: - _state, _err = _poll_kfp_run(_run_id, timeout_pipeline) - poll_results[_run_id] = ( - _state # recorded here; Phase 4 will skip it - ) - poll_errors[_run_id] = _err - print(f" [{_state}] {_run_id[:8]}...") - if _state.upper() != "SUCCEEDED": - _mobile_price_ok = False - - # ── Phase 3: MLflow KServe examples (need registered model) ─────── - print("\nPhase 3: deploying MLflow KServe InferenceServices...") - if not _mobile_price_ok: - _reason = ( - "prerequisite mlflow/mobile-price-classification skipped (MLflow credentials)" - if _mobile_price_skipped - else "prerequisite mlflow/mobile-price-classification notebook or KFP pipeline did not succeed" - ) - print(f" [SKIP] {_reason}") - for name in ( - "serving/mlflow-kserve-minimal", - "serving/mlflow-kserve-inference-protocols", - ): - results[name] = Result(name=name, status="SKIP", error=_reason) - phase3_futures = {} - else: - phase3_futures = {} - - # mlflow-kserve-minimal: deploy then immediately smoke-test - # deploy_and_test() in apply.py handles both steps - phase3_futures["serving/mlflow-kserve-minimal"] = _submit_script( - executor, - results, - "serving/mlflow-kserve-minimal", - root / "serving" / "mlflow-kserve-minimal" / "apply.py", - timeout_notebook, - ) + _phase1(ctx, include_keda, include_shadow, include_pytorch) + phase2_futures = _phase2_submit(ctx) - # inference-protocols notebook handles its own deploy + test. - # extract_run_ids=False: this notebook is not a pipeline submission; - # its output contains ISVC/request UUIDs that must not be polled - # as KFP run IDs (they return 404 from the KFP API). - phase3_futures["serving/mlflow-kserve-inference-protocols"] = ( - _submit_notebook( - executor, - results, - "serving/mlflow-kserve-inference-protocols", - root - / "serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb", - output_dir, - timeout_notebook, - extract_run_ids=False, - ) - ) + mobile_price_skipped = ( + "mlflow/mobile-price-classification" not in phase2_futures + ) + mobile_price_ok = _await_mobile_price(ctx, phase2_futures) + phase3_futures = _phase3_submit(ctx, mobile_price_ok, mobile_price_skipped) # Phase 2 remaining + phase 3 run concurrently; print as each finishes print("\nPhase 2 (remaining) + Phase 3 running concurrently...") remaining = { - **{ - k: v - for k, v in phase2_futures.items() - if k != "mlflow/mobile-price-classification" - }, - **phase3_futures, + k: v + for k, v in phase2_futures.items() + if k != "mlflow/mobile-price-classification" } - _future_to_name2 = {v: k for k, v in remaining.items()} - for f in as_completed(remaining.values()): - _print_result(results[_future_to_name2[f]]) - - # ── Phase 4: KFP run polling ────────────────────────────────────── - # Exclude run IDs already polled inline before Phase 3 - all_run_ids: list[str] = [ - rid - for r in results.values() - for rid in r.kfp_run_ids - if rid not in poll_results - ] - - if all_run_ids: - print(f"\nPhase 4: polling {len(all_run_ids)} KFP run(s)...") - poll_futures = { - run_id: executor.submit(_poll_kfp_run, run_id, timeout_pipeline) - for run_id in all_run_ids - } - for run_id, f in poll_futures.items(): - try: - poll_results[run_id], poll_errors[run_id] = f.result() - except Exception as exc: # noqa: BLE001 - poll_results[run_id] = f"POLL_ERROR: {exc}" - poll_errors[run_id] = "" - state = poll_results[run_id] - print(f" [{state}] {run_id[:8]}...") - else: - print("\nPhase 4: no KFP run IDs found, skipping poll.") + remaining.update(phase3_futures) + _drain(ctx, remaining) + + _phase4_poll(ctx) finally: - # ── Phase 5: cleanup ───────────────────────────────────────────────── - print("\nPhase 5: running cleanup scripts...") - with ThreadPoolExecutor(max_workers=4) as ex: - futs = [ex.submit(_run_cleanup, p) for p in cleanup_scripts if p.exists()] - for f in as_completed(futs): - f.result() # already swallows exceptions inside _run_cleanup + _phase5_cleanup(cleanup_scripts) # Build run_id → source notebook name map from all results run_id_to_name: dict[str, str] = {} From 91d33087f27558dd0bd7a100870215f1c47651b2 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Tue, 16 Jun 2026 14:16:20 +0200 Subject: [PATCH 34/57] refactor: declarative Example/Step registration table in CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace ad-hoc per-phase logic with a single _EXAMPLES list. Adding a new example is now one entry in the table — no phase function to find, no cleanup list to remember separately, opt-in flag and mlflow-dependency declared inline. Key changes: - Step / Example dataclasses describe what to run - _EXAMPLES table is the single source of truth for all examples, their phase, cleanup scripts, opt-in gates and mlflow dependency - _make_work() generates a work closure from a list of Steps, replacing _submit_notebook / _submit_script / _submit_chain - _run_phase() filters _EXAMPLES by phase and submits matching work - _preflight() and cleanup list are derived from _EXAMPLES - _print_dry_run() is also derived from _EXAMPLES (stays current) - Fix --dry-run bug: preflight no longer runs in dry-run mode --- ci/run_all.py | 760 ++++++++++++++++++++++++-------------------------- 1 file changed, 366 insertions(+), 394 deletions(-) diff --git a/ci/run_all.py b/ci/run_all.py index 35cd151..591b0e6 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -4,27 +4,25 @@ Execution model --------------- Phase 1 (parallel, self-contained): - notebooks/dask/dask_example.ipynb - mlflow/mlflow-quickstart-example.ipynb - mlflow/mlflow-image-example.ipynb - mlflow/mlflow-kfp-example.ipynb - serving/minimal-s3-model/minimal-s3-model.ipynb + All examples whose phase=1 in _EXAMPLES, including opt-in ones. Phase 2 (parallel, pipeline submissions — return fast): - mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb - pipelines/lightweight-components/mobile-price-classifications.ipynb - pipelines/lightweight-python-package/submit-cluster.py (Python script) + All examples whose phase=2 in _EXAMPLES. Phase 3 (depends on Phase 2 mlflow-mobile-price notebook completing): - serving/mlflow-kserve-minimal/apply.py + test_inference_service.py - serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb + All examples whose phase=3 in _EXAMPLES. Phase 4 (KFP run polling — runs alongside Phase 3): Poll all KFP run IDs extracted from Phase 2 output notebooks until Succeeded or timeout. Phase 5 — cleanup (always, in finally block): - All cleanup.py scripts in parallel. + All cleanup.py scripts derived from _EXAMPLES, in parallel. + +Adding a new example +-------------------- +Add one entry to _EXAMPLES below. Everything else (cleanup, phase +scheduling, opt-in gating, dry-run output) is derived from the table. Usage from a Jupyter notebook ------------------------------ @@ -34,7 +32,8 @@ CLI usage --------- python ci/run_all.py [--timeout-notebook 1800] [--timeout-pipeline 3600] - [--include-keda] [--dry-run] + [--include-keda] [--include-shadow] [--include-pytorch] + [--dry-run] Prerequisites ------------- @@ -53,7 +52,7 @@ from concurrent.futures import Future, ThreadPoolExecutor, as_completed from dataclasses import dataclass, field from pathlib import Path -from typing import Callable +from typing import Callable, Literal # ── Repo root ───────────────────────────────────────────────────────────────── @@ -94,6 +93,206 @@ class Result: kfp_run_ids: list[str] = field(default_factory=list) +# ── Example registry ────────────────────────────────────────────────────────── + + +@dataclass +class Step: + """One execution unit within an Example — a notebook or a Python script.""" + + kind: Literal["notebook", "script"] + path: str # relative to repo root + extra_args: list[str] = field(default_factory=list) + # True → extract KFP run IDs from output (notebooks: broad UUID regex; + # scripts: KFP_RUN_ID= lines). Set False for ISVC examples + # whose output contains UUIDs that are not KFP run IDs. + extract_run_ids: bool = True + + +@dataclass +class Example: + """One CI example. Add a new entry to _EXAMPLES to include it in CI.""" + + name: str + steps: list[Step] + phase: int # 1 = independent 2 = pipeline submit 3 = needs mlflow model + cleanup: str | None = None # relative path to cleanup.py, or None + opt_in: str | None = ( + None # argparse dest that gates this example, e.g. "include_keda" + ) + mlflow_dependent: bool = ( + False # skip automatically when MLflow credentials are unavailable + ) + + +# ───────────────────────────────────────────────────────────────────────────── +# Registration table — add new examples here. +# ───────────────────────────────────────────────────────────────────────────── +_EXAMPLES: list[Example] = [ + # ── Phase 1: independent, self-contained ───────────────────────────────── + Example( + name="notebooks/dask", + steps=[Step("notebook", "notebooks/dask/dask_example.ipynb")], + phase=1, + cleanup="notebooks/dask/cleanup.py", + ), + Example( + name="notebooks/mobile-price-classification", + steps=[ + Step( + "notebook", + "notebooks/mobile-price-classification/mobile-price-classifications.ipynb", + ) + ], + phase=1, + ), + Example( + name="mlflow/mlflow-quickstart", + steps=[Step("notebook", "mlflow/mlflow-quickstart-example.ipynb")], + phase=1, + mlflow_dependent=True, + ), + Example( + name="mlflow/mlflow-image-example", + steps=[Step("notebook", "mlflow/mlflow-image-example.ipynb")], + phase=1, + mlflow_dependent=True, + ), + Example( + name="mlflow/mlflow-kfp-example", + steps=[Step("notebook", "mlflow/mlflow-kfp-example.ipynb")], + phase=1, + mlflow_dependent=True, + ), + Example( + name="serving/minimal-s3-model", + steps=[Step("notebook", "serving/minimal-s3-model/minimal-s3-model.ipynb")], + phase=1, + cleanup="serving/minimal-s3-model/cleanup.py", + ), + Example( + name="serving/hf-vllm-completion", + steps=[ + Step("script", "serving/hf-vllm-completion/apply.py", extract_run_ids=False) + ], + phase=1, + cleanup="serving/hf-vllm-completion/cleanup.py", + ), + Example( + name="serving/kserve-keda-autoscaling", + steps=[ + Step( + "script", + "serving/kserve-keda-autoscaling/apply.py", + extract_run_ids=False, + ) + ], + phase=1, + opt_in="include_keda", + cleanup="serving/kserve-keda-autoscaling/cleanup.py", + ), + Example( + name="serving/minimal-example-shadow-deployment", + steps=[ + Step( + "script", + "serving/minimal-example-shadow-deployment/apply.py", + extract_run_ids=False, + ) + ], + phase=1, + opt_in="include_shadow", + cleanup="serving/minimal-example-shadow-deployment/cleanup.py", + ), + Example( + name="notebooks/mnist-vae", + steps=[ + Step( + "script", + "notebooks/mnist-vae/run_training.py", + extra_args=["--max_epochs", "3"], + extract_run_ids=False, + ), + Step( + "notebook", + "notebooks/mnist-vae/visualizations.ipynb", + extract_run_ids=False, + ), + ], + phase=1, + opt_in="include_pytorch", + ), + # ── Phase 2: pipeline submissions (return fast; KFP runs polled in Phase 4) + Example( + name="mlflow/mobile-price-classification", + steps=[ + Step( + "notebook", + "mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb", + ) + ], + phase=2, + mlflow_dependent=True, + ), + Example( + name="pipelines/lightweight-components", + steps=[ + Step( + "notebook", + "pipelines/lightweight-components/mobile-price-classifications.ipynb", + ) + ], + phase=2, + ), + Example( + name="pipelines/lightweight-python-package", + steps=[ + Step("script", "pipelines/lightweight-python-package/submit-cluster.py") + ], + phase=2, + ), + Example( + name="pipelines/minimal-container-components", + steps=[ + Step("script", "pipelines/minimal-container-components/submit-cluster.py") + ], + phase=2, + ), + # ── Phase 3: MLflow KServe ISVCs (need model registered by Phase 2) ────── + Example( + name="serving/mlflow-kserve-minimal", + steps=[ + Step( + "script", + "serving/mlflow-kserve-minimal/apply.py", + extract_run_ids=False, + ) + ], + phase=3, + mlflow_dependent=True, + cleanup="serving/mlflow-kserve-minimal/cleanup.py", + ), + Example( + name="serving/mlflow-kserve-inference-protocols", + steps=[ + Step( + "notebook", + "serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb", + extract_run_ids=False, + ) + ], + phase=3, + mlflow_dependent=True, + cleanup="serving/mlflow-kserve-inference-protocols/cleanup.py", + ), +] + +# Cleanup scripts for resources not covered by an Example entry above. +_EXTRA_CLEANUP_PATHS: list[str] = [ + "hparam-tuning/minimal-mnist/cleanup.py", +] + + # ── Helpers ─────────────────────────────────────────────────────────────────── @@ -115,7 +314,6 @@ def _format_papermill_error(exc: Exception) -> str: lines = [ f"Cell {exc.exec_count} raised {exc.ename}: {exc.evalue}", ] - # Show the failing cell source (first 5 lines) if exc.source: src_lines = exc.source.strip().splitlines()[:5] lines.append(" Cell source:") @@ -123,7 +321,6 @@ def _format_papermill_error(exc: Exception) -> str: lines.append(f" {src_line}") if len(exc.source.strip().splitlines()) > 5: lines.append(" ...") - # Show the innermost traceback frame (last non-empty line) if exc.traceback: tb_lines = [l for l in exc.traceback if l.strip()] if tb_lines: @@ -132,8 +329,7 @@ def _format_papermill_error(exc: Exception) -> str: def _strip_ci_skip_cells(nb_path: Path, output_dir: Path) -> Path: - """Return a copy of the notebook with 'ci-skip' tagged cells replaced by a comment. - Returns the original path unchanged if no cells are tagged.""" + """Return a copy of the notebook with 'ci-skip' tagged cells replaced by a comment.""" with open(nb_path) as fh: nb = json.load(fh) skipped = 0 @@ -165,8 +361,8 @@ def _run_notebook(nb_path: Path, output_dir: Path, timeout: int) -> Path: str(output_path), kernel_name="python3", execution_timeout=timeout, - cwd=str(nb_path.parent), # always cwd to original notebook directory - progress_bar=False, # avoid interleaved tqdm bars from parallel threads + cwd=str(nb_path.parent), + progress_bar=False, ) except PapermillExecutionError as exc: raise RuntimeError(_format_papermill_error(exc)) from exc @@ -204,7 +400,7 @@ def _extract_run_ids_from_notebook(output_nb: Path) -> list[str]: output.get("text", []) + output.get("data", {}).get("text/plain", []) ) ids.extend(_RUN_ID_RE.findall(text)) - return list(dict.fromkeys(ids)) # deduplicate, preserve order + return list(dict.fromkeys(ids)) def _extract_run_ids_from_stdout(stdout: str) -> list[str]: @@ -220,11 +416,7 @@ def _extract_run_ids_from_stdout(stdout: str) -> list[str]: def _get_failed_task_logs(run: object, namespace: str) -> str: - """Best-effort: tail logs from the pod(s) of the first failed KFP task. - - Walks run.run_details.task_details → child_tasks → pod_name and tries - kubectl logs on those pods. Silently returns "" on any failure. - """ + """Best-effort: tail logs from the pod(s) of the first failed KFP task.""" try: task_details = ( getattr(getattr(run, "run_details", None), "task_details", None) or [] @@ -283,8 +475,6 @@ def _poll_kfp_run(run_id: str, timeout: int, interval: int = 30) -> tuple[str, s deadline = time.time() + timeout while time.time() < deadline: run = client.get_run(run_id) - # KFP v2 SDK: state is on the run object directly as a string - # KFP v1 SDK: state is on run.run.status state = str( getattr(run, "state", None) or getattr(getattr(run, "run", None), "status", None) @@ -293,8 +483,6 @@ def _poll_kfp_run(run_id: str, timeout: int, interval: int = 30) -> tuple[str, s if state in _KFP_TERMINAL_STATES: error_detail = "" if state not in ("SUCCEEDED", "SKIPPED"): - # run.error is often None even for failed runs in KFP v2 — - # the real failure is in the task pod logs err = getattr(run, "error", None) error_detail = (getattr(err, "message", "") or "") if err else "" if not error_detail and namespace: @@ -318,7 +506,47 @@ def _run_cleanup(cleanup_path: Path) -> None: print(f" WARNING: cleanup {cleanup_path.name} failed: {exc}", file=sys.stderr) -# ── Executor ────────────────────────────────────────────────────────────────── +# ── Orchestration context ───────────────────────────────────────────────────── + + +@dataclass +class _Context: + """Shared mutable state threaded through every phase.""" + + executor: ThreadPoolExecutor + root: Path + output_dir: Path + timeout_notebook: int + timeout_pipeline: int + results: dict[str, Result] + poll_results: dict[str, str] + poll_errors: dict[str, str] + + +# ── Execution primitives ────────────────────────────────────────────────────── + + +def _make_work( + steps: list[Step], root: Path, output_dir: Path, timeout: int +) -> Callable[[Result], None]: + """Build a work(result) closure from a list of Steps.""" + + def work(result: Result) -> None: + for step in steps: + if step.kind == "notebook": + out = _run_notebook(root / step.path, output_dir, timeout) + if step.extract_run_ids: + result.kfp_run_ids.extend(_extract_run_ids_from_notebook(out)) + elif step.kind == "script": + stdout, _ = _run_script( + root / step.path, + timeout, + extra_args=step.extra_args or None, + ) + if step.extract_run_ids: + result.kfp_run_ids.extend(_extract_run_ids_from_stdout(stdout)) + + return work def _timed_run( @@ -327,16 +555,12 @@ def _timed_run( name: str, work: Callable[[Result], None], ) -> Future: - """Submit `work(result)` to the executor with shared timing + error handling. - - `work` mutates the Result (e.g. sets kfp_run_ids); status/duration are set - here. The first exception is recorded as FAIL. - """ + """Submit work(result) to the executor with shared timing + error handling.""" result = Result(name=name) results[name] = result print(f" [START ] {name}") - def _run(): + def _run() -> None: t0 = time.time() try: work(result) @@ -350,79 +574,23 @@ def _run(): return executor.submit(_run) -def _submit_notebook( - executor: ThreadPoolExecutor, - results: dict[str, Result], - name: str, - nb_path: Path, - output_dir: Path, - timeout: int, - extract_run_ids: bool = True, -) -> Future: - def work(result: Result) -> None: - out = _run_notebook(nb_path, output_dir, timeout) - if extract_run_ids: - result.kfp_run_ids = _extract_run_ids_from_notebook(out) - - return _timed_run(executor, results, name, work) - - -def _submit_script( - executor: ThreadPoolExecutor, - results: dict[str, Result], - name: str, - script_path: Path, - timeout: int, - extra_args: list[str] | None = None, -) -> Future: - def work(result: Result) -> None: - stdout, _ = _run_script(script_path, timeout, extra_args=extra_args) - result.kfp_run_ids = _extract_run_ids_from_stdout(stdout) - - return _timed_run(executor, results, name, work) - - -def _submit_chain( - executor: ThreadPoolExecutor, - results: dict[str, Result], - name: str, - steps: list[tuple[str, Path | None, dict]], - output_dir: Path, - timeout: int, -) -> Future: - """Submit a sequential chain of (kind, path, kwargs) steps as a single named result. - - kind is 'notebook' or 'script'. kwargs may include 'extra_args' for scripts. - All steps run in one thread; the first failure stops the chain. - """ +def _drain(ctx: _Context, futures: dict[str, Future]) -> None: + """Wait on a {name: Future} map, printing each result as it completes.""" + by_future = {v: k for k, v in futures.items()} + for f in as_completed(futures.values()): + _print_result(ctx.results[by_future[f]]) - def work(result: Result) -> None: - for kind, path, kwargs in steps: - if kind == "notebook": - out = _run_notebook(path, output_dir, timeout) - result.kfp_run_ids.extend(_extract_run_ids_from_notebook(out)) - elif kind == "script": - stdout, _ = _run_script( - path, timeout, extra_args=kwargs.get("extra_args") - ) - result.kfp_run_ids.extend(_extract_run_ids_from_stdout(stdout)) - elif kind == "pip": - subprocess.run( - [sys.executable, "-m", "pip", "install", "-q"] - + kwargs.get("packages", []), - capture_output=True, - text=True, - check=True, - ) - return _timed_run(executor, results, name, work) +def _skip(ctx: _Context, name: str, reason: str) -> None: + """Record a SKIP result and print a one-line notice.""" + print(f" [SKIP ] {name}") + ctx.results[name] = Result(name=name, status="SKIP", error=reason) # ── Report ──────────────────────────────────────────────────────────────────── def _print_result(r: Result) -> None: - """Print a single result line, with error detail if failed.""" dur = f"{r.duration:.0f}s" if r.duration else "-" print(f" [{r.status}] {r.name} ({dur})") if r.status == "FAIL" and r.error: @@ -443,13 +611,12 @@ def _print_report( failed += 1 else: passed += 1 - for run_id, status in poll_results.items(): + for status in poll_results.values(): if status.upper() == "SUCCEEDED": passed += 1 else: failed += 1 - # Failed details first so they're easy to scroll back to if failed: print("\nFailed details:") print("-" * 70) @@ -467,7 +634,6 @@ def _print_report( for line in err.splitlines(): print(f" {line}") - # Summary table last — easy to see final verdict at a glance print("\n" + "=" * 70) print(f"{'EXAMPLE':<{col}} {'STATUS':<10} {'DURATION'}") print("-" * 70) @@ -487,22 +653,34 @@ def _print_report( print() -# ── MLflow pre-flight check ─────────────────────────────────────────────────── - -# Notebooks that require working MLflow credentials -_MLFLOW_DEPENDENT = frozenset( - { - "mlflow/mlflow-quickstart", - "mlflow/mlflow-image-example", - "mlflow/mlflow-kfp-example", - "mlflow/mobile-price-classification", +def _print_dry_run() -> None: + by_phase: dict[int, list[str]] = {} + for ex in _EXAMPLES: + label = ex.name + if ex.opt_in: + label += f" (--{ex.opt_in.replace('_', '-')})" + if ex.mlflow_dependent: + label += " (mlflow)" + by_phase.setdefault(ex.phase, []).append(label) + phase_names = { + 1: "independent", + 2: "pipeline submissions", + 3: "MLflow KServe ISVCs", } -) + print("[dry-run] Would execute the following phases:") + for phase, labels in sorted(by_phase.items()): + print(f" Phase {phase} ({phase_names.get(phase, f'phase {phase}')}):") + for label in labels: + print(f" {label}") + print(" Phase 4: KFP run polling") + print(" Phase 5: cleanup") + + +# ── Pre-flight ──────────────────────────────────────────────────────────────── def _check_mlflow_credentials() -> tuple[bool, str]: - """Return (ok, reason). Checks secret existence then validates credentials - with a quick call to the MLflow REST API.""" + """Return (ok, reason). Checks secret existence then validates with MLflow API.""" import base64 as _b64 import json as _json import urllib.error @@ -519,7 +697,6 @@ def _check_mlflow_credentials() -> tuple[bool, str]: "mlflow-credentials secret not found — " "run scripts/setup_mlflow_credentials.py first" ) - try: data = _json.loads(r.stdout)["data"] uri = _b64.b64decode(data["MLFLOW_TRACKING_URI"]).decode().rstrip("/") @@ -553,235 +730,59 @@ def _check_mlflow_credentials() -> tuple[bool, str]: return False, f"Could not reach MLflow at {uri}: {exc}" -# ── Orchestration context ───────────────────────────────────────────────────── - - -@dataclass -class _Context: - """Shared mutable state threaded through every phase.""" - - executor: ThreadPoolExecutor - root: Path - output_dir: Path - timeout_notebook: int - timeout_pipeline: int - results: dict[str, Result] - poll_results: dict[str, str] - poll_errors: dict[str, str] - - -_CLEANUP_RELPATHS = [ - "notebooks/dask/cleanup.py", - "serving/minimal-s3-model/cleanup.py", - "serving/mlflow-kserve-minimal/cleanup.py", - "serving/mlflow-kserve-inference-protocols/cleanup.py", - "serving/hf-vllm-completion/cleanup.py", - "serving/kserve-keda-autoscaling/cleanup.py", - "serving/minimal-example-shadow-deployment/cleanup.py", - "hparam-tuning/minimal-mnist/cleanup.py", -] +def _preflight(results: dict[str, Result]) -> bool: + """Install papermill and validate MLflow credentials. -_DRY_RUN_PHASES = [ - "Phase 1: dask, mlflow-quickstart, mlflow-image, mlflow-kfp, minimal-s3-model, hf-vllm-completion", - "Phase 1 (opt-in): kserve-keda-autoscaling (--include-keda), minimal-example-shadow-deployment (--include-shadow)", - "Phase 2: mlflow-mobile-price, lightweight-components, lightweight-python-package", - "Phase 3: mlflow-kserve-minimal, mlflow-kserve-inference-protocols", - "Phase 4: KFP run polling", - "Phase 5: cleanup", -] - - -def _drain(ctx: _Context, futures: dict[str, Future]) -> None: - """Wait on a {name: Future} map, printing each result as it completes.""" - by_future = {v: k for k, v in futures.items()} - for f in as_completed(futures.values()): - _print_result(ctx.results[by_future[f]]) - - -def _skip(ctx: _Context, name: str, reason: str, label: str | None = None) -> None: - """Record a SKIP result and print a one-line notice.""" - print(f" [SKIP ] {label or name}") - ctx.results[name] = Result(name=name, status="SKIP", error=reason) - - -# ── Pre-flight ──────────────────────────────────────────────────────────────── - - -def _preflight(results: dict[str, Result]) -> tuple[bool, str]: - """Ensure papermill, then validate MLflow credentials. - - Returns (mlflow_ok, reason). On failure, MLflow-dependent examples are - recorded as SKIP in `results`. + MLflow-dependent examples are recorded as SKIP in `results` on failure. + Returns True if MLflow credentials are valid. """ _ensure_papermill() print("Pre-flight: checking MLflow credentials...") - mlflow_ok, mlflow_reason = _check_mlflow_credentials() + mlflow_ok, reason = _check_mlflow_credentials() if mlflow_ok: print(" [OK] MLflow credentials valid") else: - print(f" [SKIP] {mlflow_reason}") - for name in _MLFLOW_DEPENDENT: - results[name] = Result(name=name, status="SKIP", error=mlflow_reason) - return mlflow_ok, mlflow_reason + print(f" [SKIP] {reason}") + for ex in _EXAMPLES: + if ex.mlflow_dependent: + results[ex.name] = Result(name=ex.name, status="SKIP", error=reason) + return mlflow_ok -# ── Phases ────────────────────────────────────────────────────────────────── +# ── Phases ──────────────────────────────────────────────────────────────────── -def _phase1( +def _run_phase( ctx: _Context, - include_keda: bool, - include_shadow: bool, - include_pytorch: bool, -) -> None: - """Phase 1: independent notebooks + self-contained serving examples.""" - print("Phase 1: running independent notebooks in parallel...") + phase: int, + opts: dict[str, bool], +) -> dict[str, Future]: + """Submit all examples for a given phase; return {name: Future}.""" futures: dict[str, Future] = {} - for name, rel in [ - ("notebooks/dask", "notebooks/dask/dask_example.ipynb"), - ( - "notebooks/mobile-price-classification", - "notebooks/mobile-price-classification/mobile-price-classifications.ipynb", - ), - ("mlflow/mlflow-quickstart", "mlflow/mlflow-quickstart-example.ipynb"), - ("mlflow/mlflow-image-example", "mlflow/mlflow-image-example.ipynb"), - ("mlflow/mlflow-kfp-example", "mlflow/mlflow-kfp-example.ipynb"), - ("serving/minimal-s3-model", "serving/minimal-s3-model/minimal-s3-model.ipynb"), - ]: - if name in ctx.results: # already SKIP from pre-flight - print(f" [SKIP ] {name}") + for ex in _EXAMPLES: + if ex.phase != phase: continue - futures[name] = _submit_notebook( - ctx.executor, - ctx.results, - name, - ctx.root / rel, - ctx.output_dir, - ctx.timeout_notebook, - ) - - # hf-vllm-completion (CPU): self-contained, no KFP pipelines - futures["serving/hf-vllm-completion"] = _submit_script( - ctx.executor, - ctx.results, - "serving/hf-vllm-completion", - ctx.root / "serving" / "hf-vllm-completion" / "apply.py", - ctx.timeout_notebook, - ) - - # kserve-keda-autoscaling: opt-in; apply.py self-checks KEDA CRDs - # and exits 0 with a skip message if KEDA is not installed. - if include_keda: - futures["serving/kserve-keda-autoscaling"] = _submit_script( - ctx.executor, - ctx.results, - "serving/kserve-keda-autoscaling", - ctx.root / "serving" / "kserve-keda-autoscaling" / "apply.py", - ctx.timeout_notebook, - ) - else: - _skip( - ctx, - "serving/kserve-keda-autoscaling", - "opt-in: pass --include-keda to run this example", - label="serving/kserve-keda-autoscaling (pass --include-keda to enable)", - ) - - # minimal-example-shadow-deployment: opt-in; apply.py self-checks for - # the postgres-operator CRD and exits 0 if not found. Istio - # VirtualService mirroring is not verified in CI (requires domain config). - if include_shadow: - futures["serving/minimal-example-shadow-deployment"] = _submit_script( - ctx.executor, - ctx.results, - "serving/minimal-example-shadow-deployment", - ctx.root / "serving" / "minimal-example-shadow-deployment" / "apply.py", - ctx.timeout_notebook, - ) - else: - _skip( - ctx, - "serving/minimal-example-shadow-deployment", - "opt-in: pass --include-shadow to run this example", - label="serving/minimal-example-shadow-deployment (pass --include-shadow to enable)", - ) - - # mnist-vae: opt-in (requires pytorch_lightning, not in all images) - if include_pytorch: - futures["notebooks/mnist-vae"] = _submit_chain( - ctx.executor, - ctx.results, - "notebooks/mnist-vae", - steps=[ - # run_training.py self-installs pytorch-lightning if absent - ( - "script", - ctx.root / "notebooks/mnist-vae/run_training.py", - {"extra_args": ["--max_epochs", "3"]}, - ), - ("notebook", ctx.root / "notebooks/mnist-vae/visualizations.ipynb", {}), - ], - output_dir=ctx.output_dir, - timeout=ctx.timeout_notebook, - ) - else: - print(" [SKIP ] notebooks/mnist-vae (pass --include-pytorch to enable)") - - _drain(ctx, futures) - - -def _phase2_submit(ctx: _Context) -> dict[str, Future]: - """Phase 2: submit pipeline notebooks + scripts (they return fast).""" - print("\nPhase 2: submitting pipelines...") - futures: dict[str, Future] = {} - for name, rel in [ - ( - "mlflow/mobile-price-classification", - "mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb", - ), - ( - "pipelines/lightweight-components", - "pipelines/lightweight-components/mobile-price-classifications.ipynb", - ), - ]: - if name in ctx.results: # already SKIP from pre-flight - print(f" [SKIP ] {name}") + if ex.name in ctx.results: # already SKIP from pre-flight + print(f" [SKIP ] {ex.name}") continue - futures[name] = _submit_notebook( - ctx.executor, - ctx.results, - name, - ctx.root / rel, - ctx.output_dir, - ctx.timeout_notebook, - ) - - futures["pipelines/lightweight-python-package"] = _submit_script( - ctx.executor, - ctx.results, - "pipelines/lightweight-python-package", - ctx.root / "pipelines" / "lightweight-python-package" / "submit-cluster.py", - ctx.timeout_notebook, - ) - futures["pipelines/minimal-container-components"] = _submit_script( - ctx.executor, - ctx.results, - "pipelines/minimal-container-components", - ctx.root / "pipelines" / "minimal-container-components" / "submit-cluster.py", - ctx.timeout_notebook, - ) + if ex.opt_in and not opts.get(ex.opt_in, False): + _skip( + ctx, ex.name, f"opt-in: pass --{ex.opt_in.replace('_', '-')} to enable" + ) + continue + work = _make_work(ex.steps, ctx.root, ctx.output_dir, ctx.timeout_notebook) + futures[ex.name] = _timed_run(ctx.executor, ctx.results, ex.name, work) return futures def _await_mobile_price(ctx: _Context, phase2_futures: dict[str, Future]) -> bool: - """Wait for the mlflow-mobile-price notebook and poll its KFP run inline. + """Wait for mlflow-mobile-price and poll its KFP run inline. Phase 3 ISVCs need the registered model, so this must complete before Phase 3 starts. Returns True if the model is registered and ready. """ name = "mlflow/mobile-price-classification" - skipped = name not in phase2_futures - if skipped: + if name not in phase2_futures: return False phase2_futures[name].result() @@ -799,7 +800,7 @@ def _await_mobile_price(ctx: _Context, phase2_futures: dict[str, Future]) -> boo ok = True for run_id in run_ids: state, err = _poll_kfp_run(run_id, ctx.timeout_pipeline) - ctx.poll_results[run_id] = state # recorded here; Phase 4 will skip it + ctx.poll_results[run_id] = state ctx.poll_errors[run_id] = err print(f" [{state}] {run_id[:8]}...") if state.upper() != "SUCCEEDED": @@ -807,55 +808,7 @@ def _await_mobile_price(ctx: _Context, phase2_futures: dict[str, Future]) -> boo return ok -def _phase3_submit( - ctx: _Context, mobile_price_ok: bool, mobile_price_skipped: bool -) -> dict[str, Future]: - """Phase 3: deploy MLflow KServe InferenceServices (need registered model).""" - print("\nPhase 3: deploying MLflow KServe InferenceServices...") - isvc_names = ( - "serving/mlflow-kserve-minimal", - "serving/mlflow-kserve-inference-protocols", - ) - if not mobile_price_ok: - reason = ( - "prerequisite mlflow/mobile-price-classification skipped (MLflow credentials)" - if mobile_price_skipped - else "prerequisite mlflow/mobile-price-classification notebook or KFP pipeline did not succeed" - ) - print(f" [SKIP] {reason}") - for name in isvc_names: - ctx.results[name] = Result(name=name, status="SKIP", error=reason) - return {} - - futures: dict[str, Future] = {} - # mlflow-kserve-minimal: deploy then immediately smoke-test - # deploy_and_test() in apply.py handles both steps - futures["serving/mlflow-kserve-minimal"] = _submit_script( - ctx.executor, - ctx.results, - "serving/mlflow-kserve-minimal", - ctx.root / "serving" / "mlflow-kserve-minimal" / "apply.py", - ctx.timeout_notebook, - ) - # inference-protocols notebook handles its own deploy + test. - # extract_run_ids=False: this notebook is not a pipeline submission; - # its output contains ISVC/request UUIDs that must not be polled - # as KFP run IDs (they return 404 from the KFP API). - futures["serving/mlflow-kserve-inference-protocols"] = _submit_notebook( - ctx.executor, - ctx.results, - "serving/mlflow-kserve-inference-protocols", - ctx.root - / "serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb", - ctx.output_dir, - ctx.timeout_notebook, - extract_run_ids=False, - ) - return futures - - def _phase4_poll(ctx: _Context) -> None: - """Phase 4: poll all KFP run IDs not already polled inline before Phase 3.""" all_run_ids = [ rid for r in ctx.results.values() @@ -881,12 +834,11 @@ def _phase4_poll(ctx: _Context) -> None: def _phase5_cleanup(cleanup_scripts: list[Path]) -> None: - """Phase 5: run all cleanup scripts in parallel (always, even on failure).""" print("\nPhase 5: running cleanup scripts...") with ThreadPoolExecutor(max_workers=4) as ex: futs = [ex.submit(_run_cleanup, p) for p in cleanup_scripts if p.exists()] for f in as_completed(futs): - f.result() # already swallows exceptions inside _run_cleanup + f.result() # ── Main ────────────────────────────────────────────────────────────────────── @@ -903,14 +855,20 @@ def run_all( root = _REPO_ROOT results: dict[str, Result] = {} - mlflow_ok, _ = _preflight(results) - if not mlflow_ok and dry_run: - print("[dry-run] Would execute the following phases:") - for label in _DRY_RUN_PHASES: - print(f" {label}") + if dry_run: + _print_dry_run() return results - cleanup_scripts = [root / rel for rel in _CLEANUP_RELPATHS] + _preflight(results) + + opts = { + "include_keda": include_keda, + "include_pytorch": include_pytorch, + "include_shadow": include_shadow, + } + cleanup_scripts = [root / ex.cleanup for ex in _EXAMPLES if ex.cleanup] + [ + root / p for p in _EXTRA_CLEANUP_PATHS + ] poll_results: dict[str, str] = {} poll_errors: dict[str, str] = {} @@ -927,16 +885,31 @@ def run_all( poll_errors=poll_errors, ) - _phase1(ctx, include_keda, include_shadow, include_pytorch) - phase2_futures = _phase2_submit(ctx) + print("Phase 1: running independent examples in parallel...") + _drain(ctx, _run_phase(ctx, 1, opts)) - mobile_price_skipped = ( - "mlflow/mobile-price-classification" not in phase2_futures - ) + print("\nPhase 2: submitting pipelines...") + phase2_futures = _run_phase(ctx, 2, opts) mobile_price_ok = _await_mobile_price(ctx, phase2_futures) - phase3_futures = _phase3_submit(ctx, mobile_price_ok, mobile_price_skipped) - # Phase 2 remaining + phase 3 run concurrently; print as each finishes + print("\nPhase 3: deploying MLflow KServe InferenceServices...") + if mobile_price_ok: + phase3_futures = _run_phase(ctx, 3, opts) + else: + skipped = "mlflow/mobile-price-classification" not in phase2_futures + reason = ( + "prerequisite mlflow/mobile-price-classification skipped (MLflow credentials)" + if skipped + else "prerequisite mlflow/mobile-price-classification notebook or KFP pipeline did not succeed" + ) + print(f" [SKIP] {reason}") + for ex in _EXAMPLES: + if ex.phase == 3: + ctx.results[ex.name] = Result( + name=ex.name, status="SKIP", error=reason + ) + phase3_futures = {} + print("\nPhase 2 (remaining) + Phase 3 running concurrently...") remaining = { k: v @@ -951,12 +924,9 @@ def run_all( finally: _phase5_cleanup(cleanup_scripts) - # Build run_id → source notebook name map from all results - run_id_to_name: dict[str, str] = {} - for r in results.values(): - for run_id in r.kfp_run_ids: - run_id_to_name[run_id] = r.name - + run_id_to_name = { + run_id: r.name for r in results.values() for run_id in r.kfp_run_ids + } _print_report(results, poll_results, poll_errors, run_id_to_name) return results @@ -981,7 +951,7 @@ def run_all( parser.add_argument( "--include-keda", action="store_true", - help="Include KEDA autoscaling example (opt-in; requires KEDA installed in the cluster)", + help="Include KEDA autoscaling example (opt-in; requires KEDA in the cluster; runs on CPU)", ) parser.add_argument( "--include-shadow", @@ -997,7 +967,9 @@ def run_all( help="Include pytorch_lightning examples (opt-in; requires pytorch in the notebook image)", ) parser.add_argument( - "--dry-run", action="store_true", help="Print plan without executing anything" + "--dry-run", + action="store_true", + help="Print plan without executing anything", ) args = parser.parse_args() From 74a043ccf0df96eefa83f2429e299e1d93420574 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Tue, 16 Jun 2026 14:31:36 +0200 Subject: [PATCH 35/57] docs: add ci/README.md with conventions for adding examples, scripts, and ci-skip tags --- ci/README.md | 156 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 ci/README.md diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 0000000..73c697d --- /dev/null +++ b/ci/README.md @@ -0,0 +1,156 @@ +# CI — contributor guide + +> This document is written primarily for AI agents making changes to this +> repository. It describes the conventions that must be followed for a new +> example to be picked up by `ci/run_all.py` correctly. + +--- + +## Adding a new example + +Register it in the `_EXAMPLES` list in `run_all.py`. Everything else — +phase scheduling, opt-in gating, cleanup, dry-run output, MLflow-credential +skipping — is derived from that entry automatically. + +```python +Example( + name="serving/my-new-example", # display name and result key + steps=[ + Step("script", "serving/my-new-example/apply.py"), + # or: Step("notebook", "serving/my-new-example/example.ipynb") + # chain multiple steps if needed (executed sequentially) + ], + phase=1, # see Phase rules below + cleanup="serving/my-new-example/cleanup.py", # omit if no K8s resources + opt_in="include_foo", # omit if always enabled + mlflow_dependent=False, # True = skip when MLflow creds absent +) +``` + +### Phase rules + +| Phase | When to use | +|-------|-------------| +| 1 | Self-contained: does not depend on anything else in CI | +| 2 | Submits a KFP pipeline and returns fast; actual run is polled in Phase 4 | +| 3 | Requires a model already registered by the Phase 2 mlflow-mobile-price pipeline | + +### Opt-in flag + +Add `opt_in="include_foo"` and a corresponding `--include-foo` argument in +the `__main__` block of `run_all.py`. Use opt-in when the example requires +cluster add-ons (KEDA, postgres-operator, GPU nodes) that are not guaranteed +to be present. + +--- + +## apply.py and cleanup.py + +### cleanup.py — always add when K8s resources are created + +Any example that creates Kubernetes resources (InferenceService, Deployment, +Service, CRD instance, …) must have a `cleanup.py` in its directory. +CI runs all cleanup scripts in parallel in a `finally` block so they execute +even on failure. + +A cleanup script must: +- Delete resources idempotently (`--ignore-not-found`) +- Never raise on failure (print a warning at most) +- Read the namespace from the pod filesystem: + +```python +with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as f: + ns = f.read().strip() +``` + +**Do not** add a `cleanup.py` for examples that only create in-cluster +transient objects managed by KFP (pipeline runs, artifacts) — those are +cleaned up by KFP's own retention policy. + +### apply.py — only for serving examples driven by a script + +Add an `apply.py` only when the CI execution of a serving example is driven +by a Python script rather than a notebook. This is the case when: + +- The notebook's deploy/test section requires manual inputs that cannot be + automated, so a separate script owns the full deploy-wait-smoketest cycle. +- The example has no notebook at all. + +An `apply.py` must exit non-zero on failure (CI relies on the return code). +It must print a clean error message to stderr and suppress the full traceback: + +```python +if __name__ == "__main__": + try: + deploy() + except Exception as exc: + print(str(exc), file=sys.stderr) + sys.exit(1) +``` + +If you add an `apply.py`, always add the matching `cleanup.py` as well. + +--- + +## scripts/ utilities + +`scripts/` contains two cluster utilities importable from notebooks and apply +scripts. Both work identically whether called from a notebook cell or from an +`apply.py`. + +### setup_mlflow_credentials.py + +One-time, interactive. Stores MLflow credentials in the +`mlflow-credentials` K8s secret. **Requires human input** (the MLflow +Personal Access Token cannot be obtained programmatically). Run it once from +a JupyterLab terminal: + +```bash +python examples/scripts/setup_mlflow_credentials.py +``` + +Do not call this from CI. CI validates the secret exists in the preflight +check and skips MLflow-dependent examples if it does not. + +### get_or_create_api_key.py + +Idempotent, no human input. Creates or retrieves the `examples-key` +AIGatewayKey CR. Call it from any notebook cell or script that needs an +inference API key: + +```python +import sys, subprocess, os +_root = subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip() +sys.path.insert(0, os.path.join(_root, "scripts")) +from get_or_create_api_key import get_or_create_api_key + +API_KEY = get_or_create_api_key() +``` + +--- + +## ci-skip cell tag + +Tag a notebook cell with `ci-skip` to have CI replace it with a no-op comment +before execution. Use this **only** for cells that genuinely cannot run +headlessly: + +- Cells that require manual input (hardcoded paths, paste-your-token + placeholders, `input()` calls). +- Interactive widget cells (`ipywidgets`, `ipykernel` display-only code). + +**Do not** use `ci-skip` to work around a fixable automation problem. If a +cell fails because it needs credentials or a namespace, fix it to load those +automatically (see the MLflow credential pattern above). `ci-skip` is a last +resort, not a convenience. + +To tag a cell in JupyterLab: select the cell → Property Inspector (right +panel) → add `ci-skip` under Cell Tags. + +In raw notebook JSON the tag appears as: + +```json +"metadata": { + "tags": ["ci-skip"] +} +``` From 62a72ace1df507515ae8cdd0b4b428def9f8721b Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Tue, 16 Jun 2026 14:52:52 +0200 Subject: [PATCH 36/57] docs: add ci/ and scripts/ to repo overview, link to ci/README.md --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 05c3307..4e315ef 100644 --- a/README.md +++ b/README.md @@ -7,15 +7,21 @@ For full platform documentation, see [docs.prokube.ai](https://docs.prokube.ai/) ```py . ├── .github # workflows to build images +├── ci # CI orchestrator (run_all.py) and contributor conventions ├── hparam-tuning # hyperparameter tuning examples (Katib) ├── images # custom container images used by examples ├── mlflow # MLflow experiment tracking examples ├── notebooks # Jupyter notebook examples (Dask, MNIST VAE, etc.) ├── pipelines # Kubeflow Pipelines examples ├── rstudio # RStudio examples +├── scripts # shared utilities (credential setup, API key management) ├── serving # model serving examples (KServe, vLLM, shadow deployments) ``` +Many serving examples include an `apply.py` (deploy + smoke-test) and a `cleanup.py` +(teardown of Kubernetes resources) alongside the notebook. These are used by the CI +orchestrator and can also be run manually. + ## Note about storage Storage on Kubernetes is a complex topic and a deep dive is outside the scope of this repository. There are two types of storage you might encounter here — block and object storage. @@ -43,6 +49,12 @@ Some examples require a minimum prokube platform version. If an example is not l |---|---|---| | `serving/minimal-s3-model` | v1.7.0 | Requires s3creds secret with KServe support | +## CI + +Examples are tested end-to-end by `ci/run_all.py`, which runs inside a Kubeflow +notebook pod. See [`ci/README.md`](ci/README.md) for how the orchestrator works and +how to add a new example to CI. + ## Contributing All code contributions should go via pull requests. Make sure your code is clearly documented and that it adheres to established standards (e.g. PEP). From 5556e658e3639f5b6b54055be218ae4c64d9cd0f Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Tue, 16 Jun 2026 15:06:46 +0200 Subject: [PATCH 37/57] Cosmetics --- .../mlflow-mobile-price-classification.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb b/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb index ef18fd4..1c14910 100644 --- a/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb +++ b/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb @@ -55,12 +55,12 @@ "source": [ "## MLflow Setup\n", "\n", - "To use MLflow tracking in this pipeline, create a Kubernetes secret with your MLflow credentials:\n", + "To use MLflow tracking in this pipeline, create a Kubernetes secret with your MLflow credentials.\n", "\n", "Run `scripts/setup_mlflow_credentials.py` once to create the required secret:\n", "```bash\n", "# From a JupyterLab terminal:\n", - "python examples/scripts/setup_mlflow_credentials.py\n", + "cd && python examples/scripts/setup_mlflow_credentials.py\n", "```\n", "You will be prompted for your MLflow tracking URI, username (e-mail), and Personal Access Token.\n", "\n", From cd9f21f00b24630fe22dfc41f1a351ae134986e4 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Tue, 16 Jun 2026 15:09:06 +0200 Subject: [PATCH 38/57] docs: align MLflow setup instructions across all MLflow notebooks --- mlflow/mlflow-image-example.ipynb | 20 ++++++++------------ mlflow/mlflow-kfp-example.ipynb | 26 +++++++------------------- mlflow/mlflow-quickstart-example.ipynb | 20 ++++++++------------ 3 files changed, 23 insertions(+), 43 deletions(-) diff --git a/mlflow/mlflow-image-example.ipynb b/mlflow/mlflow-image-example.ipynb index 0a5b36d..041a21a 100644 --- a/mlflow/mlflow-image-example.ipynb +++ b/mlflow/mlflow-image-example.ipynb @@ -9,20 +9,16 @@ "\n", "This notebook demonstrates how to use MLflow in the prokube platform with Personal Access Token (PAT) authentication on a slightly more complex example then the Quick Start.\n", "\n", - "## Prerequisites\n", + "## MLflow Setup\n", "\n", - "- Generate your Personal Access Token:\n", - " - Open [MLflow UI](/mlflow) in your browser\n", - " - Navigate to the [Permissions](/mlflow/oidc/ui/#) page\n", - " - Click on \"Create access key\" button\n", - " - Copy the generated token and store it securely\n", - " - Note: You won't be able to see the token again!\n", + "Run `scripts/setup_mlflow_credentials.py` once to create the required secret:\n", + "```bash\n", + "# From a JupyterLab terminal:\n", + "cd && python examples/scripts/setup_mlflow_credentials.py\n", + "```\n", + "You will be prompted for your MLflow tracking URI, username (e-mail), and Personal Access Token.\n", "\n", - "- Configure the credentials below with your values\n", - "\n", - "## Authentication\n", - "\n", - "The prokube MLflow setup uses OIDC authentication with PAT support for programmatic access.\n" + "To generate a Personal Access Token, open the [MLflow UI](/mlflow) and navigate to [Permissions](/mlflow/oidc/ui/#)." ] }, { diff --git a/mlflow/mlflow-kfp-example.ipynb b/mlflow/mlflow-kfp-example.ipynb index 4477f41..23732ed 100644 --- a/mlflow/mlflow-kfp-example.ipynb +++ b/mlflow/mlflow-kfp-example.ipynb @@ -7,36 +7,24 @@ "source": [ "# Simple pipeline with MLflow model tracking\n", "\n", - "An experiment with iris dataset. In general, to use MLflow in a Kubeflow Pipeline, the necessary environment variables should be passed to the containers using the MLflow logic. This is implemented in the `add_env_vars_to_tasks` function. \n", + "An experiment with iris dataset. In general, to use MLflow in a Kubeflow Pipeline, the necessary environment variables should be passed to the containers using the MLflow logic. This is implemented in the `add_env_vars_to_tasks` function.\n", "\n", - "\n", - "## Prerequisites\n", - "\n", - "To use MLflow in Kubeflow Pipelines, you need to create a Kubernetes secret with your MLflow credentials. \n", - "\n", - "To generate \n", - "\n", - "- Generate your Personal Access Token (re-use your old one, if you have already generated it):\n", - " - Open [MLflow UI](/mlflow) in your browser\n", - " - Navigate to the [Permissions](/mlflow/oidc/ui/#) page\n", - " - Click on \"Create access key\" button\n", - " - Copy the generated token and store it securely\n", - " - Note: You won't be able to see the token again!\n", + "## MLflow Setup\n", "\n", "Run `scripts/setup_mlflow_credentials.py` once to create the required secret:\n", "```bash\n", "# From a JupyterLab terminal:\n", - "python examples/scripts/setup_mlflow_credentials.py\n", + "cd && python examples/scripts/setup_mlflow_credentials.py\n", "```\n", "You will be prompted for your MLflow tracking URI, username (e-mail), and Personal Access Token.\n", "\n", + "To generate a Personal Access Token, open the [MLflow UI](/mlflow) and navigate to [Permissions](/mlflow/oidc/ui/#).\n", + "\n", "## How it works\n", "\n", - "In this pipeline, the add_mlflow_env_vars_to_tasks function injects the MLflow credentials from the Kubernetes secret into each pipeline component. This allows the components to authenticate with\n", - "MLflow and track experiments.\n", + "In this pipeline, the add_mlflow_env_vars_to_tasks function injects the MLflow credentials from the Kubernetes secret into each pipeline component. This allows the components to authenticate with MLflow and track experiments.\n", "\n", - "This notebook defines a simple pipeline for preprocessing data, training and logging model, and prediction on test data. It also shows one way to handle MLflow experiment info inside the pipeline - by\n", - "passing run information between components." + "This notebook defines a simple pipeline for preprocessing data, training and logging model, and prediction on test data. It also shows one way to handle MLflow experiment info inside the pipeline - by passing run information between components." ] }, { diff --git a/mlflow/mlflow-quickstart-example.ipynb b/mlflow/mlflow-quickstart-example.ipynb index b60514d..7a73868 100644 --- a/mlflow/mlflow-quickstart-example.ipynb +++ b/mlflow/mlflow-quickstart-example.ipynb @@ -27,20 +27,16 @@ "\n", "This notebook demonstrates how to use MLflow in the prokube platform with Personal Access Token (PAT) authentication.\n", "\n", - "## Prerequisites\n", + "## MLflow Setup\n", "\n", - "- Generate your Personal Access Token:\n", - " - Open [MLflow UI](/mlflow) in your browser\n", - " - Navigate to the [Permissions](/mlflow/oidc/ui/#) page\n", - " - Click on \"Create access key\" button\n", - " - Copy the generated token and store it securely\n", - " - Note: You won't be able to see the token again!\n", + "Run `scripts/setup_mlflow_credentials.py` once to create the required secret:\n", + "```bash\n", + "# From a JupyterLab terminal:\n", + "cd && python examples/scripts/setup_mlflow_credentials.py\n", + "```\n", + "You will be prompted for your MLflow tracking URI, username (e-mail), and Personal Access Token.\n", "\n", - "- Configure the credentials below with your values\n", - "\n", - "## Authentication\n", - "\n", - "The prokube MLflow setup uses OIDC authentication with PAT support for programmatic access.\n" + "To generate a Personal Access Token, open the [MLflow UI](/mlflow) and navigate to [Permissions](/mlflow/oidc/ui/#)." ] }, { From bd0f9d549a856c7306e1af46206654f74d79a549 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Tue, 16 Jun 2026 15:16:47 +0200 Subject: [PATCH 39/57] fix: update MLflow permissions link and add Create Access Token hint --- mlflow/mlflow-image-example.ipynb | 2 +- mlflow/mlflow-kfp-example.ipynb | 2 +- mlflow/mlflow-quickstart-example.ipynb | 2 +- .../mlflow-mobile-price-classification.ipynb | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mlflow/mlflow-image-example.ipynb b/mlflow/mlflow-image-example.ipynb index 041a21a..19a05ab 100644 --- a/mlflow/mlflow-image-example.ipynb +++ b/mlflow/mlflow-image-example.ipynb @@ -18,7 +18,7 @@ "```\n", "You will be prompted for your MLflow tracking URI, username (e-mail), and Personal Access Token.\n", "\n", - "To generate a Personal Access Token, open the [MLflow UI](/mlflow) and navigate to [Permissions](/mlflow/oidc/ui/#)." + "To generate a Personal Access Token, open the [MLflow UI](/mlflow) and navigate to [Permissions](/mlflow/oidc/ui/user) and click \"Create Access Token\"." ] }, { diff --git a/mlflow/mlflow-kfp-example.ipynb b/mlflow/mlflow-kfp-example.ipynb index 23732ed..4fc26bf 100644 --- a/mlflow/mlflow-kfp-example.ipynb +++ b/mlflow/mlflow-kfp-example.ipynb @@ -18,7 +18,7 @@ "```\n", "You will be prompted for your MLflow tracking URI, username (e-mail), and Personal Access Token.\n", "\n", - "To generate a Personal Access Token, open the [MLflow UI](/mlflow) and navigate to [Permissions](/mlflow/oidc/ui/#).\n", + "To generate a Personal Access Token, open the [MLflow UI](/mlflow) and navigate to [Permissions](/mlflow/oidc/ui/user) and click \"Create Access Token\".\n", "\n", "## How it works\n", "\n", diff --git a/mlflow/mlflow-quickstart-example.ipynb b/mlflow/mlflow-quickstart-example.ipynb index 7a73868..7e3e289 100644 --- a/mlflow/mlflow-quickstart-example.ipynb +++ b/mlflow/mlflow-quickstart-example.ipynb @@ -36,7 +36,7 @@ "```\n", "You will be prompted for your MLflow tracking URI, username (e-mail), and Personal Access Token.\n", "\n", - "To generate a Personal Access Token, open the [MLflow UI](/mlflow) and navigate to [Permissions](/mlflow/oidc/ui/#)." + "To generate a Personal Access Token, open the [MLflow UI](/mlflow) and navigate to [Permissions](/mlflow/oidc/ui/user) and click \"Create Access Token\"." ] }, { diff --git a/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb b/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb index 1c14910..770471b 100644 --- a/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb +++ b/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb @@ -64,7 +64,7 @@ "```\n", "You will be prompted for your MLflow tracking URI, username (e-mail), and Personal Access Token.\n", "\n", - "To generate a Personal Access Token, open the [MLflow UI](/mlflow) and navigate to [Permissions](/mlflow/oidc/ui/#)." + "To generate a Personal Access Token, open the [MLflow UI](/mlflow) and navigate to [Permissions](/mlflow/oidc/ui/user) and click \"Create Access Token\"." ] }, { From c3a3b05345724783565cf0763833fce550a00a21 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Tue, 16 Jun 2026 15:30:02 +0200 Subject: [PATCH 40/57] refactor: extract inference-protocols deploy into apply.py, notebook calls deploy() --- ci/run_all.py | 4 +- .../apply.py | 219 ++++++++++++++++++ .../inference_protocol_version_example.ipynb | 71 +----- 3 files changed, 226 insertions(+), 68 deletions(-) create mode 100644 serving/mlflow-kserve-inference-protocols/apply.py diff --git a/ci/run_all.py b/ci/run_all.py index 591b0e6..63fdce6 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -276,8 +276,8 @@ class Example: name="serving/mlflow-kserve-inference-protocols", steps=[ Step( - "notebook", - "serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb", + "script", + "serving/mlflow-kserve-inference-protocols/apply.py", extract_run_ids=False, ) ], diff --git a/serving/mlflow-kserve-inference-protocols/apply.py b/serving/mlflow-kserve-inference-protocols/apply.py new file mode 100644 index 0000000..6455bf8 --- /dev/null +++ b/serving/mlflow-kserve-inference-protocols/apply.py @@ -0,0 +1,219 @@ +""" +Deploy the v1 and v2 MLflow KServe InferenceServices and smoke-test both. + +Substitutes ```` and ```` in the YAML manifests from +the cluster environment, applies them, waits for readiness, and verifies that +both endpoints return identical predictions for the same input. + +Usage from a notebook +--------------------- + import sys + sys.path.insert(0, ".") # run from the serving/mlflow-kserve-inference-protocols/ dir + from apply import deploy + + v1_uri, v2_uri, api_key = deploy() + +CLI usage +--------- + python apply.py + +Prerequisites +------------- +- ``mlflow-credentials`` secret must exist (run scripts/setup_mlflow_credentials.py). +- The MLflow model ``mobile-price-svm-`` version 1 must be registered + (run mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb first). +""" + +from __future__ import annotations + +import base64 +import json +import os +import subprocess +import sys +import urllib.error +import urllib.request + +_HERE = os.path.dirname(__file__) + +_ISVC_NAMES = { + "v1": "v1-mobile-price-classification-inference", + "v2": "v2-mobile-price-classification-inference", +} +_YAML_FILES = { + "v1": os.path.join(_HERE, "v1-InferenceService.yaml"), + "v2": os.path.join(_HERE, "v2-InferenceService.yaml"), +} +_BODY_FILES = { + "v1": os.path.join(_HERE, "v1-mlflow-inference-body.json"), + "v2": os.path.join(_HERE, "v2-mlflow-inference-body.json"), +} + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + + +def _namespace() -> str: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: + return fh.read().strip() + + +def _mlflow_username(namespace: str) -> str: + result = subprocess.run( + [ + "kubectl", + "get", + "secret", + "mlflow-credentials", + "-n", + namespace, + "-o", + "json", + ], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError( + "mlflow-credentials secret not found.\n" + "Run scripts/setup_mlflow_credentials.py to create it." + ) + data = json.loads(result.stdout)["data"] + return base64.b64decode(data["MLFLOW_TRACKING_USERNAME"]).decode().split("@")[0] + + +def _apply_yaml(yaml_file: str, namespace: str, username: str) -> None: + manifest = open(yaml_file).read() + manifest = manifest.replace("", namespace).replace( + "", username + ) + result = subprocess.run( + ["kubectl", "apply", "-n", namespace, "-f", "-"], + input=manifest, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError( + f"kubectl apply failed for {os.path.basename(yaml_file)}:\n{result.stderr}" + ) + print(result.stdout.strip()) + + +def _wait_ready(name: str, namespace: str, timeout: int = 600) -> None: + print(f" Waiting for {name} (timeout {timeout}s)...") + result = subprocess.run( + [ + "kubectl", + "wait", + "inferenceservice", + name, + "-n", + namespace, + "--for=condition=Ready", + f"--timeout={timeout}s", + ], + capture_output=True, + text=True, + ) + if result.returncode != 0: + subprocess.run( + ["kubectl", "describe", "inferenceservice", name, "-n", namespace] + ) + raise RuntimeError(f"{name} did not become ready:\n{result.stderr}") + + +def _isvc_url(name: str, namespace: str) -> str: + return subprocess.check_output( + [ + "kubectl", + "get", + "inferenceservice", + name, + "-n", + namespace, + "-o", + "jsonpath={.status.url}", + ], + text=True, + ).strip() + + +def _get_api_key() -> str: + root = subprocess.check_output( + ["git", "rev-parse", "--show-toplevel"], text=True + ).strip() + sys.path.insert(0, os.path.join(root, "scripts")) + from get_or_create_api_key import get_or_create_api_key + + return get_or_create_api_key() + + +def _smoke_test(uri: str, name: str, protocol: str, api_key: str) -> None: + """POST one inference request; raise on non-2xx or missing predictions.""" + body = json.load(open(_BODY_FILES[protocol])) + if protocol == "v1": + url = f"{uri}/v1/models/{name}:predict" + pred_key = "predictions" + else: + url = f"{uri}/v2/models/{name}/infer" + pred_key = "outputs" + + data = json.dumps(body).encode() + req = urllib.request.Request( + url, + data=data, + headers={"Content-Type": "application/json", "X-Api-Key": api_key}, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=30) as resp: + result = json.loads(resp.read()) + except urllib.error.HTTPError as exc: + raise RuntimeError( + f"Smoke test {protocol} returned HTTP {exc.code}: {exc.read().decode()}" + ) from exc + + if pred_key not in result: + raise RuntimeError(f"Smoke test {protocol}: unexpected response: {result}") + print(f" [{protocol}] smoke test passed — {len(result[pred_key])} prediction(s)") + + +# ── Entry point ─────────────────────────────────────────────────────────────── + + +def deploy(timeout: int = 600) -> tuple[str, str, str]: + """Deploy both ISVCs, wait for readiness, and return (v1_uri, v2_uri, api_key).""" + ns = _namespace() + username = _mlflow_username(ns) + + print("Applying InferenceService manifests...") + for proto, yaml_file in _YAML_FILES.items(): + _apply_yaml(yaml_file, ns, username) + + print("Waiting for InferenceServices to become ready...") + for proto, name in _ISVC_NAMES.items(): + _wait_ready(name, ns, timeout) + + v1_uri = _isvc_url(_ISVC_NAMES["v1"], ns) + v2_uri = _isvc_url(_ISVC_NAMES["v2"], ns) + api_key = _get_api_key() + + print(f"v1 URI: {v1_uri}") + print(f"v2 URI: {v2_uri}") + + print("Running smoke tests...") + _smoke_test(v1_uri, _ISVC_NAMES["v1"], "v1", api_key) + _smoke_test(v2_uri, _ISVC_NAMES["v2"], "v2", api_key) + + return v1_uri, v2_uri, api_key + + +if __name__ == "__main__": + try: + v1_uri, v2_uri, api_key = deploy() + print(f"ISVC_URI_V1={v1_uri}") + print(f"ISVC_URI_V2={v2_uri}") + except Exception as exc: + print(str(exc), file=sys.stderr) + sys.exit(1) diff --git a/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb b/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb index 0d07609..710c07b 100644 --- a/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb +++ b/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb @@ -356,72 +356,11 @@ "metadata": {}, "outputs": [], "source": [ - "import base64, json, os, subprocess, sys\n", - "\n", - "# ── 1. Namespace & MLflow username ───────────────────────────────────────────\n", - "with open(\"/var/run/secrets/kubernetes.io/serviceaccount/namespace\") as _f:\n", - " _ns = _f.read().strip()\n", - "\n", - "_secret = subprocess.run(\n", - " [\"kubectl\", \"get\", \"secret\", \"mlflow-credentials\", \"-n\", _ns, \"-o\", \"json\"],\n", - " capture_output=True, text=True,\n", - ")\n", - "if _secret.returncode != 0:\n", - " raise RuntimeError(\n", - " \"mlflow-credentials secret not found.\\n\"\n", - " \"Run scripts/setup_mlflow_credentials.py to create it.\"\n", - " )\n", - "_creds = json.loads(_secret.stdout)[\"data\"]\n", - "_username = base64.b64decode(_creds[\"MLFLOW_TRACKING_USERNAME\"]).decode().split(\"@\")[0]\n", - "\n", - "# ── 2. Apply InferenceService YAMLs with substituted namespace & username ────\n", - "_isvc_names = [\n", - " (\"v1-InferenceService.yaml\", \"v1-mobile-price-classification-inference\"),\n", - " (\"v2-InferenceService.yaml\", \"v2-mobile-price-classification-inference\"),\n", - "]\n", - "for _yaml_file, _ in _isvc_names:\n", - " _manifest = open(_yaml_file).read()\n", - " _manifest = _manifest.replace(\"\", _ns).replace(\"\", _username)\n", - " _r = subprocess.run(\n", - " [\"kubectl\", \"apply\", \"-n\", _ns, \"-f\", \"-\"],\n", - " input=_manifest, capture_output=True, text=True,\n", - " )\n", - " if _r.returncode != 0:\n", - " raise RuntimeError(f\"Failed to apply {_yaml_file}:\\n{_r.stderr}\")\n", - " print(f\"Applied {_yaml_file}\")\n", - "\n", - "# ── 3. Wait for both ISVCs to be ready (up to 10 min each) ───────────────────\n", - "print(\"Waiting for InferenceServices to become ready...\")\n", - "for _, _name in _isvc_names:\n", - " _r = subprocess.run(\n", - " [\"kubectl\", \"wait\", \"inferenceservice\", _name, \"-n\", _ns,\n", - " \"--for=condition=Ready\", \"--timeout=600s\"],\n", - " capture_output=True, text=True,\n", - " )\n", - " if _r.returncode != 0:\n", - " raise RuntimeError(f\"InferenceService {_name} did not become ready:\\n{_r.stderr}\")\n", - " print(f\" {_name}: Ready\")\n", - "\n", - "# ── 4. Fetch external URLs from ISVC status ───────────────────────────────────\n", - "def _isvc_url(name):\n", - " return subprocess.check_output(\n", - " [\"kubectl\", \"get\", \"inferenceservice\", name, \"-n\", _ns,\n", - " \"-o\", \"jsonpath={.status.url}\"],\n", - " text=True,\n", - " ).strip()\n", - "\n", - "_uri_v1 = _isvc_url(\"v1-mobile-price-classification-inference\")\n", - "_uri_v2 = _isvc_url(\"v2-mobile-price-classification-inference\")\n", - "\n", - "# ── 5. Get / create inference API key ────────────────────────────────────────\n", - "_root = subprocess.check_output([\"git\", \"rev-parse\", \"--show-toplevel\"], text=True).strip()\n", - "sys.path.insert(0, os.path.join(_root, \"scripts\"))\n", - "from get_or_create_api_key import get_or_create_api_key\n", - "_api_key = get_or_create_api_key()\n", - "\n", - "print(f\"\\nv1 URI : {_uri_v1}\")\n", - "print(f\"v2 URI : {_uri_v2}\")\n", - "print(\"API key: [set]\")\n" + "import sys\n", + "sys.path.insert(0, \".\")\n", + "from apply import deploy\n", + "\n", + "_uri_v1, _uri_v2, _api_key = deploy()\n" ] }, { From 5c81be1eac6a5b5161b052dde2a82ccd5ede3bd6 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Tue, 16 Jun 2026 15:33:47 +0200 Subject: [PATCH 41/57] fix: remove hardcoded namespace from ServiceAccount YAMLs --- serving/mlflow-kserve-inference-protocols/README.md | 5 ++--- .../mlflow-kserve-inference-protocols/ServiceAccount.yaml | 1 - serving/mlflow-kserve-minimal/README.md | 7 ++----- serving/mlflow-kserve-minimal/ServiceAccount.yaml | 1 - 4 files changed, 4 insertions(+), 10 deletions(-) diff --git a/serving/mlflow-kserve-inference-protocols/README.md b/serving/mlflow-kserve-inference-protocols/README.md index 04feceb..7c14762 100644 --- a/serving/mlflow-kserve-inference-protocols/README.md +++ b/serving/mlflow-kserve-inference-protocols/README.md @@ -64,11 +64,10 @@ reference this SA. ## Deploy -1. Replace `` in `ServiceAccount.yaml` with your namespace - and apply it (once per namespace): +1. Apply the ServiceAccount (once per namespace): ```sh - kubectl apply -f ServiceAccount.yaml -n + kubectl apply -f ServiceAccount.yaml ``` 2. Replace the placeholder values in both ISVC YAMLs: diff --git a/serving/mlflow-kserve-inference-protocols/ServiceAccount.yaml b/serving/mlflow-kserve-inference-protocols/ServiceAccount.yaml index ae26f66..1f2b812 100644 --- a/serving/mlflow-kserve-inference-protocols/ServiceAccount.yaml +++ b/serving/mlflow-kserve-inference-protocols/ServiceAccount.yaml @@ -2,7 +2,6 @@ apiVersion: v1 kind: ServiceAccount metadata: name: mlflow-isvc-sa - namespace: "" # imagePullSecrets grant access to the private prokube container registry # where the custom MLflow storage-initializer image lives. # These secrets are provisioned automatically by the prokube platform. diff --git a/serving/mlflow-kserve-minimal/README.md b/serving/mlflow-kserve-minimal/README.md index cdd0e24..32465cb 100644 --- a/serving/mlflow-kserve-minimal/README.md +++ b/serving/mlflow-kserve-minimal/README.md @@ -54,15 +54,12 @@ Both `InferenceService.yaml` and `apply.py` already reference this SA. ## Deploy the InferenceService -1. Open `ServiceAccount.yaml` and replace `` with your - Kubeflow namespace, then apply it: +1. Apply the ServiceAccount (once per namespace): ```sh - kubectl apply -f ServiceAccount.yaml -n + kubectl apply -f ServiceAccount.yaml ``` - You only need to do this once per namespace. - 2. Open `InferenceService.yaml` and replace the placeholder values: | Placeholder | Description | diff --git a/serving/mlflow-kserve-minimal/ServiceAccount.yaml b/serving/mlflow-kserve-minimal/ServiceAccount.yaml index ae26f66..1f2b812 100644 --- a/serving/mlflow-kserve-minimal/ServiceAccount.yaml +++ b/serving/mlflow-kserve-minimal/ServiceAccount.yaml @@ -2,7 +2,6 @@ apiVersion: v1 kind: ServiceAccount metadata: name: mlflow-isvc-sa - namespace: "" # imagePullSecrets grant access to the private prokube container registry # where the custom MLflow storage-initializer image lives. # These secrets are provisioned automatically by the prokube platform. From efd4a9632adcffc616951d8eaeb1247c6988019e Mon Sep 17 00:00:00 2001 From: Korel Date: Thu, 18 Jun 2026 17:42:40 +0200 Subject: [PATCH 42/57] Update GitHub Actions versions in mobile-price-classification workflow * Bump `actions/checkout` from v4 to v7 * Bump `google-github-actions/auth` from v2 to v3 * Bump `docker/login-action` from v3 to v4 --- .../workflows/mobile-price-classification-components.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/mobile-price-classification-components.yaml b/.github/workflows/mobile-price-classification-components.yaml index 612d5fe..6c17a48 100644 --- a/.github/workflows/mobile-price-classification-components.yaml +++ b/.github/workflows/mobile-price-classification-components.yaml @@ -23,15 +23,15 @@ jobs: run: rm -rf /opt/hostedtoolcache - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Auth GCloud CLI - uses: google-github-actions/auth@v2 + uses: google-github-actions/auth@v3 with: credentials_json: ${{ env.IMAGE_PUSH_SECRET }} - name: Docker Login to Google Artifact Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: europe-west3-docker.pkg.dev username: _json_key From 4229ae6de66e7382bda40292301242068bf68535 Mon Sep 17 00:00:00 2001 From: Korel Date: Thu, 18 Jun 2026 18:41:15 +0200 Subject: [PATCH 43/57] Add optional --namespace flag to get_or_create_api_key script * Accept an optional `ns` parameter in `get_or_create_api_key()`, falling back to the auto-detected namespace when not provided * Add `argparse`-based CLI with `--namespace`/`-n` flag when the script is run directly * Allows callers to override the target Kubernetes namespace without changing the default behavior --- scripts/get_or_create_api_key.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/scripts/get_or_create_api_key.py b/scripts/get_or_create_api_key.py index a9a9528..a0085a2 100644 --- a/scripts/get_or_create_api_key.py +++ b/scripts/get_or_create_api_key.py @@ -22,6 +22,7 @@ from __future__ import annotations +import argparse import base64 import json import secrets @@ -123,9 +124,9 @@ def _create_key(namespace: str) -> str: return key_value -def get_or_create_api_key() -> str: +def get_or_create_api_key(ns=None) -> str: """Return the API key for the current namespace, creating it if needed.""" - ns = _namespace() + ns = ns or _namespace() if _key_exists(ns): return _read_key_from_secret(ns) return _create_key(ns) @@ -133,7 +134,14 @@ def get_or_create_api_key() -> str: if __name__ == "__main__": try: - print(get_or_create_api_key()) + argparser = argparse.ArgumentParser(description="Get or create a prokube API key") + argparser.add_argument( + "--namespace", + "-n", + help="The Kubernetes namespace to use (defaults to the pods namespace)", + ) + args = argparser.parse_args() + print(get_or_create_api_key(args.namespace)) except Exception as exc: # noqa: BLE001 print(f"ERROR: {exc}", file=sys.stderr) sys.exit(1) From e89ef2ceb0c22cec1dfd6b21744fb0d2e43db955 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Tue, 16 Jun 2026 17:43:04 +0200 Subject: [PATCH 44/57] fix: apply ServiceAccount in inference-protocols apply.py, drop dead SA namespace replace in minimal --- serving/mlflow-kserve-inference-protocols/apply.py | 4 ++++ serving/mlflow-kserve-minimal/apply.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/serving/mlflow-kserve-inference-protocols/apply.py b/serving/mlflow-kserve-inference-protocols/apply.py index 6455bf8..eb83970 100644 --- a/serving/mlflow-kserve-inference-protocols/apply.py +++ b/serving/mlflow-kserve-inference-protocols/apply.py @@ -36,6 +36,7 @@ _HERE = os.path.dirname(__file__) +_SA_YAML = os.path.join(_HERE, "ServiceAccount.yaml") _ISVC_NAMES = { "v1": "v1-mobile-price-classification-inference", "v2": "v2-mobile-price-classification-inference", @@ -187,6 +188,9 @@ def deploy(timeout: int = 600) -> tuple[str, str, str]: ns = _namespace() username = _mlflow_username(ns) + print("Applying ServiceAccount...") + _apply_yaml(_SA_YAML, ns, username) + print("Applying InferenceService manifests...") for proto, yaml_file in _YAML_FILES.items(): _apply_yaml(yaml_file, ns, username) diff --git a/serving/mlflow-kserve-minimal/apply.py b/serving/mlflow-kserve-minimal/apply.py index fd259b8..339185f 100644 --- a/serving/mlflow-kserve-minimal/apply.py +++ b/serving/mlflow-kserve-minimal/apply.py @@ -130,7 +130,7 @@ def deploy(timeout: int = 600) -> str: username = _mlflow_username(ns) with open(_SA_YAML_TEMPLATE) as fh: - sa_manifest = fh.read().replace("", ns) + sa_manifest = fh.read() _kubectl_apply(sa_manifest, ns) print(f"Applied ServiceAccount 'mlflow-isvc-sa' in namespace '{ns}'.") From e130b8bbe7a2b35b515c97f9b68ea2c26445c010 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Thu, 18 Jun 2026 19:24:42 +0200 Subject: [PATCH 45/57] feat: support older deployments in get_or_create_api_key Auto-detect the AIGatewayKey CRD; if absent (older clusters using a hardcoded EnvoyFilter), fall back to the INFERENCE_SERVICE_API_KEY env var injected by the admin, and finally to an interactive getpass prompt so notebooks degrade gracefully instead of crashing. --- scripts/get_or_create_api_key.py | 90 ++++++++++++++++++++++++++++---- 1 file changed, 80 insertions(+), 10 deletions(-) diff --git a/scripts/get_or_create_api_key.py b/scripts/get_or_create_api_key.py index a0085a2..2869b91 100644 --- a/scripts/get_or_create_api_key.py +++ b/scripts/get_or_create_api_key.py @@ -1,5 +1,5 @@ """ -Utility to create or retrieve a prokube AIGatewayKey for use in examples. +Utility to create or retrieve a prokube API key for use in examples. Usage from a notebook --------------------- @@ -15,9 +15,23 @@ python get_or_create_api_key.py # prints the key to stdout -The key is stored as an AIGatewayKey CR named ``examples-key`` backed by a -Secret named ``examples-key-secret`` in the notebook's namespace. Both -resources are created if they do not already exist. Re-running is safe. +Resolution order +---------------- +The function returns a usable model-serving API key, choosing the mechanism +that fits the deployment it runs on: + +1. **New deployments (AI Gateway).** If the ``AIGatewayKey`` CRD is present, + the key is stored as an AIGatewayKey CR named ``examples-key`` backed by a + Secret named ``examples-key-secret`` in the notebook's namespace. Both + resources are created if they do not already exist. Re-running is safe. + +2. **Older deployments (hardcoded EnvoyFilter).** Older clusters do not have + the AIGatewayKey CRD; instead an admin pre-provisions a single API key and + injects it into the notebook pod via the ``INFERENCE_SERVICE_API_KEY`` + environment variable. When the CRD is absent, that env var is used. + +3. **Fallback.** If neither is available, the caller is prompted to paste the + key interactively, so the example degrades gracefully instead of crashing. """ from __future__ import annotations @@ -25,13 +39,19 @@ import argparse import base64 import json +import os import secrets import subprocess import sys +from getpass import getpass _KEY_NAME = "examples-key" _SECRET_NAME = "examples-key-secret" +# Env var an admin injects into the notebook pod on older (pre-AI-Gateway) +# deployments that rely on a hardcoded EnvoyFilter for auth. +_API_KEY_ENV_VAR = "INFERENCE_SERVICE_API_KEY" + def _namespace() -> str: with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: @@ -47,6 +67,24 @@ def _kubectl(*args: str, input: str | None = None) -> subprocess.CompletedProces ) +def _aigatewaykey_crd_available() -> bool: + """Return True if the AIGatewayKey CRD is registered in the cluster. + + Used to distinguish new AI-Gateway deployments (CRD present) from older + deployments that rely on a hardcoded EnvoyFilter and an admin-provisioned + API key. + """ + result = _kubectl( + "get", + "crd", + "aigatewaykeys.prokube.ai", + "--ignore-not-found", + "-o", + "name", + ) + return result.returncode == 0 and bool(result.stdout.strip()) + + def _key_exists(namespace: str) -> bool: result = _kubectl( "get", "aigatewaykey", _KEY_NAME, "-n", namespace, "--ignore-not-found" @@ -124,17 +162,49 @@ def _create_key(namespace: str) -> str: return key_value +def _key_from_env() -> str | None: + """Return the admin-provisioned API key from the environment, if set.""" + value = os.environ.get(_API_KEY_ENV_VAR, "").strip() + return value or None + + def get_or_create_api_key(ns=None) -> str: - """Return the API key for the current namespace, creating it if needed.""" - ns = ns or _namespace() - if _key_exists(ns): - return _read_key_from_secret(ns) - return _create_key(ns) + """Return a model-serving API key for the current namespace. + + See the module docstring for the resolution order: AIGatewayKey CR on new + deployments, the ``INFERENCE_SERVICE_API_KEY`` env var on older ones, and an + interactive prompt as a last resort. + """ + # New deployments: manage an AIGatewayKey CR (fully automated). + if _aigatewaykey_crd_available(): + ns = ns or _namespace() + if _key_exists(ns): + return _read_key_from_secret(ns) + return _create_key(ns) + + # Older deployments: admin injects the key via an env var. + env_key = _key_from_env() + if env_key: + return env_key + + # Fallback: prompt instead of crashing. + key = getpass( + f"AIGatewayKey CRD not found and ${_API_KEY_ENV_VAR} is unset. " + "Please enter the model-serving API key (ask your cluster admin): " + ).strip() + if not key: + raise RuntimeError( + "No API key available: the AIGatewayKey CRD is absent, " + f"${_API_KEY_ENV_VAR} is unset, and no key was entered." + ) + return key if __name__ == "__main__": try: - argparser = argparse.ArgumentParser(description="Get or create a prokube API key") + argparser = argparse.ArgumentParser( + description="Get or create a prokube API key" + ) argparser.add_argument( "--namespace", "-n", From b0907e85cd382445319e44f54ed59672c6badb69 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Thu, 18 Jun 2026 19:29:39 +0200 Subject: [PATCH 46/57] fix: detect AIGatewayKey CRD via api-resources, not get crd Notebook pod service accounts lack RBAC permission to read cluster-scoped CRD objects, so the previous 'kubectl get crd' check always returned False in-cluster. API discovery (kubectl api-resources) is allowed for all authenticated users and reliably detects whether the CRD is registered. --- scripts/get_or_create_api_key.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/get_or_create_api_key.py b/scripts/get_or_create_api_key.py index 2869b91..e2eb076 100644 --- a/scripts/get_or_create_api_key.py +++ b/scripts/get_or_create_api_key.py @@ -70,19 +70,19 @@ def _kubectl(*args: str, input: str | None = None) -> subprocess.CompletedProces def _aigatewaykey_crd_available() -> bool: """Return True if the AIGatewayKey CRD is registered in the cluster. - Used to distinguish new AI-Gateway deployments (CRD present) from older - deployments that rely on a hardcoded EnvoyFilter and an admin-provisioned - API key. + Uses the API discovery endpoint (kubectl api-resources) rather than + directly reading the CRD object, because notebook pod service accounts + typically lack RBAC permission to get cluster-scoped CRD resources. + API discovery is allowed for all authenticated Kubernetes users. """ result = _kubectl( - "get", - "crd", - "aigatewaykeys.prokube.ai", - "--ignore-not-found", + "api-resources", + "--api-group=prokube.ai", "-o", "name", + "--no-headers", ) - return result.returncode == 0 and bool(result.stdout.strip()) + return result.returncode == 0 and "aigatewaykeys" in result.stdout def _key_exists(namespace: str) -> bool: From 463b2ab1165097d7f041559b8ddb6f175c9f2cfb Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Thu, 18 Jun 2026 19:38:34 +0200 Subject: [PATCH 47/57] docs: add tip about auto-created examples key and manual override in s3 notebook --- serving/minimal-s3-model/minimal-s3-model.ipynb | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/serving/minimal-s3-model/minimal-s3-model.ipynb b/serving/minimal-s3-model/minimal-s3-model.ipynb index f1b5e91..df52e86 100644 --- a/serving/minimal-s3-model/minimal-s3-model.ipynb +++ b/serving/minimal-s3-model/minimal-s3-model.ipynb @@ -297,7 +297,21 @@ "id": "2df5cbae-f406-4c33-be90-dbb0b61cf81d", "metadata": {}, "source": [ - "Enter the model serving API key in the cell below. If not known, ask the cluster administrator for it." + "
\n", + " 💡 Tip:\n", + "
\n", + " The cell below automatically creates a dedicated examples-key AIGatewayKey and its backing Secret in your namespace — no manual steps required.\n", + "

\n", + " If you'd rather use your own key, replace the cell contents with:\n", + " 
INFERENCE_SERVICE_API_KEY = \"your-key-here\"
\n", + "
\n", + "
" ] }, { From 86a2f4b8f711199fb9940748388c1d1556dc67ac Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Thu, 18 Jun 2026 19:39:51 +0200 Subject: [PATCH 48/57] docs: document auto-created examples-key and manual override in serving READMEs --- .../mlflow-kserve-inference-protocols/README.md | 9 +++++++++ serving/mlflow-kserve-minimal/README.md | 14 +++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/serving/mlflow-kserve-inference-protocols/README.md b/serving/mlflow-kserve-inference-protocols/README.md index 7c14762..ddea2de 100644 --- a/serving/mlflow-kserve-inference-protocols/README.md +++ b/serving/mlflow-kserve-inference-protocols/README.md @@ -62,6 +62,15 @@ exactly that injection and work correctly with the `default` SA. Both `v1-InferenceService.yaml` and `v2-InferenceService.yaml` already reference this SA. +## API key + +> [!TIP] +> `apply.py` (and the notebook's `deploy()` call) automatically creates a +> dedicated `examples-key` AIGatewayKey and its backing Secret +> (`examples-key-secret`) in your namespace. If you'd rather use your own key, +> pass it to the smoke-test step directly or set `API_KEY` in the environment +> before calling `apply.py`. + ## Deploy 1. Apply the ServiceAccount (once per namespace): diff --git a/serving/mlflow-kserve-minimal/README.md b/serving/mlflow-kserve-minimal/README.md index 32465cb..2a8ac2a 100644 --- a/serving/mlflow-kserve-minimal/README.md +++ b/serving/mlflow-kserve-minimal/README.md @@ -95,14 +95,22 @@ Alternatively, `apply.py` handles steps 1–4 automatically (see below). A test script and sample request body are provided to verify the deployment. +> [!TIP] +> `apply.py` automatically creates a dedicated `examples-key` AIGatewayKey and +> its backing Secret (`examples-key-secret`) in your namespace — no manual key +> management required. If you'd rather supply your own key, set `API_KEY` +> before running the test script: +> ```sh +> export API_KEY=your-key-here +> ``` + 1. Set the required environment variables (optional): ```sh - export API_KEY= + export API_KEY= # auto-created by apply.py if omitted export INFERENCE_SERVICE_URI= export PROTOCOL_VERSION=v2 ``` - You can find the inference service URL in the Kubeflow Endpoints UI. If you - don't know your API key, reach out to your prokube admin. + You can find the inference service URL in the Kubeflow Endpoints UI. 2. Run the test script: ```sh From 1724100204917bc367f8bd8154f3b0054ba2a17f Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Wed, 5 Aug 2026 10:18:22 +0200 Subject: [PATCH 49/57] remove: drop AIGatewayKey CRD / AI Gateway auth, use env var or manual key entry --- ci/README.md | 7 +- scripts/get_or_create_api_key.py | 183 +++--------------- serving/minimal-s3-model/cleanup.py | 17 +- .../minimal-s3-model/minimal-s3-model.ipynb | 4 +- .../README.md | 9 +- serving/mlflow-kserve-minimal/README.md | 8 +- 6 files changed, 42 insertions(+), 186 deletions(-) diff --git a/ci/README.md b/ci/README.md index 73c697d..393fafa 100644 --- a/ci/README.md +++ b/ci/README.md @@ -114,9 +114,10 @@ check and skips MLflow-dependent examples if it does not. ### get_or_create_api_key.py -Idempotent, no human input. Creates or retrieves the `examples-key` -AIGatewayKey CR. Call it from any notebook cell or script that needs an -inference API key: +Returns a model-serving API key: the `INFERENCE_SERVICE_API_KEY` env var if +an admin has injected one into the pod, otherwise an interactive prompt +(ask your cluster admin, or use pkui if available on your platform). Call it +from any notebook cell or script that needs an inference API key: ```python import sys, subprocess, os diff --git a/scripts/get_or_create_api_key.py b/scripts/get_or_create_api_key.py index e2eb076..4a12492 100644 --- a/scripts/get_or_create_api_key.py +++ b/scripts/get_or_create_api_key.py @@ -1,5 +1,5 @@ """ -Utility to create or retrieve a prokube API key for use in examples. +Utility to obtain a prokube model-serving API key for use in examples. Usage from a notebook --------------------- @@ -17,185 +17,57 @@ Resolution order ---------------- -The function returns a usable model-serving API key, choosing the mechanism -that fits the deployment it runs on: +There is no automated way to provision a model-serving API key from within a +notebook or script. This helper only looks for a key that has already been +made available in the environment, and otherwise prompts for one: -1. **New deployments (AI Gateway).** If the ``AIGatewayKey`` CRD is present, - the key is stored as an AIGatewayKey CR named ``examples-key`` backed by a - Secret named ``examples-key-secret`` in the notebook's namespace. Both - resources are created if they do not already exist. Re-running is safe. +1. **Environment variable.** If an admin has pre-provisioned a key and + injected it into the notebook pod via the ``INFERENCE_SERVICE_API_KEY`` + environment variable, that value is used. -2. **Older deployments (hardcoded EnvoyFilter).** Older clusters do not have - the AIGatewayKey CRD; instead an admin pre-provisions a single API key and - injects it into the notebook pod via the ``INFERENCE_SERVICE_API_KEY`` - environment variable. When the CRD is absent, that env var is used. - -3. **Fallback.** If neither is available, the caller is prompted to paste the - key interactively, so the example degrades gracefully instead of crashing. +2. **Interactive prompt.** Otherwise, the caller is prompted to paste a key. + Obtain a key from your cluster administrator, or via the prokube UI + (``pkui``) if it is available on your platform. """ from __future__ import annotations import argparse -import base64 -import json import os -import secrets -import subprocess import sys from getpass import getpass -_KEY_NAME = "examples-key" -_SECRET_NAME = "examples-key-secret" - -# Env var an admin injects into the notebook pod on older (pre-AI-Gateway) -# deployments that rely on a hardcoded EnvoyFilter for auth. +# Env var an admin injects into the notebook pod with a pre-provisioned +# model-serving API key. _API_KEY_ENV_VAR = "INFERENCE_SERVICE_API_KEY" -def _namespace() -> str: - with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as fh: - return fh.read().strip() - - -def _kubectl(*args: str, input: str | None = None) -> subprocess.CompletedProcess: - return subprocess.run( - ["kubectl", *args], - input=input, - capture_output=True, - text=True, - ) - - -def _aigatewaykey_crd_available() -> bool: - """Return True if the AIGatewayKey CRD is registered in the cluster. - - Uses the API discovery endpoint (kubectl api-resources) rather than - directly reading the CRD object, because notebook pod service accounts - typically lack RBAC permission to get cluster-scoped CRD resources. - API discovery is allowed for all authenticated Kubernetes users. - """ - result = _kubectl( - "api-resources", - "--api-group=prokube.ai", - "-o", - "name", - "--no-headers", - ) - return result.returncode == 0 and "aigatewaykeys" in result.stdout - - -def _key_exists(namespace: str) -> bool: - result = _kubectl( - "get", "aigatewaykey", _KEY_NAME, "-n", namespace, "--ignore-not-found" - ) - return bool(result.stdout.strip()) - - -def _read_key_from_secret(namespace: str) -> str: - result = _kubectl("get", "secret", _SECRET_NAME, "-n", namespace, "-o", "json") - if result.returncode != 0: - raise RuntimeError(f"Could not read secret {_SECRET_NAME}:\n{result.stderr}") - data = json.loads(result.stdout)["data"] - return base64.b64decode(data["token"]).decode() - - -def _apply(manifest: str, namespace: str) -> None: - result = _kubectl("apply", "-n", namespace, "-f", "-", input=manifest) - if result.returncode != 0: - raise RuntimeError(f"kubectl apply failed:\n{result.stderr}") - - -def _owner(namespace: str) -> str: - """Determine the key owner. - - Tries to read MLFLOW_TRACKING_USERNAME from the mlflow-credentials secret - (the canonical identity in the prokube platform). Falls back to a - namespace-scoped placeholder if the secret is absent. - """ - result = _kubectl( - "get", - "secret", - "mlflow-credentials", - "-n", - namespace, - "-o", - "jsonpath={.data.MLFLOW_TRACKING_USERNAME}", - ) - if result.returncode == 0 and result.stdout.strip(): - return base64.b64decode(result.stdout.strip()).decode() - return f"notebook-examples@{namespace}" - - -def _create_key(namespace: str) -> str: - key_value = f"pk_live_{secrets.token_hex(20)}" - owner = _owner(namespace) - - _apply( - f"apiVersion: v1\n" - f"kind: Secret\n" - f"metadata:\n" - f" name: {_SECRET_NAME}\n" - f" namespace: {namespace}\n" - f"stringData:\n" - f" token: {key_value}\n", - namespace, - ) - - _apply( - f"apiVersion: prokube.ai/v1alpha1\n" - f"kind: AIGatewayKey\n" - f"metadata:\n" - f" name: {_KEY_NAME}\n" - f" namespace: {namespace}\n" - f"spec:\n" - f" displayName: Examples API key\n" - f" owner: {owner}\n" - f" secretRef:\n" - f" name: {_SECRET_NAME}\n" - f" key: token\n" - f" scopes:\n" - f" - type: workspace\n", - namespace, - ) - - return key_value - - def _key_from_env() -> str | None: """Return the admin-provisioned API key from the environment, if set.""" value = os.environ.get(_API_KEY_ENV_VAR, "").strip() return value or None -def get_or_create_api_key(ns=None) -> str: - """Return a model-serving API key for the current namespace. +def get_or_create_api_key() -> str: + """Return a model-serving API key. - See the module docstring for the resolution order: AIGatewayKey CR on new - deployments, the ``INFERENCE_SERVICE_API_KEY`` env var on older ones, and an - interactive prompt as a last resort. + See the module docstring for the resolution order: the + ``INFERENCE_SERVICE_API_KEY`` env var if set, otherwise an interactive + prompt. """ - # New deployments: manage an AIGatewayKey CR (fully automated). - if _aigatewaykey_crd_available(): - ns = ns or _namespace() - if _key_exists(ns): - return _read_key_from_secret(ns) - return _create_key(ns) - - # Older deployments: admin injects the key via an env var. env_key = _key_from_env() if env_key: return env_key - # Fallback: prompt instead of crashing. key = getpass( - f"AIGatewayKey CRD not found and ${_API_KEY_ENV_VAR} is unset. " - "Please enter the model-serving API key (ask your cluster admin): " + f"${_API_KEY_ENV_VAR} is unset. Please enter your model-serving API " + "key (ask your cluster admin, or use pkui if available on your " + "platform): " ).strip() if not key: raise RuntimeError( - "No API key available: the AIGatewayKey CRD is absent, " - f"${_API_KEY_ENV_VAR} is unset, and no key was entered." + f"No API key available: ${_API_KEY_ENV_VAR} is unset and no key " + "was entered." ) return key @@ -203,15 +75,10 @@ def get_or_create_api_key(ns=None) -> str: if __name__ == "__main__": try: argparser = argparse.ArgumentParser( - description="Get or create a prokube API key" - ) - argparser.add_argument( - "--namespace", - "-n", - help="The Kubernetes namespace to use (defaults to the pods namespace)", + description="Get a prokube model-serving API key" ) - args = argparser.parse_args() - print(get_or_create_api_key(args.namespace)) + argparser.parse_args() + print(get_or_create_api_key()) except Exception as exc: # noqa: BLE001 print(f"ERROR: {exc}", file=sys.stderr) sys.exit(1) diff --git a/serving/minimal-s3-model/cleanup.py b/serving/minimal-s3-model/cleanup.py index 9059dd3..0508e16 100644 --- a/serving/minimal-s3-model/cleanup.py +++ b/serving/minimal-s3-model/cleanup.py @@ -3,12 +3,10 @@ Deletes (idempotently): - InferenceService ``kserve-object-storage-test`` - - AIGatewayKey ``examples-key`` and its backing Secret ``examples-key-secret`` - (only when --include-api-key is passed, since the key is shared across examples) Usage ----- - python cleanup.py [--include-api-key] [--dry-run] + python cleanup.py [--dry-run] """ from __future__ import annotations @@ -35,29 +33,20 @@ def _kubectl_delete(*args: str, dry_run: bool = False) -> None: print(result.stdout.strip() or f"deleted (or not found): {' '.join(args)}") -def cleanup(include_api_key: bool = False, dry_run: bool = False) -> None: +def cleanup(dry_run: bool = False) -> None: ns = _namespace() _kubectl_delete( "inferenceservice", "kserve-object-storage-test", "-n", ns, dry_run=dry_run ) - if include_api_key: - _kubectl_delete("aigatewaykey", "examples-key", "-n", ns, dry_run=dry_run) - _kubectl_delete("secret", "examples-key-secret", "-n", ns, dry_run=dry_run) - if __name__ == "__main__": parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) - parser.add_argument( - "--include-api-key", - action="store_true", - help="Also delete the shared AIGatewayKey", - ) parser.add_argument( "--dry-run", action="store_true", help="Print commands without executing them" ) args = parser.parse_args() - cleanup(include_api_key=args.include_api_key, dry_run=args.dry_run) + cleanup(dry_run=args.dry_run) diff --git a/serving/minimal-s3-model/minimal-s3-model.ipynb b/serving/minimal-s3-model/minimal-s3-model.ipynb index df52e86..2117185 100644 --- a/serving/minimal-s3-model/minimal-s3-model.ipynb +++ b/serving/minimal-s3-model/minimal-s3-model.ipynb @@ -306,9 +306,9 @@ "\">\n", " 💡 Tip:\n", "
\n", - " The cell below automatically creates a dedicated examples-key AIGatewayKey and its backing Secret in your namespace — no manual steps required.\n", + " The cell below prompts for your model-serving API key (or reads it from the INFERENCE_SERVICE_API_KEY env var if your admin already injected one). Ask your cluster administrator for a key, or use pkui if it is available on your platform.\n", "

\n", - " If you'd rather use your own key, replace the cell contents with:\n", + " If you'd rather set it directly, replace the cell contents with:\n", " 
INFERENCE_SERVICE_API_KEY = \"your-key-here\"
\n", "
\n", "" diff --git a/serving/mlflow-kserve-inference-protocols/README.md b/serving/mlflow-kserve-inference-protocols/README.md index ddea2de..e1ccb14 100644 --- a/serving/mlflow-kserve-inference-protocols/README.md +++ b/serving/mlflow-kserve-inference-protocols/README.md @@ -65,11 +65,10 @@ reference this SA. ## API key > [!TIP] -> `apply.py` (and the notebook's `deploy()` call) automatically creates a -> dedicated `examples-key` AIGatewayKey and its backing Secret -> (`examples-key-secret`) in your namespace. If you'd rather use your own key, -> pass it to the smoke-test step directly or set `API_KEY` in the environment -> before calling `apply.py`. +> `apply.py` (and the notebook's `deploy()` call) need a model-serving API +> key. Ask your cluster administrator for one, or use pkui if it is +> available on your platform. Set `API_KEY` in the environment before +> calling `apply.py`, or you'll be prompted for it interactively. ## Deploy diff --git a/serving/mlflow-kserve-minimal/README.md b/serving/mlflow-kserve-minimal/README.md index 2a8ac2a..3889eaf 100644 --- a/serving/mlflow-kserve-minimal/README.md +++ b/serving/mlflow-kserve-minimal/README.md @@ -96,17 +96,17 @@ Alternatively, `apply.py` handles steps 1–4 automatically (see below). A test script and sample request body are provided to verify the deployment. > [!TIP] -> `apply.py` automatically creates a dedicated `examples-key` AIGatewayKey and -> its backing Secret (`examples-key-secret`) in your namespace — no manual key -> management required. If you'd rather supply your own key, set `API_KEY` +> `apply.py` needs a model-serving API key. Ask your cluster administrator +> for one, or use pkui if it is available on your platform, then set it > before running the test script: > ```sh > export API_KEY=your-key-here > ``` +> If `API_KEY` is not set, `apply.py` will prompt for it interactively. 1. Set the required environment variables (optional): ```sh - export API_KEY= # auto-created by apply.py if omitted + export API_KEY= # ask your admin, or use pkui if available export INFERENCE_SERVICE_URI= export PROTOCOL_VERSION=v2 ``` From 49b9e61cb3da390981d1fd25c4e09bc7553954ca Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Wed, 5 Aug 2026 10:50:09 +0200 Subject: [PATCH 50/57] feat: add INFERENCE_SERVICE_API_KEY preflight check, skip api-key-dependent examples --- ci/README.md | 15 +++++++++++++- ci/run_all.py | 57 +++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 65 insertions(+), 7 deletions(-) diff --git a/ci/README.md b/ci/README.md index 393fafa..70b8e25 100644 --- a/ci/README.md +++ b/ci/README.md @@ -10,7 +10,7 @@ Register it in the `_EXAMPLES` list in `run_all.py`. Everything else — phase scheduling, opt-in gating, cleanup, dry-run output, MLflow-credential -skipping — is derived from that entry automatically. +and API-key skipping — is derived from that entry automatically. ```python Example( @@ -24,6 +24,7 @@ Example( cleanup="serving/my-new-example/cleanup.py", # omit if no K8s resources opt_in="include_foo", # omit if always enabled mlflow_dependent=False, # True = skip when MLflow creds absent + api_key_dependent=False, # True = skip when INFERENCE_SERVICE_API_KEY is unset ) ``` @@ -128,6 +129,18 @@ from get_or_create_api_key import get_or_create_api_key API_KEY = get_or_create_api_key() ``` +CI runs headlessly and cannot answer the interactive prompt, so +`INFERENCE_SERVICE_API_KEY` **must** be exported before running +`ci/run_all.py`: + +```bash +export INFERENCE_SERVICE_API_KEY= # ask your admin, or use pkui if available +``` + +CI validates this in the preflight check and skips `api_key_dependent` +examples if it is unset — mark any new example that calls +`get_or_create_api_key()` with `api_key_dependent=True`. + --- ## ci-skip cell tag diff --git a/ci/run_all.py b/ci/run_all.py index 63fdce6..6a5da7b 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -39,12 +39,17 @@ ------------- pip install papermill kfp python scripts/setup_mlflow_credentials.py + export INFERENCE_SERVICE_API_KEY= # ask your + # cluster admin, or use pkui if available on your platform. Examples + # that call get_or_create_api_key() are skipped if this is unset, + # since CI runs headlessly and cannot prompt for it interactively. """ from __future__ import annotations import argparse import json +import os import re import subprocess import sys @@ -123,6 +128,9 @@ class Example: mlflow_dependent: bool = ( False # skip automatically when MLflow credentials are unavailable ) + api_key_dependent: bool = ( + False # skip automatically when INFERENCE_SERVICE_API_KEY is unset + ) # ───────────────────────────────────────────────────────────────────────────── @@ -169,6 +177,7 @@ class Example: steps=[Step("notebook", "serving/minimal-s3-model/minimal-s3-model.ipynb")], phase=1, cleanup="serving/minimal-s3-model/cleanup.py", + api_key_dependent=True, ), Example( name="serving/hf-vllm-completion", @@ -270,6 +279,7 @@ class Example: ], phase=3, mlflow_dependent=True, + api_key_dependent=True, cleanup="serving/mlflow-kserve-minimal/cleanup.py", ), Example( @@ -283,6 +293,7 @@ class Example: ], phase=3, mlflow_dependent=True, + api_key_dependent=True, cleanup="serving/mlflow-kserve-inference-protocols/cleanup.py", ), ] @@ -661,6 +672,8 @@ def _print_dry_run() -> None: label += f" (--{ex.opt_in.replace('_', '-')})" if ex.mlflow_dependent: label += " (mlflow)" + if ex.api_key_dependent: + label += " (api-key)" by_phase.setdefault(ex.phase, []).append(label) phase_names = { 1: "independent", @@ -730,22 +743,54 @@ def _check_mlflow_credentials() -> tuple[bool, str]: return False, f"Could not reach MLflow at {uri}: {exc}" +def _check_kserve_api_key() -> tuple[bool, str]: + """Return (ok, reason). Checks that INFERENCE_SERVICE_API_KEY is set. + + CI runs headlessly, so get_or_create_api_key() cannot fall back to its + interactive getpass() prompt — the env var must be pre-populated. Ask + your cluster admin for a model-serving API key, or use pkui if it is + available on your platform. + """ + if os.environ.get("INFERENCE_SERVICE_API_KEY", "").strip(): + return True, "OK" + return False, ( + "INFERENCE_SERVICE_API_KEY is unset — export a model-serving API key " + "(ask your cluster admin, or use pkui if available) before running CI" + ) + + def _preflight(results: dict[str, Result]) -> bool: - """Install papermill and validate MLflow credentials. + """Install papermill and validate MLflow credentials and the kserve API key. - MLflow-dependent examples are recorded as SKIP in `results` on failure. - Returns True if MLflow credentials are valid. + MLflow-dependent and API-key-dependent examples are recorded as SKIP in + `results` on failure. Returns True if MLflow credentials are valid. """ _ensure_papermill() + print("Pre-flight: checking MLflow credentials...") - mlflow_ok, reason = _check_mlflow_credentials() + mlflow_ok, mlflow_reason = _check_mlflow_credentials() if mlflow_ok: print(" [OK] MLflow credentials valid") else: - print(f" [SKIP] {reason}") + print(f" [SKIP] {mlflow_reason}") for ex in _EXAMPLES: if ex.mlflow_dependent: - results[ex.name] = Result(name=ex.name, status="SKIP", error=reason) + results[ex.name] = Result( + name=ex.name, status="SKIP", error=mlflow_reason + ) + + print("Pre-flight: checking kserve API key...") + api_key_ok, api_key_reason = _check_kserve_api_key() + if api_key_ok: + print(" [OK] INFERENCE_SERVICE_API_KEY is set") + else: + print(f" [SKIP] {api_key_reason}") + for ex in _EXAMPLES: + if ex.api_key_dependent and ex.name not in results: + results[ex.name] = Result( + name=ex.name, status="SKIP", error=api_key_reason + ) + return mlflow_ok From 639a279bcec898d37be06ce0f9e2eaa2d6f6b2bd Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Thu, 6 Aug 2026 09:59:02 +0200 Subject: [PATCH 51/57] fix: serialize env-mutating notebooks in Phase 1, retry internal predict call for pod-readiness race --- ci/README.md | 25 ++++++++++- ci/run_all.py | 42 +++++++++++++++++-- .../minimal-s3-model/minimal-s3-model.ipynb | 26 +++++++++--- 3 files changed, 83 insertions(+), 10 deletions(-) diff --git a/ci/README.md b/ci/README.md index 70b8e25..35f2ec1 100644 --- a/ci/README.md +++ b/ci/README.md @@ -10,7 +10,8 @@ Register it in the `_EXAMPLES` list in `run_all.py`. Everything else — phase scheduling, opt-in gating, cleanup, dry-run output, MLflow-credential -and API-key skipping — is derived from that entry automatically. +and API-key skipping, env-mutating ordering — is derived from that entry +automatically. ```python Example( @@ -25,9 +26,31 @@ Example( opt_in="include_foo", # omit if always enabled mlflow_dependent=False, # True = skip when MLflow creds absent api_key_dependent=False, # True = skip when INFERENCE_SERVICE_API_KEY is unset + env_mutating=False, # True = pip installs/upgrades packages; + # runs before the rest of its phase (see below) ) ``` +### env_mutating flag + +Every notebook in a phase is executed via papermill against the **same** +`python3` kernel/site-packages — there is no per-notebook virtualenv. If a +notebook runs `pip install --upgrade ` while another notebook in the +same phase is concurrently importing that package, the concurrent +reinstall can corrupt the import (e.g. `ModuleNotFoundError: +No module named 'pandas._libs.internals'` from a partially-replaced +compiled extension). + +Set `env_mutating=True` on any example whose steps run `pip install` / +`%pip install` (uncommented, not `-q`-only-metadata, actually mutating +installed packages) against packages that other examples in the same +phase might import. `run_all.py` runs all `env_mutating` examples in a +phase to completion **before** starting the rest of that phase, instead of +throwing everything into the same parallel batch. Prefer avoiding +`pip install --upgrade` in new examples entirely (pin/bake deps into the +notebook image) — reach for `env_mutating=True` only when that isn't +possible. + ### Phase rules | Phase | When to use | diff --git a/ci/run_all.py b/ci/run_all.py index 6a5da7b..24b1c15 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -3,8 +3,15 @@ Execution model --------------- +Phase 1a (sequential, env-mutating): + Phase-1 examples with env_mutating=True (e.g. ones that `pip install + --upgrade` into the shared notebook kernel env), run to completion one + at a time before Phase 1 starts, to avoid a concurrent pip install + corrupting another notebook's in-progress import of the same package. + Phase 1 (parallel, self-contained): - All examples whose phase=1 in _EXAMPLES, including opt-in ones. + All remaining examples whose phase=1 in _EXAMPLES, including opt-in + ones. Phase 2 (parallel, pipeline submissions — return fast): All examples whose phase=2 in _EXAMPLES. @@ -131,6 +138,16 @@ class Example: api_key_dependent: bool = ( False # skip automatically when INFERENCE_SERVICE_API_KEY is unset ) + env_mutating: bool = ( + False # True = pip installs/upgrades packages into the shared + # notebook kernel environment. All papermill notebooks execute + # against the same site-packages (kernel_name="python3"), so an + # env_mutating example must finish before other same-phase + # examples start, or a concurrent `pip install` can corrupt an + # in-progress import in another notebook (e.g. partially replaced + # compiled extension modules). Run first within its phase, not + # concurrently with the rest. + ) # ───────────────────────────────────────────────────────────────────────────── @@ -143,6 +160,7 @@ class Example: steps=[Step("notebook", "notebooks/dask/dask_example.ipynb")], phase=1, cleanup="notebooks/dask/cleanup.py", + env_mutating=True, ), Example( name="notebooks/mobile-price-classification", @@ -674,6 +692,8 @@ def _print_dry_run() -> None: label += " (mlflow)" if ex.api_key_dependent: label += " (api-key)" + if ex.env_mutating: + label += " (env-mutating, runs first in its phase)" by_phase.setdefault(ex.phase, []).append(label) phase_names = { 1: "independent", @@ -801,12 +821,19 @@ def _run_phase( ctx: _Context, phase: int, opts: dict[str, bool], + predicate: Callable[[Example], bool] | None = None, ) -> dict[str, Future]: - """Submit all examples for a given phase; return {name: Future}.""" + """Submit all examples for a given phase; return {name: Future}. + + If `predicate` is given, only examples for which it returns True are + submitted (used to run env_mutating examples in their own sub-step). + """ futures: dict[str, Future] = {} for ex in _EXAMPLES: if ex.phase != phase: continue + if predicate is not None and not predicate(ex): + continue if ex.name in ctx.results: # already SKIP from pre-flight print(f" [SKIP ] {ex.name}") continue @@ -930,8 +957,17 @@ def run_all( poll_errors=poll_errors, ) + print( + "Phase 1a: running environment-mutating examples " + "(pip install into the shared kernel env) sequentially first..." + ) + _drain(ctx, _run_phase(ctx, 1, opts, predicate=lambda ex: ex.env_mutating)) + print("Phase 1: running independent examples in parallel...") - _drain(ctx, _run_phase(ctx, 1, opts)) + _drain( + ctx, + _run_phase(ctx, 1, opts, predicate=lambda ex: not ex.env_mutating), + ) print("\nPhase 2: submitting pipelines...") phase2_futures = _run_phase(ctx, 2, opts) diff --git a/serving/minimal-s3-model/minimal-s3-model.ipynb b/serving/minimal-s3-model/minimal-s3-model.ipynb index 2117185..3eee6ed 100644 --- a/serving/minimal-s3-model/minimal-s3-model.ipynb +++ b/serving/minimal-s3-model/minimal-s3-model.ipynb @@ -274,14 +274,28 @@ "metadata": {}, "outputs": [], "source": [ + "import time\n", "import requests\n", "isvc_internal_url = f\"{inference_service_name}-predictor.{namespace}\" # KServe internal endpoint is always built like this\n", - "response = requests.post(\n", - " f\"http://{isvc_internal_url}/v1/models/{inference_service_name}:predict\",\n", - " # an iris instance is [sepal_length, sepal_width, petal_length, petal_width]\n", - " json={\"instances\": [[6.8, 2.8, 4.8, 1.4], [5.1, 3.5, 1.4, 0.2]]}\n", - ")\n", - "print(response.json())" + "predict_url = f\"http://{isvc_internal_url}/v1/models/{inference_service_name}:predict\"\n", + "# an iris instance is [sepal_length, sepal_width, petal_length, petal_width]\n", + "payload = {\"instances\": [[6.8, 2.8, 4.8, 1.4], [5.1, 3.5, 1.4, 0.2]]}\n", + "\n", + "# `kubectl wait --for=condition=ready` above only checks the\n", + "# InferenceService CR; the predictor Pod can take a few more seconds to\n", + "# register as a ready Service endpoint, so retry briefly instead of\n", + "# failing on the very first request.\n", + "for attempt in range(6):\n", + " try:\n", + " response = requests.post(predict_url, json=payload, timeout=10)\n", + " response.raise_for_status()\n", + " print(response.json())\n", + " break\n", + " except Exception as exc:\n", + " if attempt == 5:\n", + " raise\n", + " print(f\" attempt {attempt + 1}/6 failed ({exc}); retrying in 5s...\")\n", + " time.sleep(5)" ] }, { From 70ef8063bca2d041d95c4f0feb04152f4970b219 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Thu, 6 Aug 2026 10:00:58 +0200 Subject: [PATCH 52/57] fix: drop unnecessary pandas pin from dask notebook pip install --- notebooks/dask/dask_example.ipynb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/notebooks/dask/dask_example.ipynb b/notebooks/dask/dask_example.ipynb index b8ac14b..6aec986 100644 --- a/notebooks/dask/dask_example.ipynb +++ b/notebooks/dask/dask_example.ipynb @@ -40,7 +40,10 @@ }, "outputs": [], "source": [ - "!pip install --upgrade ipywidgets \"dask-kubernetes==2026.3.0\" pandas \"dask[complete]==2026.3.0\" \"anyio>=4.12\"\n" + "# pandas is a transitive dependency of dask[complete] already, no need to\n", + "# force-upgrade it separately (that was needlessly bumping pandas to the\n", + "# latest release and mutating the shared notebook kernel env).\n", + "!pip install --upgrade ipywidgets \"dask-kubernetes==2026.3.0\" \"dask[complete]==2026.3.0\" \"anyio>=4.12\"\n" ] }, { From 62bc43f1fa3ff60d6f65b01d5954f3dacc8e4eba Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Thu, 6 Aug 2026 10:15:16 +0200 Subject: [PATCH 53/57] fix: use -predictor internal Service DNS name in smoke tests --- serving/hf-vllm-completion/apply.py | 6 +++++- serving/minimal-example-shadow-deployment/apply.py | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/serving/hf-vllm-completion/apply.py b/serving/hf-vllm-completion/apply.py index 8b13032..25123b1 100644 --- a/serving/hf-vllm-completion/apply.py +++ b/serving/hf-vllm-completion/apply.py @@ -77,8 +77,12 @@ def _wait_isvc_ready(name: str, namespace: str, timeout: int) -> None: def _smoke_test(namespace: str, timeout: int = 120) -> None: """POST to the internal cluster URL; retries until the model responds.""" + # KServe's predictor Service is always named "-predictor", + # never the bare ISVC name (see PredictorServiceName in kserve's + # pkg/constants/constants.go) — hitting "{_ISVC_NAME}.{namespace}" + # directly resolves to nothing. url = ( - f"http://{_ISVC_NAME}.{namespace}.svc.cluster.local" + f"http://{_ISVC_NAME}-predictor.{namespace}.svc.cluster.local" f"/v1/models/{_MODEL_NAME}:predict" ) payload = json.dumps({"instances": ["KServe is wonderful!"]}).encode() diff --git a/serving/minimal-example-shadow-deployment/apply.py b/serving/minimal-example-shadow-deployment/apply.py index 1bbe915..dabe96c 100644 --- a/serving/minimal-example-shadow-deployment/apply.py +++ b/serving/minimal-example-shadow-deployment/apply.py @@ -238,8 +238,11 @@ def _smoke_test(namespace: str, timeout: int = 120) -> None: The doubler predictor multiplies each input value by FACTOR=2, so [1.0, 2.0, 3.0] must produce predictions [2.0, 4.0, 6.0]. """ + # KServe's predictor Service is always named "-predictor", + # never the bare ISVC name. url = ( - f"http://{_DOUBLER_ISVC}.{namespace}.svc.cluster.local/v1/models/model:predict" + f"http://{_DOUBLER_ISVC}-predictor.{namespace}.svc.cluster.local" + "/v1/models/model:predict" ) inputs = [1.0, 2.0, 3.0] expected = [2.0, 4.0, 6.0] From 88c0268b33998bffa45eeecc1aae7c30afa75784 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Thu, 6 Aug 2026 10:52:59 +0200 Subject: [PATCH 54/57] feat: add kserve_internal_url helper, auto-detect agentgateway vs direct predictor Service --- ci/README.md | 34 +++++++- scripts/kserve_internal_url.py | 80 +++++++++++++++++++ serving/hf-vllm-completion/apply.py | 15 ++-- .../apply.py | 13 +-- .../minimal-s3-model/minimal-s3-model.ipynb | 11 ++- 5 files changed, 134 insertions(+), 19 deletions(-) create mode 100644 scripts/kserve_internal_url.py diff --git a/ci/README.md b/ci/README.md index 35f2ec1..0b4ed0d 100644 --- a/ci/README.md +++ b/ci/README.md @@ -118,9 +118,9 @@ If you add an `apply.py`, always add the matching `cleanup.py` as well. ## scripts/ utilities -`scripts/` contains two cluster utilities importable from notebooks and apply -scripts. Both work identically whether called from a notebook cell or from an -`apply.py`. +`scripts/` contains cluster utilities importable from notebooks and apply +scripts. All of them work identically whether called from a notebook cell +or from an `apply.py`. ### setup_mlflow_credentials.py @@ -164,6 +164,34 @@ CI validates this in the preflight check and skips `api_key_dependent` examples if it is unset — mark any new example that calls `get_or_create_api_key()` with `api_key_dependent=True`. +### kserve_internal_url.py + +Returns the correct **internal, in-cluster** predict URL for a KServe +InferenceService, compatible with both prokube generations: + +- Currently released: hits the predictor Service directly — + `http://-predictor..svc.cluster.local/v1/models/:predict`. +- Upcoming (agentgateway-based, not yet released): routes through the + shared `agentgateway-proxy` Service instead — + `http://agentgateway-proxy.agentgateway-system.svc.cluster.local/_platform/serving///v2/models//infer`. + +It picks the URL by probing for the `agentgateway-proxy` Service once +(cached for the process) — no config flag needed. The request/response +payload format is unchanged either way (plain KServe V1 JSON, e.g. +`{"instances": [...]}`) — only the URL differs. Any example that predicts +against an InferenceService via its internal cluster URL (not the external +gateway URL) should use this instead of hardcoding the `-predictor` +pattern: + +```python +import sys, subprocess, os +_root = subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip() +sys.path.insert(0, os.path.join(_root, "scripts")) +from kserve_internal_url import internal_predict_url + +url = internal_predict_url(isvc_name, namespace, model_name) +``` + --- ## ci-skip cell tag diff --git a/scripts/kserve_internal_url.py b/scripts/kserve_internal_url.py new file mode 100644 index 0000000..54fd2ba --- /dev/null +++ b/scripts/kserve_internal_url.py @@ -0,0 +1,80 @@ +""" +Resolve the internal in-cluster predict URL for a KServe InferenceService, +compatible with both the currently-released prokube platform and the +upcoming agentgateway-based platform. + +Currently-released prokube versions expose the predictor directly as a +plain Kubernetes Service named ``-predictor`` in the caller's +namespace, using the KServe V1 protocol:: + + http://-predictor..svc.cluster.local/v1/models/:predict + +Upcoming (not yet released) prokube versions route ALL serving traffic — +internal and external alike — through a shared ``agentgateway-proxy`` +Service in the ``agentgateway-system`` namespace:: + + http://agentgateway-proxy.agentgateway-system.svc.cluster.local/_platform/serving///v2/models//infer + +The request/response payload is unchanged between the two (plain KServe V1 +JSON, e.g. ``{"instances": [...]}``) — only the URL differs, so callers +don't need to change how they build the request body. + +Usage +----- + import sys, subprocess, os + _root = subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip() + sys.path.insert(0, os.path.join(_root, "scripts")) + from kserve_internal_url import internal_predict_url + + url = internal_predict_url(isvc_name, namespace, model_name) +""" + +from __future__ import annotations + +import subprocess +from functools import lru_cache + +_AGENTGATEWAY_NAMESPACE = "agentgateway-system" +_AGENTGATEWAY_SERVICE = "agentgateway-proxy" + + +@lru_cache(maxsize=1) +def _agentgateway_available() -> bool: + """Return True if the new agentgateway-proxy Service exists in the cluster. + + Cached for the lifetime of the process — this is a cheap probe, but + there's no need to repeat it for every predict call in a run. + """ + result = subprocess.run( + [ + "kubectl", + "get", + "service", + _AGENTGATEWAY_SERVICE, + "-n", + _AGENTGATEWAY_NAMESPACE, + "--ignore-not-found", + ], + capture_output=True, + text=True, + ) + return result.returncode == 0 and bool(result.stdout.strip()) + + +def internal_predict_url(isvc_name: str, namespace: str, model_name: str) -> str: + """Return the internal in-cluster V1-protocol predict URL for an ISVC. + + Detects whether the new agentgateway-proxy is present (upcoming prokube + release) and returns the matching URL; falls back to hitting the KServe + predictor Service directly on clusters that don't have it yet. The + request/response payload format is unchanged either way. + """ + if _agentgateway_available(): + return ( + f"http://{_AGENTGATEWAY_SERVICE}.{_AGENTGATEWAY_NAMESPACE}.svc.cluster.local" + f"/_platform/serving/{namespace}/{isvc_name}/v2/models/{model_name}/infer" + ) + return ( + f"http://{isvc_name}-predictor.{namespace}.svc.cluster.local" + f"/v1/models/{model_name}:predict" + ) diff --git a/serving/hf-vllm-completion/apply.py b/serving/hf-vllm-completion/apply.py index 25123b1..a7aa239 100644 --- a/serving/hf-vllm-completion/apply.py +++ b/serving/hf-vllm-completion/apply.py @@ -77,14 +77,13 @@ def _wait_isvc_ready(name: str, namespace: str, timeout: int) -> None: def _smoke_test(namespace: str, timeout: int = 120) -> None: """POST to the internal cluster URL; retries until the model responds.""" - # KServe's predictor Service is always named "-predictor", - # never the bare ISVC name (see PredictorServiceName in kserve's - # pkg/constants/constants.go) — hitting "{_ISVC_NAME}.{namespace}" - # directly resolves to nothing. - url = ( - f"http://{_ISVC_NAME}-predictor.{namespace}.svc.cluster.local" - f"/v1/models/{_MODEL_NAME}:predict" - ) + _root = subprocess.check_output( + ["git", "rev-parse", "--show-toplevel"], text=True + ).strip() + sys.path.insert(0, os.path.join(_root, "scripts")) + from kserve_internal_url import internal_predict_url # noqa: PLC0415 + + url = internal_predict_url(_ISVC_NAME, namespace, _MODEL_NAME) payload = json.dumps({"instances": ["KServe is wonderful!"]}).encode() deadline = time.time() + timeout last_err: Exception | None = None diff --git a/serving/minimal-example-shadow-deployment/apply.py b/serving/minimal-example-shadow-deployment/apply.py index dabe96c..fc5c02a 100644 --- a/serving/minimal-example-shadow-deployment/apply.py +++ b/serving/minimal-example-shadow-deployment/apply.py @@ -238,12 +238,13 @@ def _smoke_test(namespace: str, timeout: int = 120) -> None: The doubler predictor multiplies each input value by FACTOR=2, so [1.0, 2.0, 3.0] must produce predictions [2.0, 4.0, 6.0]. """ - # KServe's predictor Service is always named "-predictor", - # never the bare ISVC name. - url = ( - f"http://{_DOUBLER_ISVC}-predictor.{namespace}.svc.cluster.local" - "/v1/models/model:predict" - ) + _root = subprocess.check_output( + ["git", "rev-parse", "--show-toplevel"], text=True + ).strip() + sys.path.insert(0, os.path.join(_root, "scripts")) + from kserve_internal_url import internal_predict_url # noqa: PLC0415 + + url = internal_predict_url(_DOUBLER_ISVC, namespace, "model") inputs = [1.0, 2.0, 3.0] expected = [2.0, 4.0, 6.0] payload = json.dumps({"values": inputs}).encode() diff --git a/serving/minimal-s3-model/minimal-s3-model.ipynb b/serving/minimal-s3-model/minimal-s3-model.ipynb index 3eee6ed..39dabd6 100644 --- a/serving/minimal-s3-model/minimal-s3-model.ipynb +++ b/serving/minimal-s3-model/minimal-s3-model.ipynb @@ -275,9 +275,16 @@ "outputs": [], "source": [ "import time\n", + "import sys, subprocess, os\n", "import requests\n", - "isvc_internal_url = f\"{inference_service_name}-predictor.{namespace}\" # KServe internal endpoint is always built like this\n", - "predict_url = f\"http://{isvc_internal_url}/v1/models/{inference_service_name}:predict\"\n", + "_root = subprocess.check_output([\"git\", \"rev-parse\", \"--show-toplevel\"], text=True).strip()\n", + "sys.path.insert(0, os.path.join(_root, \"scripts\"))\n", + "from kserve_internal_url import internal_predict_url\n", + "\n", + "# Resolves to the plain predictor Service URL on the currently-released\n", + "# prokube platform, or to the agentgateway-proxy URL on newer platforms\n", + "# where all serving traffic (internal and external) is routed through it.\n", + "predict_url = internal_predict_url(inference_service_name, namespace, inference_service_name)\n", "# an iris instance is [sepal_length, sepal_width, petal_length, petal_width]\n", "payload = {\"instances\": [[6.8, 2.8, 4.8, 1.4], [5.1, 3.5, 1.4, 0.2]]}\n", "\n", From 8aaa14f39fcf014809272e46817598a522f84715 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Thu, 6 Aug 2026 11:06:32 +0200 Subject: [PATCH 55/57] fix: probe agentgateway via DNS instead of kubectl (RBAC forbidden cross-namespace) --- ci/README.md | 7 +++++-- scripts/kserve_internal_url.py | 34 ++++++++++++++++------------------ 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/ci/README.md b/ci/README.md index 0b4ed0d..d07e2d6 100644 --- a/ci/README.md +++ b/ci/README.md @@ -175,8 +175,11 @@ InferenceService, compatible with both prokube generations: shared `agentgateway-proxy` Service instead — `http://agentgateway-proxy.agentgateway-system.svc.cluster.local/_platform/serving///v2/models//infer`. -It picks the URL by probing for the `agentgateway-proxy` Service once -(cached for the process) — no config flag needed. The request/response +It picks the URL via a plain DNS lookup for the `agentgateway-proxy` Service +(cached for the process) — no config flag needed, and no RBAC required +(unlike `kubectl get service`, which notebook pod service accounts +typically can't do cross-namespace; DNS resolution needs no permissions, +and a non-existent Service just fails to resolve). The request/response payload format is unchanged either way (plain KServe V1 JSON, e.g. `{"instances": [...]}`) — only the URL differs. Any example that predicts against an InferenceService via its internal cluster URL (not the external diff --git a/scripts/kserve_internal_url.py b/scripts/kserve_internal_url.py index 54fd2ba..9fcf8a1 100644 --- a/scripts/kserve_internal_url.py +++ b/scripts/kserve_internal_url.py @@ -31,34 +31,32 @@ from __future__ import annotations -import subprocess +import socket from functools import lru_cache _AGENTGATEWAY_NAMESPACE = "agentgateway-system" _AGENTGATEWAY_SERVICE = "agentgateway-proxy" +_AGENTGATEWAY_HOST = f"{_AGENTGATEWAY_SERVICE}.{_AGENTGATEWAY_NAMESPACE}.svc.cluster.local" @lru_cache(maxsize=1) def _agentgateway_available() -> bool: """Return True if the new agentgateway-proxy Service exists in the cluster. - Cached for the lifetime of the process — this is a cheap probe, but - there's no need to repeat it for every predict call in a run. + Uses a plain DNS lookup rather than `kubectl get service`: notebook pod + service accounts typically lack RBAC to read Services in another + namespace (e.g. `default-editor` gets a 403 Forbidden on + `agentgateway-system`), but every Kubernetes Service gets a DNS record + regardless of RBAC, and a non-existent Service fails resolution outright + (`socket.gaierror`) rather than returning an HTTP-level error. Cached for + the lifetime of the process — no need to repeat it for every predict + call in a run. """ - result = subprocess.run( - [ - "kubectl", - "get", - "service", - _AGENTGATEWAY_SERVICE, - "-n", - _AGENTGATEWAY_NAMESPACE, - "--ignore-not-found", - ], - capture_output=True, - text=True, - ) - return result.returncode == 0 and bool(result.stdout.strip()) + try: + socket.gethostbyname(_AGENTGATEWAY_HOST) + return True + except socket.gaierror: + return False def internal_predict_url(isvc_name: str, namespace: str, model_name: str) -> str: @@ -71,7 +69,7 @@ def internal_predict_url(isvc_name: str, namespace: str, model_name: str) -> str """ if _agentgateway_available(): return ( - f"http://{_AGENTGATEWAY_SERVICE}.{_AGENTGATEWAY_NAMESPACE}.svc.cluster.local" + f"http://{_AGENTGATEWAY_HOST}" f"/_platform/serving/{namespace}/{isvc_name}/v2/models/{model_name}/infer" ) return ( From 78216e47a7fdc3d6dd8e605d6b644cf80e295297 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Thu, 6 Aug 2026 11:38:52 +0200 Subject: [PATCH 56/57] refactor: trim verbose docstrings --- ci/run_all.py | 73 +------------------ hparam-tuning/minimal-mnist/cleanup.py | 13 +--- notebooks/dask/cleanup.py | 13 +--- notebooks/mnist-vae/run_training.py | 15 +--- .../lightweight-python-package/pipeline.py | 13 +--- scripts/get_or_create_api_key.py | 41 +---------- scripts/kserve_internal_url.py | 51 +------------ scripts/setup_mlflow_credentials.py | 32 +------- serving/hf-vllm-completion/apply.py | 12 +-- serving/hf-vllm-completion/cleanup.py | 11 +-- serving/kserve-keda-autoscaling/apply.py | 22 +----- serving/kserve-keda-autoscaling/cleanup.py | 12 +-- .../apply.py | 28 +------ .../cleanup.py | 13 +--- serving/minimal-s3-model/cleanup.py | 11 +-- .../apply.py | 26 +------ .../cleanup.py | 12 +-- serving/mlflow-kserve-minimal/apply.py | 32 +------- serving/mlflow-kserve-minimal/cleanup.py | 11 +-- 19 files changed, 27 insertions(+), 414 deletions(-) diff --git a/ci/run_all.py b/ci/run_all.py index 24b1c15..2c9ab3e 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -1,56 +1,4 @@ -""" -CI orchestrator — runs all automatable examples and reports results. - -Execution model ---------------- -Phase 1a (sequential, env-mutating): - Phase-1 examples with env_mutating=True (e.g. ones that `pip install - --upgrade` into the shared notebook kernel env), run to completion one - at a time before Phase 1 starts, to avoid a concurrent pip install - corrupting another notebook's in-progress import of the same package. - -Phase 1 (parallel, self-contained): - All remaining examples whose phase=1 in _EXAMPLES, including opt-in - ones. - -Phase 2 (parallel, pipeline submissions — return fast): - All examples whose phase=2 in _EXAMPLES. - -Phase 3 (depends on Phase 2 mlflow-mobile-price notebook completing): - All examples whose phase=3 in _EXAMPLES. - -Phase 4 (KFP run polling — runs alongside Phase 3): - Poll all KFP run IDs extracted from Phase 2 output notebooks until - Succeeded or timeout. - -Phase 5 — cleanup (always, in finally block): - All cleanup.py scripts derived from _EXAMPLES, in parallel. - -Adding a new example --------------------- -Add one entry to _EXAMPLES below. Everything else (cleanup, phase -scheduling, opt-in gating, dry-run output) is derived from the table. - -Usage from a Jupyter notebook ------------------------------- - import subprocess - subprocess.run(["python", "ci/run_all.py"], check=True) - -CLI usage ---------- - python ci/run_all.py [--timeout-notebook 1800] [--timeout-pipeline 3600] - [--include-keda] [--include-shadow] [--include-pytorch] - [--dry-run] - -Prerequisites -------------- - pip install papermill kfp - python scripts/setup_mlflow_credentials.py - export INFERENCE_SERVICE_API_KEY= # ask your - # cluster admin, or use pkui if available on your platform. Examples - # that call get_or_create_api_key() are skipped if this is unset, - # since CI runs headlessly and cannot prompt for it interactively. -""" +"""Run registered examples by phase, poll KFP runs, and always clean up.""" from __future__ import annotations @@ -138,16 +86,7 @@ class Example: api_key_dependent: bool = ( False # skip automatically when INFERENCE_SERVICE_API_KEY is unset ) - env_mutating: bool = ( - False # True = pip installs/upgrades packages into the shared - # notebook kernel environment. All papermill notebooks execute - # against the same site-packages (kernel_name="python3"), so an - # env_mutating example must finish before other same-phase - # examples start, or a concurrent `pip install` can corrupt an - # in-progress import in another notebook (e.g. partially replaced - # compiled extension modules). Run first within its phase, not - # concurrently with the rest. - ) + env_mutating: bool = False # run first; modifies the shared Python environment # ───────────────────────────────────────────────────────────────────────────── @@ -764,13 +703,7 @@ def _check_mlflow_credentials() -> tuple[bool, str]: def _check_kserve_api_key() -> tuple[bool, str]: - """Return (ok, reason). Checks that INFERENCE_SERVICE_API_KEY is set. - - CI runs headlessly, so get_or_create_api_key() cannot fall back to its - interactive getpass() prompt — the env var must be pre-populated. Ask - your cluster admin for a model-serving API key, or use pkui if it is - available on your platform. - """ + """Check that headless CI has an inference API key.""" if os.environ.get("INFERENCE_SERVICE_API_KEY", "").strip(): return True, "OK" return False, ( diff --git a/hparam-tuning/minimal-mnist/cleanup.py b/hparam-tuning/minimal-mnist/cleanup.py index 32d6475..9e8aab2 100644 --- a/hparam-tuning/minimal-mnist/cleanup.py +++ b/hparam-tuning/minimal-mnist/cleanup.py @@ -1,15 +1,4 @@ -""" -Cleanup script for the hparam-tuning/minimal-mnist example. - -Deletes (idempotently): - - Katib Experiment ``minimal-mnist`` - - Any Trial pods created by the Experiment (Katib garbage-collects these - automatically, but this speeds things up if you need to clean up quickly) - -Usage ------ - python cleanup.py [--dry-run] -""" +"""Delete the minimal MNIST Katib experiment and its trial pods.""" from __future__ import annotations diff --git a/notebooks/dask/cleanup.py b/notebooks/dask/cleanup.py index 0269852..9bff01a 100644 --- a/notebooks/dask/cleanup.py +++ b/notebooks/dask/cleanup.py @@ -1,15 +1,4 @@ -""" -Cleanup script for the dask example. - -Deletes (idempotently): - - Any DaskCluster CRs left running in the current namespace - (dask-kubernetes names them after the KubeCluster object; the example - uses the default name ``dask-cluster``) - -Usage ------ - python cleanup.py [--dry-run] -""" +"""Delete the Dask cluster created by the example.""" from __future__ import annotations diff --git a/notebooks/mnist-vae/run_training.py b/notebooks/mnist-vae/run_training.py index afa0cdd..27a0664 100644 --- a/notebooks/mnist-vae/run_training.py +++ b/notebooks/mnist-vae/run_training.py @@ -2,11 +2,7 @@ import subprocess import sys -# Re-exec with conda's libstdc++ prepended to LD_LIBRARY_PATH when needed. -# scipy 1.17+ requires CXXABI_1.3.15 which the system libstdc++ may lack; the -# conda-bundled libstdc++ provides it. LD_LIBRARY_PATH must be set before the -# process starts (the dynamic linker reads it once), so we re-exec if the -# conda lib dir is not already in the path. +# Re-exec with Conda's libstdc++ so SciPy can resolve its CXXABI symbols. _conda_lib = os.path.normpath( os.path.join(os.path.dirname(sys.executable), "..", "lib") ) @@ -55,14 +51,7 @@ ) @click.option("--max_epochs", default=50, type=int, help="Number of training epochs.") def run(hidden_dim: int, latent_dim: int, max_epochs: int) -> None: - """ - Train a VAE model on the MNIST dataset using PyTorch Lightning. - - Args: - hidden_dim (int): Dimension of the hidden layer. - latent_dim (int): Dimension of the latent space. - max_epochs (int): Number of training epochs. - """ + """Train a VAE on MNIST.""" # Initialize data module dm = MNISTDataModule(data_path="./data", num_workers=0, batch_size=32) diff --git a/pipelines/lightweight-python-package/pipeline.py b/pipelines/lightweight-python-package/pipeline.py index f770aaf..f0acbb5 100644 --- a/pipelines/lightweight-python-package/pipeline.py +++ b/pipelines/lightweight-python-package/pipeline.py @@ -179,18 +179,7 @@ def mobile_price_classification_pipeline( scatter_plot_column_y: str = "battery_power", seed: int = 42, ): - """ - Mobile price classification pipeline using containerized components. - - This pipeline covers the following steps: - 1. Read data from the specified paths. - 2. Split the data into training and validation sets. - 3. Fit the MinMax scaler. - 4. Tune hyperparameters for the SVM model. - 5. Train the SVM model with the best hyperparameters. - 6. Evaluate the trained model. - 7. Test the model and visualize the results with a scatter plot. - """ + """Train, tune, evaluate, and visualize a mobile-price classifier.""" from kfp import kubernetes # Step 1: Read the data diff --git a/scripts/get_or_create_api_key.py b/scripts/get_or_create_api_key.py index 4a12492..0588a02 100644 --- a/scripts/get_or_create_api_key.py +++ b/scripts/get_or_create_api_key.py @@ -1,34 +1,4 @@ -""" -Utility to obtain a prokube model-serving API key for use in examples. - -Usage from a notebook ---------------------- - import sys, subprocess, os - _root = subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip() - sys.path.insert(0, os.path.join(_root, "scripts")) - from get_or_create_api_key import get_or_create_api_key - - API_KEY = get_or_create_api_key() - -CLI usage ---------- - python get_or_create_api_key.py - # prints the key to stdout - -Resolution order ----------------- -There is no automated way to provision a model-serving API key from within a -notebook or script. This helper only looks for a key that has already been -made available in the environment, and otherwise prompts for one: - -1. **Environment variable.** If an admin has pre-provisioned a key and - injected it into the notebook pod via the ``INFERENCE_SERVICE_API_KEY`` - environment variable, that value is used. - -2. **Interactive prompt.** Otherwise, the caller is prompted to paste a key. - Obtain a key from your cluster administrator, or via the prokube UI - (``pkui``) if it is available on your platform. -""" +"""Read an inference API key from the environment or prompt for one.""" from __future__ import annotations @@ -37,8 +7,6 @@ import sys from getpass import getpass -# Env var an admin injects into the notebook pod with a pre-provisioned -# model-serving API key. _API_KEY_ENV_VAR = "INFERENCE_SERVICE_API_KEY" @@ -49,12 +17,7 @@ def _key_from_env() -> str | None: def get_or_create_api_key() -> str: - """Return a model-serving API key. - - See the module docstring for the resolution order: the - ``INFERENCE_SERVICE_API_KEY`` env var if set, otherwise an interactive - prompt. - """ + """Return the configured API key, prompting if necessary.""" env_key = _key_from_env() if env_key: return env_key diff --git a/scripts/kserve_internal_url.py b/scripts/kserve_internal_url.py index 9fcf8a1..eb21e7f 100644 --- a/scripts/kserve_internal_url.py +++ b/scripts/kserve_internal_url.py @@ -1,33 +1,4 @@ -""" -Resolve the internal in-cluster predict URL for a KServe InferenceService, -compatible with both the currently-released prokube platform and the -upcoming agentgateway-based platform. - -Currently-released prokube versions expose the predictor directly as a -plain Kubernetes Service named ``-predictor`` in the caller's -namespace, using the KServe V1 protocol:: - - http://-predictor..svc.cluster.local/v1/models/:predict - -Upcoming (not yet released) prokube versions route ALL serving traffic — -internal and external alike — through a shared ``agentgateway-proxy`` -Service in the ``agentgateway-system`` namespace:: - - http://agentgateway-proxy.agentgateway-system.svc.cluster.local/_platform/serving///v2/models//infer - -The request/response payload is unchanged between the two (plain KServe V1 -JSON, e.g. ``{"instances": [...]}``) — only the URL differs, so callers -don't need to change how they build the request body. - -Usage ------ - import sys, subprocess, os - _root = subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip() - sys.path.insert(0, os.path.join(_root, "scripts")) - from kserve_internal_url import internal_predict_url - - url = internal_predict_url(isvc_name, namespace, model_name) -""" +"""Build an internal KServe prediction URL for the available routing mode.""" from __future__ import annotations @@ -41,17 +12,7 @@ @lru_cache(maxsize=1) def _agentgateway_available() -> bool: - """Return True if the new agentgateway-proxy Service exists in the cluster. - - Uses a plain DNS lookup rather than `kubectl get service`: notebook pod - service accounts typically lack RBAC to read Services in another - namespace (e.g. `default-editor` gets a 403 Forbidden on - `agentgateway-system`), but every Kubernetes Service gets a DNS record - regardless of RBAC, and a non-existent Service fails resolution outright - (`socket.gaierror`) rather than returning an HTTP-level error. Cached for - the lifetime of the process — no need to repeat it for every predict - call in a run. - """ + """Return whether agentgateway resolves through cluster DNS.""" try: socket.gethostbyname(_AGENTGATEWAY_HOST) return True @@ -60,13 +21,7 @@ def _agentgateway_available() -> bool: def internal_predict_url(isvc_name: str, namespace: str, model_name: str) -> str: - """Return the internal in-cluster V1-protocol predict URL for an ISVC. - - Detects whether the new agentgateway-proxy is present (upcoming prokube - release) and returns the matching URL; falls back to hitting the KServe - predictor Service directly on clusters that don't have it yet. The - request/response payload format is unchanged either way. - """ + """Return the prediction URL for the detected in-cluster route.""" if _agentgateway_available(): return ( f"http://{_AGENTGATEWAY_HOST}" diff --git a/scripts/setup_mlflow_credentials.py b/scripts/setup_mlflow_credentials.py index b1b3011..b79b216 100644 --- a/scripts/setup_mlflow_credentials.py +++ b/scripts/setup_mlflow_credentials.py @@ -1,34 +1,4 @@ -""" -One-time setup: store MLflow credentials in a Kubernetes secret so that -example notebooks can read them without hardcoded values. - -This is the only step in the automation plan that requires human input — -the MLflow Personal Access Token (PAT) must be created via the MLflow UI -(Permissions → Create access key) and cannot be obtained programmatically. - -Usage from a notebook ---------------------- - import sys, subprocess, os - _root = subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip() - sys.path.insert(0, os.path.join(_root, "scripts")) - from setup_mlflow_credentials import setup_mlflow_credentials - - setup_mlflow_credentials( - uri="https://my-cluster.example.com/mlflow/", - username="alice@example.com", - password="", - ) - -CLI usage (any argument can be omitted; you will be prompted for it) ---------------------------------------------------------------------- - python setup_mlflow_credentials.py \\ - --uri https://my-cluster.example.com/mlflow/ \\ - --username alice@example.com \\ - --password - -The credentials are stored in a Secret named ``mlflow-credentials`` in the -notebook's namespace. Re-running updates the secret in place. -""" +"""Create or update the namespace's MLflow credentials Secret.""" from __future__ import annotations diff --git a/serving/hf-vllm-completion/apply.py b/serving/hf-vllm-completion/apply.py index a7aa239..aeeadfa 100644 --- a/serving/hf-vllm-completion/apply.py +++ b/serving/hf-vllm-completion/apply.py @@ -1,14 +1,4 @@ -""" -Deploy the DistilBERT CPU InferenceService (HuggingFace backend) and smoke-test it. - -The model (distilbert-base-uncased-finetuned-sst-2-english, ~250 MB) is -downloaded from the HuggingFace Hub on first start, so the ISVC may take a -few minutes to become ready. - -Usage ------ - python apply.py -""" +"""Deploy and smoke-test the DistilBERT InferenceService.""" from __future__ import annotations diff --git a/serving/hf-vllm-completion/cleanup.py b/serving/hf-vllm-completion/cleanup.py index a874d37..4760584 100644 --- a/serving/hf-vllm-completion/cleanup.py +++ b/serving/hf-vllm-completion/cleanup.py @@ -1,13 +1,4 @@ -""" -Cleanup script for the hf-vllm-completion example. - -Deletes (idempotently): - - InferenceService ``distilbert-inf-serv`` - -Usage ------ - python cleanup.py [--dry-run] -""" +"""Delete the DistilBERT InferenceService.""" from __future__ import annotations diff --git a/serving/kserve-keda-autoscaling/apply.py b/serving/kserve-keda-autoscaling/apply.py index 5f928f0..30070e0 100644 --- a/serving/kserve-keda-autoscaling/apply.py +++ b/serving/kserve-keda-autoscaling/apply.py @@ -1,17 +1,4 @@ -""" -Deploy the KEDA autoscaling example (OPT-125M, vLLM backend, CPU) and verify -that both the InferenceService and the ScaledObject become active. - -KEDA CRD check --------------- -If the ``scaledobjects.keda.sh`` CRD is absent the script exits with 0 and a -SKIP message rather than failing — the example is opt-in because KEDA is not -installed in every prokube cluster. - -Usage ------ - python apply.py -""" +"""Deploy and verify KEDA autoscaling, skipping when KEDA is unavailable.""" from __future__ import annotations @@ -50,12 +37,7 @@ def _kubectl_apply(manifest: str, namespace: str) -> None: def _try_apply_scaledobject(manifest: str, namespace: str) -> str | None: - """Apply the ScaledObject manifest; return an error string if KEDA is absent. - - Returns None on success, or a non-empty string if kubectl rejected the - manifest because the ScaledObject CRD is not installed. Any other error - is raised so CI marks the example as FAIL instead of silently skipping. - """ + """Apply the ScaledObject, returning an error only when KEDA is absent.""" result = subprocess.run( ["kubectl", "apply", "-f", "-", "-n", namespace], input=manifest, diff --git a/serving/kserve-keda-autoscaling/cleanup.py b/serving/kserve-keda-autoscaling/cleanup.py index d0c5946..ef45978 100644 --- a/serving/kserve-keda-autoscaling/cleanup.py +++ b/serving/kserve-keda-autoscaling/cleanup.py @@ -1,14 +1,4 @@ -""" -Cleanup script for the kserve-keda-autoscaling example. - -Deletes (idempotently, ScaledObject first to stop KEDA activity): - - ScaledObject ``opt-125m-scaledobject`` - - InferenceService ``opt-125m`` - -Usage ------ - python cleanup.py [--dry-run] -""" +"""Delete the KEDA ScaledObject and InferenceService.""" from __future__ import annotations diff --git a/serving/minimal-example-shadow-deployment/apply.py b/serving/minimal-example-shadow-deployment/apply.py index fc5c02a..1ea6dd6 100644 --- a/serving/minimal-example-shadow-deployment/apply.py +++ b/serving/minimal-example-shadow-deployment/apply.py @@ -1,30 +1,4 @@ -""" -Deploy the shadow deployment example and smoke-test the primary InferenceService. - -What this script does ---------------------- -1. Checks for the CrunchyData postgres-operator CRD — exits 0 with a SKIP - message if not found (it is installed by default in prokube clusters). -2. Deploys the PostgresCluster (postgres-cluster.yaml) and waits for the - primary pod to become ready. -3. Creates the two tables the transformer needs (inference_requests and - inference_response) via a short-lived psql pod. -4. Deploys the primary (doubler) and shadow (tripler) InferenceServices and - waits for both to become ready. -5. Sends a smoke-test request to the primary via its cluster-internal URL - and verifies the response contains predictions. - -What this script does NOT do ------------------------------ -- Apply the Istio VirtualService manifests — those embed namespace and domain - name and must be configured manually for each environment. -- Verify the shadow mirroring behaviour (that requires traffic via the - VirtualService, which is out of scope for automated CI). - -Usage ------ - python apply.py -""" +"""Deploy Postgres and the doubler/tripler services, then test the primary.""" from __future__ import annotations diff --git a/serving/minimal-example-shadow-deployment/cleanup.py b/serving/minimal-example-shadow-deployment/cleanup.py index eafd42b..c276ff5 100644 --- a/serving/minimal-example-shadow-deployment/cleanup.py +++ b/serving/minimal-example-shadow-deployment/cleanup.py @@ -1,15 +1,4 @@ -""" -Cleanup script for the minimal-example-shadow-deployment example. - -Deletes (idempotently): - - InferenceService ``double-minimal-custom-inference`` - - InferenceService ``triple-minimal-custom-inference`` - - PostgresCluster ``inferencing-postgres`` - -Usage ------ - python cleanup.py [--dry-run] -""" +"""Delete the shadow example's InferenceServices and Postgres cluster.""" from __future__ import annotations diff --git a/serving/minimal-s3-model/cleanup.py b/serving/minimal-s3-model/cleanup.py index 0508e16..52177f6 100644 --- a/serving/minimal-s3-model/cleanup.py +++ b/serving/minimal-s3-model/cleanup.py @@ -1,13 +1,4 @@ -""" -Cleanup script for the minimal-s3-model example. - -Deletes (idempotently): - - InferenceService ``kserve-object-storage-test`` - -Usage ------ - python cleanup.py [--dry-run] -""" +"""Delete the object-storage InferenceService.""" from __future__ import annotations diff --git a/serving/mlflow-kserve-inference-protocols/apply.py b/serving/mlflow-kserve-inference-protocols/apply.py index eb83970..7026f8c 100644 --- a/serving/mlflow-kserve-inference-protocols/apply.py +++ b/serving/mlflow-kserve-inference-protocols/apply.py @@ -1,28 +1,4 @@ -""" -Deploy the v1 and v2 MLflow KServe InferenceServices and smoke-test both. - -Substitutes ```` and ```` in the YAML manifests from -the cluster environment, applies them, waits for readiness, and verifies that -both endpoints return identical predictions for the same input. - -Usage from a notebook ---------------------- - import sys - sys.path.insert(0, ".") # run from the serving/mlflow-kserve-inference-protocols/ dir - from apply import deploy - - v1_uri, v2_uri, api_key = deploy() - -CLI usage ---------- - python apply.py - -Prerequisites -------------- -- ``mlflow-credentials`` secret must exist (run scripts/setup_mlflow_credentials.py). -- The MLflow model ``mobile-price-svm-`` version 1 must be registered - (run mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb first). -""" +"""Deploy and smoke-test the v1 and v2 MLflow KServe services.""" from __future__ import annotations diff --git a/serving/mlflow-kserve-inference-protocols/cleanup.py b/serving/mlflow-kserve-inference-protocols/cleanup.py index f4fb07d..9bf653b 100644 --- a/serving/mlflow-kserve-inference-protocols/cleanup.py +++ b/serving/mlflow-kserve-inference-protocols/cleanup.py @@ -1,14 +1,4 @@ -""" -Cleanup script for the mlflow-kserve-inference-protocols example. - -Deletes (idempotently): - - InferenceService ``v1-mobile-price-classification-inference`` - - InferenceService ``v2-mobile-price-classification-inference`` - -Usage ------ - python cleanup.py [--dry-run] -""" +"""Delete the v1 and v2 MLflow InferenceServices.""" from __future__ import annotations diff --git a/serving/mlflow-kserve-minimal/apply.py b/serving/mlflow-kserve-minimal/apply.py index 339185f..d81a863 100644 --- a/serving/mlflow-kserve-minimal/apply.py +++ b/serving/mlflow-kserve-minimal/apply.py @@ -1,30 +1,4 @@ -""" -Deploy the mlflow-kserve-minimal InferenceService and run an inference test. - -Substitutes ```` and ```` in InferenceService.yaml -from the cluster environment, applies it, waits for readiness, runs a smoke -test via ``test_inference_service.py``, and prints the external URL. - -Usage from a notebook ---------------------- - import sys, subprocess, os - _root = subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip() - sys.path.insert(0, os.path.join(_root, "serving", "mlflow-kserve-minimal")) - from apply import deploy - - isvc_uri = deploy() - -CLI usage ---------- - python apply.py - # prints ISVC_URI= and runs inference smoke test - -Prerequisites -------------- -- ``mlflow-credentials`` secret must exist (run scripts/setup_mlflow_credentials.py). -- The MLflow model ``mobile-price-svm-`` version 1 must be registered - (run mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb first). -""" +"""Deploy and smoke-test the MLflow-backed InferenceService.""" from __future__ import annotations @@ -66,9 +40,7 @@ def _mlflow_username(namespace: str) -> str: ) data = json.loads(result.stdout)["data"] full = base64.b64decode(data["MLFLOW_TRACKING_USERNAME"]).decode() - # Pipeline registers models as "mobile-price-svm-{local_user}" where - # local_user = MLFLOW_TRACKING_USERNAME.split('@')[0], e.g. - # "developer1@prokube.cloud" → "developer1" + # Model names use the username portion before "@". return full.split("@")[0] diff --git a/serving/mlflow-kserve-minimal/cleanup.py b/serving/mlflow-kserve-minimal/cleanup.py index 96c341c..3f8ea59 100644 --- a/serving/mlflow-kserve-minimal/cleanup.py +++ b/serving/mlflow-kserve-minimal/cleanup.py @@ -1,13 +1,4 @@ -""" -Cleanup script for the mlflow-kserve-minimal example. - -Deletes (idempotently): - - InferenceService ``mobile-price-svm`` - -Usage ------ - python cleanup.py [--dry-run] -""" +"""Delete the minimal MLflow InferenceService.""" from __future__ import annotations From b51d925e3fdd882d263600f0f3bd462b91f1e8e5 Mon Sep 17 00:00:00 2001 From: Igor Kvachenok Date: Thu, 6 Aug 2026 12:45:04 +0200 Subject: [PATCH 57/57] refactor: package shared utilities as installable pk_helpers Replace the scripts/ dir + git rev-parse/sys.path boilerplate with a real installable package (src/pk_helpers). Notebooks install it via a '%pip install -e ~/kubeflow-examples' setup cell; apply.py scripts import it directly. CI editable-installs it in preflight so subprocess imports resolve. Interactive tools are now console scripts (pk-setup-mlflow-credentials, pk-get-api-key). --- README.md | 6 ++- ci/README.md | 41 +++++++++++-------- ci/run_all.py | 25 +++++++++-- mlflow/mlflow-image-example.ipynb | 8 ++-- mlflow/mlflow-kfp-example.ipynb | 4 +- mlflow/mlflow-quickstart-example.ipynb | 8 ++-- .../mlflow-mobile-price-classification.ipynb | 4 +- pyproject.toml | 18 ++++++++ serving/hf-vllm-completion/apply.py | 6 +-- .../apply.py | 6 +-- .../minimal-s3-model/minimal-s3-model.ipynb | 18 ++++---- .../apply.py | 8 +--- .../inference_protocol_version_example.ipynb | 2 +- serving/mlflow-kserve-minimal/apply.py | 8 +--- src/pk_helpers/__init__.py | 26 ++++++++++++ .../pk_helpers/api_key.py | 7 +++- .../pk_helpers/kserve_url.py | 0 .../pk_helpers/mlflow_credentials.py | 7 +++- 18 files changed, 135 insertions(+), 67 deletions(-) create mode 100644 pyproject.toml create mode 100644 src/pk_helpers/__init__.py rename scripts/get_or_create_api_key.py => src/pk_helpers/api_key.py (92%) rename scripts/kserve_internal_url.py => src/pk_helpers/kserve_url.py (100%) rename scripts/setup_mlflow_credentials.py => src/pk_helpers/mlflow_credentials.py (95%) diff --git a/README.md b/README.md index 4e315ef..4d54463 100644 --- a/README.md +++ b/README.md @@ -14,10 +14,14 @@ For full platform documentation, see [docs.prokube.ai](https://docs.prokube.ai/) ├── notebooks # Jupyter notebook examples (Dask, MNIST VAE, etc.) ├── pipelines # Kubeflow Pipelines examples ├── rstudio # RStudio examples -├── scripts # shared utilities (credential setup, API key management) ├── serving # model serving examples (KServe, vLLM, shadow deployments) +└── src/pk_helpers # installable prokube helpers (credential setup, API key, KServe URLs) ``` +The shared `pk_helpers` package is installable from the repo root +(`pip install -e .`); notebooks install it via a `%pip install -q -e ~/kubeflow-examples` +setup cell and import helpers with `from pk_helpers import ...`. + Many serving examples include an `apply.py` (deploy + smoke-test) and a `cleanup.py` (teardown of Kubernetes resources) alongside the notebook. These are used by the CI orchestrator and can also be run manually. diff --git a/ci/README.md b/ci/README.md index d07e2d6..03f438e 100644 --- a/ci/README.md +++ b/ci/README.md @@ -116,27 +116,42 @@ If you add an `apply.py`, always add the matching `cleanup.py` as well. --- -## scripts/ utilities +## pk_helpers package -`scripts/` contains cluster utilities importable from notebooks and apply -scripts. All of them work identically whether called from a notebook cell +`pk_helpers` (source in `src/pk_helpers/`) bundles the prokube-platform +utilities importable from notebooks and apply scripts. It is a regular, +installable Python package — install it once, editable, from the repo root: + +```bash +pip install -e . +``` + +In notebooks, do this from a setup cell near the top: + +```python +%pip install -q -e ~/kubeflow-examples +``` + +CI installs it automatically in the preflight step (`_ensure_pk_helpers`), +so `apply.py` scripts can `from pk_helpers import ...` without any path +wiring. Every helper works identically whether called from a notebook cell or from an `apply.py`. -### setup_mlflow_credentials.py +### setup_mlflow_credentials One-time, interactive. Stores MLflow credentials in the `mlflow-credentials` K8s secret. **Requires human input** (the MLflow Personal Access Token cannot be obtained programmatically). Run it once from -a JupyterLab terminal: +a JupyterLab terminal via the installed console script: ```bash -python examples/scripts/setup_mlflow_credentials.py +pk-setup-mlflow-credentials ``` Do not call this from CI. CI validates the secret exists in the preflight check and skips MLflow-dependent examples if it does not. -### get_or_create_api_key.py +### get_or_create_api_key Returns a model-serving API key: the `INFERENCE_SERVICE_API_KEY` env var if an admin has injected one into the pod, otherwise an interactive prompt @@ -144,10 +159,7 @@ an admin has injected one into the pod, otherwise an interactive prompt from any notebook cell or script that needs an inference API key: ```python -import sys, subprocess, os -_root = subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip() -sys.path.insert(0, os.path.join(_root, "scripts")) -from get_or_create_api_key import get_or_create_api_key +from pk_helpers import get_or_create_api_key API_KEY = get_or_create_api_key() ``` @@ -164,7 +176,7 @@ CI validates this in the preflight check and skips `api_key_dependent` examples if it is unset — mark any new example that calls `get_or_create_api_key()` with `api_key_dependent=True`. -### kserve_internal_url.py +### internal_predict_url Returns the correct **internal, in-cluster** predict URL for a KServe InferenceService, compatible with both prokube generations: @@ -187,10 +199,7 @@ gateway URL) should use this instead of hardcoding the `-predictor` pattern: ```python -import sys, subprocess, os -_root = subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip() -sys.path.insert(0, os.path.join(_root, "scripts")) -from kserve_internal_url import internal_predict_url +from pk_helpers import internal_predict_url url = internal_predict_url(isvc_name, namespace, model_name) ``` diff --git a/ci/run_all.py b/ci/run_all.py index 2c9ab3e..8a818c9 100644 --- a/ci/run_all.py +++ b/ci/run_all.py @@ -37,6 +37,24 @@ def _ensure_papermill() -> None: print("papermill installed.") +def _ensure_pk_helpers() -> None: + """Editable-install the pk_helpers package if it is not importable. + + Notebooks and serving apply.py scripts import shared prokube helpers via + ``from pk_helpers import ...``. Installing the repo editable makes the same + interpreter used by the CI subprocesses able to resolve those imports. + """ + try: + import pk_helpers # noqa: F401 + except ImportError: + print("pk_helpers not found — installing repo editable...") + subprocess.run( + [sys.executable, "-m", "pip", "install", "-e", str(_REPO_ROOT)], + check=True, + ) + print("pk_helpers installed.") + + # ── KFP run ID pattern (UUID v4) ───────────────────────────────────────────── _RUN_ID_RE = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") @@ -667,7 +685,7 @@ def _check_mlflow_credentials() -> tuple[bool, str]: if r.returncode != 0: return False, ( "mlflow-credentials secret not found — " - "run scripts/setup_mlflow_credentials.py first" + "run `pk-setup-mlflow-credentials` first" ) try: data = _json.loads(r.stdout)["data"] @@ -677,7 +695,7 @@ def _check_mlflow_credentials() -> tuple[bool, str]: except KeyError as exc: return ( False, - f"mlflow-credentials secret is missing key {exc} — re-run setup_mlflow_credentials.py", + f"mlflow-credentials secret is missing key {exc} — re-run `pk-setup-mlflow-credentials`", ) creds = _b64.b64encode(f"{username}:{password}".encode()).decode() @@ -692,7 +710,7 @@ def _check_mlflow_credentials() -> tuple[bool, str]: if exc.code in (401, 403): return False, ( "MLflow credentials are invalid or the PAT has expired — " - "re-run scripts/setup_mlflow_credentials.py" + "re-run `pk-setup-mlflow-credentials`" ) return ( False, @@ -719,6 +737,7 @@ def _preflight(results: dict[str, Result]) -> bool: `results` on failure. Returns True if MLflow credentials are valid. """ _ensure_papermill() + _ensure_pk_helpers() print("Pre-flight: checking MLflow credentials...") mlflow_ok, mlflow_reason = _check_mlflow_credentials() diff --git a/mlflow/mlflow-image-example.ipynb b/mlflow/mlflow-image-example.ipynb index 19a05ab..058a001 100644 --- a/mlflow/mlflow-image-example.ipynb +++ b/mlflow/mlflow-image-example.ipynb @@ -11,10 +11,10 @@ "\n", "## MLflow Setup\n", "\n", - "Run `scripts/setup_mlflow_credentials.py` once to create the required secret:\n", + "Run `pk-setup-mlflow-credentials` once to create the required secret:\n", "```bash\n", "# From a JupyterLab terminal:\n", - "cd && python examples/scripts/setup_mlflow_credentials.py\n", + "pk-setup-mlflow-credentials\n", "```\n", "You will be prompted for your MLflow tracking URI, username (e-mail), and Personal Access Token.\n", "\n", @@ -41,7 +41,7 @@ "import base64, json, os, subprocess\n", "\n", "# Load MLflow credentials from the mlflow-credentials Kubernetes secret.\n", - "# Run scripts/setup_mlflow_credentials.py once to create this secret.\n", + "# Run `pk-setup-mlflow-credentials` once to create this secret.\n", "with open(\"/var/run/secrets/kubernetes.io/serviceaccount/namespace\") as _f:\n", " _ns = _f.read().strip()\n", "\n", @@ -52,7 +52,7 @@ "if _result.returncode != 0:\n", " raise RuntimeError(\n", " \"mlflow-credentials secret not found.\\n\"\n", - " \"Run scripts/setup_mlflow_credentials.py to create it.\"\n", + " \"Run `pk-setup-mlflow-credentials` to create it.\"\n", " )\n", "_data = json.loads(_result.stdout)[\"data\"]\n", "os.environ[\"MLFLOW_TRACKING_URI\"] = base64.b64decode(_data[\"MLFLOW_TRACKING_URI\"]).decode()\n", diff --git a/mlflow/mlflow-kfp-example.ipynb b/mlflow/mlflow-kfp-example.ipynb index 4fc26bf..58d43b1 100644 --- a/mlflow/mlflow-kfp-example.ipynb +++ b/mlflow/mlflow-kfp-example.ipynb @@ -11,10 +11,10 @@ "\n", "## MLflow Setup\n", "\n", - "Run `scripts/setup_mlflow_credentials.py` once to create the required secret:\n", + "Run `pk-setup-mlflow-credentials` once to create the required secret:\n", "```bash\n", "# From a JupyterLab terminal:\n", - "cd && python examples/scripts/setup_mlflow_credentials.py\n", + "pk-setup-mlflow-credentials\n", "```\n", "You will be prompted for your MLflow tracking URI, username (e-mail), and Personal Access Token.\n", "\n", diff --git a/mlflow/mlflow-quickstart-example.ipynb b/mlflow/mlflow-quickstart-example.ipynb index 7e3e289..6b56716 100644 --- a/mlflow/mlflow-quickstart-example.ipynb +++ b/mlflow/mlflow-quickstart-example.ipynb @@ -29,10 +29,10 @@ "\n", "## MLflow Setup\n", "\n", - "Run `scripts/setup_mlflow_credentials.py` once to create the required secret:\n", + "Run `pk-setup-mlflow-credentials` once to create the required secret:\n", "```bash\n", "# From a JupyterLab terminal:\n", - "cd && python examples/scripts/setup_mlflow_credentials.py\n", + "pk-setup-mlflow-credentials\n", "```\n", "You will be prompted for your MLflow tracking URI, username (e-mail), and Personal Access Token.\n", "\n", @@ -51,7 +51,7 @@ "import base64, json, os, subprocess\n", "\n", "# Load MLflow credentials from the mlflow-credentials Kubernetes secret.\n", - "# Run scripts/setup_mlflow_credentials.py once to create this secret.\n", + "# Run `pk-setup-mlflow-credentials` once to create this secret.\n", "with open(\"/var/run/secrets/kubernetes.io/serviceaccount/namespace\") as _f:\n", " _ns = _f.read().strip()\n", "\n", @@ -62,7 +62,7 @@ "if _result.returncode != 0:\n", " raise RuntimeError(\n", " \"mlflow-credentials secret not found.\\n\"\n", - " \"Run scripts/setup_mlflow_credentials.py to create it.\"\n", + " \"Run `pk-setup-mlflow-credentials` to create it.\"\n", " )\n", "_data = json.loads(_result.stdout)[\"data\"]\n", "os.environ[\"MLFLOW_TRACKING_URI\"] = base64.b64decode(_data[\"MLFLOW_TRACKING_URI\"]).decode()\n", diff --git a/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb b/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb index 770471b..8ea48e6 100644 --- a/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb +++ b/mlflow/mobile-price-classification/mlflow-mobile-price-classification.ipynb @@ -57,10 +57,10 @@ "\n", "To use MLflow tracking in this pipeline, create a Kubernetes secret with your MLflow credentials.\n", "\n", - "Run `scripts/setup_mlflow_credentials.py` once to create the required secret:\n", + "Run `pk-setup-mlflow-credentials` once to create the required secret:\n", "```bash\n", "# From a JupyterLab terminal:\n", - "cd && python examples/scripts/setup_mlflow_credentials.py\n", + "pk-setup-mlflow-credentials\n", "```\n", "You will be prompted for your MLflow tracking URI, username (e-mail), and Personal Access Token.\n", "\n", diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..7d07dd1 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,18 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "pk-helpers" +version = "0.1.0" +description = "Prokube platform helpers shared across the Kubeflow examples." +readme = "README.md" +requires-python = ">=3.10" +dependencies = [] + +[project.scripts] +pk-get-api-key = "pk_helpers.api_key:main" +pk-setup-mlflow-credentials = "pk_helpers.mlflow_credentials:main" + +[tool.hatch.build.targets.wheel] +packages = ["src/pk_helpers"] diff --git a/serving/hf-vllm-completion/apply.py b/serving/hf-vllm-completion/apply.py index aeeadfa..f12b4d4 100644 --- a/serving/hf-vllm-completion/apply.py +++ b/serving/hf-vllm-completion/apply.py @@ -67,11 +67,7 @@ def _wait_isvc_ready(name: str, namespace: str, timeout: int) -> None: def _smoke_test(namespace: str, timeout: int = 120) -> None: """POST to the internal cluster URL; retries until the model responds.""" - _root = subprocess.check_output( - ["git", "rev-parse", "--show-toplevel"], text=True - ).strip() - sys.path.insert(0, os.path.join(_root, "scripts")) - from kserve_internal_url import internal_predict_url # noqa: PLC0415 + from pk_helpers import internal_predict_url url = internal_predict_url(_ISVC_NAME, namespace, _MODEL_NAME) payload = json.dumps({"instances": ["KServe is wonderful!"]}).encode() diff --git a/serving/minimal-example-shadow-deployment/apply.py b/serving/minimal-example-shadow-deployment/apply.py index 1ea6dd6..8cc28f2 100644 --- a/serving/minimal-example-shadow-deployment/apply.py +++ b/serving/minimal-example-shadow-deployment/apply.py @@ -212,11 +212,7 @@ def _smoke_test(namespace: str, timeout: int = 120) -> None: The doubler predictor multiplies each input value by FACTOR=2, so [1.0, 2.0, 3.0] must produce predictions [2.0, 4.0, 6.0]. """ - _root = subprocess.check_output( - ["git", "rev-parse", "--show-toplevel"], text=True - ).strip() - sys.path.insert(0, os.path.join(_root, "scripts")) - from kserve_internal_url import internal_predict_url # noqa: PLC0415 + from pk_helpers import internal_predict_url url = internal_predict_url(_DOUBLER_ISVC, namespace, "model") inputs = [1.0, 2.0, 3.0] diff --git a/serving/minimal-s3-model/minimal-s3-model.ipynb b/serving/minimal-s3-model/minimal-s3-model.ipynb index 39dabd6..64ea38f 100644 --- a/serving/minimal-s3-model/minimal-s3-model.ipynb +++ b/serving/minimal-s3-model/minimal-s3-model.ipynb @@ -16,7 +16,11 @@ "outputs": [], "source": [ "# sklearn, joblib, s3fs already come with the notebook image\n", - "# %pip install sklearn joblib s3fs" + "# %pip install sklearn joblib s3fs\n", + "\n", + "# Install the prokube platform helpers (pk_helpers) shared by these examples.\n", + "# The kubeflow-examples repo is cloned into $HOME at notebook startup.\n", + "%pip install -q -e ~/kubeflow-examples\n" ] }, { @@ -275,11 +279,8 @@ "outputs": [], "source": [ "import time\n", - "import sys, subprocess, os\n", "import requests\n", - "_root = subprocess.check_output([\"git\", \"rev-parse\", \"--show-toplevel\"], text=True).strip()\n", - "sys.path.insert(0, os.path.join(_root, \"scripts\"))\n", - "from kserve_internal_url import internal_predict_url\n", + "from pk_helpers import internal_predict_url\n", "\n", "# Resolves to the plain predictor Service URL on the currently-released\n", "# prokube platform, or to the agentgateway-proxy URL on newer platforms\n", @@ -342,12 +343,9 @@ "metadata": {}, "outputs": [], "source": [ - "import sys, subprocess, os\n", - "_root = subprocess.check_output([\"git\", \"rev-parse\", \"--show-toplevel\"], text=True).strip()\n", - "sys.path.insert(0, os.path.join(_root, \"scripts\"))\n", - "from get_or_create_api_key import get_or_create_api_key\n", + "from pk_helpers import get_or_create_api_key\n", "\n", - "INFERENCE_SERVICE_API_KEY = get_or_create_api_key()\n" + "INFERENCE_SERVICE_API_KEY = get_or_create_api_key()" ] }, { diff --git a/serving/mlflow-kserve-inference-protocols/apply.py b/serving/mlflow-kserve-inference-protocols/apply.py index 7026f8c..0a069f4 100644 --- a/serving/mlflow-kserve-inference-protocols/apply.py +++ b/serving/mlflow-kserve-inference-protocols/apply.py @@ -53,7 +53,7 @@ def _mlflow_username(namespace: str) -> str: if result.returncode != 0: raise RuntimeError( "mlflow-credentials secret not found.\n" - "Run scripts/setup_mlflow_credentials.py to create it." + "Run `pk-setup-mlflow-credentials` to create it." ) data = json.loads(result.stdout)["data"] return base64.b64decode(data["MLFLOW_TRACKING_USERNAME"]).decode().split("@")[0] @@ -117,11 +117,7 @@ def _isvc_url(name: str, namespace: str) -> str: def _get_api_key() -> str: - root = subprocess.check_output( - ["git", "rev-parse", "--show-toplevel"], text=True - ).strip() - sys.path.insert(0, os.path.join(root, "scripts")) - from get_or_create_api_key import get_or_create_api_key + from pk_helpers import get_or_create_api_key return get_or_create_api_key() diff --git a/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb b/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb index 710c07b..6217534 100644 --- a/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb +++ b/serving/mlflow-kserve-inference-protocols/inference_protocol_version_example.ipynb @@ -346,7 +346,7 @@ "and run inference requests automatically.\n", "\n", "**Prerequisite:** the `mlflow-credentials` secret must exist in your namespace\n", - "(run `scripts/setup_mlflow_credentials.py` once if you have not done so).\n" + "(run `pk-setup-mlflow-credentials` once if you have not done so).\n" ] }, { diff --git a/serving/mlflow-kserve-minimal/apply.py b/serving/mlflow-kserve-minimal/apply.py index d81a863..e832445 100644 --- a/serving/mlflow-kserve-minimal/apply.py +++ b/serving/mlflow-kserve-minimal/apply.py @@ -36,7 +36,7 @@ def _mlflow_username(namespace: str) -> str: if result.returncode != 0: raise RuntimeError( "mlflow-credentials secret not found.\n" - "Run scripts/setup_mlflow_credentials.py to create it." + "Run `pk-setup-mlflow-credentials` to create it." ) data = json.loads(result.stdout)["data"] full = base64.b64decode(data["MLFLOW_TRACKING_USERNAME"]).decode() @@ -133,11 +133,7 @@ def deploy(timeout: int = 600) -> str: def test(uri: str) -> None: """Run the inference smoke test against the deployed ISVC.""" # Get / create the API key - _root = subprocess.check_output( - ["git", "rev-parse", "--show-toplevel"], text=True - ).strip() - sys.path.insert(0, os.path.join(_root, "scripts")) - from get_or_create_api_key import get_or_create_api_key # noqa: PLC0415 + from pk_helpers import get_or_create_api_key api_key = get_or_create_api_key() diff --git a/src/pk_helpers/__init__.py b/src/pk_helpers/__init__.py new file mode 100644 index 0000000..43379ea --- /dev/null +++ b/src/pk_helpers/__init__.py @@ -0,0 +1,26 @@ +"""Prokube platform helpers shared across the Kubeflow examples. + +These utilities abstract away logic specific to the prokube platform so that +notebooks and ``apply.py`` scripts can import them directly instead of wiring +up ``sys.path`` by hand. + +Install once (editable) from the repo root:: + + pip install -e . + +Then import what you need:: + + from pk_helpers import get_or_create_api_key, internal_predict_url +""" + +from __future__ import annotations + +from pk_helpers.api_key import get_or_create_api_key +from pk_helpers.kserve_url import internal_predict_url +from pk_helpers.mlflow_credentials import setup_mlflow_credentials + +__all__ = [ + "get_or_create_api_key", + "internal_predict_url", + "setup_mlflow_credentials", +] diff --git a/scripts/get_or_create_api_key.py b/src/pk_helpers/api_key.py similarity index 92% rename from scripts/get_or_create_api_key.py rename to src/pk_helpers/api_key.py index 0588a02..e819e27 100644 --- a/scripts/get_or_create_api_key.py +++ b/src/pk_helpers/api_key.py @@ -35,7 +35,8 @@ def get_or_create_api_key() -> str: return key -if __name__ == "__main__": +def main() -> None: + """Console-script entry point: print the resolved API key.""" try: argparser = argparse.ArgumentParser( description="Get a prokube model-serving API key" @@ -45,3 +46,7 @@ def get_or_create_api_key() -> str: except Exception as exc: # noqa: BLE001 print(f"ERROR: {exc}", file=sys.stderr) sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/kserve_internal_url.py b/src/pk_helpers/kserve_url.py similarity index 100% rename from scripts/kserve_internal_url.py rename to src/pk_helpers/kserve_url.py diff --git a/scripts/setup_mlflow_credentials.py b/src/pk_helpers/mlflow_credentials.py similarity index 95% rename from scripts/setup_mlflow_credentials.py rename to src/pk_helpers/mlflow_credentials.py index b79b216..a6c1a10 100644 --- a/scripts/setup_mlflow_credentials.py +++ b/src/pk_helpers/mlflow_credentials.py @@ -77,7 +77,8 @@ def setup_mlflow_credentials( print(f"Secret '{_SECRET_NAME}' created/updated in namespace '{ns}'.") -if __name__ == "__main__": +def main() -> None: + """Console-script entry point: create/update the MLflow credentials secret.""" parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) @@ -93,3 +94,7 @@ def setup_mlflow_credentials( except Exception as exc: # noqa: BLE001 print(f"ERROR: {exc}", file=sys.stderr) sys.exit(1) + + +if __name__ == "__main__": + main()