Feature- Tests

builders/test_runner.py

@@ -0,0 +1,44 @@
+import subprocess
+import sys
+
+from fastapi import FastAPI, Response
+from ray import serve
+
+
+fastapi = FastAPI()
+
+
+@serve.deployment(num_replicas=1)
+@serve.ingress(fastapi)
+class TestRunner:
+    def __init__(self) -> None:
+        subprocess.run(
+            [sys.executable, "-m", "pip", "install", "pytest", "-q"],
+            check=True,
+        )
+
+    @fastapi.post("/")
+    def run(self) -> Response:
+        result = subprocess.run(
+            [
+                sys.executable,
+                "-m",
+                "pytest",
+                "tests/model_snapshots/",
+                "-v",
+                "--tb=short",
+                "--no-header",
+                "-s",
+                "--color=no",
+            ],
+            capture_output=True,
+            text=True,
+        )
+
+        output = result.stdout
+        if result.returncode != 0:
+            output += f"\nSTDERR:\n{result.stderr}"
+        return Response(content=output, media_type="text/plain")
+
+
+app = TestRunner.bind()  # type: ignore[attr-defined]

builders/throughput_runner.py

@@ -0,0 +1,58 @@
+import subprocess
+import sys
+
+from fastapi import FastAPI, Response
+from ray import serve
+
+
+fastapi = FastAPI()
+
+
+@serve.deployment(num_replicas=1)
+@serve.ingress(fastapi)
+class ThroughputRunner:
+    def __init__(self) -> None:
+        subprocess.run(
+            [sys.executable, "-m", "pip", "install", "pytest", "-q"],
+            check=True,
+        )
+
+    @fastapi.post("/")
+    def run(
+        self,
+        duration_s: float = 300.0,
+        concurrency: int = 8,
+        timeout: float = 60.0,
+        wait_ready: bool = True,
+        wait_timeout_s: float = 0.0,
+        wait_interval_s: float = 10.0,
+    ) -> Response:
+        cmd = [
+            sys.executable,
+            "tests/perf_throughput.py",
+            "--duration-s",
+            str(duration_s),
+            "--concurrency",
+            str(concurrency),
+            "--timeout",
+            str(timeout),
+        ]
+        if wait_ready:
+            cmd.extend(
+                [
+                    "--wait-ready",
+                    "--wait-timeout-s",
+                    str(wait_timeout_s),
+                    "--wait-interval-s",
+                    str(wait_interval_s),
+                ]
+            )
+
+        result = subprocess.run(cmd, capture_output=True, text=True)
+        output = result.stdout + (
+            f"\nSTDERR:\n{result.stderr}" if result.returncode != 0 else ""
+        )
+        return Response(content=output, media_type="text/plain")
+
+
+app = ThroughputRunner.bind()  # type: ignore[attr-defined]

helm/rayservice/applications/test-runner.yaml

@@ -0,0 +1,16 @@
+- name: test-runner
+  import_path: builders.test_runner:app
+  route_prefix: /run-tests
+  runtime_env:
+    working_dir: https://github.com/RationAI/model-service/archive/refs/heads/main.zip
+    pip:
+      - git+https://github.com/RationAI/rationai-sdk-python.git
+  deployments:
+    - name: TestRunner
+      autoscaling_config:
+        min_replicas: 0
+        max_replicas: 1
+        target_ongoing_requests: 1
+      ray_actor_options:
+        num_cpus: 1
+        memory: 2147483648

helm/rayservice/applications/throughput-test.yaml

@@ -0,0 +1,16 @@
+- name: throughput-runner
+  import_path: builders.throughput_runner:app
+  route_prefix: /run-throughput
+  runtime_env:
+    working_dir: https://github.com/RationAI/model-service/archive/refs/heads/main.zip
+    pip:
+      - git+https://github.com/RationAI/rationai-sdk-python.git
+  deployments:
+    - name: ThroughputRunner
+      autoscaling_config:
+        min_replicas: 0
+        max_replicas: 1
+        target_ongoing_requests: 1
+      ray_actor_options:
+        num_cpus: 1
+        memory: 2147483648

helm/rayservice/values.yaml

@@ -8,3 +8,5 @@ applications:
   - prostate-classifier-1
   - prov-gigapath
   - virchow2
+  - test-runner
+  - throughput-test

helm/rayservice/workers/cpu-workers.yaml

@@ -45,6 +45,8 @@ template:
             mountPath: /mnt/cache
           - name: huggingface-cache
             mountPath: /mnt/huggingface_cache
+          - name: test-refs
+            mountPath: /mnt/test_refs
     volumes:
       - name: data
         persistentVolumeClaim:
@@ -64,3 +66,6 @@ template:
       - name: huggingface-cache
         persistentVolumeClaim:
           claimName: huggingface-cache-pvc
+      - name: test-refs
+        persistentVolumeClaim:
+          claimName: model-test-refs-pvc

helm/rayservice/workers/mig20-workers.yaml

@@ -55,6 +55,8 @@ template:
             mountPath: /mnt/cache
           - name: huggingface-cache
             mountPath: /mnt/huggingface_cache
+          - name: test-refs
+            mountPath: /mnt/test_refs
     volumes:
       - name: data
         persistentVolumeClaim:
@@ -74,3 +76,6 @@ template:
       - name: huggingface-cache
         persistentVolumeClaim:
           claimName: huggingface-cache-pvc
+      - name: test-refs
+        persistentVolumeClaim:
+          claimName: model-test-refs-pvc

pvc/model-test-refs-pvc.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: model-test-refs-pvc
+  namespace: rationai-jobs-ns
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 5Gi
+  storageClassName: nfs-csi

pyproject.toml

@@ -18,5 +18,8 @@ dependencies = [
 ]
 
 [dependency-groups]
-dev = ["mypy>=1.18.2", "ruff>=0.14.6"]
+dev = ["mypy>=1.18.2", "ruff>=0.14.6", "pytest>=8.4.2"]
 docs = ["mkdocs>=1.6.0", "mkdocs-material>=9.6.0", "pymdown-extensions>=10.0"]
+
+[tool.pytest.ini_options]
+pythonpath = ["tests/model_snapshots"]