diff --git a/data/rubric_manifest.json b/data/rubric_manifest.json new file mode 100644 index 00000000..7b1697f2 --- /dev/null +++ b/data/rubric_manifest.json @@ -0,0 +1,6 @@ +{ + "rubric_file": "rubric.tsv", + "rubric_prompt_beginning_file": "rubric_prompt_beginning.txt", + "question_prompt_file": "question_prompt.txt", + "personas": ["data/personas.tsv"] +} diff --git a/judge/rubric_config.py b/judge/rubric_config.py index 47f3304d..13013919 100644 --- a/judge/rubric_config.py +++ b/judge/rubric_config.py @@ -6,6 +6,7 @@ """ import asyncio +import json from dataclasses import dataclass from pathlib import Path from typing import Any, Dict, List, Optional @@ -124,6 +125,65 @@ async def load( question_prompt_template=question_prompt_template, ) + @classmethod + async def load_bundle(cls, manifest_path: str) -> "RubricConfig": + """Load a rubric from a rubric bundle manifest. + + A rubric bundle manifest is a JSON file describing what a rubric + *is* -- its files and (informational only) intended personas -- as + opposed to how to run it, which belongs in a run config, not here. + + Manifest shape: + { + "rubric_file": "rubric.tsv", + "rubric_prompt_beginning_file": "rubric_prompt_beginning.txt", + "question_prompt_file": "question_prompt.txt", + "personas": ["data/personas.tsv"] + } + + `personas` is informational only -- it documents which personas this + rubric is intended/validated for; it is not read by this loader. + File paths in the manifest are relative to the manifest's own folder. + + Args: + manifest_path: Path to the rubric bundle manifest JSON file + + Returns: + Loaded RubricConfig with all data + + Raises: + FileNotFoundError: If the manifest or any file it references + doesn't exist + ValueError: If the manifest is missing a required key + """ + manifest_file = Path(manifest_path) + if not manifest_file.exists(): + raise FileNotFoundError( + f"Rubric bundle manifest not found: {manifest_file}" + ) + + async with aiofiles.open(manifest_file, "r", encoding="utf-8") as f: + manifest = json.loads(await f.read()) + + required_keys = ( + "rubric_file", + "rubric_prompt_beginning_file", + "question_prompt_file", + ) + missing_keys = [key for key in required_keys if key not in manifest] + if missing_keys: + raise ValueError( + f"Rubric bundle manifest {manifest_file} is missing required " + f"key(s): {missing_keys}" + ) + + return await cls.load( + rubric_folder=str(manifest_file.parent), + rubric_file=manifest["rubric_file"], + rubric_prompt_beginning_file=manifest["rubric_prompt_beginning_file"], + question_prompt_file=manifest["question_prompt_file"], + ) + @staticmethod async def _read_file(file_path: Path) -> str: """Read text file asynchronously. diff --git a/tests/fixtures/rubric_manifest_simple.json b/tests/fixtures/rubric_manifest_simple.json new file mode 100644 index 00000000..0c5d5ec9 --- /dev/null +++ b/tests/fixtures/rubric_manifest_simple.json @@ -0,0 +1,5 @@ +{ + "rubric_file": "rubric_simple.tsv", + "rubric_prompt_beginning_file": "rubric_prompt_beginning.txt", + "question_prompt_file": "question_prompt.txt" +} diff --git a/tests/unit/judge/test_rubric_config.py b/tests/unit/judge/test_rubric_config.py index 5c463827..2a022070 100644 --- a/tests/unit/judge/test_rubric_config.py +++ b/tests/unit/judge/test_rubric_config.py @@ -1,5 +1,6 @@ """Unit tests for judge rubric configuration.""" +import json from pathlib import Path import pandas as pd @@ -16,6 +17,7 @@ COL_SEVERITY, EXPECTED_DIMENSION_NAMES, IGNORE_COLUMNS, + RubricConfig, ) @@ -149,3 +151,33 @@ def test_no_duplicate_dimensions(self): f"Number of unique dimensions in rubric ({len(unique_dimensions)}) " f"doesn't match EXPECTED_DIMENSION_NAMES ({len(EXPECTED_DIMENSION_NAMES)})" ) + + +@pytest.mark.unit +class TestLoadBundle: + """Tests for RubricConfig.load_bundle().""" + + async def test_load_bundle_success(self): + """Test loading a rubric via a valid bundle manifest.""" + rubric_config = await RubricConfig.load_bundle( + "tests/fixtures/rubric_manifest_simple.json" + ) + assert rubric_config.question_flow_data + assert rubric_config.question_order + assert rubric_config.rubric_prompt_beginning + assert rubric_config.question_prompt_template + + async def test_load_bundle_missing_manifest(self): + """Test that a missing manifest file raises FileNotFoundError.""" + with pytest.raises(FileNotFoundError): + await RubricConfig.load_bundle("tests/fixtures/does_not_exist.json") + + async def test_load_bundle_missing_required_key(self, tmp_path): + """Test that a manifest missing a required key raises ValueError.""" + manifest_path = tmp_path / "incomplete_manifest.json" + manifest_path.write_text( + json.dumps({"rubric_file": "rubric_simple.tsv"}), encoding="utf-8" + ) + + with pytest.raises(ValueError): + await RubricConfig.load_bundle(str(manifest_path))