diff --git a/Makefile b/Makefile index abff0778..ba86c4ec 100644 --- a/Makefile +++ b/Makefile @@ -100,13 +100,17 @@ start-autograder: # Sandbox Management -sandbox-build-all: sandbox-build-python sandbox-build-java sandbox-build-node sandbox-build-cpp +sandbox-build-all: sandbox-build-python sandbox-build-pyds sandbox-build-java sandbox-build-node sandbox-build-cpp @echo "✅ All sandbox images built successfully!" sandbox-build-python: @echo "Building Python sandbox image..." docker build -t sandbox-py:latest -f sandbox_manager/images/Dockerfile.python sandbox_manager/images/ +sandbox-build-pyds: + @echo "Building Python Data Science sandbox image..." + docker build -t sandbox-pyds:latest -f sandbox_manager/images/Dockerfile.python-ds sandbox_manager/images/ + sandbox-build-java: @echo "Building Java sandbox image..." docker build -t sandbox-java:latest -f sandbox_manager/images/Dockerfile.java sandbox_manager/images/ diff --git a/README.md b/README.md index c5a13433..9fa71882 100644 --- a/README.md +++ b/README.md @@ -221,7 +221,21 @@ Validates HTML, CSS, and JavaScript files. | `check_css_property` | Validate CSS rules | `selector`, `property`, `expected_value` | And much more! Check the [WebDev Template Documentation](docs/templates/web_dev.md) for the full list of tests. -#### 4. Custom Templates + +#### 4. Data Science Template +Validates data science assignments including ML model metrics, JSON/CSV outputs, and generated artifacts. Requires Python DS sandbox. + +| Test Name | Description | Key Parameters | +|-----------|-------------|----------------| +| `expect_stdout_value` | Extract a value from stdout and compare to an expected value | `extraction_pattern`, `expected_value`, `tolerance` | +| `expect_metric` | Extract metric from stdout and validate vs threshold | `metric_pattern`, `condition`, `threshold` | +| `expect_csv_output` | Validate generated CSV file contents | `artifact_path`, `expected_columns`, `expected_shape`, `expected_values` | +| `expect_json_output` | Validate generated JSON structure and keys | `artifact_path`, `required_keys`, `expected_values` | +| `expect_model_artifact` | Verify a model file was produced | `artifact_path`, `min_size_bytes` | + +Check the [Data Science Template Documentation](docs/template-library/data_science.md) for full details. + +#### 5. Custom Templates Upload your own test functions for specialized grading contexts: ```python diff --git a/autograder/services/command_resolver.py b/autograder/services/command_resolver.py index ad7351fe..bce1b40f 100644 --- a/autograder/services/command_resolver.py +++ b/autograder/services/command_resolver.py @@ -15,6 +15,7 @@ class CommandResolver: # Default command patterns for each language DEFAULT_COMMANDS = { Language.PYTHON: "python3 {filename}", + Language.PYTHON_DS: "python3 {filename}", Language.JAVA: "java {classname}", Language.NODE: "node {filename}", Language.CPP: "./{executable}", @@ -105,7 +106,7 @@ def _auto_resolve_command( This is used when program_command is "CMD" placeholder. """ command = None - if language == Language.PYTHON: + if language in (Language.PYTHON, Language.PYTHON_DS): command = f"python3 {fallback_filename}" if fallback_filename else "python3 main.py" elif language == Language.JAVA: # Java requires class name without .java extension diff --git a/autograder/services/template_library_service.py b/autograder/services/template_library_service.py index 300bfb78..3d4fac42 100644 --- a/autograder/services/template_library_service.py +++ b/autograder/services/template_library_service.py @@ -14,6 +14,7 @@ from autograder.template_library.api_testing import ApiTestingTemplate from autograder.template_library.input_output import InputOutputTemplate from autograder.template_library.static_analysis import StaticAnalysisTemplate +from autograder.template_library.data_science import DataScienceTemplate logger = logging.getLogger(__name__) @@ -37,6 +38,7 @@ class TemplateLibraryService: "api": ApiTestingTemplate, "input_output": InputOutputTemplate, "static_analysis": StaticAnalysisTemplate, + "data_science": DataScienceTemplate, } def __new__(cls): diff --git a/autograder/template_library/__init__.py b/autograder/template_library/__init__.py index e1970d33..fb77d69d 100644 --- a/autograder/template_library/__init__.py +++ b/autograder/template_library/__init__.py @@ -3,9 +3,12 @@ from autograder.template_library.web_dev.template import WebDevTemplate from autograder.template_library.api_testing import ApiTestingTemplate from autograder.template_library.input_output import InputOutputTemplate +from autograder.template_library.data_science import DataScienceTemplate __all__ = [ "WebDevTemplate", "ApiTestingTemplate", "InputOutputTemplate", + "DataScienceTemplate", ] + diff --git a/autograder/template_library/api_testing.py b/autograder/template_library/api_testing.py index bf07fb6f..4fbe5444 100644 --- a/autograder/template_library/api_testing.py +++ b/autograder/template_library/api_testing.py @@ -109,9 +109,9 @@ def execute(self, files, sandbox: SandboxContainer, endpoint: str = "", expected data = response.json() if data.get(expected_key) == expected_value: score = 100 - report = t("api_testing.check_response_json.report.success", locale=locale, endpoint=endpoint, key=expected_key, value=expected_value) + report = t("api_testing.check_response_json.report.success", locale=locale, endpoint=endpoint, field_key=expected_key, value=expected_value) else: - report = t("api_testing.check_response_json.report.failure", locale=locale, endpoint=endpoint, key=expected_key, expected=expected_value, actual=data.get(expected_key)) + report = t("api_testing.check_response_json.report.failure", locale=locale, endpoint=endpoint, field_key=expected_key, expected=expected_value, actual=data.get(expected_key)) except json.JSONDecodeError: report = t("api_testing.check_response_json.report.invalid_json", locale=locale, endpoint=endpoint) diff --git a/autograder/template_library/data_science.py b/autograder/template_library/data_science.py new file mode 100644 index 00000000..1dfa4ca7 --- /dev/null +++ b/autograder/template_library/data_science.py @@ -0,0 +1,752 @@ + +import csv +import io +import json +import logging +import re +from typing import Optional + +from autograder.models.abstract.template import Template +from autograder.models.dataclass.param_description import ParamDescription +from autograder.models.dataclass.test_result import TestResult +from autograder.template_library.execution_base import BaseExecutionTest +from autograder.translations import t +from sandbox_manager.sandbox_container import SandboxContainer + +def validate_artifact_path(artifact_path: str) -> Optional[str]: + """Return an error message if the path is unsafe, else None.""" + if not artifact_path: + return "artifact_path is required" + if artifact_path.startswith("/") or ".." in artifact_path.split("/"): + return f"Invalid artifact_path (absolute or traversal): {artifact_path}" + return None + +def values_match(actual, expected, tolerance: float) -> bool: + """Compare two values with numeric tolerance if both are numbers.""" + try: + actual_num = float(actual) + expected_num = float(expected) + return abs(actual_num - expected_num) <= tolerance + except (ValueError, TypeError): + return str(actual).strip() == str(expected).strip() + + +class ExpectStdoutValueTest(BaseExecutionTest): + """ + Runs a command, extracts a value from stdout via regex + (using a named group 'value'), and compares it to an expected + value with optional numeric tolerance. + """ + + @property + def name(self): + return "expect_stdout_value" + + @property + def description(self): + return t("data_science.expect_stdout_value.description") + + @property + def required_file(self): + """No specific file is required before execution.""" + return None + + @property + def parameter_description(self): + return [ + ParamDescription("program_command", t("data_science.expect_stdout_value.params.program_command"), "string"), + ParamDescription("extraction_pattern", t("data_science.expect_stdout_value.params.extraction_pattern"), "string"), + ParamDescription("expected_value", t("data_science.expect_stdout_value.params.expected_value"), "number"), + ParamDescription("tolerance", t("data_science.expect_stdout_value.params.tolerance"), "number"), + ] + + # pylint: disable=too-many-locals + def execute(self, files, sandbox: SandboxContainer, *args, **kwargs) -> TestResult: + program_command = kwargs.get("program_command") + extraction_pattern = kwargs.get("extraction_pattern", "") + expected_value = kwargs.get("expected_value") + tolerance = kwargs.get("tolerance", 0) + locale = kwargs.get("locale") + + try: + output = self.run_sandbox_execution( + sandbox=sandbox, + program_command=program_command, + ) + + error_result = self.check_for_base_errors(output, locale=locale) + if error_result: + return error_result + + stdout = output.stdout.strip() + + # Extract value using regex with named group 'value' + match = re.search(extraction_pattern, stdout) + if not match or "value" not in match.groupdict(): + return TestResult( + test_name=self.name, + score=0.0, + report=t("data_science.expect_stdout_value.report.no_match", + locale=locale, pattern=extraction_pattern, stdout=stdout) + ) + + actual_str = match.group("value") + try: + actual_value = float(actual_str) + except (ValueError, TypeError): + return TestResult( + test_name=self.name, + score=0.0, + report=t("data_science.expect_stdout_value.report.not_numeric", + locale=locale, value=actual_str) + ) + + expected_float = float(expected_value) + tol = float(tolerance) + + if abs(actual_value - expected_float) <= tol: + return TestResult( + test_name=self.name, + score=100.0, + report=t("data_science.expect_stdout_value.report.success", + locale=locale, expected=expected_float, actual=actual_value, tolerance=tol) + ) + + return TestResult( + test_name=self.name, + score=0.0, + report=t("data_science.expect_stdout_value.report.failure", + locale=locale, expected=expected_float, actual=actual_value, tolerance=tol) + ) + + except (ValueError, TimeoutError, RuntimeError) as e: + return TestResult( + test_name=self.name, + score=0.0, + report=t("data_science.expect_stdout_value.report.internal_error", + locale=locale, error=str(e)) + ) + + +class ExpectMetricTest(BaseExecutionTest): + """ + Runs a command, extracts a numeric metric from stdout via regex, + and validates it against a threshold using a comparison operator. + + Designed for ML metrics like accuracy, RMSE, F1-score, etc. + """ + + VALID_CONDITIONS = {">=", "<=", ">", "<", "=="} + + @property + def name(self): + return "expect_metric" + + @property + def description(self): + return t("data_science.expect_metric.description") + + @property + def required_file(self): + """No specific file is required before execution.""" + return None + + @property + def parameter_description(self): + return [ + ParamDescription("program_command", t("data_science.expect_metric.params.program_command"), "string"), + ParamDescription("metric_pattern", t("data_science.expect_metric.params.metric_pattern"), "string"), + ParamDescription("condition", t("data_science.expect_metric.params.condition"), "string"), + ParamDescription("threshold", t("data_science.expect_metric.params.threshold"), "number"), + ] + + @staticmethod + def _evaluate_condition(actual: float, condition: str, threshold: float) -> bool: + """Evaluate a comparison condition between actual and threshold.""" + if condition == ">=": + return actual >= threshold + if condition == "<=": + return actual <= threshold + if condition == ">": + return actual > threshold + if condition == "<": + return actual < threshold + if condition == "==": + return actual == threshold + return False + + # pylint: disable=too-many-locals,too-many-return-statements + def execute(self, files, sandbox: SandboxContainer, *args, **kwargs) -> TestResult: + program_command = kwargs.get("program_command") + metric_pattern = kwargs.get("metric_pattern", "") + condition = kwargs.get("condition", ">=") + threshold = kwargs.get("threshold") + locale = kwargs.get("locale") + + # Validate condition operator + if condition not in self.VALID_CONDITIONS: + return TestResult( + test_name=self.name, + score=0.0, + report=t("data_science.expect_metric.report.invalid_condition", + locale=locale, condition=condition, + valid=", ".join(sorted(self.VALID_CONDITIONS))) + ) + + try: + output = self.run_sandbox_execution( + sandbox=sandbox, + program_command=program_command, + ) + + error_result = self.check_for_base_errors(output, locale=locale) + if error_result: + return error_result + + stdout = output.stdout.strip() + + # Extract metric using regex with named group 'value' + match = re.search(metric_pattern, stdout) + if not match or "value" not in match.groupdict(): + return TestResult( + test_name=self.name, + score=0.0, + report=t("data_science.expect_metric.report.no_match", + locale=locale, pattern=metric_pattern, stdout=stdout) + ) + + actual_str = match.group("value") + try: + actual_value = float(actual_str) + except (ValueError, TypeError): + return TestResult( + test_name=self.name, + score=0.0, + report=t("data_science.expect_metric.report.not_numeric", + locale=locale, value=actual_str) + ) + + threshold_float = float(threshold) + + if self._evaluate_condition(actual_value, condition, threshold_float): + return TestResult( + test_name=self.name, + score=100.0, + report=t("data_science.expect_metric.report.success", + locale=locale, metric=actual_value, + condition=condition, threshold=threshold_float) + ) + + return TestResult( + test_name=self.name, + score=0.0, + report=t("data_science.expect_metric.report.failure", + locale=locale, metric=actual_value, + condition=condition, threshold=threshold_float) + ) + + except (ValueError, TimeoutError, RuntimeError) as e: + return TestResult( + test_name=self.name, + score=0.0, + report=t("data_science.expect_metric.report.internal_error", + locale=locale, error=str(e)) + ) + + +class ExpectCsvOutputTest(BaseExecutionTest): + """ + Runs a command, extracts a CSV file from the sandbox, and validates: + - Column names (presence and order) + - Shape (rows x cols) + - Cell values with numeric tolerance + + Scoring is proportional: each check contributes to the final score. + """ + + @property + def name(self): + return "expect_csv_output" + + @property + def description(self): + return t("data_science.expect_csv_output.description") + + @property + def required_file(self): + """No specific file is required before execution.""" + return None + + @property + def parameter_description(self): + return [ + ParamDescription("program_command", t("data_science.expect_csv_output.params.program_command"), "string"), + ParamDescription("artifact_path", t("data_science.expect_csv_output.params.artifact_path"), "string"), + ParamDescription("expected_columns", t("data_science.expect_csv_output.params.expected_columns"), "list of strings"), + ParamDescription("expected_shape", t("data_science.expect_csv_output.params.expected_shape"), "list [rows, cols]"), + ParamDescription("tolerance", t("data_science.expect_csv_output.params.tolerance"), "number"), + ParamDescription("expected_values", t("data_science.expect_csv_output.params.expected_values"), "list of lists"), + ] + + + @staticmethod + def _parse_csv(content: str) -> tuple: + """Parse CSV content, return (headers, rows) or raise ValueError.""" + reader = csv.reader(io.StringIO(content)) + rows = list(reader) + if not rows: + raise ValueError("CSV file is empty") + headers = rows[0] + data_rows = rows[1:] + return headers, data_rows + + + # pylint: disable=too-many-locals,too-many-return-statements,too-many-branches,too-many-statements + def execute(self, files, sandbox: SandboxContainer, *args, **kwargs) -> TestResult: + program_command = kwargs.get("program_command") + artifact_path = kwargs.get("artifact_path", "") + expected_columns = kwargs.get("expected_columns") + expected_shape = kwargs.get("expected_shape") + tolerance = float(kwargs.get("tolerance", 0)) + expected_values = kwargs.get("expected_values") + locale = kwargs.get("locale") + + # Validate artifact path + path_error = validate_artifact_path(artifact_path) + if path_error: + return TestResult( + test_name=self.name, score=0.0, + report=t("data_science.expect_csv_output.report.invalid_path", + locale=locale, error=path_error) + ) + + # Execute program + try: + output = self.run_sandbox_execution( + sandbox=sandbox, + program_command=program_command, + ) + error_result = self.check_for_base_errors(output, locale=locale) + if error_result: + return error_result + except (ValueError, TimeoutError, RuntimeError) as e: + return TestResult( + test_name=self.name, score=0.0, + report=t("data_science.expect_csv_output.report.internal_error", + locale=locale, error=str(e)) + ) + + # Extract file + try: + extracted = sandbox.extract_file(f"/app/{artifact_path}") + except FileNotFoundError: + return TestResult( + test_name=self.name, score=0.0, + report=t("data_science.expect_csv_output.report.file_not_found", + locale=locale, path=artifact_path) + ) + except (ValueError, RuntimeError) as e: + return TestResult( + test_name=self.name, score=0.0, + report=t("data_science.expect_csv_output.report.extraction_error", + locale=locale, error=str(e)) + ) + + # Parse CSV + try: + headers, data_rows = self._parse_csv(extracted.content_text) + except (ValueError, csv.Error) as e: + return TestResult( + test_name=self.name, score=0.0, + report=t("data_science.expect_csv_output.report.parse_error", + locale=locale, error=str(e)) + ) + + # Proportional scoring: each active check contributes equally + checks = [] + details = [] + + # Check columns + if expected_columns is not None: + columns_match = headers == expected_columns + checks.append(columns_match) + if columns_match: + details.append(t("data_science.expect_csv_output.report.columns_ok", locale=locale)) + else: + details.append(t("data_science.expect_csv_output.report.columns_mismatch", + locale=locale, expected=expected_columns, actual=headers)) + + # Check shape + if expected_shape is not None: + expected_rows, expected_cols = expected_shape + actual_rows = len(data_rows) + actual_cols = len(headers) + shape_match = (actual_rows == expected_rows and actual_cols == expected_cols) + checks.append(shape_match) + if shape_match: + details.append(t("data_science.expect_csv_output.report.shape_ok", + locale=locale, rows=actual_rows, cols=actual_cols)) + else: + details.append(t("data_science.expect_csv_output.report.shape_mismatch", + locale=locale, + expected_rows=expected_rows, expected_cols=expected_cols, + actual_rows=actual_rows, actual_cols=actual_cols)) + + # Check values with tolerance + if expected_values is not None: + total_cells = 0 + matching_cells = 0 + for row_idx, expected_row in enumerate(expected_values): + if row_idx >= len(data_rows): + total_cells += len(expected_row) + continue + actual_row = data_rows[row_idx] + for col_idx, expected_val in enumerate(expected_row): + total_cells += 1 + if col_idx < len(actual_row): + if values_match(actual_row[col_idx], expected_val, tolerance): + matching_cells += 1 + + if total_cells > 0: + value_ratio = matching_cells / total_cells + checks.append(value_ratio) + details.append(t("data_science.expect_csv_output.report.values_check", + locale=locale, matching=matching_cells, + total=total_cells, ratio=f"{value_ratio:.0%}")) + else: + checks.append(True) + + # Calculate final score + if not checks: + # No checks specified — file exists, that's a pass + return TestResult( + test_name=self.name, score=100.0, + report=t("data_science.expect_csv_output.report.success", + locale=locale, path=artifact_path) + ) + + # Proportional: bool checks count as 1.0/0.0, ratio checks as their value + score_parts = [] + for check in checks: + if isinstance(check, bool): + score_parts.append(100.0 if check else 0.0) + else: + score_parts.append(float(check) * 100.0) + + final_score = sum(score_parts) / len(score_parts) + report = "\n".join(details) + + if final_score >= 100.0: + report = t("data_science.expect_csv_output.report.success", + locale=locale, path=artifact_path) + "\n" + report + + return TestResult( + test_name=self.name, + score=round(final_score, 2), + report=report + ) + + +class ExpectJsonOutputTest(BaseExecutionTest): + """ + Runs a command, extracts a JSON file from the sandbox, and validates: + - Required keys exist + - Value equality (with tolerance for numeric values) + - Nested key access via dot notation (e.g., "metrics.accuracy") + """ + + @property + def name(self): + return "expect_json_output" + + @property + def description(self): + return t("data_science.expect_json_output.description") + + @property + def required_file(self): + """No specific file is required before execution.""" + return None + + @property + def parameter_description(self): + return [ + ParamDescription("program_command", t("data_science.expect_json_output.params.program_command"), "string"), + ParamDescription("artifact_path", t("data_science.expect_json_output.params.artifact_path"), "string"), + ParamDescription("required_keys", t("data_science.expect_json_output.params.required_keys"), "list of strings"), + ParamDescription("expected_values", t("data_science.expect_json_output.params.expected_values"), "dict"), + ParamDescription("tolerance", t("data_science.expect_json_output.params.tolerance"), "number"), + ] + + + @staticmethod + def _get_nested_value(data: dict, dotted_key: str): + """ + Access a nested value using dot notation. + Returns (value, True) if found, (None, False) if not. + """ + keys = dotted_key.split(".") + current = data + for key in keys: + if isinstance(current, dict) and key in current: + current = current[key] + else: + return None, False + return current, True + + @staticmethod + def _values_match(actual, expected, tolerance: float) -> bool: + """Compare two values with numeric tolerance if both are numbers.""" + try: + actual_num = float(actual) + expected_num = float(expected) + return abs(actual_num - expected_num) <= tolerance + except (ValueError, TypeError): + return actual == expected + + # pylint: disable=too-many-locals,too-many-return-statements,too-many-branches + def execute(self, files, sandbox: SandboxContainer, *args, **kwargs) -> TestResult: + program_command = kwargs.get("program_command") + artifact_path = kwargs.get("artifact_path", "") + required_keys = kwargs.get("required_keys") + expected_values = kwargs.get("expected_values") + tolerance = float(kwargs.get("tolerance", 0)) + locale = kwargs.get("locale") + + # Validate artifact path + path_error = validate_artifact_path(artifact_path) + if path_error: + return TestResult( + test_name=self.name, score=0.0, + report=t("data_science.expect_json_output.report.invalid_path", + locale=locale, error=path_error) + ) + + # Execute program + try: + output = self.run_sandbox_execution( + sandbox=sandbox, + program_command=program_command, + ) + error_result = self.check_for_base_errors(output, locale=locale) + if error_result: + return error_result + except (ValueError, TimeoutError, RuntimeError) as e: + return TestResult( + test_name=self.name, score=0.0, + report=t("data_science.expect_json_output.report.internal_error", + locale=locale, error=str(e)) + ) + + # Extract file + try: + extracted = sandbox.extract_file(f"/app/{artifact_path}") + except FileNotFoundError: + return TestResult( + test_name=self.name, score=0.0, + report=t("data_science.expect_json_output.report.file_not_found", + locale=locale, path=artifact_path) + ) + except (ValueError, RuntimeError) as e: + return TestResult( + test_name=self.name, score=0.0, + report=t("data_science.expect_json_output.report.extraction_error", + locale=locale, error=str(e)) + ) + + # Parse JSON + try: + data = json.loads(extracted.content_text) + except json.JSONDecodeError as e: + return TestResult( + test_name=self.name, score=0.0, + report=t("data_science.expect_json_output.report.parse_error", + locale=locale, error=str(e)) + ) + + # Proportional scoring + checks = [] + details = [] + + # Check required keys + if required_keys is not None: + for key in required_keys: + _, found = self._get_nested_value(data, key) + checks.append(found) + if found: + details.append(t("data_science.expect_json_output.report.key_found", + locale=locale, field_key=key)) + else: + details.append(t("data_science.expect_json_output.report.key_missing", + locale=locale, field_key=key)) + + # Check expected values + if expected_values is not None: + for key, expected_val in expected_values.items(): + actual_val, found = self._get_nested_value(data, key) + if not found: + checks.append(False) + details.append(t("data_science.expect_json_output.report.value_key_missing", + locale=locale, field_key=key)) + elif values_match(actual_val, expected_val, tolerance): + checks.append(True) + details.append(t("data_science.expect_json_output.report.value_match", + locale=locale, field_key=key, expected=expected_val, actual=actual_val)) + else: + checks.append(False) + details.append(t("data_science.expect_json_output.report.value_mismatch", + locale=locale, field_key=key, expected=expected_val, actual=actual_val)) + + # Calculate final score + if not checks: + return TestResult( + test_name=self.name, score=100.0, + report=t("data_science.expect_json_output.report.success", + locale=locale, path=artifact_path) + ) + + passed = sum(1 for c in checks if c) + total = len(checks) + final_score = (passed / total) * 100.0 + report = "\n".join(details) + + if final_score >= 100.0: + report = t("data_science.expect_json_output.report.success", + locale=locale, path=artifact_path) + "\n" + report + + return TestResult( + test_name=self.name, + score=round(final_score, 2), + report=report + ) + + +class ExpectModelArtifactTest(BaseExecutionTest): + """ + Runs a command and verifies that a trained model file was produced + in the sandbox with a minimum file size. + """ + + @property + def name(self): + return "expect_model_artifact" + + @property + def description(self): + return t("data_science.expect_model_artifact.description") + + @property + def required_file(self): + """No specific file is required before execution.""" + return None + + @property + def parameter_description(self): + return [ + ParamDescription("program_command", t("data_science.expect_model_artifact.params.program_command"), "string"), + ParamDescription("artifact_path", t("data_science.expect_model_artifact.params.artifact_path"), "string"), + ParamDescription("min_size_bytes", t("data_science.expect_model_artifact.params.min_size_bytes"), "integer"), + ] + + + # pylint: disable=too-many-return-statements + def execute(self, files, sandbox: SandboxContainer, *args, **kwargs) -> TestResult: + program_command = kwargs.get("program_command") + artifact_path = kwargs.get("artifact_path", "") + min_size_bytes = int(kwargs.get("min_size_bytes", 0)) + locale = kwargs.get("locale") + + # Validate artifact path + path_error = validate_artifact_path(artifact_path) + if path_error: + return TestResult( + test_name=self.name, score=0.0, + report=t("data_science.expect_model_artifact.report.invalid_path", + locale=locale, error=path_error) + ) + + # Execute program + try: + output = self.run_sandbox_execution( + sandbox=sandbox, + program_command=program_command, + ) + error_result = self.check_for_base_errors(output, locale=locale) + if error_result: + return error_result + except (ValueError, TimeoutError, RuntimeError) as e: + return TestResult( + test_name=self.name, score=0.0, + report=t("data_science.expect_model_artifact.report.internal_error", + locale=locale, error=str(e)) + ) + + # Extract file + try: + extracted = sandbox.extract_file(f"/app/{artifact_path}") + except FileNotFoundError: + return TestResult( + test_name=self.name, score=0.0, + report=t("data_science.expect_model_artifact.report.file_not_found", + locale=locale, path=artifact_path) + ) + except (ValueError, RuntimeError) as e: + return TestResult( + test_name=self.name, score=0.0, + report=t("data_science.expect_model_artifact.report.extraction_error", + locale=locale, error=str(e)) + ) + + # Validate file size + actual_size = extracted.size + if actual_size < min_size_bytes: + return TestResult( + test_name=self.name, score=0.0, + report=t("data_science.expect_model_artifact.report.too_small", + locale=locale, path=artifact_path, + min_size=min_size_bytes, actual_size=actual_size) + ) + + return TestResult( + test_name=self.name, score=100.0, + report=t("data_science.expect_model_artifact.report.success", + locale=locale, path=artifact_path, size=actual_size) + ) + + +class DataScienceTemplate(Template): + """ + A template for data science assignments. Validates data manipulation + results, model outputs, statistical metrics, and generated artifacts. + + Requires sandbox execution (requires_sandbox = True). + """ + + def __init__(self): + self.logger = logging.getLogger(__name__) + + self.tests = { + "expect_stdout_value": ExpectStdoutValueTest(), + "expect_metric": ExpectMetricTest(), + "expect_csv_output": ExpectCsvOutputTest(), + "expect_json_output": ExpectJsonOutputTest(), + "expect_model_artifact": ExpectModelArtifactTest(), + } + + @property + def template_name(self): + return t("data_science.template.name") + + @property + def template_description(self): + return t("data_science.template.description") + + @property + def requires_sandbox(self) -> bool: + return True + + def get_test(self, name: str): + test_function = self.tests.get(name) + if not test_function: + raise AttributeError(f"Test '{name}' not found in the '{self.template_name}' template.") + return test_function diff --git a/autograder/template_library/execution_base.py b/autograder/template_library/execution_base.py new file mode 100644 index 00000000..6147f32b --- /dev/null +++ b/autograder/template_library/execution_base.py @@ -0,0 +1,67 @@ + +from typing import Optional + +from autograder.models.abstract.test_function import TestFunction +from autograder.models.dataclass.test_result import TestResult +from autograder.translations import t +from sandbox_manager.sandbox_container import SandboxContainer +from sandbox_manager.models.sandbox_models import ResponseCategory + + +class BaseExecutionTest(TestFunction): + """ + Abstract base class for tests that involve running a student's code + in a sandbox and handling basic execution results (timeouts, crashes). + """ + + def run_sandbox_execution(self, sandbox: SandboxContainer, inputs: list = None, + program_command: Optional[str] = None): + """ + Executes the command inside the sandbox. + `program_command` must already be a resolved string (or None). + Returns the raw `output` from the sandbox run. + """ + # Run with a pre-resolved command + if program_command: + safe_inputs = inputs if inputs is not None else [] + return sandbox.run_commands(safe_inputs, program_command=program_command) + + if inputs is None: + raise ValueError("inputs parameter is required if no program_command is provided") + command = ' '.join(inputs) if isinstance(inputs, list) else str(inputs) + return sandbox.run_command(command) + + def check_for_base_errors(self, output, **kwargs) -> TestResult: + """ + Checks for Timeout, Compilation Error, or Runtime Error in the sandbox output. + Returns a TestResult with score 0.0 if an error is found, or None if successful. + """ + if output.category == ResponseCategory.TIMEOUT: + return TestResult( + test_name=self.name, + score=0.0, + report=t("io.execution.timeout", locale=kwargs.get("locale"), time=output.execution_time) + ) + + if output.category == ResponseCategory.COMPILATION_ERROR: + return TestResult( + test_name=self.name, + score=0.0, + report=t("io.execution.compilation_error", locale=kwargs.get("locale"), error=output.stderr) + ) + + if output.category == ResponseCategory.RUNTIME_ERROR: + return TestResult( + test_name=self.name, + score=0.0, + report=t("io.execution.runtime_error", locale=kwargs.get("locale"), error=output.stderr) + ) + + if output.category == ResponseCategory.SYSTEM_ERROR: + return TestResult( + test_name=self.name, + score=0.0, + report=t("io.execution.system_error", locale=kwargs.get("locale"), error=output.stderr) + ) + + return None diff --git a/autograder/template_library/input_output.py b/autograder/template_library/input_output.py index d8c64dfd..ca067a4b 100644 --- a/autograder/template_library/input_output.py +++ b/autograder/template_library/input_output.py @@ -11,72 +11,10 @@ from autograder.models.dataclass.test_result import TestResult from autograder.models.dataclass.structural_analysis_result import StructuralAnalysisResult from autograder.translations import t +from autograder.template_library.execution_base import BaseExecutionTest from sandbox_manager.sandbox_container import SandboxContainer from sandbox_manager.models.sandbox_models import Language, ResponseCategory - -# =============================================================== -# Base TestFunction for Executable Validations -# =============================================================== - -class BaseExecutionTest(TestFunction): - """ - Abstract base class for tests that involve running a student's code - in a sandbox and handling basic execution results (timeouts, crashes). - """ - - def run_sandbox_execution(self, sandbox: SandboxContainer, inputs: list = None, - program_command: Optional[str] = None): - """ - Executes the command inside the sandbox. - `program_command` must already be a resolved string (or None). - Returns the raw `output` from the sandbox run. - """ - # Run with a pre-resolved command - if program_command: - safe_inputs = inputs if inputs is not None else [] - return sandbox.run_commands(safe_inputs, program_command=program_command) - - if inputs is None: - raise ValueError("inputs parameter is required if no program_command is provided") - command = ' '.join(inputs) if isinstance(inputs, list) else str(inputs) - return sandbox.run_command(command) - - def check_for_base_errors(self, output, **kwargs) -> TestResult: - """ - Checks for Timeout, Compilation Error, or Runtime Error in the sandbox output. - Returns a TestResult with score 0.0 if an error is found, or None if successful. - """ - if output.category == ResponseCategory.TIMEOUT: - return TestResult( - test_name=self.name, - score=0.0, - report=t("io.execution.timeout", locale=kwargs.get("locale"), time=output.execution_time) - ) - - if output.category == ResponseCategory.COMPILATION_ERROR: - return TestResult( - test_name=self.name, - score=0.0, - report=t("io.execution.compilation_error", locale=kwargs.get("locale"), error=output.stderr) - ) - - if output.category == ResponseCategory.RUNTIME_ERROR: - return TestResult( - test_name=self.name, - score=0.0, - report=t("io.execution.runtime_error", locale=kwargs.get("locale"), error=output.stderr) - ) - - if output.category == ResponseCategory.SYSTEM_ERROR: - return TestResult( - test_name=self.name, - score=0.0, - report=t("io.execution.system_error", locale=kwargs.get("locale"), error=output.stderr) - ) - - return None - # =============================================================== # TestFunction for Input/Output Validation # =============================================================== diff --git a/autograder/translations/en.json b/autograder/translations/en.json index 713710b0..dd1d4422 100644 --- a/autograder/translations/en.json +++ b/autograder/translations/en.json @@ -551,8 +551,8 @@ }, "report": { "request_failed": "Request failed with status code {code}.", - "success": "Success! Response from '{endpoint}' contained the expected key-value pair ('{key}': '{value}').", - "failure": "JSON response from '{endpoint}' did not contain the expected value. Expected '{expected}' for key '{key}', but got '{actual}'.", + "success": "Success! Response from '{endpoint}' contained the expected key-value pair ('{field_key}': '{value}').", + "failure": "JSON response from '{endpoint}' did not contain the expected value. Expected '{expected}' for key '{field_key}', but got '{actual}'.", "invalid_json": "Response from '{endpoint}' was not valid JSON." } }, @@ -565,5 +565,107 @@ "error": { "step_interrupted": "An unexpected error occurred during step execution: {error}" } + }, + "data_science": { + "template": { + "name": "Data Science", + "description": "A template for evaluating data science assignments involving data manipulation, model training, and metric validation." + }, + "expect_stdout_value": { + "description": "Extracts a numeric value from program stdout using a regex pattern and compares it to an expected value with optional tolerance.", + "params": { + "program_command": "Command to execute the student program.", + "extraction_pattern": "Regex pattern with a named group 'value' to extract the numeric output (e.g., 'total_rows:\\\\s*(?P\\\\d+)').", + "expected_value": "The expected numeric value.", + "tolerance": "Acceptable absolute difference between actual and expected values (default: 0)." + }, + "report": { + "success": "Success: Extracted value {actual} matches expected {expected} (tolerance: {tolerance}).", + "failure": "FAILURE: Extracted value {actual} does not match expected {expected} (tolerance: {tolerance}).", + "no_match": "FAILURE: Pattern '{pattern}' did not match any value in stdout.\nOutput:\n{stdout}", + "not_numeric": "FAILURE: Extracted value '{value}' is not a valid number.", + "internal_error": "SYSTEM ERROR: Internal autograder failure: {error}" + } + }, + "expect_metric": { + "description": "Extracts a numeric metric from program stdout and validates it against a threshold using a comparison operator (>=, <=, >, <, ==).", + "params": { + "program_command": "Command to execute the student program.", + "metric_pattern": "Regex pattern with a named group 'value' to extract the metric (e.g., 'accuracy:\\\\s*(?P[\\\\d.]+)').", + "condition": "Comparison operator: '>=', '<=', '>', '<', or '=='.", + "threshold": "The threshold value for comparison." + }, + "report": { + "success": "Success: Metric {metric} satisfies condition {condition} {threshold}.", + "failure": "FAILURE: Metric {metric} does not satisfy condition {condition} {threshold}.", + "no_match": "FAILURE: Pattern '{pattern}' did not match any metric in stdout.\nOutput:\n{stdout}", + "not_numeric": "FAILURE: Extracted metric '{value}' is not a valid number.", + "invalid_condition": "CONFIGURATION ERROR: Invalid condition '{condition}'. Valid conditions: {valid}.", + "internal_error": "SYSTEM ERROR: Internal autograder failure: {error}" + } + }, + "expect_csv_output": { + "description": "Validates a CSV file generated by the student program, checking columns, shape, and cell values with tolerance.", + "params": { + "program_command": "Command to execute the student program.", + "artifact_path": "Relative path (under /app) of the CSV file to validate.", + "expected_columns": "List of expected column names in order.", + "expected_shape": "Expected shape as [rows, cols] (excluding header).", + "tolerance": "Acceptable absolute difference for numeric value comparisons (default: 0).", + "expected_values": "List of lists containing expected cell values for comparison." + }, + "report": { + "success": "Success: CSV artifact '{path}' passed all checks.", + "columns_ok": "✓ Columns match expected order and names.", + "columns_mismatch": "✗ Column mismatch. Expected: {expected}, Actual: {actual}", + "shape_ok": "✓ Shape matches: {rows} rows × {cols} columns.", + "shape_mismatch": "✗ Shape mismatch. Expected: {expected_rows}×{expected_cols}, Actual: {actual_rows}×{actual_cols}", + "values_check": "Values: {matching}/{total} cells match ({ratio}).", + "file_not_found": "FAILURE: Expected CSV artifact '{path}' was not found. Make sure your program creates this file.", + "extraction_error": "FAILURE: Could not read artifact from sandbox.\nDetails: {error}", + "parse_error": "FAILURE: Could not parse CSV file.\nDetails: {error}", + "invalid_path": "CONFIGURATION ERROR: Invalid artifact path.\nDetails: {error}", + "internal_error": "SYSTEM ERROR: Internal autograder failure: {error}" + } + }, + "expect_json_output": { + "description": "Validates a JSON file generated by the student program, checking required keys and values with tolerance.", + "params": { + "program_command": "Command to execute the student program.", + "artifact_path": "Relative path (under /app) of the JSON file to validate.", + "required_keys": "List of keys that must exist (supports dot notation for nested keys).", + "expected_values": "Dictionary of key-value pairs to validate (supports dot notation).", + "tolerance": "Acceptable absolute difference for numeric value comparisons (default: 0)." + }, + "report": { + "success": "Success: JSON artifact '{path}' passed all checks.", + "key_found": "✓ Key '{field_key}' found.", + "key_missing": "✗ Required key '{field_key}' not found.", + "value_match": "✓ Key '{field_key}': expected {expected}, got {actual}.", + "value_mismatch": "✗ Key '{field_key}': expected {expected}, got {actual}.", + "value_key_missing": "✗ Key '{field_key}' not found (cannot check value).", + "file_not_found": "FAILURE: Expected JSON artifact '{path}' was not found. Make sure your program creates this file.", + "extraction_error": "FAILURE: Could not read artifact from sandbox.\nDetails: {error}", + "parse_error": "FAILURE: Could not parse JSON file.\nDetails: {error}", + "invalid_path": "CONFIGURATION ERROR: Invalid artifact path.\nDetails: {error}", + "internal_error": "SYSTEM ERROR: Internal autograder failure: {error}" + } + }, + "expect_model_artifact": { + "description": "Validates that a trained model file was produced by the student program with a minimum file size.", + "params": { + "program_command": "Command to execute the student program.", + "artifact_path": "Relative path (under /app) of the model file to validate (e.g., 'model.pkl').", + "min_size_bytes": "Minimum expected file size in bytes (to detect empty/corrupt files)." + }, + "report": { + "success": "Success: Model artifact '{path}' exists ({size} bytes).", + "file_not_found": "FAILURE: Expected model artifact '{path}' was not found. Make sure your program saves the trained model.", + "too_small": "FAILURE: Model artifact '{path}' is too small ({actual_size} bytes, minimum: {min_size} bytes). The model may not have been trained correctly.", + "extraction_error": "FAILURE: Could not read artifact from sandbox.\nDetails: {error}", + "invalid_path": "CONFIGURATION ERROR: Invalid artifact path.\nDetails: {error}", + "internal_error": "SYSTEM ERROR: Internal autograder failure: {error}" + } + } } } diff --git a/autograder/translations/pt_br.json b/autograder/translations/pt_br.json index 1784d4f2..cd66d6db 100644 --- a/autograder/translations/pt_br.json +++ b/autograder/translations/pt_br.json @@ -551,8 +551,8 @@ }, "report": { "request_failed": "A requisição falhou com status code {code}.", - "success": "Sucesso! A resposta de '{endpoint}' continha o par chave-valor esperado ('{key}': '{value}').", - "failure": "A resposta JSON de '{endpoint}' não continha o valor esperado. Esperado '{expected}' para a chave '{key}', mas obteve '{actual}'.", + "success": "Sucesso! A resposta de '{endpoint}' continha o par chave-valor esperado ('{field_key}': '{value}').", + "failure": "A resposta JSON de '{endpoint}' não continha o valor esperado. Esperado '{expected}' para a chave '{field_key}', mas obteve '{actual}'.", "invalid_json": "A resposta de '{endpoint}' não é um JSON válido." } }, @@ -565,5 +565,107 @@ "error": { "step_interrupted": "Ocorreu um erro inesperado durante a execução da etapa: {error}" } + }, + "data_science": { + "template": { + "name": "Ciência de Dados", + "description": "Um modelo para avaliar trabalhos de ciência de dados envolvendo manipulação de dados, treinamento de modelos e validação de métricas." + }, + "expect_stdout_value": { + "description": "Extrai um valor numérico da saída padrão do programa usando um padrão regex e compara com um valor esperado com tolerância opcional.", + "params": { + "program_command": "Comando para executar o programa do aluno.", + "extraction_pattern": "Padrão regex com um grupo nomeado 'value' para extrair o valor numérico (ex: 'total_rows:\\\\s*(?P\\\\d+)').", + "expected_value": "O valor numérico esperado.", + "tolerance": "Diferença absoluta aceitável entre os valores real e esperado (padrão: 0)." + }, + "report": { + "success": "Sucesso: Valor extraído {actual} corresponde ao esperado {expected} (tolerância: {tolerance}).", + "failure": "FALHA: Valor extraído {actual} não corresponde ao esperado {expected} (tolerância: {tolerance}).", + "no_match": "FALHA: O padrão '{pattern}' não encontrou nenhum valor na saída.\nSaída:\n{stdout}", + "not_numeric": "FALHA: O valor extraído '{value}' não é um número válido.", + "internal_error": "ERRO DE SISTEMA: Falha interna do autograder: {error}" + } + }, + "expect_metric": { + "description": "Extrai uma métrica numérica da saída padrão do programa e valida contra um limiar usando um operador de comparação (>=, <=, >, <, ==).", + "params": { + "program_command": "Comando para executar o programa do aluno.", + "metric_pattern": "Padrão regex com um grupo nomeado 'value' para extrair a métrica (ex: 'accuracy:\\\\s*(?P[\\\\d.]+)').", + "condition": "Operador de comparação: '>=', '<=', '>', '<' ou '=='.", + "threshold": "O valor limiar para comparação." + }, + "report": { + "success": "Sucesso: Métrica {metric} satisfaz a condição {condition} {threshold}.", + "failure": "FALHA: Métrica {metric} não satisfaz a condição {condition} {threshold}.", + "no_match": "FALHA: O padrão '{pattern}' não encontrou nenhuma métrica na saída.\nSaída:\n{stdout}", + "not_numeric": "FALHA: A métrica extraída '{value}' não é um número válido.", + "invalid_condition": "ERRO DE CONFIGURAÇÃO: Condição inválida '{condition}'. Condições válidas: {valid}.", + "internal_error": "ERRO DE SISTEMA: Falha interna do autograder: {error}" + } + }, + "expect_csv_output": { + "description": "Valida um arquivo CSV gerado pelo programa do aluno, verificando colunas, dimensões e valores das células com tolerância.", + "params": { + "program_command": "Comando para executar o programa do aluno.", + "artifact_path": "Caminho relativo (sob /app) do arquivo CSV a ser validado.", + "expected_columns": "Lista de nomes de colunas esperados em ordem.", + "expected_shape": "Dimensão esperada como [linhas, colunas] (excluindo cabeçalho).", + "tolerance": "Diferença absoluta aceitável para comparações de valores numéricos (padrão: 0).", + "expected_values": "Lista de listas contendo valores esperados das células para comparação." + }, + "report": { + "success": "Sucesso: O artefato CSV '{path}' passou em todas as verificações.", + "columns_ok": "✓ Colunas correspondem à ordem e nomes esperados.", + "columns_mismatch": "✗ Colunas divergentes. Esperado: {expected}, Obtido: {actual}", + "shape_ok": "✓ Dimensão correta: {rows} linhas × {cols} colunas.", + "shape_mismatch": "✗ Dimensão divergente. Esperado: {expected_rows}×{expected_cols}, Obtido: {actual_rows}×{actual_cols}", + "values_check": "Valores: {matching}/{total} células correspondem ({ratio}).", + "file_not_found": "FALHA: O artefato CSV '{path}' não foi encontrado. Verifique se o seu programa cria este arquivo.", + "extraction_error": "FALHA: Não foi possível ler o artefato do sandbox.\nDetalhes: {error}", + "parse_error": "FALHA: Não foi possível analisar o arquivo CSV.\nDetalhes: {error}", + "invalid_path": "ERRO DE CONFIGURAÇÃO: Caminho de artefato inválido.\nDetalhes: {error}", + "internal_error": "ERRO DE SISTEMA: Falha interna do autograder: {error}" + } + }, + "expect_json_output": { + "description": "Valida um arquivo JSON gerado pelo programa do aluno, verificando chaves obrigatórias e valores com tolerância.", + "params": { + "program_command": "Comando para executar o programa do aluno.", + "artifact_path": "Caminho relativo (sob /app) do arquivo JSON a ser validado.", + "required_keys": "Lista de chaves que devem existir (suporta notação de ponto para chaves aninhadas).", + "expected_values": "Dicionário de pares chave-valor para validar (suporta notação de ponto).", + "tolerance": "Diferença absoluta aceitável para comparações de valores numéricos (padrão: 0)." + }, + "report": { + "success": "Sucesso: O artefato JSON '{path}' passou em todas as verificações.", + "key_found": "✓ Chave '{field_key}' encontrada.", + "key_missing": "✗ Chave obrigatória '{field_key}' não encontrada.", + "value_match": "✓ Chave '{field_key}': esperado {expected}, obtido {actual}.", + "value_mismatch": "✗ Chave '{field_key}': esperado {expected}, obtido {actual}.", + "value_key_missing": "✗ Chave '{field_key}' não encontrada (não foi possível verificar o valor).", + "file_not_found": "FALHA: O artefato JSON '{path}' não foi encontrado. Verifique se o seu programa cria este arquivo.", + "extraction_error": "FALHA: Não foi possível ler o artefato do sandbox.\nDetalhes: {error}", + "parse_error": "FALHA: Não foi possível analisar o arquivo JSON.\nDetalhes: {error}", + "invalid_path": "ERRO DE CONFIGURAÇÃO: Caminho de artefato inválido.\nDetalhes: {error}", + "internal_error": "ERRO DE SISTEMA: Falha interna do autograder: {error}" + } + }, + "expect_model_artifact": { + "description": "Valida que um arquivo de modelo treinado foi produzido pelo programa do aluno com um tamanho mínimo de arquivo.", + "params": { + "program_command": "Comando para executar o programa do aluno.", + "artifact_path": "Caminho relativo (sob /app) do arquivo de modelo a ser validado (ex: 'model.pkl').", + "min_size_bytes": "Tamanho mínimo esperado do arquivo em bytes (para detectar arquivos vazios/corrompidos)." + }, + "report": { + "success": "Sucesso: O artefato de modelo '{path}' existe ({size} bytes).", + "file_not_found": "FALHA: O artefato de modelo '{path}' não foi encontrado. Verifique se o seu programa salva o modelo treinado.", + "too_small": "FALHA: O artefato de modelo '{path}' é muito pequeno ({actual_size} bytes, mínimo: {min_size} bytes). O modelo pode não ter sido treinado corretamente.", + "extraction_error": "FALHA: Não foi possível ler o artefato do sandbox.\nDetalhes: {error}", + "invalid_path": "ERRO DE CONFIGURAÇÃO: Caminho de artefato inválido.\nDetalhes: {error}", + "internal_error": "ERRO DE SISTEMA: Falha interna do autograder: {error}" + } + } } } diff --git a/docs/template-library/data_science.md b/docs/template-library/data_science.md new file mode 100644 index 00000000..9c278e11 --- /dev/null +++ b/docs/template-library/data_science.md @@ -0,0 +1,244 @@ +# Data Science Template (`data_science`) + +The Data Science template tests Python assignments that involve reading datasets, performing transformations, training models, and producing predictions or analytical outputs. It leverages the sandbox environment with pre-installed scientific packages (e.g., pandas, numpy, scikit-learn). + +> **Template name for configs:** `data_science` +> **Requires sandbox:** Yes +> **Supported languages:** Python (Data Science variant: `PYTHON_DS`) + +--- + +## Prerequisites + +Before using this template, you must build the data science Docker image on the host machine running the autograder: + +```bash +docker build -t sandbox-pyds:latest -f sandbox_manager/images/Dockerfile.python-ds . +``` + +Assignments should specify the language as `PYTHON_DS` to utilize this environment. + +--- + +## Test Functions + +### `expect_stdout_value` + +Executes a program, extracts a numeric value from stdout using a regular expression (with a named group `value`), and compares it to an expected value with an optional numeric tolerance. + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `program_command` | string | ✓ | Command to execute the student program | +| `extraction_pattern` | string | ✓ | Regex pattern with a `(?P...)` named group | +| `expected_value` | number | ✓ | The numeric value expected | +| `tolerance` | number | ✗ | Acceptable absolute difference (default: `0`) | + +**Scoring:** 100 if the extracted value is within the tolerance, 0 otherwise. + +**Example:** +```json +{ + "name": "expect_stdout_value", + "parameters": { + "program_command": "python3 analysis.py", + "extraction_pattern": "Mean absolute error:\\s*(?P[\\d.]+)", + "expected_value": 0.5, + "tolerance": 0.1 + }, + "weight": 100 +} +``` + +--- + +### `expect_metric` + +Executes a program, extracts a metric from stdout via regex, and validates it against a threshold using a comparison operator (`>=`, `<=`, `>`, `<`, `==`). + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `program_command` | string | ✓ | Command to execute the student program | +| `metric_pattern` | string | ✓ | Regex pattern with a `(?P...)` named group | +| `condition` | string | ✓ | Comparison operator (`>=`, `<=`, `>`, `<`, `==`) | +| `threshold` | number | ✓ | The threshold value for the metric | + +**Scoring:** 100 if the condition is met, 0 otherwise. + +**Example:** +```json +{ + "name": "expect_metric", + "parameters": { + "program_command": "python3 train.py", + "metric_pattern": "Accuracy:\\s*(?P[\\d.]+)", + "condition": ">=", + "threshold": 0.85 + }, + "weight": 100 +} +``` + +--- + +### `expect_csv_output` + +Validates a generated CSV file by checking column names, shape (rows x cols), and specific cell values (with numeric tolerance). Uses proportional scoring based on how many checks pass. + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `program_command` | string | ✓ | Command to execute the student program | +| `artifact_path` | string | ✓ | Relative path to the generated CSV file | +| `expected_columns` | list[string] | ✗ | Expected column names in exact order | +| `expected_shape` | list[number] | ✗ | Expected `[rows, columns]` including header row | +| `expected_values` | list[list] | ✗ | 2D array of expected values | +| `tolerance` | number | ✗ | Tolerance for numeric comparisons | + +**Scoring:** Proportional based on successful checks (e.g., matching columns counts as 1 check, correct shape as 1, and the ratio of correctly matched cell values counts as 1). + +**Example:** +```json +{ + "name": "expect_csv_output", + "parameters": { + "program_command": "python3 script.py", + "artifact_path": "predictions.csv", + "expected_columns": ["id", "prediction"], + "expected_shape": [101, 2] + }, + "weight": 100 +} +``` + +--- + +### `expect_json_output` + +Validates a generated JSON file by checking for the presence of required keys and expected values. Supports dot-notation for nested keys (e.g., `metrics.accuracy`). + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `program_command` | string | ✓ | Command to execute the student program | +| `artifact_path` | string | ✓ | Relative path to the generated JSON file | +| `required_keys` | list[string] | ✗ | Keys that must exist in the JSON | +| `expected_values` | dict | ✗ | Dictionary of key-value pairs that must match | +| `tolerance` | number | ✗ | Tolerance for numeric comparisons | + +**Scoring:** Proportional based on successful key findings and value matches. + +**Example:** +```json +{ + "name": "expect_json_output", + "parameters": { + "program_command": "python3 eval.py", + "artifact_path": "results.json", + "required_keys": ["status", "metrics.loss"], + "expected_values": { + "status": "success", + "metrics.accuracy": 0.95 + }, + "tolerance": 0.02 + }, + "weight": 100 +} +``` + +--- + +### `expect_model_artifact` + +Verifies that the student program produced a model file (or any artifact) in the sandbox and that it meets a minimum file size constraint. + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `program_command` | string | ✓ | Command to execute the student program | +| `artifact_path` | string | ✓ | Relative path to the generated model file | +| `min_size_bytes` | number | ✗ | Minimum required size of the file in bytes | + +**Scoring:** 100 if the file exists and its size >= `min_size_bytes`, 0 otherwise. + +**Example:** +```json +{ + "name": "expect_model_artifact", + "parameters": { + "program_command": "python3 train_model.py", + "artifact_path": "model.pkl", + "min_size_bytes": 10240 + }, + "weight": 100 +} +``` + +--- + +## Assignment Example + +Here is a full assignment configuration that uses the data science template to inject a dataset via assets, train a model, and validate its outputs: + +```json +{ + "external_assignment_id": "ds-regression-task", + "template_name": "data_science", + "languages": ["PYTHON_DS"], + "setup_config": { + "required_files": ["train.py"], + "setup_commands": [], + "assets": [ + { + "source": "url", + "url": "https://example.com/dataset.csv", + "destination_path": "/tmp/dataset.csv", + "extract": false + } + ] + }, + "criteria_config": { + "base": { + "weight": 100, + "subjects": [ + { + "subject_name": "Model Evaluation", + "weight": 50, + "tests": [ + { + "name": "expect_metric", + "parameters": { + "program_command": "python3 train.py --data /tmp/dataset.csv", + "metric_pattern": "R2 Score:\\s*(?P[\\d.]+)", + "condition": ">=", + "threshold": 0.80 + }, + "weight": 100 + } + ] + }, + { + "subject_name": "Artifact Generation", + "weight": 50, + "tests": [ + { + "name": "expect_model_artifact", + "parameters": { + "program_command": "python3 train.py --data /tmp/dataset.csv", + "artifact_path": "model.pkl", + "min_size_bytes": 5000 + }, + "weight": 50 + }, + { + "name": "expect_csv_output", + "parameters": { + "program_command": "python3 train.py --data /tmp/dataset.csv", + "artifact_path": "predictions.csv", + "expected_columns": ["id", "pred_value"] + }, + "weight": 50 + } + ] + } + ] + } + } +} +``` diff --git a/sandbox_manager/images/Dockerfile.python-ds b/sandbox_manager/images/Dockerfile.python-ds new file mode 100644 index 00000000..5c72b1e8 --- /dev/null +++ b/sandbox_manager/images/Dockerfile.python-ds @@ -0,0 +1,29 @@ +FROM python:3.11-slim + +# 1. Create a non-root user +RUN groupadd -r sandbox && useradd -r -g sandbox sandbox + +# 2. Install scientific Python packages for data science assignments +RUN pip install --no-cache-dir \ + numpy==1.26.4 \ + pandas==2.2.1 \ + scikit-learn==1.4.1.post1 \ + matplotlib==3.8.3 \ + seaborn==0.13.2 + +# 3. Security Hardening: Remove network tools and dangerous binaries +RUN rm -f /usr/bin/wget /usr/bin/curl +RUN find / -perm /6000 -type f -exec chmod a-s {} \; || true + +# 4. Setup Workspace +WORKDIR /app +RUN chown sandbox:sandbox /app + +# 5. Make /app a volume so it's writable even when container is read-only +VOLUME ["/app"] + +# 6. Switch to user +USER sandbox + +# 7. Default entrypoint +CMD ["python3"] diff --git a/sandbox_manager/models/sandbox_models.py b/sandbox_manager/models/sandbox_models.py index 905939b7..21b5cb70 100644 --- a/sandbox_manager/models/sandbox_models.py +++ b/sandbox_manager/models/sandbox_models.py @@ -7,6 +7,7 @@ class Language(Enum): """Supported programming languages in the sandbox.""" PYTHON = ("python", "sandbox-py:latest") + PYTHON_DS = ("python_ds", "sandbox-pyds:latest") JAVA = ("java", "sandbox-java:latest") NODE = ("node", "sandbox-node:latest") CPP = ("cpp", "sandbox-cpp:latest") diff --git a/tests/unit/test_data_science.py b/tests/unit/test_data_science.py new file mode 100644 index 00000000..79cff0e0 --- /dev/null +++ b/tests/unit/test_data_science.py @@ -0,0 +1,906 @@ +""" +Unit tests for DataScienceTemplate and its test functions. + +Tests cover: +- ExpectStdoutValueTest: regex extraction, tolerance, no match, non-numeric, execution errors +- ExpectMetricTest: all condition operators, threshold pass/fail, no match, invalid condition +- ExpectCsvOutputTest: columns, shape, values with tolerance, proportional scoring, missing file +- ExpectJsonOutputTest: required keys, expected values, nested access, tolerance, missing file +- ExpectModelArtifactTest: file exists, min size pass/fail, missing file +- DataScienceTemplate: contract validation, test registration +""" + +import json +import unittest +from unittest.mock import MagicMock + +from autograder.template_library.data_science import ( + DataScienceTemplate, + ExpectStdoutValueTest, + ExpectMetricTest, + ExpectCsvOutputTest, + ExpectJsonOutputTest, + ExpectModelArtifactTest, +) +from sandbox_manager.models.sandbox_models import ( + CommandResponse, ResponseCategory, ExtractedFile, +) + + +def _make_sandbox(stdout="", stderr="", exit_code=0, category=ResponseCategory.SUCCESS): + """Return a mock sandbox whose run_commands returns the given CommandResponse.""" + sandbox = MagicMock() + sandbox.run_commands.return_value = CommandResponse( + stdout=stdout, stderr=stderr, exit_code=exit_code, + execution_time=0.1, category=category, + ) + sandbox.run_command.return_value = CommandResponse( + stdout=stdout, stderr=stderr, exit_code=exit_code, + execution_time=0.1, category=category, + ) + return sandbox + + +def _make_extracted(content: str, path="/app/output.csv"): + return ExtractedFile( + path=path, + content_bytes=content.encode("utf-8"), + size=len(content.encode("utf-8")), + content_text=content, + encoding="utf-8", + ) + + +def _make_extracted_bytes(content_bytes: bytes, path="/app/model.pkl"): + return ExtractedFile( + path=path, + content_bytes=content_bytes, + size=len(content_bytes), + content_text="", + encoding="utf-8", + ) + + +# =============================================================== +# TestExpectStdoutValue +# =============================================================== + +class TestExpectStdoutValueSuccess(unittest.TestCase): + """Tests for successful value extraction and comparison.""" + + def setUp(self): + self.test = ExpectStdoutValueTest() + + def test_exact_match(self): + """Extracted value exactly matches expected value.""" + sandbox = _make_sandbox(stdout="total_rows: 1500") + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 analysis.py stats", + extraction_pattern=r"total_rows:\s*(?P\d+)", + expected_value=1500, + tolerance=0, + ) + self.assertEqual(result.score, 100.0) + + def test_match_with_tolerance(self): + """Extracted value within tolerance of expected value.""" + sandbox = _make_sandbox(stdout="accuracy: 0.847") + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 analysis.py", + extraction_pattern=r"accuracy:\s*(?P[\d.]+)", + expected_value=0.85, + tolerance=0.01, + ) + self.assertEqual(result.score, 100.0) + + def test_match_outside_tolerance(self): + """Extracted value outside tolerance of expected value.""" + sandbox = _make_sandbox(stdout="accuracy: 0.70") + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 analysis.py", + extraction_pattern=r"accuracy:\s*(?P[\d.]+)", + expected_value=0.85, + tolerance=0.01, + ) + self.assertEqual(result.score, 0.0) + + +class TestExpectStdoutValueErrors(unittest.TestCase): + """Tests for error handling in ExpectStdoutValueTest.""" + + def setUp(self): + self.test = ExpectStdoutValueTest() + + def test_no_match_in_stdout(self): + """Pattern doesn't match anything in stdout.""" + sandbox = _make_sandbox(stdout="no relevant output here") + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 analysis.py", + extraction_pattern=r"total_rows:\s*(?P\d+)", + expected_value=1500, + ) + self.assertEqual(result.score, 0.0) + self.assertIn("total_rows", result.report) + + def test_non_numeric_value(self): + """Extracted value is not a valid number.""" + sandbox = _make_sandbox(stdout="total_rows: abc") + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 analysis.py", + extraction_pattern=r"total_rows:\s*(?P\w+)", + expected_value=1500, + ) + self.assertEqual(result.score, 0.0) + self.assertIn("abc", result.report) + + def test_timeout(self): + """Program times out.""" + sandbox = _make_sandbox( + stderr="Execution timed out", + exit_code=124, + category=ResponseCategory.TIMEOUT, + ) + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 analysis.py", + extraction_pattern=r"total_rows:\s*(?P\d+)", + expected_value=1500, + ) + self.assertEqual(result.score, 0.0) + + def test_runtime_error(self): + """Program crashes with runtime error.""" + sandbox = _make_sandbox( + stderr="ImportError: No module named 'pandas'", + exit_code=1, + category=ResponseCategory.RUNTIME_ERROR, + ) + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 analysis.py", + extraction_pattern=r"total_rows:\s*(?P\d+)", + expected_value=1500, + ) + self.assertEqual(result.score, 0.0) + + +# =============================================================== +# TestExpectMetric +# =============================================================== + +class TestExpectMetricConditions(unittest.TestCase): + """Tests for each comparison condition in ExpectMetricTest.""" + + def setUp(self): + self.test = ExpectMetricTest() + + def test_gte_pass(self): + """Metric >= threshold.""" + sandbox = _make_sandbox(stdout="accuracy: 0.90") + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 train.py", + metric_pattern=r"accuracy:\s*(?P[\d.]+)", + condition=">=", + threshold=0.85, + ) + self.assertEqual(result.score, 100.0) + + def test_gte_fail(self): + """Metric < threshold.""" + sandbox = _make_sandbox(stdout="accuracy: 0.70") + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 train.py", + metric_pattern=r"accuracy:\s*(?P[\d.]+)", + condition=">=", + threshold=0.85, + ) + self.assertEqual(result.score, 0.0) + + def test_lte_pass(self): + """Metric <= threshold (e.g., RMSE).""" + sandbox = _make_sandbox(stdout="rmse: 2.3") + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 train.py", + metric_pattern=r"rmse:\s*(?P[\d.]+)", + condition="<=", + threshold=3.0, + ) + self.assertEqual(result.score, 100.0) + + def test_gt_pass(self): + """Metric > threshold.""" + sandbox = _make_sandbox(stdout="f1: 0.91") + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 train.py", + metric_pattern=r"f1:\s*(?P[\d.]+)", + condition=">", + threshold=0.90, + ) + self.assertEqual(result.score, 100.0) + + def test_lt_pass(self): + """Metric < threshold.""" + sandbox = _make_sandbox(stdout="loss: 0.05") + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 train.py", + metric_pattern=r"loss:\s*(?P[\d.]+)", + condition="<", + threshold=0.1, + ) + self.assertEqual(result.score, 100.0) + + def test_eq_pass(self): + """Metric == threshold.""" + sandbox = _make_sandbox(stdout="epochs: 10") + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 train.py", + metric_pattern=r"epochs:\s*(?P[\d.]+)", + condition="==", + threshold=10.0, + ) + self.assertEqual(result.score, 100.0) + + def test_eq_fail(self): + """Metric != threshold.""" + sandbox = _make_sandbox(stdout="epochs: 5") + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 train.py", + metric_pattern=r"epochs:\s*(?P[\d.]+)", + condition="==", + threshold=10.0, + ) + self.assertEqual(result.score, 0.0) + + +class TestExpectMetricErrors(unittest.TestCase): + """Tests for error handling in ExpectMetricTest.""" + + def setUp(self): + self.test = ExpectMetricTest() + + def test_invalid_condition(self): + """Invalid condition operator is rejected.""" + sandbox = _make_sandbox(stdout="accuracy: 0.90") + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 train.py", + metric_pattern=r"accuracy:\s*(?P[\d.]+)", + condition="!=", + threshold=0.85, + ) + self.assertEqual(result.score, 0.0) + self.assertIn("!=", result.report) + + def test_no_match_in_stdout(self): + """Pattern doesn't match anything in stdout.""" + sandbox = _make_sandbox(stdout="no metrics here") + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 train.py", + metric_pattern=r"accuracy:\s*(?P[\d.]+)", + condition=">=", + threshold=0.85, + ) + self.assertEqual(result.score, 0.0) + + def test_non_numeric_metric(self): + """Extracted metric is not a valid number.""" + sandbox = _make_sandbox(stdout="accuracy: N/A") + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 train.py", + metric_pattern=r"accuracy:\s*(?P\S+)", + condition=">=", + threshold=0.85, + ) + self.assertEqual(result.score, 0.0) + + +# =============================================================== +# TestExpectCsvOutput +# =============================================================== + +class TestExpectCsvOutputColumns(unittest.TestCase): + """Tests for column validation in ExpectCsvOutputTest.""" + + def setUp(self): + self.test = ExpectCsvOutputTest() + + def test_columns_match(self): + """Columns match expected names and order.""" + csv_content = "name,cuisine,rating\nPizza Place,Italian,4.5\n" + sandbox = _make_sandbox() + sandbox.extract_file.return_value = _make_extracted(csv_content) + + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 analysis.py preprocess", + artifact_path="output.csv", + expected_columns=["name", "cuisine", "rating"], + ) + self.assertEqual(result.score, 100.0) + + def test_columns_mismatch(self): + """Columns don't match expected.""" + csv_content = "name,type,score\nPizza Place,Italian,4.5\n" + sandbox = _make_sandbox() + sandbox.extract_file.return_value = _make_extracted(csv_content) + + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 analysis.py preprocess", + artifact_path="output.csv", + expected_columns=["name", "cuisine", "rating"], + ) + self.assertEqual(result.score, 0.0) + + +class TestExpectCsvOutputShape(unittest.TestCase): + """Tests for shape validation in ExpectCsvOutputTest.""" + + def setUp(self): + self.test = ExpectCsvOutputTest() + + def test_shape_match(self): + """Shape matches expected rows and columns.""" + csv_content = "a,b,c\n1,2,3\n4,5,6\n" + sandbox = _make_sandbox() + sandbox.extract_file.return_value = _make_extracted(csv_content) + + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 analysis.py", + artifact_path="output.csv", + expected_shape=[2, 3], + ) + self.assertEqual(result.score, 100.0) + + def test_shape_mismatch(self): + """Shape doesn't match expected.""" + csv_content = "a,b\n1,2\n" + sandbox = _make_sandbox() + sandbox.extract_file.return_value = _make_extracted(csv_content) + + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 analysis.py", + artifact_path="output.csv", + expected_shape=[3, 3], + ) + self.assertEqual(result.score, 0.0) + + +class TestExpectCsvOutputValues(unittest.TestCase): + """Tests for value validation with tolerance in ExpectCsvOutputTest.""" + + def setUp(self): + self.test = ExpectCsvOutputTest() + + def test_values_exact_match(self): + """Values match exactly.""" + csv_content = "id,score\n1,95.0\n2,87.5\n" + sandbox = _make_sandbox() + sandbox.extract_file.return_value = _make_extracted(csv_content) + + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 analysis.py", + artifact_path="output.csv", + expected_values=[["1", "95.0"], ["2", "87.5"]], + ) + self.assertEqual(result.score, 100.0) + + def test_values_within_tolerance(self): + """Numeric values within tolerance.""" + csv_content = "id,score\n1,95.3\n2,87.2\n" + sandbox = _make_sandbox() + sandbox.extract_file.return_value = _make_extracted(csv_content) + + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 analysis.py", + artifact_path="output.csv", + expected_values=[["1", "95.0"], ["2", "87.5"]], + tolerance=0.5, + ) + self.assertEqual(result.score, 100.0) + + def test_proportional_scoring(self): + """Partial match gives proportional score.""" + csv_content = "id,score\n1,95.0\n2,0.0\n" + sandbox = _make_sandbox() + sandbox.extract_file.return_value = _make_extracted(csv_content) + + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 analysis.py", + artifact_path="output.csv", + expected_values=[["1", "95.0"], ["2", "87.5"]], + tolerance=0, + ) + # 3 out of 4 cells match (id=1, score=95.0, id=2 match; score=0.0 doesn't) + self.assertGreater(result.score, 0.0) + self.assertLess(result.score, 100.0) + + +class TestExpectCsvOutputErrors(unittest.TestCase): + """Tests for error handling in ExpectCsvOutputTest.""" + + def setUp(self): + self.test = ExpectCsvOutputTest() + + def test_file_not_found(self): + """CSV artifact not found in sandbox.""" + sandbox = _make_sandbox() + sandbox.extract_file.side_effect = FileNotFoundError("File not found") + + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 analysis.py", + artifact_path="output.csv", + expected_columns=["a", "b"], + ) + self.assertEqual(result.score, 0.0) + self.assertIn("output.csv", result.report) + + def test_invalid_path(self): + """Absolute path is rejected.""" + sandbox = _make_sandbox() + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 analysis.py", + artifact_path="/etc/passwd", + expected_columns=["a"], + ) + self.assertEqual(result.score, 0.0) + + def test_empty_artifact_path(self): + """Empty artifact path is rejected.""" + sandbox = _make_sandbox() + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 analysis.py", + artifact_path="", + ) + self.assertEqual(result.score, 0.0) + + def test_no_checks_specified(self): + """File exists with no specific checks = pass.""" + csv_content = "a,b\n1,2\n" + sandbox = _make_sandbox() + sandbox.extract_file.return_value = _make_extracted(csv_content) + + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 analysis.py", + artifact_path="output.csv", + ) + self.assertEqual(result.score, 100.0) + + def test_combined_checks_proportional(self): + """Multiple checks contribute proportionally to score.""" + csv_content = "name,score\nAlice,95\nBob,80\n" + sandbox = _make_sandbox() + sandbox.extract_file.return_value = _make_extracted(csv_content) + + # Columns match, shape doesn't + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 analysis.py", + artifact_path="output.csv", + expected_columns=["name", "score"], + expected_shape=[5, 2], # Wrong rows + ) + self.assertEqual(result.score, 50.0) + + +# =============================================================== +# TestExpectJsonOutput +# =============================================================== + +class TestExpectJsonOutputKeys(unittest.TestCase): + """Tests for key validation in ExpectJsonOutputTest.""" + + def setUp(self): + self.test = ExpectJsonOutputTest() + + def test_required_keys_found(self): + """All required keys found.""" + data = {"accuracy": 0.9, "loss": 0.1, "epochs": 10} + sandbox = _make_sandbox() + sandbox.extract_file.return_value = _make_extracted(json.dumps(data), path="/app/metrics.json") + + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 train.py", + artifact_path="metrics.json", + required_keys=["accuracy", "loss", "epochs"], + ) + self.assertEqual(result.score, 100.0) + + def test_required_key_missing(self): + """One required key missing.""" + data = {"accuracy": 0.9, "loss": 0.1} + sandbox = _make_sandbox() + sandbox.extract_file.return_value = _make_extracted(json.dumps(data), path="/app/metrics.json") + + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 train.py", + artifact_path="metrics.json", + required_keys=["accuracy", "loss", "epochs"], + ) + self.assertGreater(result.score, 0.0) + self.assertLess(result.score, 100.0) + + def test_nested_key_found(self): + """Nested key access via dot notation.""" + data = {"metrics": {"accuracy": 0.9}} + sandbox = _make_sandbox() + sandbox.extract_file.return_value = _make_extracted(json.dumps(data), path="/app/results.json") + + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 train.py", + artifact_path="results.json", + required_keys=["metrics.accuracy"], + ) + self.assertEqual(result.score, 100.0) + + +class TestExpectJsonOutputValues(unittest.TestCase): + """Tests for value validation in ExpectJsonOutputTest.""" + + def setUp(self): + self.test = ExpectJsonOutputTest() + + def test_value_match(self): + """Expected value matches actual value.""" + data = {"accuracy": 0.9, "model": "rf"} + sandbox = _make_sandbox() + sandbox.extract_file.return_value = _make_extracted(json.dumps(data), path="/app/metrics.json") + + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 train.py", + artifact_path="metrics.json", + expected_values={"model": "rf"}, + ) + self.assertEqual(result.score, 100.0) + + def test_numeric_value_with_tolerance(self): + """Numeric value within tolerance.""" + data = {"accuracy": 0.847} + sandbox = _make_sandbox() + sandbox.extract_file.return_value = _make_extracted(json.dumps(data), path="/app/metrics.json") + + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 train.py", + artifact_path="metrics.json", + expected_values={"accuracy": 0.85}, + tolerance=0.01, + ) + self.assertEqual(result.score, 100.0) + + def test_value_mismatch(self): + """Expected value doesn't match actual value.""" + data = {"accuracy": 0.5} + sandbox = _make_sandbox() + sandbox.extract_file.return_value = _make_extracted(json.dumps(data), path="/app/metrics.json") + + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 train.py", + artifact_path="metrics.json", + expected_values={"accuracy": 0.9}, + tolerance=0.01, + ) + self.assertEqual(result.score, 0.0) + + +class TestExpectJsonOutputErrors(unittest.TestCase): + """Tests for error handling in ExpectJsonOutputTest.""" + + def setUp(self): + self.test = ExpectJsonOutputTest() + + def test_file_not_found(self): + """JSON file not found.""" + sandbox = _make_sandbox() + sandbox.extract_file.side_effect = FileNotFoundError("File not found") + + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 train.py", + artifact_path="metrics.json", + required_keys=["accuracy"], + ) + self.assertEqual(result.score, 0.0) + + def test_invalid_json(self): + """File content is not valid JSON.""" + sandbox = _make_sandbox() + sandbox.extract_file.return_value = _make_extracted("not json {{{", path="/app/metrics.json") + + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 train.py", + artifact_path="metrics.json", + required_keys=["accuracy"], + ) + self.assertEqual(result.score, 0.0) + + def test_invalid_path(self): + """Absolute path is rejected.""" + sandbox = _make_sandbox() + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 train.py", + artifact_path="/etc/passwd", + ) + self.assertEqual(result.score, 0.0) + + +# =============================================================== +# TestExpectModelArtifact +# =============================================================== + +class TestExpectModelArtifactSuccess(unittest.TestCase): + """Tests for successful model artifact validation.""" + + def setUp(self): + self.test = ExpectModelArtifactTest() + + def test_model_exists_sufficient_size(self): + """Model file exists and meets minimum size.""" + content = b"\x80\x04" + b"\x00" * 4096 # Simulate a pickle file + sandbox = _make_sandbox() + sandbox.extract_file.return_value = _make_extracted_bytes(content, path="/app/model.pkl") + + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 train.py", + artifact_path="model.pkl", + min_size_bytes=2048, + ) + self.assertEqual(result.score, 100.0) + + def test_model_exists_no_min_size(self): + """Model file exists with no min_size requirement.""" + content = b"\x80\x04\x95" + sandbox = _make_sandbox() + sandbox.extract_file.return_value = _make_extracted_bytes(content, path="/app/model.pkl") + + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 train.py", + artifact_path="model.pkl", + ) + self.assertEqual(result.score, 100.0) + + +class TestExpectModelArtifactErrors(unittest.TestCase): + """Tests for error handling in ExpectModelArtifactTest.""" + + def setUp(self): + self.test = ExpectModelArtifactTest() + + def test_model_too_small(self): + """Model file exists but is too small.""" + content = b"\x80" # 1 byte + sandbox = _make_sandbox() + sandbox.extract_file.return_value = _make_extracted_bytes(content, path="/app/model.pkl") + + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 train.py", + artifact_path="model.pkl", + min_size_bytes=2048, + ) + self.assertEqual(result.score, 0.0) + self.assertIn("too small", result.report.lower()) + + def test_model_not_found(self): + """Model file not found.""" + sandbox = _make_sandbox() + sandbox.extract_file.side_effect = FileNotFoundError("File not found") + + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 train.py", + artifact_path="model.pkl", + min_size_bytes=2048, + ) + self.assertEqual(result.score, 0.0) + self.assertIn("model.pkl", result.report) + + def test_invalid_path(self): + """Absolute path is rejected.""" + sandbox = _make_sandbox() + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 train.py", + artifact_path="/tmp/model.pkl", + min_size_bytes=2048, + ) + self.assertEqual(result.score, 0.0) + + def test_traversal_path_rejected(self): + """Path traversal is rejected.""" + sandbox = _make_sandbox() + result = self.test.execute( + files=None, sandbox=sandbox, + program_command="python3 train.py", + artifact_path="../model.pkl", + min_size_bytes=2048, + ) + self.assertEqual(result.score, 0.0) + + +# =============================================================== +# TestDataScienceTemplate +# =============================================================== + +class TestDataScienceTemplate(unittest.TestCase): + """Tests for the DataScienceTemplate class.""" + + def setUp(self): + self.template = DataScienceTemplate() + + def test_contract_validation(self): + """Template passes contract validation.""" + self.template.validate_contract() + self.assertIsInstance(self.template.get_tests(), dict) + + def test_requires_sandbox(self): + """Template requires sandbox.""" + self.assertTrue(self.template.requires_sandbox) + + def test_template_name_not_empty(self): + """Template name is not empty.""" + self.assertTrue(len(self.template.template_name) > 0) + + def test_template_description_not_empty(self): + """Template description is not empty.""" + self.assertTrue(len(self.template.template_description) > 0) + + def test_get_test_expect_stdout_value(self): + """Can retrieve expect_stdout_value test.""" + test = self.template.get_test("expect_stdout_value") + self.assertIsInstance(test, ExpectStdoutValueTest) + + def test_get_test_expect_metric(self): + """Can retrieve expect_metric test.""" + test = self.template.get_test("expect_metric") + self.assertIsInstance(test, ExpectMetricTest) + + def test_get_test_expect_csv_output(self): + """Can retrieve expect_csv_output test.""" + test = self.template.get_test("expect_csv_output") + self.assertIsInstance(test, ExpectCsvOutputTest) + + def test_get_test_expect_json_output(self): + """Can retrieve expect_json_output test.""" + test = self.template.get_test("expect_json_output") + self.assertIsInstance(test, ExpectJsonOutputTest) + + def test_get_test_expect_model_artifact(self): + """Can retrieve expect_model_artifact test.""" + test = self.template.get_test("expect_model_artifact") + self.assertIsInstance(test, ExpectModelArtifactTest) + + def test_get_nonexistent_test_raises(self): + """Getting a non-existent test raises AttributeError.""" + with self.assertRaises(AttributeError): + self.template.get_test("nonexistent_test") + + def test_all_tests_registered(self): + """All 5 test functions are registered.""" + tests = self.template.get_tests() + expected_names = { + "expect_stdout_value", + "expect_metric", + "expect_csv_output", + "expect_json_output", + "expect_model_artifact", + } + self.assertEqual(set(tests.keys()), expected_names) + + +# =============================================================== +# TestExpectStdoutValueMetadata +# =============================================================== + +class TestExpectStdoutValueMetadata(unittest.TestCase): + """Tests for metadata properties of ExpectStdoutValueTest.""" + + def test_name(self): + """Test the template name.""" + self.assertEqual(ExpectStdoutValueTest().name, "expect_stdout_value") + + def test_description_not_empty(self): + """Test the description is not empty.""" + self.assertTrue(len(ExpectStdoutValueTest().description) > 0) + + def test_parameter_descriptions(self): + """Test parameter descriptions exist.""" + params = ExpectStdoutValueTest().parameter_description + names = [p.name for p in params] + self.assertIn("program_command", names) + self.assertIn("extraction_pattern", names) + self.assertIn("expected_value", names) + self.assertIn("tolerance", names) + + +class TestExpectMetricMetadata(unittest.TestCase): + """Tests for metadata properties of ExpectMetricTest.""" + + def test_name(self): + """Test the template name.""" + self.assertEqual(ExpectMetricTest().name, "expect_metric") + + def test_description_not_empty(self): + """Test the description is not empty.""" + self.assertTrue(len(ExpectMetricTest().description) > 0) + + def test_parameter_descriptions(self): + """Test parameter descriptions exist.""" + params = ExpectMetricTest().parameter_description + names = [p.name for p in params] + self.assertIn("program_command", names) + self.assertIn("metric_pattern", names) + self.assertIn("condition", names) + self.assertIn("threshold", names) + + +class TestExpectCsvOutputMetadata(unittest.TestCase): + """Tests for metadata properties of ExpectCsvOutputTest.""" + + def test_name(self): + """Test the template name.""" + self.assertEqual(ExpectCsvOutputTest().name, "expect_csv_output") + + def test_description_not_empty(self): + """Test the description is not empty.""" + self.assertTrue(len(ExpectCsvOutputTest().description) > 0) + + +class TestExpectJsonOutputMetadata(unittest.TestCase): + """Tests for metadata properties of ExpectJsonOutputTest.""" + + def test_name(self): + """Test the template name.""" + self.assertEqual(ExpectJsonOutputTest().name, "expect_json_output") + + def test_description_not_empty(self): + """Test the description is not empty.""" + self.assertTrue(len(ExpectJsonOutputTest().description) > 0) + + +class TestExpectModelArtifactMetadata(unittest.TestCase): + """Tests for metadata properties of ExpectModelArtifactTest.""" + + def test_name(self): + """Test the template name.""" + self.assertEqual(ExpectModelArtifactTest().name, "expect_model_artifact") + + def test_description_not_empty(self): + """Test the description is not empty.""" + self.assertTrue(len(ExpectModelArtifactTest().description) > 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_template_contract.py b/tests/unit/test_template_contract.py index fbe1c194..7e01b26c 100644 --- a/tests/unit/test_template_contract.py +++ b/tests/unit/test_template_contract.py @@ -6,6 +6,7 @@ from autograder.models.dataclass.test_result import TestResult from autograder.template_library.api_testing import ApiTestingTemplate from autograder.template_library.input_output import InputOutputTemplate +from autograder.template_library.data_science import DataScienceTemplate from autograder.template_library.web_dev.template import WebDevTemplate @@ -44,7 +45,7 @@ def get_test(self, name: str): def test_builtin_templates_validate_contract(): - for template in [WebDevTemplate(), InputOutputTemplate(), ApiTestingTemplate()]: + for template in [WebDevTemplate(), InputOutputTemplate(), ApiTestingTemplate(), DataScienceTemplate()]: template.validate_contract() assert isinstance(template.get_tests(), dict)