feat(pipeline): split PreFlight into granular setup steps (#302)

README.md

@@ -7,11 +7,9 @@
 **An educational-standards-driven autograding tool that transforms assignment grading into an engaging learning experience.**
 
 [![Python](https://img.shields.io/badge/Python-3.9+-blue.svg)](https://www.python.org/downloads/)
-[![FastAPI](https://img.shields.io/badge/FastAPI-0.115.0-009688.svg)](https://fastapi.tiangolo.com/)
 [![Docker](https://img.shields.io/badge/Docker-Enabled-2496ED.svg)](https://www.docker.com/)
 [![Tests](https://github.com/webtech-network/autograder/actions/workflows/pytest.yml/badge.svg)](https://github.com/webtech-network/autograder/actions/workflows/pytest.yml)
 [![Lint](https://github.com/webtech-network/autograder/actions/workflows/pylint.yml/badge.svg)](https://github.com/webtech-network/autograder/actions/workflows/pylint.yml)
-[![Docs](https://github.com/webtech-network/autograder/actions/workflows/validate-docs.yml/badge.svg)](https://github.com/webtech-network/autograder/actions/workflows/validate-docs.yml)
 [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](LICENSE)
 
 [Features](#features) • [Architecture](#architecture) • [Quick Start](#quick-start) • [Templates](#grading-templates) • [Pipeline](#pipeline-workflow) • [API](#rest-api) • [GitHub Action](#github-action) • [Docs](https://webtech-network.github.io/autograder/) • [Community](#community-standards)

autograder/autograder.py

@@ -168,7 +168,7 @@ def build_pipeline(  # pylint: disable=too-many-arguments,too-many-locals
         "include_feedback": include_feedback,
         "grading_criteria": grading_criteria,
         "feedback_config": feedback_config,
-        "setup_config": setup_config,
+        "setup_config": _merge_setup_config(setup_config, templates),
         "custom_template": custom_template,
         "feedback_mode": feedback_mode,
         "export_results": export_results,
@@ -180,8 +180,10 @@ def build_pipeline(  # pylint: disable=too-many-arguments,too-many-locals
     execution_order = [
         StepName.LOAD_TEMPLATE,
         StepName.BUILD_TREE,
+        StepName.FILE_CHECK,
         StepName.SANDBOX,
-        StepName.PRE_FLIGHT,
+        StepName.ASSET_INJECTION,
+        StepName.SETUP_COMMANDS,
         StepName.AI_BATCH,
         StepName.STRUCTURAL_ANALYSIS,
         StepName.GRADE,
@@ -195,6 +197,49 @@ def build_pipeline(  # pylint: disable=too-many-arguments,too-many-locals
         if step_instance is not None:
             pipeline.add_step(step_name, step_instance)
 
+    return pipeline
+
+
+def _merge_setup_config(assignment_config, templates) -> dict:
+    """
+    Merge template-level setup requirements into the assignment setup config.
+    Assignment config values take precedence for shared structure.
+    """
+    import copy
 
+    if assignment_config is None:
+        merged = {}
+    elif isinstance(assignment_config, dict):
+        merged = copy.deepcopy(assignment_config)
+    else:
+        # Handle SetupConfig object if passed
+        merged = assignment_config.model_dump(exclude_none=True)
+
+    for template in templates:
+        # Merge required_files
+        for lang, files in template.required_files.items():
+            if not files:
+                continue
+            if lang not in merged:
+                merged[lang] = {}
+            if "required_files" not in merged[lang]:
+                merged[lang]["required_files"] = []
+
+            for f in files:
+                if f not in merged[lang]["required_files"]:
+                    merged[lang]["required_files"].append(f)
+
+        # Merge setup_commands
+        for lang, commands in template.setup_commands.items():
+            if not commands:
+                continue
+            if lang not in merged:
+                merged[lang] = {}
+            if "setup_commands" not in merged[lang]:
+                merged[lang]["setup_commands"] = []
+
+            for cmd in commands:
+                if cmd not in merged[lang]["setup_commands"]:
+                    merged[lang]["setup_commands"].append(cmd)
 
-    return pipeline
+    return merged

autograder/models/abstract/step.py

@@ -2,7 +2,7 @@
 from abc import ABC, abstractmethod
 from autograder.translations import t
 
-from autograder.models.dataclass.step_result import StepResult, StepStatus, StepName
+from autograder.models.dataclass.step_result import StepResult, StepStatus, StepName, StepCategory
 from autograder.models.pipeline_execution import PipelineExecution
 
 logger = logging.getLogger(__name__)
@@ -17,6 +17,12 @@ class Step(ABC):
     def step_name(self) -> StepName:
         """Return the name of the step (e.g. StepName.GRADE)."""
 
+    @property
+    @abstractmethod
+    def category(self) -> StepCategory:
+        """Return the category of the step (e.g. StepCategory.GRADING)."""
+        pass
+
     def execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution:
         """
         Execute the step on the pipeline execution context.

autograder/models/abstract/template.py

@@ -25,6 +25,16 @@ def requires_sandbox(self) -> bool:
         """Declare whether template tests require sandbox execution."""
         raise NotImplementedError
 
+    @property
+    def required_files(self) -> Dict[str, list]:
+        """Return a dictionary of required files per language."""
+        return {}
+
+    @property
+    def setup_commands(self) -> Dict[str, list]:
+        """Return a dictionary of setup commands per language."""
+        return {}
+
     @abstractmethod
     def get_test(self, name: str) -> TestFunction:
         """Return a test function instance by its registry name."""

autograder/models/dataclass/step_result.py

@@ -20,11 +20,21 @@ class StepStatus(Enum):
     INTERRUPTED = "interrupted"
 
 
+class StepCategory(Enum):
+    """Enumeration of pipeline step categories."""
+    SETUP = "setup"
+    GRADING = "grading"
+    REPORTING = "reporting"
+
+
 class StepName(Enum):
     """Enumeration of all available pipeline steps."""
     BOOTSTRAP = "BootstrapStep"
     LOAD_TEMPLATE = "LoadTemplateStep"
     BUILD_TREE = "BuildTreeStep"
+    FILE_CHECK = "FileCheckStep"
+    ASSET_INJECTION = "AssetInjectionStep"
+    SETUP_COMMANDS = "SetupCommandsStep"
     PRE_FLIGHT = "PreFlightStep"
     SANDBOX = "SandboxStep"
     AI_BATCH = "AiBatchStep"

autograder/serializers/pipeline_execution_serializer.py

@@ -56,7 +56,9 @@ def serialize(cls, execution: PipelineExecution) -> Dict[str, Any]:
             steps.append(step_info)
 
         # Count planned vs completed steps
-        total_planned = 7  # BOOTSTRAP, LOAD_TEMPLATE, BUILD_TREE, PRE_FLIGHT, GRADE, FEEDBACK, EXPORT
+        # If the execution context has a planned steps count, use it.
+        # Otherwise fall back to a reasonable estimate.
+        total_planned = getattr(execution, "total_steps_planned", 7)
         completed = len([s for s in execution.step_results if s.step != StepName.BOOTSTRAP])
 
         return {
@@ -75,6 +77,14 @@ def _get_success_message(cls, step: StepResult, execution: PipelineExecution) ->
             return "Template loaded successfully"
         elif step.step == StepName.BUILD_TREE:
             return "Criteria tree built successfully"
+        elif step.step == StepName.FILE_CHECK:
+            return "All required files are present"
+        elif step.step == StepName.SANDBOX:
+            return "Sandbox environment allocated"
+        elif step.step == StepName.ASSET_INJECTION:
+            return "Assets injected successfully"
+        elif step.step == StepName.SETUP_COMMANDS:
+            return "Setup commands executed successfully"
         elif step.step == StepName.PRE_FLIGHT:
             return "All preflight checks passed"
         elif step.step == StepName.GRADE:
@@ -95,23 +105,22 @@ def _extract_error_details(cls, step: StepResult) -> Dict[str, Any]:
         """
         error_details = {}
 
-        if step.step == StepName.PRE_FLIGHT and step.error_data:
-            # error_data is a list of PreflightError objects. 
-            # We map properties from the first critical error for legacy backward compatibility.
+        if step.step in [StepName.FILE_CHECK, StepName.ASSET_INJECTION, StepName.SETUP_COMMANDS, StepName.PRE_FLIGHT] and step.error_data:
+            # error_data is a list of PreflightError objects or similar. 
             if isinstance(step.error_data, list) and len(step.error_data) > 0:
                 first_error = step.error_data[0]
 
                 # Check for Dictionary or Object access depending on how it's passed at runtime
                 err_type = first_error.type if hasattr(first_error, 'type') else first_error.get('type')
                 err_details = first_error.details if hasattr(first_error, 'details') else first_error.get('details', {})
 
-                if err_type == PreflightCheckType.FILE_CHECK:
+                if err_type == PreflightCheckType.FILE_CHECK or step.step == StepName.FILE_CHECK:
                     error_details["error_type"] = "required_file_missing"
                     error_details["phase"] = "required_files"
                     if err_details and "missing_file" in err_details:
                         error_details["missing_file"] = err_details["missing_file"]
 
-                elif err_type == PreflightCheckType.SETUP_COMMAND:
+                elif err_type == PreflightCheckType.SETUP_COMMAND or step.step == StepName.SETUP_COMMANDS:
                     error_details["error_type"] = "setup_command_failed"
                     error_details["phase"] = "setup_commands"
                     if err_details:
@@ -131,5 +140,10 @@ def _extract_error_details(cls, step: StepResult) -> Dict[str, Any]:
                         # Preserve legacy nested structure
                         if failed_command:
                             error_details["failed_command"] = failed_command
+
+                elif step.step == StepName.ASSET_INJECTION:
+                    error_details["error_type"] = "asset_injection_failed"
+                    error_details["phase"] = "asset_injection"
+                    error_details["message"] = step.error
 
         return error_details

autograder/steps/ai_batch_step.py

@@ -3,7 +3,7 @@
 
 from autograder.models.abstract.step import Step
 from autograder.models.criteria_tree import CriteriaTree, TestNode
-from autograder.models.dataclass.step_result import StepName, StepResult, StepStatus
+from autograder.models.dataclass.step_result import StepName, StepResult, StepStatus, StepCategory
 from autograder.models.dataclass.submission import SubmissionFile
 from autograder.models.dataclass.test_result import TestResult
 from autograder.models.pipeline_execution import PipelineExecution
@@ -36,6 +36,10 @@ class AiBatchStep(Step):
     def step_name(self) -> StepName:
         return StepName.AI_BATCH
 
+    @property
+    def category(self) -> StepCategory:
+        return StepCategory.GRADING
+
     def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution:
         """
         Collect all AI test functions from the criteria tree, send a single

autograder/steps/asset_injection_step.py

@@ -0,0 +1,58 @@
+import logging
+from autograder.models.abstract.step import Step
+from autograder.models.pipeline_execution import PipelineExecution
+from autograder.models.dataclass.step_result import StepResult, StepName, StepCategory
+from autograder.services.assets.resolver import AssetSourceResolver
+from autograder.models.config.setup import SetupConfig
+from autograder.translations import t
+
+logger = logging.getLogger(__name__)
+
+
+class AssetInjectionStep(Step):
+    """
+    Step responsible for injecting static assets into the sandbox.
+    Requires an existing sandbox in the pipeline execution context.
+    """
+
+    def __init__(self, setup_config):
+        if isinstance(setup_config, dict):
+            self._setup_config = SetupConfig.from_dict(setup_config)
+        else:
+            self._setup_config = setup_config or SetupConfig()
+        self._asset_resolver = AssetSourceResolver() if self._setup_config.assets else None
+
+    @property
+    def step_name(self) -> StepName:
+        return StepName.ASSET_INJECTION
+
+    @property
+    def category(self) -> StepCategory:
+        return StepCategory.SETUP
+
+    def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution:
+        if not self._setup_config.assets:
+            return pipeline_exec.add_step_result(StepResult.success(self.step_name, None))
+
+        sandbox = pipeline_exec.sandbox
+        if not sandbox:
+            error_msg = t("preflight.error.setup_command_missing_sandbox", locale=pipeline_exec.locale)
+            logger.error("Sandbox required for asset injection but was not found in pipeline execution.")
+            return pipeline_exec.add_step_result(StepResult.fail(
+                step=self.step_name,
+                error=error_msg
+            ))
+
+        logger.info("Injecting assets into sandbox (external_user_id=%s)", pipeline_exec.submission.user_id)
+        try:
+            resolved_assets = self._asset_resolver.resolve_assets(self._setup_config.assets)
+            sandbox.inject_assets(resolved_assets)
+        except Exception as e:
+            error_msg = f"Failed to inject assets: {str(e)}"
+            logger.error("Asset injection failed (external_user_id=%s): %s", pipeline_exec.submission.user_id, error_msg)
+            return pipeline_exec.add_step_result(StepResult.fail(
+                step=self.step_name,
+                error=error_msg
+            ))
+
+        return pipeline_exec.add_step_result(StepResult.success(self.step_name, None))

autograder/steps/build_tree_step.py

@@ -4,7 +4,7 @@
 from autograder.services.criteria_tree_service import CriteriaTreeService
 from autograder.models.abstract.step import Step
 from autograder.models.config.criteria import CriteriaConfig
-from autograder.models.dataclass.step_result import StepResult, StepStatus, StepName
+from autograder.models.dataclass.step_result import StepResult, StepStatus, StepName, StepCategory
 
 logger = logging.getLogger(__name__)
 
@@ -30,6 +30,10 @@ def __init__(self, criteria_json: dict):
     def step_name(self) -> StepName:
         return StepName.BUILD_TREE
 
+    @property
+    def category(self) -> StepCategory:
+        return StepCategory.SETUP
+
     def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution:
         """
         Build a criteria tree from the configuration and template.

autograder/steps/export_step.py

@@ -3,7 +3,7 @@
 from autograder.models.abstract.exporter import Exporter
 from autograder.models.abstract.step import Step
 from autograder.models.pipeline_execution import PipelineExecution
-from autograder.models.dataclass.step_result import StepResult, StepStatus, StepName
+from autograder.models.dataclass.step_result import StepResult, StepStatus, StepName, StepCategory
 
 logger = logging.getLogger(__name__)
 
@@ -21,6 +21,10 @@ def __init__(self, exporter_service: Exporter):
     def step_name(self) -> StepName:
         return StepName.EXPORTER
 
+    @property
+    def category(self) -> StepCategory:
+        return StepCategory.REPORTING
+
     def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution:
         """
         Export the final grading result to an external system.

autograder/steps/feedback_step.py

@@ -2,7 +2,7 @@
 
 from autograder.models.abstract.step import Step
 from autograder.models.dataclass.focus import Focus 
-from autograder.models.dataclass.step_result import StepName, StepResult, StepStatus
+from autograder.models.dataclass.step_result import StepName, StepResult, StepStatus, StepCategory
 from autograder.models.pipeline_execution import PipelineExecution
 from autograder.services.report.reporter_service import ReporterService
 
@@ -25,6 +25,10 @@ def __init__(self,
     def step_name(self) -> StepName:
         return StepName.FEEDBACK
 
+    @property
+    def category(self) -> StepCategory:
+        return StepCategory.REPORTING
+
     def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution:
         """Adds feedback to the grading result using the reporter service."""
         logger.info("Generating feedback (external_user_id=%s)", pipeline_exec.submission.user_id)

autograder/steps/file_check_step.py

@@ -0,0 +1,46 @@
+import logging
+from autograder.models.abstract.step import Step
+from autograder.models.pipeline_execution import PipelineExecution
+from autograder.models.dataclass.step_result import StepResult, StepName, StepCategory
+from autograder.services.pre_flight_service import PreFlightService
+
+logger = logging.getLogger(__name__)
+
+
+class FileCheckStep(Step):
+    """
+    Step responsible for validating that all required files exist in the submission.
+    This step runs before sandbox creation.
+    """
+
+    def __init__(self, setup_config):
+        self._setup_config = setup_config
+
+    @property
+    def step_name(self) -> StepName:
+        return StepName.FILE_CHECK
+
+    @property
+    def category(self) -> StepCategory:
+        return StepCategory.SETUP
+
+    def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution:
+        submission_language = pipeline_exec.submission.language
+        pre_flight_service = PreFlightService(self._setup_config, submission_language, locale=pipeline_exec.locale)
+
+        if not pre_flight_service.required_files:
+            return pipeline_exec.add_step_result(StepResult.success(self.step_name, None))
+
+        logger.info("Checking required files for submission (external_user_id=%s)", pipeline_exec.submission.user_id)
+        files_ok = pre_flight_service.check_required_files(pipeline_exec.submission.submission_files)
+
+        if not files_ok:
+            error_msg = "\n".join(pre_flight_service.get_error_messages())
+            logger.warning("Required files check failed (external_user_id=%s): %s", pipeline_exec.submission.user_id, error_msg)
+            return pipeline_exec.add_step_result(StepResult.fail(
+                step=self.step_name,
+                error=error_msg,
+                error_data=pre_flight_service.fatal_errors
+            ))
+
+        return pipeline_exec.add_step_result(StepResult.success(self.step_name, None))