diff --git a/README.md b/README.md index c5a13433..b29b2bbd 100644 --- a/README.md +++ b/README.md @@ -7,11 +7,9 @@ **An educational-standards-driven autograding tool that transforms assignment grading into an engaging learning experience.** [![Python](https://img.shields.io/badge/Python-3.9+-blue.svg)](https://www.python.org/downloads/) -[![FastAPI](https://img.shields.io/badge/FastAPI-0.115.0-009688.svg)](https://fastapi.tiangolo.com/) [![Docker](https://img.shields.io/badge/Docker-Enabled-2496ED.svg)](https://www.docker.com/) [![Tests](https://github.com/webtech-network/autograder/actions/workflows/pytest.yml/badge.svg)](https://github.com/webtech-network/autograder/actions/workflows/pytest.yml) [![Lint](https://github.com/webtech-network/autograder/actions/workflows/pylint.yml/badge.svg)](https://github.com/webtech-network/autograder/actions/workflows/pylint.yml) -[![Docs](https://github.com/webtech-network/autograder/actions/workflows/validate-docs.yml/badge.svg)](https://github.com/webtech-network/autograder/actions/workflows/validate-docs.yml) [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](LICENSE) [Features](#features) • [Architecture](#architecture) • [Quick Start](#quick-start) • [Templates](#grading-templates) • [Pipeline](#pipeline-workflow) • [API](#rest-api) • [GitHub Action](#github-action) • [Docs](https://webtech-network.github.io/autograder/) • [Community](#community-standards) diff --git a/autograder/autograder.py b/autograder/autograder.py index 8ec924d0..f6d01d15 100644 --- a/autograder/autograder.py +++ b/autograder/autograder.py @@ -168,7 +168,7 @@ def build_pipeline( # pylint: disable=too-many-arguments,too-many-locals "include_feedback": include_feedback, "grading_criteria": grading_criteria, "feedback_config": feedback_config, - "setup_config": setup_config, + "setup_config": _merge_setup_config(setup_config, templates), "custom_template": custom_template, "feedback_mode": feedback_mode, "export_results": export_results, @@ -180,8 +180,10 @@ def build_pipeline( # pylint: disable=too-many-arguments,too-many-locals execution_order = [ StepName.LOAD_TEMPLATE, StepName.BUILD_TREE, + StepName.FILE_CHECK, StepName.SANDBOX, - StepName.PRE_FLIGHT, + StepName.ASSET_INJECTION, + StepName.SETUP_COMMANDS, StepName.AI_BATCH, StepName.STRUCTURAL_ANALYSIS, StepName.GRADE, @@ -195,6 +197,49 @@ def build_pipeline( # pylint: disable=too-many-arguments,too-many-locals if step_instance is not None: pipeline.add_step(step_name, step_instance) + return pipeline + + +def _merge_setup_config(assignment_config, templates) -> dict: + """ + Merge template-level setup requirements into the assignment setup config. + Assignment config values take precedence for shared structure. + """ + import copy + if assignment_config is None: + merged = {} + elif isinstance(assignment_config, dict): + merged = copy.deepcopy(assignment_config) + else: + # Handle SetupConfig object if passed + merged = assignment_config.model_dump(exclude_none=True) + + for template in templates: + # Merge required_files + for lang, files in template.required_files.items(): + if not files: + continue + if lang not in merged: + merged[lang] = {} + if "required_files" not in merged[lang]: + merged[lang]["required_files"] = [] + + for f in files: + if f not in merged[lang]["required_files"]: + merged[lang]["required_files"].append(f) + + # Merge setup_commands + for lang, commands in template.setup_commands.items(): + if not commands: + continue + if lang not in merged: + merged[lang] = {} + if "setup_commands" not in merged[lang]: + merged[lang]["setup_commands"] = [] + + for cmd in commands: + if cmd not in merged[lang]["setup_commands"]: + merged[lang]["setup_commands"].append(cmd) - return pipeline + return merged diff --git a/autograder/models/abstract/step.py b/autograder/models/abstract/step.py index 86deb0ee..d4fef39f 100644 --- a/autograder/models/abstract/step.py +++ b/autograder/models/abstract/step.py @@ -2,7 +2,7 @@ from abc import ABC, abstractmethod from autograder.translations import t -from autograder.models.dataclass.step_result import StepResult, StepStatus, StepName +from autograder.models.dataclass.step_result import StepResult, StepStatus, StepName, StepCategory from autograder.models.pipeline_execution import PipelineExecution logger = logging.getLogger(__name__) @@ -17,6 +17,12 @@ class Step(ABC): def step_name(self) -> StepName: """Return the name of the step (e.g. StepName.GRADE).""" + @property + @abstractmethod + def category(self) -> StepCategory: + """Return the category of the step (e.g. StepCategory.GRADING).""" + pass + def execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution: """ Execute the step on the pipeline execution context. diff --git a/autograder/models/abstract/template.py b/autograder/models/abstract/template.py index fe666546..6804b1d1 100644 --- a/autograder/models/abstract/template.py +++ b/autograder/models/abstract/template.py @@ -25,6 +25,16 @@ def requires_sandbox(self) -> bool: """Declare whether template tests require sandbox execution.""" raise NotImplementedError + @property + def required_files(self) -> Dict[str, list]: + """Return a dictionary of required files per language.""" + return {} + + @property + def setup_commands(self) -> Dict[str, list]: + """Return a dictionary of setup commands per language.""" + return {} + @abstractmethod def get_test(self, name: str) -> TestFunction: """Return a test function instance by its registry name.""" diff --git a/autograder/models/dataclass/step_result.py b/autograder/models/dataclass/step_result.py index ab14b4d7..62a2e965 100644 --- a/autograder/models/dataclass/step_result.py +++ b/autograder/models/dataclass/step_result.py @@ -20,11 +20,21 @@ class StepStatus(Enum): INTERRUPTED = "interrupted" +class StepCategory(Enum): + """Enumeration of pipeline step categories.""" + SETUP = "setup" + GRADING = "grading" + REPORTING = "reporting" + + class StepName(Enum): """Enumeration of all available pipeline steps.""" BOOTSTRAP = "BootstrapStep" LOAD_TEMPLATE = "LoadTemplateStep" BUILD_TREE = "BuildTreeStep" + FILE_CHECK = "FileCheckStep" + ASSET_INJECTION = "AssetInjectionStep" + SETUP_COMMANDS = "SetupCommandsStep" PRE_FLIGHT = "PreFlightStep" SANDBOX = "SandboxStep" AI_BATCH = "AiBatchStep" diff --git a/autograder/serializers/pipeline_execution_serializer.py b/autograder/serializers/pipeline_execution_serializer.py index 13854f75..53b22239 100644 --- a/autograder/serializers/pipeline_execution_serializer.py +++ b/autograder/serializers/pipeline_execution_serializer.py @@ -56,7 +56,9 @@ def serialize(cls, execution: PipelineExecution) -> Dict[str, Any]: steps.append(step_info) # Count planned vs completed steps - total_planned = 7 # BOOTSTRAP, LOAD_TEMPLATE, BUILD_TREE, PRE_FLIGHT, GRADE, FEEDBACK, EXPORT + # If the execution context has a planned steps count, use it. + # Otherwise fall back to a reasonable estimate. + total_planned = getattr(execution, "total_steps_planned", 7) completed = len([s for s in execution.step_results if s.step != StepName.BOOTSTRAP]) return { @@ -75,6 +77,14 @@ def _get_success_message(cls, step: StepResult, execution: PipelineExecution) -> return "Template loaded successfully" elif step.step == StepName.BUILD_TREE: return "Criteria tree built successfully" + elif step.step == StepName.FILE_CHECK: + return "All required files are present" + elif step.step == StepName.SANDBOX: + return "Sandbox environment allocated" + elif step.step == StepName.ASSET_INJECTION: + return "Assets injected successfully" + elif step.step == StepName.SETUP_COMMANDS: + return "Setup commands executed successfully" elif step.step == StepName.PRE_FLIGHT: return "All preflight checks passed" elif step.step == StepName.GRADE: @@ -95,9 +105,8 @@ def _extract_error_details(cls, step: StepResult) -> Dict[str, Any]: """ error_details = {} - if step.step == StepName.PRE_FLIGHT and step.error_data: - # error_data is a list of PreflightError objects. - # We map properties from the first critical error for legacy backward compatibility. + if step.step in [StepName.FILE_CHECK, StepName.ASSET_INJECTION, StepName.SETUP_COMMANDS, StepName.PRE_FLIGHT] and step.error_data: + # error_data is a list of PreflightError objects or similar. if isinstance(step.error_data, list) and len(step.error_data) > 0: first_error = step.error_data[0] @@ -105,13 +114,13 @@ def _extract_error_details(cls, step: StepResult) -> Dict[str, Any]: err_type = first_error.type if hasattr(first_error, 'type') else first_error.get('type') err_details = first_error.details if hasattr(first_error, 'details') else first_error.get('details', {}) - if err_type == PreflightCheckType.FILE_CHECK: + if err_type == PreflightCheckType.FILE_CHECK or step.step == StepName.FILE_CHECK: error_details["error_type"] = "required_file_missing" error_details["phase"] = "required_files" if err_details and "missing_file" in err_details: error_details["missing_file"] = err_details["missing_file"] - elif err_type == PreflightCheckType.SETUP_COMMAND: + elif err_type == PreflightCheckType.SETUP_COMMAND or step.step == StepName.SETUP_COMMANDS: error_details["error_type"] = "setup_command_failed" error_details["phase"] = "setup_commands" if err_details: @@ -131,5 +140,10 @@ def _extract_error_details(cls, step: StepResult) -> Dict[str, Any]: # Preserve legacy nested structure if failed_command: error_details["failed_command"] = failed_command + + elif step.step == StepName.ASSET_INJECTION: + error_details["error_type"] = "asset_injection_failed" + error_details["phase"] = "asset_injection" + error_details["message"] = step.error return error_details diff --git a/autograder/steps/ai_batch_step.py b/autograder/steps/ai_batch_step.py index 7f739a0e..82daa5a5 100644 --- a/autograder/steps/ai_batch_step.py +++ b/autograder/steps/ai_batch_step.py @@ -3,7 +3,7 @@ from autograder.models.abstract.step import Step from autograder.models.criteria_tree import CriteriaTree, TestNode -from autograder.models.dataclass.step_result import StepName, StepResult, StepStatus +from autograder.models.dataclass.step_result import StepName, StepResult, StepStatus, StepCategory from autograder.models.dataclass.submission import SubmissionFile from autograder.models.dataclass.test_result import TestResult from autograder.models.pipeline_execution import PipelineExecution @@ -36,6 +36,10 @@ class AiBatchStep(Step): def step_name(self) -> StepName: return StepName.AI_BATCH + @property + def category(self) -> StepCategory: + return StepCategory.GRADING + def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution: """ Collect all AI test functions from the criteria tree, send a single diff --git a/autograder/steps/asset_injection_step.py b/autograder/steps/asset_injection_step.py new file mode 100644 index 00000000..d3e52c18 --- /dev/null +++ b/autograder/steps/asset_injection_step.py @@ -0,0 +1,58 @@ +import logging +from autograder.models.abstract.step import Step +from autograder.models.pipeline_execution import PipelineExecution +from autograder.models.dataclass.step_result import StepResult, StepName, StepCategory +from autograder.services.assets.resolver import AssetSourceResolver +from autograder.models.config.setup import SetupConfig +from autograder.translations import t + +logger = logging.getLogger(__name__) + + +class AssetInjectionStep(Step): + """ + Step responsible for injecting static assets into the sandbox. + Requires an existing sandbox in the pipeline execution context. + """ + + def __init__(self, setup_config): + if isinstance(setup_config, dict): + self._setup_config = SetupConfig.from_dict(setup_config) + else: + self._setup_config = setup_config or SetupConfig() + self._asset_resolver = AssetSourceResolver() if self._setup_config.assets else None + + @property + def step_name(self) -> StepName: + return StepName.ASSET_INJECTION + + @property + def category(self) -> StepCategory: + return StepCategory.SETUP + + def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution: + if not self._setup_config.assets: + return pipeline_exec.add_step_result(StepResult.success(self.step_name, None)) + + sandbox = pipeline_exec.sandbox + if not sandbox: + error_msg = t("preflight.error.setup_command_missing_sandbox", locale=pipeline_exec.locale) + logger.error("Sandbox required for asset injection but was not found in pipeline execution.") + return pipeline_exec.add_step_result(StepResult.fail( + step=self.step_name, + error=error_msg + )) + + logger.info("Injecting assets into sandbox (external_user_id=%s)", pipeline_exec.submission.user_id) + try: + resolved_assets = self._asset_resolver.resolve_assets(self._setup_config.assets) + sandbox.inject_assets(resolved_assets) + except Exception as e: + error_msg = f"Failed to inject assets: {str(e)}" + logger.error("Asset injection failed (external_user_id=%s): %s", pipeline_exec.submission.user_id, error_msg) + return pipeline_exec.add_step_result(StepResult.fail( + step=self.step_name, + error=error_msg + )) + + return pipeline_exec.add_step_result(StepResult.success(self.step_name, None)) diff --git a/autograder/steps/build_tree_step.py b/autograder/steps/build_tree_step.py index ca05c0e6..e828c710 100644 --- a/autograder/steps/build_tree_step.py +++ b/autograder/steps/build_tree_step.py @@ -4,7 +4,7 @@ from autograder.services.criteria_tree_service import CriteriaTreeService from autograder.models.abstract.step import Step from autograder.models.config.criteria import CriteriaConfig -from autograder.models.dataclass.step_result import StepResult, StepStatus, StepName +from autograder.models.dataclass.step_result import StepResult, StepStatus, StepName, StepCategory logger = logging.getLogger(__name__) @@ -30,6 +30,10 @@ def __init__(self, criteria_json: dict): def step_name(self) -> StepName: return StepName.BUILD_TREE + @property + def category(self) -> StepCategory: + return StepCategory.SETUP + def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution: """ Build a criteria tree from the configuration and template. diff --git a/autograder/steps/export_step.py b/autograder/steps/export_step.py index 2b73e164..3f25ef77 100644 --- a/autograder/steps/export_step.py +++ b/autograder/steps/export_step.py @@ -3,7 +3,7 @@ from autograder.models.abstract.exporter import Exporter from autograder.models.abstract.step import Step from autograder.models.pipeline_execution import PipelineExecution -from autograder.models.dataclass.step_result import StepResult, StepStatus, StepName +from autograder.models.dataclass.step_result import StepResult, StepStatus, StepName, StepCategory logger = logging.getLogger(__name__) @@ -21,6 +21,10 @@ def __init__(self, exporter_service: Exporter): def step_name(self) -> StepName: return StepName.EXPORTER + @property + def category(self) -> StepCategory: + return StepCategory.REPORTING + def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution: """ Export the final grading result to an external system. diff --git a/autograder/steps/feedback_step.py b/autograder/steps/feedback_step.py index 1c9f888e..3ba820c1 100644 --- a/autograder/steps/feedback_step.py +++ b/autograder/steps/feedback_step.py @@ -2,7 +2,7 @@ from autograder.models.abstract.step import Step from autograder.models.dataclass.focus import Focus -from autograder.models.dataclass.step_result import StepName, StepResult, StepStatus +from autograder.models.dataclass.step_result import StepName, StepResult, StepStatus, StepCategory from autograder.models.pipeline_execution import PipelineExecution from autograder.services.report.reporter_service import ReporterService @@ -25,6 +25,10 @@ def __init__(self, def step_name(self) -> StepName: return StepName.FEEDBACK + @property + def category(self) -> StepCategory: + return StepCategory.REPORTING + def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution: """Adds feedback to the grading result using the reporter service.""" logger.info("Generating feedback (external_user_id=%s)", pipeline_exec.submission.user_id) diff --git a/autograder/steps/file_check_step.py b/autograder/steps/file_check_step.py new file mode 100644 index 00000000..47d63c09 --- /dev/null +++ b/autograder/steps/file_check_step.py @@ -0,0 +1,46 @@ +import logging +from autograder.models.abstract.step import Step +from autograder.models.pipeline_execution import PipelineExecution +from autograder.models.dataclass.step_result import StepResult, StepName, StepCategory +from autograder.services.pre_flight_service import PreFlightService + +logger = logging.getLogger(__name__) + + +class FileCheckStep(Step): + """ + Step responsible for validating that all required files exist in the submission. + This step runs before sandbox creation. + """ + + def __init__(self, setup_config): + self._setup_config = setup_config + + @property + def step_name(self) -> StepName: + return StepName.FILE_CHECK + + @property + def category(self) -> StepCategory: + return StepCategory.SETUP + + def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution: + submission_language = pipeline_exec.submission.language + pre_flight_service = PreFlightService(self._setup_config, submission_language, locale=pipeline_exec.locale) + + if not pre_flight_service.required_files: + return pipeline_exec.add_step_result(StepResult.success(self.step_name, None)) + + logger.info("Checking required files for submission (external_user_id=%s)", pipeline_exec.submission.user_id) + files_ok = pre_flight_service.check_required_files(pipeline_exec.submission.submission_files) + + if not files_ok: + error_msg = "\n".join(pre_flight_service.get_error_messages()) + logger.warning("Required files check failed (external_user_id=%s): %s", pipeline_exec.submission.user_id, error_msg) + return pipeline_exec.add_step_result(StepResult.fail( + step=self.step_name, + error=error_msg, + error_data=pre_flight_service.fatal_errors + )) + + return pipeline_exec.add_step_result(StepResult.success(self.step_name, None)) diff --git a/autograder/steps/focus_step.py b/autograder/steps/focus_step.py index a2615bf1..b9236185 100644 --- a/autograder/steps/focus_step.py +++ b/autograder/steps/focus_step.py @@ -1,7 +1,7 @@ import logging from autograder.models.abstract.step import Step -from autograder.models.dataclass.step_result import StepName, StepResult, StepStatus +from autograder.models.dataclass.step_result import StepName, StepResult, StepStatus, StepCategory from autograder.models.pipeline_execution import PipelineExecution from autograder.services.focus_service import FocusService @@ -20,6 +20,10 @@ def __init__(self, trim_service: FocusService) -> None: def step_name(self) -> StepName: return StepName.FOCUS + @property + def category(self) -> StepCategory: + return StepCategory.GRADING + def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution: """ Trim the result tree to get the main points of failure diff --git a/autograder/steps/grade_step.py b/autograder/steps/grade_step.py index e6adf5d2..d2b6c0a9 100644 --- a/autograder/steps/grade_step.py +++ b/autograder/steps/grade_step.py @@ -2,7 +2,7 @@ from autograder.models.dataclass.grade_step_result import GradeStepResult from autograder.models.pipeline_execution import PipelineExecution -from autograder.models.dataclass.step_result import StepResult, StepStatus, StepName +from autograder.models.dataclass.step_result import StepResult, StepStatus, StepName, StepCategory from autograder.models.abstract.step import Step from autograder.services.grader.grader_service import GraderService @@ -28,6 +28,10 @@ def __init__( def step_name(self) -> StepName: return StepName.GRADE + @property + def category(self) -> StepCategory: + return StepCategory.GRADING + def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution: """ Grade a submission based on the criteria tree and template. diff --git a/autograder/steps/load_template_step.py b/autograder/steps/load_template_step.py index 6662c386..d7bbe2a3 100644 --- a/autograder/steps/load_template_step.py +++ b/autograder/steps/load_template_step.py @@ -1,7 +1,7 @@ import logging from typing import List, Union, Any, Optional -from autograder.models.dataclass.step_result import StepResult, StepName, StepStatus +from autograder.models.dataclass.step_result import StepResult, StepName, StepStatus, StepCategory from autograder.models.pipeline_execution import PipelineExecution from autograder.services.template_library_service import TemplateLibraryService from autograder.models.abstract.step import Step @@ -59,6 +59,10 @@ def normalize_template_names(template_name: Union[str, List[str], None]) -> List def step_name(self) -> StepName: return StepName.LOAD_TEMPLATE + @property + def category(self) -> StepCategory: + return StepCategory.SETUP + def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution: """ Load the grading templates and return them as part of the step result. diff --git a/autograder/steps/pre_flight_step.py b/autograder/steps/pre_flight_step.py index bd42cfc5..b4fe2c78 100644 --- a/autograder/steps/pre_flight_step.py +++ b/autograder/steps/pre_flight_step.py @@ -2,7 +2,7 @@ from autograder.models.abstract.step import Step from autograder.models.pipeline_execution import PipelineExecution -from autograder.models.dataclass.step_result import StepResult, StepName +from autograder.models.dataclass.step_result import StepResult, StepName, StepCategory from autograder.services.pre_flight_service import PreFlightService from autograder.translations import t from autograder.models.config.setup import SetupConfig @@ -35,6 +35,10 @@ def __init__(self, setup_config): def step_name(self) -> StepName: return StepName.PRE_FLIGHT + @property + def category(self) -> StepCategory: + return StepCategory.SETUP + def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution: """ Execute pre-flight checks (required files, assets, setup commands). diff --git a/autograder/steps/sandbox_step.py b/autograder/steps/sandbox_step.py index df421c08..e56fc514 100644 --- a/autograder/steps/sandbox_step.py +++ b/autograder/steps/sandbox_step.py @@ -2,7 +2,7 @@ from autograder.models.abstract.step import Step from autograder.models.pipeline_execution import PipelineExecution -from autograder.models.dataclass.step_result import StepResult, StepName +from autograder.models.dataclass.step_result import StepResult, StepName, StepCategory from autograder.services.sandbox_service import SandboxService from autograder.translations import t @@ -26,6 +26,10 @@ def __init__(self): def step_name(self) -> StepName: return StepName.SANDBOX + @property + def category(self) -> StepCategory: + return StepCategory.SETUP + def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution: """ Execute sandbox acquisition and preparation. diff --git a/autograder/steps/setup_commands_step.py b/autograder/steps/setup_commands_step.py new file mode 100644 index 00000000..8ab5fce4 --- /dev/null +++ b/autograder/steps/setup_commands_step.py @@ -0,0 +1,56 @@ +import logging +from autograder.models.abstract.step import Step +from autograder.models.pipeline_execution import PipelineExecution +from autograder.models.dataclass.step_result import StepResult, StepName, StepCategory +from autograder.services.pre_flight_service import PreFlightService +from autograder.translations import t + +logger = logging.getLogger(__name__) + + +class SetupCommandsStep(Step): + """ + Step responsible for executing setup commands (e.g., compilation) in the sandbox. + Requires an existing sandbox in the pipeline execution context. + """ + + def __init__(self, setup_config): + self._setup_config = setup_config + + @property + def step_name(self) -> StepName: + return StepName.SETUP_COMMANDS + + @property + def category(self) -> StepCategory: + return StepCategory.SETUP + + def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution: + submission_language = pipeline_exec.submission.language + pre_flight_service = PreFlightService(self._setup_config, submission_language, locale=pipeline_exec.locale) + + if not pre_flight_service.setup_commands: + return pipeline_exec.add_step_result(StepResult.success(self.step_name, None)) + + sandbox = pipeline_exec.sandbox + if not sandbox: + error_msg = t("preflight.error.missing_sandbox", locale=pipeline_exec.locale) + logger.error("Sandbox required for setup commands but was not found in pipeline execution.") + return pipeline_exec.add_step_result(StepResult.fail( + step=self.step_name, + error=error_msg + )) + + logger.info("Running setup commands in sandbox (external_user_id=%s)", pipeline_exec.submission.user_id) + setup_ok = pre_flight_service.check_setup_commands(sandbox) + + if not setup_ok: + error_msg = "\n".join(pre_flight_service.get_error_messages()) + logger.warning("Setup commands failed (external_user_id=%s): %s", pipeline_exec.submission.user_id, error_msg) + return pipeline_exec.add_step_result(StepResult.fail( + step=self.step_name, + error=error_msg, + error_data=pre_flight_service.fatal_errors + )) + + return pipeline_exec.add_step_result(StepResult.success(self.step_name, None)) diff --git a/autograder/steps/step_registry.py b/autograder/steps/step_registry.py index e243aa6b..edead094 100644 --- a/autograder/steps/step_registry.py +++ b/autograder/steps/step_registry.py @@ -7,7 +7,9 @@ # Imports of all steps from autograder.steps.load_template_step import TemplateLoaderStep from autograder.steps.build_tree_step import BuildTreeStep -from autograder.steps.pre_flight_step import PreFlightStep +from autograder.steps.file_check_step import FileCheckStep +from autograder.steps.asset_injection_step import AssetInjectionStep +from autograder.steps.setup_commands_step import SetupCommandsStep from autograder.steps.sandbox_step import SandboxStep from autograder.steps.ai_batch_step import AiBatchStep from autograder.steps.structural_analysis_step import StructuralAnalysisStep @@ -36,7 +38,9 @@ def __init__(self, config: Dict[str, Any], templates: Optional[List[Any]] = None self._builders: Dict[StepName, Callable[[], Optional[Step]]] = { StepName.LOAD_TEMPLATE: self._build_load_template, StepName.BUILD_TREE: self._build_build_tree, - StepName.PRE_FLIGHT: self._build_pre_flight, + StepName.FILE_CHECK: self._build_file_check, + StepName.ASSET_INJECTION: self._build_asset_injection, + StepName.SETUP_COMMANDS: self._build_setup_commands, StepName.SANDBOX: self._build_sandbox, StepName.AI_BATCH: self._build_ai_batch, StepName.STRUCTURAL_ANALYSIS: self._build_structural_analysis, @@ -54,18 +58,42 @@ def _build_load_template(self) -> Optional[Step]: def _build_build_tree(self) -> Optional[Step]: return BuildTreeStep(self.config.get("grading_criteria")) - def _build_pre_flight(self) -> Optional[Step]: - # Only return PreFlightStep if there is a setup_config to process. + def _build_file_check(self) -> Optional[Step]: setup_config = self.config.get("setup_config") if not setup_config: return None - return PreFlightStep(setup_config) + return FileCheckStep(setup_config) - def _build_sandbox(self) -> Optional[Step]: - # Only return SandboxStep if at least one template requires it. - if not any(t.requires_sandbox for t in self.templates): + def _build_asset_injection(self) -> Optional[Step]: + setup_config = self.config.get("setup_config") + if not setup_config: + return None + return AssetInjectionStep(setup_config) + + def _build_setup_commands(self) -> Optional[Step]: + setup_config = self.config.get("setup_config") + if not setup_config: return None - return SandboxStep() + return SetupCommandsStep(setup_config) + + def _build_sandbox(self) -> Optional[Step]: + # Only return SandboxStep if at least one template requires it + # OR if there are assets or setup commands in the merged setup_config. + if any(t.requires_sandbox for t in self.templates): + return SandboxStep() + + setup_config = self.config.get("setup_config") + if setup_config and isinstance(setup_config, dict): + # Check for assets + if setup_config.get("assets"): + return SandboxStep() + + # Check for setup commands in any language block + for key, value in setup_config.items(): + if isinstance(value, dict) and value.get("setup_commands"): + return SandboxStep() + + return None def _build_ai_batch(self) -> Optional[Step]: # Check if any of the loaded templates have AI test functions, or if the criteria tree diff --git a/autograder/steps/structural_analysis_step.py b/autograder/steps/structural_analysis_step.py index 0d64413e..ce3467ae 100644 --- a/autograder/steps/structural_analysis_step.py +++ b/autograder/steps/structural_analysis_step.py @@ -3,7 +3,7 @@ from autograder.models.abstract.step import Step from autograder.models.pipeline_execution import PipelineExecution -from autograder.models.dataclass.step_result import StepResult, StepName +from autograder.models.dataclass.step_result import StepResult, StepName, StepCategory from autograder.models.dataclass.structural_analysis_result import StructuralAnalysisResult from sandbox_manager.models.sandbox_models import Language @@ -24,6 +24,10 @@ class StructuralAnalysisStep(Step): def step_name(self) -> StepName: return StepName.STRUCTURAL_ANALYSIS + @property + def category(self) -> StepCategory: + return StepCategory.GRADING + def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution: submission = pipeline_exec.submission language = submission.language diff --git a/autograder/utils/feedback_generator.py b/autograder/utils/feedback_generator.py index 1e50edf4..7332eae5 100644 --- a/autograder/utils/feedback_generator.py +++ b/autograder/utils/feedback_generator.py @@ -15,17 +15,18 @@ def generate_preflight_feedback(pipeline_execution_summary: Dict[str, Any], loca Returns: Markdown-formatted feedback string """ - # Find the failed preflight step - preflight_step = None + # Find the failed preflight/setup step + setup_steps = ["FileCheckStep", "AssetInjectionStep", "SetupCommandsStep", "PreFlightStep"] + failed_step = None for step in pipeline_execution_summary.get("steps", []): - if step["name"] == "PreFlightStep" and step["status"] == "fail": - preflight_step = step + if step["name"] in setup_steps and step["status"] == "fail": + failed_step = step break - if not preflight_step: + if not failed_step: return t("feedback.preflight.failed_title", locale=locale) + "\n\n" + t("feedback.preflight.failed_subtitle", locale=locale) - error_details = preflight_step.get("error_details", {}) + error_details = failed_step.get("error_details", {}) error_type = error_details.get("error_type", "unknown") feedback = t("feedback.preflight.failed_title", locale=locale) + "\n\n" @@ -84,6 +85,13 @@ def generate_preflight_feedback(pipeline_execution_summary: Dict[str, Any], loca feedback += f"{t('feedback.preflight.review_error_instruction', locale=locale)}\n" feedback += f"{t('feedback.preflight.generic_fix_instruction', locale=locale)}\n" feedback += f"{t('feedback.preflight.generic_resubmit_instruction', locale=locale)}\n" + + elif error_type == "asset_injection_failed": + feedback += f"Asset injection failed.\n\n" + if error_details.get("message"): + feedback += f"**Details:** {error_details['message']}\n\n" + feedback += f"This is likely a system configuration issue. Please contact your instructor.\n" + else: feedback += f"{t('feedback.preflight.what_to_do', locale=locale)}\n" feedback += f"{t('feedback.preflight.generic_error_instruction', locale=locale)}\n" @@ -91,4 +99,3 @@ def generate_preflight_feedback(pipeline_execution_summary: Dict[str, Any], loca feedback += f"{t('feedback.preflight.generic_resubmit_instruction', locale=locale)}\n" return feedback - diff --git a/docs/architecture/pipeline_execution_tracking.md b/docs/architecture/pipeline_execution_tracking.md index f030f634..39c79a91 100644 --- a/docs/architecture/pipeline_execution_tracking.md +++ b/docs/architecture/pipeline_execution_tracking.md @@ -11,7 +11,7 @@ As of February 2026, the autograder now provides comprehensive pipeline executio Every submission response now includes a `pipeline_execution` field that contains: - Execution status and timing -- All executed pipeline steps +- All executed pipeline steps (including granular setup steps) - Detailed error information for failures - Human-readable error messages in the `feedback` field @@ -54,8 +54,8 @@ The response now has three distinct fields for different purposes: "pipeline_execution": { "status": "success", "failed_at_step": null, - "total_steps_planned": 7, - "steps_completed": 7, + "total_steps_planned": 10, + "steps_completed": 10, "execution_time_ms": 4521, "steps": [ { @@ -69,9 +69,24 @@ The response now has three distinct fields for different purposes: "message": "Criteria tree built successfully" }, { - "name": "PRE_FLIGHT", + "name": "FILE_CHECK", "status": "success", - "message": "All preflight checks passed" + "message": "All required files are present" + }, + { + "name": "SANDBOX", + "status": "success", + "message": "Sandbox environment allocated" + }, + { + "name": "ASSET_INJECTION", + "status": "success", + "message": "Assets injected successfully" + }, + { + "name": "SETUP_COMMANDS", + "status": "success", + "message": "Setup commands executed successfully" }, { "name": "GRADE", @@ -88,7 +103,7 @@ The response now has three distinct fields for different purposes: } ``` -### Response When Preflight Fails +### Response When Setup Fails (e.g. Compilation) ```json { @@ -102,9 +117,9 @@ The response now has three distinct fields for different purposes: "pipeline_execution": { "status": "failed", - "failed_at_step": "PRE_FLIGHT", - "total_steps_planned": 7, - "steps_completed": 3, + "failed_at_step": "SetupCommandsStep", + "total_steps_planned": 10, + "steps_completed": 6, "execution_time_ms": 1523, "steps": [ { @@ -118,7 +133,22 @@ The response now has three distinct fields for different purposes: "message": "Criteria tree built successfully" }, { - "name": "PRE_FLIGHT", + "name": "FILE_CHECK", + "status": "success", + "message": "All required files are present" + }, + { + "name": "SANDBOX", + "status": "success", + "message": "Sandbox environment allocated" + }, + { + "name": "ASSET_INJECTION", + "status": "success", + "message": "Assets injected successfully" + }, + { + "name": "SETUP_COMMANDS", "status": "fail", "message": "Setup command 'Compile Calculator.java' failed with exit code 1", "error_details": { @@ -139,26 +169,26 @@ The response now has three distinct fields for different purposes: ## Using Pipeline Execution Data -### Check for Preflight Failures +### Check for Setup Failures ```python -def check_preflight_status(submission): - """Check if submission failed during preflight.""" +def check_setup_status(submission): + """Check if submission failed during setup (file check, asset injection, setup commands).""" pipeline = submission.get('pipeline_execution', {}) + setup_steps = ['FileCheckStep', 'AssetInjectionStep', 'SetupCommandsStep'] - if pipeline.get('failed_at_step') == 'PRE_FLIGHT': - print("Preflight check failed!") + failed_at = pipeline.get('failed_at_step') + if failed_at in setup_steps: + print(f"Setup failed at {failed_at}!") # Get error details for step in pipeline.get('steps', []): - if step['name'] == 'PRE_FLIGHT' and step['status'] == 'fail': + if step['name'] == failed_at and step['status'] == 'fail': error_details = step.get('error_details', {}) if error_details.get('error_type') == 'setup_command_failed': cmd = error_details.get('failed_command', {}) print(f"Command failed: {cmd.get('command')}") - print(f"Exit code: {cmd.get('exit_code')}") - print(f"Error: {cmd.get('stderr')}") elif error_details.get('error_type') == 'required_file_missing': print(f"Missing file: {error_details.get('missing_file')}") @@ -188,47 +218,11 @@ def show_execution_timeline(submission): print(f" Error Type: {step['error_details'].get('error_type')}") ``` -### Aggregate Statistics - -```python -def get_failure_statistics(submissions): - """Analyze where submissions are failing.""" - stats = { - 'total': len(submissions), - 'preflight_failures': 0, - 'grading_failures': 0, - 'compilation_errors': 0, - 'missing_files': 0, - } - - for sub in submissions: - pipeline = sub.get('pipeline_execution', {}) - - if pipeline.get('status') == 'failed': - failed_step = pipeline.get('failed_at_step') - - if failed_step == 'PRE_FLIGHT': - stats['preflight_failures'] += 1 - - # Get error type - for step in pipeline.get('steps', []): - if step['name'] == 'PRE_FLIGHT': - error_type = step.get('error_details', {}).get('error_type') - if error_type == 'setup_command_failed': - stats['compilation_errors'] += 1 - elif error_type == 'required_file_missing': - stats['missing_files'] += 1 - else: - stats['grading_failures'] += 1 - - return stats -``` - ## Error Types -### Preflight Errors +### Granular Setup Errors -#### 1. Required File Missing +#### 1. Required File Missing (`FileCheckStep`) **Error Details Structure:** ```json @@ -239,12 +233,18 @@ def get_failure_statistics(submissions): } ``` -**Generated Feedback:** -- Lists the missing file name -- Reminds about case-sensitivity -- Provides actionable steps to fix +#### 2. Asset Injection Failed (`AssetInjectionStep`) -#### 2. Setup Command Failed +**Error Details Structure:** +```json +{ + "error_type": "asset_injection_failed", + "phase": "asset_injection", + "message": "Failed to download asset from S3" +} +``` + +#### 3. Setup Command Failed (`SetupCommandsStep`) **Error Details Structure:** ```json @@ -261,91 +261,27 @@ def get_failure_statistics(submissions): } ``` -**Generated Feedback:** -- Shows the command that was executed -- Displays exit code -- Includes full compiler/error output -- Provides language-specific guidance (Java, C++, etc.) - ## Migration Guide ### For API Consumers -If you're consuming the autograder API, you should update your code to handle the new `pipeline_execution` field: +The legacy `PRE_FLIGHT` step has been replaced by `FileCheckStep`, `AssetInjectionStep`, and `SetupCommandsStep`. API consumers should update their logic to check for any of these steps when identifying setup failures. **Before:** ```python -result = get_submission(submission_id) -if result['status'] == 'failed': - print("Submission failed") # No details available +if failed_step == 'PRE_FLIGHT': + # handle error ``` **After:** ```python -result = get_submission(submission_id) -if result['status'] == 'failed': - # Check what went wrong - pipeline = result.get('pipeline_execution', {}) - failed_step = pipeline.get('failed_at_step') - - if failed_step == 'PRE_FLIGHT': - # Show the detailed preflight error from feedback - print(result.get('feedback', 'Preflight check failed')) - else: - print(f"Failed at step: {failed_step}") +setup_steps = ['FileCheckStep', 'AssetInjectionStep', 'SetupCommandsStep'] +if failed_step in setup_steps: + # handle error ``` -### Database Changes - -A new column `pipeline_execution` (JSON type) was added to the `submission_results` table. - -**Migration:** -```bash -alembic upgrade head -``` - -This migration is backward compatible - existing submissions will have `pipeline_execution: null`. - -## Best Practices - -### For Students - -1. **Read the Feedback**: The `feedback` field now contains detailed, human-readable error messages -2. **Check Compilation First**: If you see "Preflight Check Failed", fix compilation errors before resubmitting -3. **Verify File Names**: Case-sensitive file name checks happen in preflight - -### For Instructors - -1. **Monitor Preflight Failures**: High preflight failure rates may indicate unclear instructions -2. **Use Error Statistics**: Aggregate `pipeline_execution` data to identify common student issues -3. **Customize Feedback**: Consider the auto-generated feedback when designing rubrics - -### For Developers - -1. **Always Check `pipeline_execution`**: Even successful submissions include execution details -2. **Parse Error Details**: The `error_details` object provides machine-readable error information -3. **Display Execution Time**: Help users understand performance characteristics - -## FAQ - -### Q: Will `result_tree` ever be populated if preflight fails? -**A:** No. If preflight fails, `result_tree` will always be `null` because grading never occurred. - -### Q: Is `pipeline_execution` always present? -**A:** Yes, for all new submissions. Old submissions (before this feature) will have `pipeline_execution: null`. - -### Q: Can I disable pipeline execution tracking? -**A:** No, it's always included. It adds minimal overhead and provides valuable debugging information. - -### Q: What if a step is skipped? -**A:** Skipped steps don't appear in the `steps` array. Only executed steps are included. - -### Q: How do I know if compilation failed? -**A:** Check if `failed_at_step == "PRE_FLIGHT"` and look for `error_type: "setup_command_failed"` in the error details. - ## See Also - [API Documentation](../API.md) - [Core Structures](core_structures.md) -- [Setup Config Feature](../features/setup_config_feature.md) - +- [Pipeline README](../pipeline/README.md) diff --git a/docs/pipeline/04-file-check.md b/docs/pipeline/04-file-check.md new file mode 100644 index 00000000..26892be1 --- /dev/null +++ b/docs/pipeline/04-file-check.md @@ -0,0 +1,68 @@ +# Step 4: File Check + +## Purpose + +The File Check step validates that all required files exist in the student's submission before proceeding with sandbox execution. This provides a fast-fail mechanism that gives students immediate feedback about missing files without consuming sandbox resources. + +## How It Works + +The step execution follows these logic gates: + +1. **Load Configuration**: The step reads the `setup_config` for the submission's language and extracts the `required_files` list. If no required files are configured, the step immediately succeeds. + +2. **Check Submission Files**: The `PreFlightService` compares the list of required files against the actual files present in the submission. + +3. **Report Results**: If any required files are missing, the step fails with a descriptive error listing all missing files. Otherwise, it succeeds. + +## Dependencies + +| Step | What It Needs | +|------|---------------| +| None | This step only needs the submission files from the pipeline context | + +## Input + +| Source | Data | +|--------|------| +| Constructor | `setup_config: dict` — language-keyed setup configuration | +| Pipeline | `pipeline_exec.submission` → submission files and language | + +## Output + +| Field | Type | Description | +|-------|------|-------------| +| `data` | `None` | This step does not produce data for downstream steps | +| `status` | `StepStatus.SUCCESS` | All required files are present | + +## Language-Specific Configuration + +The `required_files` list is defined per language in the `setup_config`: + +```json +{ + "python": { + "required_files": ["main.py"] + }, + "java": { + "required_files": ["Calculator.java"] + } +} +``` + +Templates can also contribute required files via `template.required_files`, which are merged with the assignment-level configuration during pipeline assembly. + +## Failure Scenarios + +- **Missing required file** → `StepStatus.FAIL` with an error message listing all missing files. + +## Next Step + +After file validation, the pipeline proceeds to **[Step 3: Sandbox](03-sandbox.md)** to prepare the execution environment (if the submission requires it). + +--- + +## Source + +`autograder/steps/file_check_step.py` → `FileCheckStep` + +`autograder/services/pre_flight_service.py` → `PreFlightService` diff --git a/docs/pipeline/04.1-asset-injection.md b/docs/pipeline/04.1-asset-injection.md new file mode 100644 index 00000000..8184487d --- /dev/null +++ b/docs/pipeline/04.1-asset-injection.md @@ -0,0 +1,66 @@ +# Step 4.1: Asset Injection + +## Purpose + +The Asset Injection step is responsible for injecting grader-owned static assets (datasets, test fixtures, configuration files) from external sources (e.g., S3) into the sandbox workspace. These assets are required by the grading template but are not part of the student's submission. + +## How It Works + +The step execution follows these logic gates: + +1. **Check Configuration**: The step reads the `setup_config` for an `assets` list. If no assets are configured, the step immediately succeeds with no action. + +2. **Resolve Assets**: The `AssetSourceResolver` resolves each asset entry, fetching the file content from the configured source (typically S3). + +3. **Inject into Sandbox**: Resolved assets are injected into the sandbox container's workspace using the sandbox's `inject_assets` method. + +## Dependencies + +| Step | What It Needs | +|------|---------------| +| **Sandbox** | Requires the `SandboxContainer` to inject files into | + +## Input + +| Source | Data | +|--------|------| +| Constructor | `setup_config: dict` — setup configuration containing `assets` list | +| Pipeline | `pipeline_exec.sandbox` → the active sandbox container | + +## Output + +| Field | Type | Description | +|-------|------|-------------| +| `data` | `None` | This step does not produce data for downstream steps | +| `status` | `StepStatus.SUCCESS` | All assets were successfully resolved and injected | + +## Asset Configuration + +Assets are defined in the `setup_config` as a list of source references: + +```json +{ + "assets": [ + { "source": "s3", "key": "datasets/test_input.csv", "target": "/tmp/test_input.csv" }, + { "source": "s3", "key": "fixtures/expected_output.json", "target": "/tmp/expected.json" } + ] +} +``` + +## Failure Scenarios + +- **No Sandbox** → `StepStatus.FAIL` if the sandbox was not created in a previous step but asset injection is required. +- **Asset Resolution Error** → `StepStatus.FAIL` if an asset cannot be fetched from its source (e.g., S3 object not found, network failure). +- **Injection Error** → `StepStatus.FAIL` if writing the asset into the container fails. + +## Next Step + +After asset injection, the pipeline proceeds to **[Step 4.2: Setup Commands](04.2-setup-commands.md)** to execute compilation or initialization scripts. + +--- + +## Source + +`autograder/steps/asset_injection_step.py` → `AssetInjectionStep` + +`autograder/services/assets/resolver.py` → `AssetSourceResolver` diff --git a/docs/pipeline/04.2-setup-commands.md b/docs/pipeline/04.2-setup-commands.md new file mode 100644 index 00000000..da2938d6 --- /dev/null +++ b/docs/pipeline/04.2-setup-commands.md @@ -0,0 +1,78 @@ +# Step 4.2: Setup Commands + +## Purpose + +The Setup Commands step executes compilation or initialization scripts inside the sandbox. This handles language-specific build steps (e.g., `javac`, `gcc`, `npm install`) that must complete successfully before grading can begin. + +## How It Works + +The step execution follows these logic gates: + +1. **Load Configuration**: The step reads the `setup_config` for the submission's language and extracts the `setup_commands` list. If no commands are configured, the step immediately succeeds. + +2. **Validate Sandbox**: The step verifies that a sandbox container exists in the pipeline context. If setup commands are defined but no sandbox is available, the step fails immediately. + +3. **Execute Commands Sequentially**: Each command is executed in order within the sandbox via the `PreFlightService`. Execution is **stop-on-failure** — if any command returns a non-zero exit code, the step stops immediately without running remaining commands. + +4. **Report Results**: If all commands succeed, the step succeeds. If any command fails, the step fails with details including exit code, stdout, and stderr. + +## Dependencies + +| Step | What It Needs | +|------|---------------| +| **Sandbox** | Requires the `SandboxContainer` to execute commands in | + +## Input + +| Source | Data | +|--------|------| +| Constructor | `setup_config: dict` — language-keyed setup configuration | +| Pipeline | `pipeline_exec.sandbox` → the active sandbox container | +| Pipeline | `pipeline_exec.submission` → submission language | + +## Output + +| Field | Type | Description | +|-------|------|-------------| +| `data` | `None` | This step does not produce data for downstream steps | +| `status` | `StepStatus.SUCCESS` | All setup commands executed successfully | + +## Language-Specific Configuration + +Setup commands are defined per language in the `setup_config`: + +```json +{ + "java": { + "setup_commands": ["javac Calculator.java"] + }, + "c": { + "setup_commands": ["gcc -o main main.c -lm"] + }, + "javascript": { + "setup_commands": ["npm install"] + } +} +``` + +Templates can also contribute setup commands via `template.setup_commands`, which are merged with the assignment-level configuration during pipeline assembly. + +## Failure Scenarios + +- **No Sandbox** → `StepStatus.FAIL` if the sandbox was not created but setup commands are required. +- **Command Failed** → `StepStatus.FAIL` with the failing command's exit code, stdout, and stderr. Remaining commands are not executed. +- **Execution Error** → `StepStatus.FAIL` if the sandbox fails to respond or the connection is lost. + +## Next Step + +After setup commands complete, the pipeline proceeds to the grading phase starting with **[Step 5: Grade](05-grade.md)** to execute the actual test functions. + +--- + +## Source + +`autograder/steps/setup_commands_step.py` → `SetupCommandsStep` + +`autograder/services/pre_flight_service.py` → `PreFlightService` + +`autograder/services/sandbox_service.py` → `SandboxService` diff --git a/docs/pipeline/README.md b/docs/pipeline/README.md index 88e0f1e1..53169c68 100644 --- a/docs/pipeline/README.md +++ b/docs/pipeline/README.md @@ -10,7 +10,7 @@ The pipeline is: - **Fail-fast**: If any step fails, the pipeline stops immediately and reports the failure point. ``` -Submission ──▶ [LOAD_TEMPLATE] ──▶ [BUILD_TREE] ──▶ [SANDBOX] ──▶ [PRE_FLIGHT] ──▶ [AI_BATCH] ──▶ [STRUCTURAL_ANALYSIS] ──▶ [GRADE] ──▶ [FOCUS] ──▶ [FEEDBACK] ──▶ [EXPORT] +Submission ──▶ [LOAD_TEMPLATE] ──▶ [BUILD_TREE] ──▶ [FILE_CHECK] ──▶ [SANDBOX] ──▶ [ASSET_INJECTION] ──▶ [SETUP_COMMANDS] ──▶ [AI_BATCH] ──▶ [STRUCTURAL_ANALYSIS] ──▶ [GRADE] ──▶ [FOCUS] ──▶ [FEEDBACK] ──▶ [EXPORT] │ PipelineExecution (with GradingResult) @@ -94,8 +94,10 @@ The `data` field is polymorphic — each step stores a different type: | BOOTSTRAP | `Submission` | | LOAD_TEMPLATE | `Template` | | BUILD_TREE | `CriteriaTree` | -| PRE_FLIGHT | `None` | +| FILE_CHECK | `None` | | SANDBOX | `SandboxContainer | None` | +| ASSET_INJECTION | `None` | +| SETUP_COMMANDS | `None` | | STRUCTURAL_ANALYSIS | `StructuralAnalysisResult` | | GRADE | `GradeStepResult` | | FOCUS | `Focus` | @@ -108,6 +110,18 @@ All steps implement the `Step` abstract class: ```python class Step(ABC): + @property + @abstractmethod + def step_name(self) -> StepName: + """Return the name of the step.""" + pass + + @property + @abstractmethod + def category(self) -> StepCategory: + """Return the category of the step.""" + pass + @abstractmethod def execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution: """Execute the step on the pipeline execution context.""" @@ -125,8 +139,10 @@ Steps read data from previous steps via `pipeline_exec.get_step_result(StepName. |------|----------| | **Load Template** | — (no dependencies) | | **Build Tree** | Load Template (needs the `Template` to match test functions) | +| **File Check** | Load Template (checks for template-required files) | | **Sandbox** | Load Template (checks if template requires sandbox) | -| **Pre-Flight** | Load Template, Sandbox (requires sandbox to run setup commands) | +| **Asset Injection** | Sandbox (requires container to inject assets) | +| **Setup Commands** | Sandbox (requires container to run commands) | | **Structural Analysis** | — (needs `Submission` from context) | | **Grade** | Load Template, Build Tree, Sandbox*, Structural Analysis* | | **Focus** | Grade (needs the `ResultTree` from grading) | @@ -156,7 +172,7 @@ pipeline = build_pipeline( **Assembly rules:** The `StepRegistry` applies the specific conditionality parameters natively for step constructions: -1. **Load Template**, **Build Tree**, **Sandbox**, and **Pre-Flight** are unconditionally instantiated. +1. **Load Template**, **Build Tree**, **File Check**, **Sandbox**, **Asset Injection**, and **Setup Commands** are instantiated based on configuration and template requirements. 3. **Grade** and **Focus** always evaluate into their respective steps. 4. **Feedback** strictly resolves if `include_feedback=True`, consuming the `ReporterService` using the designated `feedback_mode`. 5. **Export** compiles optionally through `export_results=True`. @@ -164,7 +180,7 @@ The `StepRegistry` applies the specific conditionality parameters natively for s These instantiated elements are natively pushed into the `AutograderPipeline` linearly along their static `execution_order` mapping: ``` -LOAD_TEMPLATE → BUILD_TREE → SANDBOX → PRE_FLIGHT → STRUCTURAL_ANALYSIS → GRADE → FOCUS → FEEDBACK → EXPORT +LOAD_TEMPLATE → BUILD_TREE → FILE_CHECK → SANDBOX → ASSET_INJECTION → SETUP_COMMANDS → AI_BATCH → STRUCTURAL_ANALYSIS → GRADE → FOCUS → FEEDBACK → EXPORTER ``` --- @@ -175,8 +191,10 @@ LOAD_TEMPLATE → BUILD_TREE → SANDBOX → PRE_FLIGHT → STRUCTURAL_ANALYSIS |------|------|-------------|----------------| | [Load Template](01-load-template.md) | `load_template_step.py` | Loads the grading template (built-in or custom) containing test functions | None | | [Build Tree](02-build-tree.md) | `build_tree_step.py` | Constructs the `CriteriaTree` from JSON config, matching test functions from the template | Load Template | +| [File Check](04-file-check.md) | `file_check_step.py` | Validates that required files (from config or template) exist in submission | None | | [Sandbox](03-sandbox.md) | `sandbox_step.py` | Creates the sandbox environment and prepares the workspace | Load Template | -| [Pre-Flight](04-pre-flight.md) | `pre_flight_step.py` | Validates required files and executes setup commands (compilation, etc.) | Sandbox | +| [Asset Injection](04.1-asset-injection.md) | `asset_injection_step.py` | Injects static assets into the sandbox workspace | Sandbox | +| [Setup Commands](04.2-setup-commands.md) | `setup_commands_step.py` | Executes setup commands (compilation, etc.) in the sandbox | Sandbox | | [Structural Analysis](04.8-structural-analysis.md) | `structural_analysis_step.py` | Parses submission files into ast-grep SgRoot objects | None | | [Grade](05-grade.md) | `grade_step.py` | Executes all tests against the submission and produces the scored `ResultTree` | Load Template, Build Tree, Sandbox* | | [Focus](06-focus.md) | `focus_step.py` | Ranks all tests by their impact on the final score | Grade | diff --git a/issue-302-implementation-spec.md b/issue-302-implementation-spec.md deleted file mode 100644 index c535ee6e..00000000 --- a/issue-302-implementation-spec.md +++ /dev/null @@ -1,245 +0,0 @@ -# Issue 302 Implementation Spec: Split PreFlight into granular setup steps - -## Goal - -Replace the current `PreFlightStep` "god step" with three single-purpose setup steps while keeping the rest of the grading pipeline behavior stable: - -1. `FileCheckStep` for required file validation -2. `AssetInjectionStep` for sandbox asset copy-in -3. `SetupCommandsStep` for setup/compilation command execution - -Also add step categorization, allow templates to contribute setup requirements, and update every consumer that still assumes one monolithic `PreFlightStep`. - -## Current state - -The current flow is: - -`LOAD_TEMPLATE -> BUILD_TREE -> SANDBOX -> PRE_FLIGHT -> AI_BATCH -> STRUCTURAL_ANALYSIS -> GRADE -> FOCUS -> FEEDBACK -> EXPORTER` - -Relevant code paths: - -- `autograder/autograder.py` builds the pipeline order. -- `autograder/steps/step_registry.py` decides which steps are included. -- `autograder/steps/pre_flight_step.py` currently handles: - - required file validation - - asset injection - - setup commands -- `autograder/services/pre_flight_service.py` resolves language-specific setup config and executes setup commands. -- `autograder/serializers/pipeline_execution_serializer.py` still special-cases `PRE_FLIGHT`. -- `autograder/utils/feedback_generator.py` and `web/service/grading_service.py` still look for `PreFlightStep`. - -## Target behavior - -### Final pipeline order - -`LOAD_TEMPLATE -> BUILD_TREE -> FILE_CHECK -> SANDBOX -> ASSET_INJECTION -> SETUP_COMMANDS -> AI_BATCH -> STRUCTURAL_ANALYSIS -> GRADE -> FOCUS -> FEEDBACK -> EXPORTER` - -Step inclusion rules: - -| Step | Include when | Notes | -|------|--------------|-------| -| `FileCheckStep` | merged required files are non-empty | Must run before sandbox allocation | -| `SandboxStep` | any template requires sandbox OR any merged setup assets/commands exist | Sandbox is needed for asset injection and setup commands | -| `AssetInjectionStep` | merged assets are non-empty | Runs only after sandbox exists | -| `SetupCommandsStep` | merged setup commands are non-empty | Runs only after sandbox exists | - -### Step categories - -Add a `category`/`step_type` attribute to `Step` and expose a shared enum: - -| Category | Intended steps | -|----------|----------------| -| `SETUP` | bootstrap, load template, build tree, file check, sandbox, asset injection, setup commands | -| `GRADING` | AI batch, structural analysis, grade, focus | -| `REPORTING` | feedback, exporter | - -The category is primarily metadata, but it should be available on every step and registered centrally so later pipeline tooling can filter by phase. - -### Template-level setup requirements - -Templates should be able to contribute setup requirements in addition to assignment-level `setup_config`. - -The merged setup payload should: - -- keep assignment `setup_config` as the base -- merge template-provided `required_files` -- merge template-provided `setup_commands` -- preserve assignment-level `assets` -- keep language-specific resolution behavior unchanged - -Important: merge raw config before step construction, but keep language resolution inside the existing `SetupConfig` / `PreFlightService` flow so the runtime still picks the correct language block from the submission. - -## Design by component - -### 1) Step model and registry - -Files: - -- `autograder/models/dataclass/step_result.py` -- `autograder/models/abstract/step.py` -- `autograder/steps/step_registry.py` - -Required changes: - -- Add `StepCategory` enum with `SETUP`, `GRADING`, `REPORTING`. -- Add a `category` property to `Step`. -- Update every existing step class to return a category. -- Teach `StepRegistry` to build the new setup steps and to stop registering `PRE_FLIGHT`. - -### 2) New step classes - -Files to add: - -- `autograder/steps/file_check_step.py` -- `autograder/steps/asset_injection_step.py` -- `autograder/steps/setup_commands_step.py` - -Behavior: - -#### `FileCheckStep` - -- Reads the merged required files for the current submission language. -- Fails fast before sandbox creation. -- Emits structured error data for missing files. -- Should reuse the existing file-check logic from `PreFlightService` rather than duplicating resolution rules. - -#### `AssetInjectionStep` - -- Requires an existing sandbox. -- Resolves assets with `AssetSourceResolver`. -- Injects resolved assets with `SandboxContainer.inject_assets(...)`. -- Fails with a distinct structured error type for asset injection failures. - -#### `SetupCommandsStep` - -- Requires an existing sandbox. -- Runs commands sequentially. -- Stops on the first failing command. -- Reuses the current command execution/error formatting logic from `PreFlightService` and `SandboxService`. - -### 3) Setup config merging - -Files: - -- `autograder/autograder.py` -- `autograder/models/abstract/template.py` -- `autograder/services/template_library_service.py` if template metadata is surfaced - -Required changes: - -- Add optional template-level setup requirement accessors to `Template`. -- Keep default behavior empty so existing templates do not need changes beyond the interface. -- Merge all template contributions into the assignment `setup_config` before step construction. -- If multiple templates contribute the same file or command, de-duplicate while preserving order. - -Recommended merge rules: - -1. assignment config wins for shared structure -2. template values are appended -3. duplicates are removed -4. empty values are ignored - -### 4) Pipeline execution reporting - -Files: - -- `autograder/serializers/pipeline_execution_serializer.py` -- `autograder/utils/feedback_generator.py` -- `web/service/grading_service.py` - -Required changes: - -- Stop special-casing `PreFlightStep` only. -- Treat any of the three setup steps as preflight/setup failures. -- Update `failed_at_step`, step messages, and `error_details` mapping. -- Replace the hard-coded `total_steps_planned = 7` with a value derived from the actual registered steps. -- Add category-aware success messages for: - - `FILE_CHECK` - - `ASSET_INJECTION` - - `SETUP_COMMANDS` - -### 5) Compatibility/deprecation - -Files: - -- `autograder/steps/pre_flight_step.py` -- `autograder/models/dataclass/step_result.py` -- docs under `autograder/docs/` - -Required changes: - -- `PreFlightStep` should no longer be part of the pipeline. -- `StepName.PRE_FLIGHT` should not be emitted by new runs. -- Keep compatibility shims only if needed for imports or old tests, but new orchestration must use the split steps. - -## Error model guidance - -Keep the existing structured error approach, but extend it so the new steps are distinguishable. - -Suggested mapping: - -| Failure source | Structured type | -|----------------|-----------------| -| missing required file | `FILE_CHECK` | -| asset copy failure | `ASSET_INJECTION` | -| setup command failure | `SETUP_COMMAND` | -| missing sandbox for setup work | `SANDBOX_CREATION` or a dedicated setup/sandbox type | - -The serializer and feedback generator should read these types instead of parsing step names. - -## Docs to update - -Update docs that currently describe `PreFlightStep` as a single stage: - -- `autograder/docs/pipeline/04-pre-flight.md` -- `autograder/docs/pipeline/README.md` -- `autograder/docs/architecture/pipeline_execution_tracking.md` -- `autograder/docs/architecture/core_structures.md` -- `autograder/docs/features/setup_config_feature.md` -- `autograder/docs/API.md` -- `autograder/web/README.md` - -## Tests that should exist after implementation - -### Unit - -- `tests/unit/test_language_specific_setup_config.py` -- new tests for `FileCheckStep`, `AssetInjectionStep`, `SetupCommandsStep` -- serializer tests for the new setup step names and failure details -- feedback generation tests for all setup failure types -- pipeline builder tests that verify sandbox inclusion when assets/commands exist - -### Integration - -- full local-sandbox pipeline test covering: - - assignment creation - - submission creation - - file check failure - - asset injection success - - setup command success/failure - - final submission result payload - -### Web/API - -- grading service tests updated for new setup step names -- submission result serialization tests updated for new pipeline summary shape - -## Implementation order - -1. Add step category metadata and new step names. -2. Split `PreFlightStep` into the three setup steps. -3. Merge template-level setup requirements into the pipeline config. -4. Update registry and pipeline ordering. -5. Update serializer, feedback, and web service consumers. -6. Refresh tests and docs. - -## Validation expectations - -After implementation, the code should still satisfy: - -- `pytest` passes -- local autograder API runs in local sandbox mode -- assignment creation and submission grading still work end-to-end -- pipeline logs show the new step breakdown -- submission result payloads expose the full execution trail, including step-by-step setup failures and command output - diff --git a/sandbox_manager/manager.py b/sandbox_manager/manager.py index d7eb50fe..25491a35 100644 --- a/sandbox_manager/manager.py +++ b/sandbox_manager/manager.py @@ -14,10 +14,17 @@ _CLIENT: Optional[docker.DockerClient] = None _SHUTDOWN_REGISTERED = False -def _get_client() -> docker.DockerClient: + +def _get_docker_client() -> docker.DockerClient: global _CLIENT if _CLIENT is None: - _CLIENT = docker.from_env() + try: + _CLIENT = docker.from_env() + except Exception as e: + raise RuntimeError( + "Failed to initialize Docker client. Make sure Docker is running " + "and the socket is accessible." + ) from e return _CLIENT def initialize_sandbox_manager( @@ -53,7 +60,7 @@ def initialize_sandbox_manager( # Clean up orphaned containers before initializing new pools print("[SandboxManager] Cleaning up orphaned containers from previous runs...") - client = _get_client() + client = _get_docker_client() _cleanup_orphaned_containers(client) language_pools = {config.language: LanguagePool(config.language, config, client) for config in pool_configs} diff --git a/tests/unit/pipeline/test_ai_batch_step.py b/tests/unit/pipeline/test_ai_batch_step.py index b6c70717..572338b0 100644 --- a/tests/unit/pipeline/test_ai_batch_step.py +++ b/tests/unit/pipeline/test_ai_batch_step.py @@ -396,6 +396,14 @@ def template_description(self): def requires_sandbox(self): return False + @property + def required_files(self): + return {} + + @property + def setup_commands(self): + return {} + def get_test(self, name): return None @@ -441,6 +449,14 @@ def template_description(self): def requires_sandbox(self): return False + @property + def required_files(self): + return {} + + @property + def setup_commands(self): + return {} + def get_test(self, name): return None diff --git a/tests/unit/pipeline/test_issue_331_pipeline_builder.py b/tests/unit/pipeline/test_issue_331_pipeline_builder.py index cdec0a76..39604fcd 100644 --- a/tests/unit/pipeline/test_issue_331_pipeline_builder.py +++ b/tests/unit/pipeline/test_issue_331_pipeline_builder.py @@ -19,6 +19,8 @@ def test_build_pipeline_minimal_steps(self, mock_template_service_class): mock_service = mock_template_service_class.get_instance.return_value mock_template = MagicMock(spec=Template) mock_template.requires_sandbox = False + mock_template.required_files = {} + mock_template.setup_commands = {} mock_template.get_tests.return_value = {} # No AI tests mock_service.load_builtin_template.return_value = mock_template @@ -34,13 +36,15 @@ def test_build_pipeline_minimal_steps(self, mock_template_service_class): # Check steps step_names = list(pipeline._steps.keys()) - # Should NOT include SANDBOX, PRE_FLIGHT, AI_BATCH + # Should NOT include SANDBOX, FILE_CHECK, ASSET_INJECTION, SETUP_COMMANDS, AI_BATCH self.assertIn(StepName.LOAD_TEMPLATE, step_names) self.assertIn(StepName.BUILD_TREE, step_names) self.assertIn(StepName.GRADE, step_names) self.assertNotIn(StepName.SANDBOX, step_names) - self.assertNotIn(StepName.PRE_FLIGHT, step_names) + self.assertNotIn(StepName.FILE_CHECK, step_names) + self.assertNotIn(StepName.ASSET_INJECTION, step_names) + self.assertNotIn(StepName.SETUP_COMMANDS, step_names) self.assertNotIn(StepName.AI_BATCH, step_names) @patch("autograder.autograder.TemplateLibraryService") @@ -49,6 +53,8 @@ def test_build_pipeline_with_sandbox(self, mock_template_service_class): mock_service = mock_template_service_class.get_instance.return_value mock_template = MagicMock(spec=Template) mock_template.requires_sandbox = True + mock_template.required_files = {} + mock_template.setup_commands = {} mock_template.get_tests.return_value = {} mock_service.load_builtin_template.return_value = mock_template @@ -61,16 +67,17 @@ def test_build_pipeline_with_sandbox(self, mock_template_service_class): step_names = list(pipeline._steps.keys()) self.assertIn(StepName.SANDBOX, step_names) - # PRE_FLIGHT is still not included if no setup_config - self.assertNotIn(StepName.PRE_FLIGHT, step_names) + # FILE_CHECK is still not included if no setup_config + self.assertNotIn(StepName.FILE_CHECK, step_names) @patch("autograder.autograder.TemplateLibraryService") def test_build_pipeline_with_setup_config(self, mock_template_service_class): - # Mock Template NOT requiring sandbox (but setup_config might require it, - # actually PreFlightStep requires sandbox if it has assets or commands) + # Mock Template NOT requiring sandbox mock_service = mock_template_service_class.get_instance.return_value mock_template = MagicMock(spec=Template) mock_template.requires_sandbox = False + mock_template.required_files = {} + mock_template.setup_commands = {} mock_template.get_tests.return_value = {} mock_service.load_builtin_template.return_value = mock_template @@ -79,11 +86,12 @@ def test_build_pipeline_with_setup_config(self, mock_template_service_class): include_feedback=False, grading_criteria={"base": []}, feedback_config={}, - setup_config={"required_files": ["main.py"]} + setup_config={"python": {"required_files": ["main.py"]}} ) step_names = list(pipeline._steps.keys()) - self.assertIn(StepName.PRE_FLIGHT, step_names) + self.assertIn(StepName.FILE_CHECK, step_names) + self.assertNotIn(StepName.PRE_FLIGHT, step_names) @patch("autograder.autograder.TemplateLibraryService") def test_build_pipeline_with_ai(self, mock_template_service_class): diff --git a/tests/unit/pipeline/test_pipeline_execution_serializer.py b/tests/unit/pipeline/test_pipeline_execution_serializer.py index 135a926b..f1c1da53 100644 --- a/tests/unit/pipeline/test_pipeline_execution_serializer.py +++ b/tests/unit/pipeline/test_pipeline_execution_serializer.py @@ -22,7 +22,7 @@ def test_serialize_legacy_file_missing_error(): ] step_result = StepResult.fail( - step=StepName.PRE_FLIGHT, + step=StepName.FILE_CHECK, error="Arquivo ou diretório obrigatório não encontrado: `\'Main.java\'`", error_data=mock_error_data ) @@ -34,10 +34,10 @@ def test_serialize_legacy_file_missing_error(): # Assert assert summary["status"] == "failed" - assert summary["failed_at_step"] == "PreFlightStep" + assert summary["failed_at_step"] == "FileCheckStep" preflight_step = summary["steps"][0] - assert preflight_step["name"] == "PreFlightStep" + assert preflight_step["name"] == "FileCheckStep" assert preflight_step["status"] == "fail" assert preflight_step["error_details"] == { "error_type": "required_file_missing", @@ -68,7 +68,7 @@ def test_serialize_setup_command_failed(): ] step_result = StepResult.fail( - step=StepName.PRE_FLIGHT, + step=StepName.SETUP_COMMANDS, error="Setup command \'Compile\' failed...", error_data=mock_error_data ) @@ -80,10 +80,10 @@ def test_serialize_setup_command_failed(): # Assert assert summary["status"] == "failed" - assert summary["failed_at_step"] == "PreFlightStep" + assert summary["failed_at_step"] == "SetupCommandsStep" preflight_step = summary["steps"][0] - assert preflight_step["name"] == "PreFlightStep" + assert preflight_step["name"] == "SetupCommandsStep" assert preflight_step["status"] == "fail" assert preflight_step["error_details"] == { "error_type": "setup_command_failed", diff --git a/tests/unit/pipeline/test_pipeline_steps.py b/tests/unit/pipeline/test_pipeline_steps.py index 7be04837..24e555a0 100644 --- a/tests/unit/pipeline/test_pipeline_steps.py +++ b/tests/unit/pipeline/test_pipeline_steps.py @@ -14,7 +14,7 @@ from autograder.models.abstract.template import Template from autograder.models.abstract.test_function import TestFunction from autograder.models.dataclass.param_description import ParamDescription -from autograder.models.dataclass.step_result import StepName, StepResult, StepStatus +from autograder.models.dataclass.step_result import StepName, StepResult, StepStatus, StepCategory from autograder.models.dataclass.submission import Submission, SubmissionFile from autograder.models.dataclass.test_result import TestResult from autograder.models.pipeline_execution import PipelineExecution @@ -80,6 +80,14 @@ def requires_sandbox(self) -> bool: """Mock templates don't require sandboxes.""" return False + @property + def required_files(self): + return {} + + @property + def setup_commands(self): + return {} + def stop(self): """No cleanup needed for mock templates.""" @@ -103,6 +111,11 @@ def step_name(self) -> StepName: """Return the step name.""" return StepName.LOAD_TEMPLATE + @property + def category(self) -> StepCategory: + """Return the step category.""" + return StepCategory.SETUP + def _execute(self, pipeline_exec: PipelineExecution, locale=None): """Inject the template into the pipeline execution.""" return pipeline_exec.add_step_result(StepResult( diff --git a/tests/unit/test_command_resolution_at_build_time.py b/tests/unit/test_command_resolution_at_build_time.py index 878ab403..4007dfa7 100644 --- a/tests/unit/test_command_resolution_at_build_time.py +++ b/tests/unit/test_command_resolution_at_build_time.py @@ -64,6 +64,14 @@ def template_description(self): def requires_sandbox(self): return False + @property + def required_files(self): + return {} + + @property + def setup_commands(self): + return {} + def get_test(self, name): raise KeyError(name) diff --git a/tests/unit/test_template_contract.py b/tests/unit/test_template_contract.py index fbe1c194..97da3450 100644 --- a/tests/unit/test_template_contract.py +++ b/tests/unit/test_template_contract.py @@ -1,4 +1,5 @@ import pytest +from typing import Dict, Any from autograder.models.abstract.template import Template from autograder.models.abstract.test_function import TestFunction @@ -39,6 +40,14 @@ def template_description(self) -> str: def requires_sandbox(self) -> bool: return False + @property + def required_files(self) -> Dict[str, list]: + return {} + + @property + def setup_commands(self) -> Dict[str, list]: + return {} + def get_test(self, name: str): return DummyTest() diff --git a/tests/unit/test_test_function_validation.py b/tests/unit/test_test_function_validation.py index e3b4812c..923177cf 100644 --- a/tests/unit/test_test_function_validation.py +++ b/tests/unit/test_test_function_validation.py @@ -42,6 +42,10 @@ def template_name(self): return "Mock Template" def template_description(self): return "Mock" @property def requires_sandbox(self): return False + @property + def required_files(self): return {} + @property + def setup_commands(self): return {} def get_test(self, name): return self.tests.get(name) diff --git a/tests/web/test_grading_service.py b/tests/web/test_grading_service.py index b9eff52f..2c3192eb 100644 --- a/tests/web/test_grading_service.py +++ b/tests/web/test_grading_service.py @@ -95,7 +95,7 @@ async def test_grade_submission_pipeline_failure(): mock_failed_step = Mock() mock_failed_step.status = StepStatus.FAIL - mock_failed_step.step = StepName.PRE_FLIGHT + mock_failed_step.step = StepName.FILE_CHECK mock_failed_step.error = "Syntax error" mock_failed_step.error_data = [] @@ -103,7 +103,7 @@ async def test_grade_submission_pipeline_failure(): mock_execution.get_previous_step = Mock(return_value=mock_failed_step) mock_execution.get_pipeline_execution_summary = Mock(return_value={ "status": "failed", - "failed_at_step": "PreFlightStep" + "failed_at_step": "FileCheckStep" }) mock_pipeline.run = Mock(return_value=mock_execution) @@ -145,7 +145,7 @@ async def test_grade_submission_pipeline_failure(): create_call = mock_result_repo.create.call_args[1] assert create_call["final_score"] == 0.0 assert create_call["pipeline_status"] == PipelineStatus.FAILED - assert create_call["failed_at_step"] == "PreFlightStep" + assert create_call["failed_at_step"] == "FileCheckStep" def test_node_to_dict(): diff --git a/web/service/grading_service.py b/web/service/grading_service.py index b72f6a50..fe6b3b9e 100644 --- a/web/service/grading_service.py +++ b/web/service/grading_service.py @@ -164,7 +164,8 @@ async def _persist_failure(result_repo, submission_repo, request: GradingRequest feedback = None failed_step_name = pipeline_summary.get("failed_at_step") - if failed_step_name == "PreFlightStep": + setup_steps = ["FileCheckStep", "AssetInjectionStep", "SetupCommandsStep", "PreFlightStep"] + if failed_step_name in setup_steps: feedback = generate_preflight_feedback(pipeline_summary, locale=request.locale) await result_repo.create(