From 510ce69b7177ceee0c43829ab120075a3661707b Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Fri, 15 May 2026 17:51:33 -0700 Subject: [PATCH] fix: Orchestrator - Only setting execution node status to CANCELLED after the container execution is terminated Fixes: https://github.com/TangleML/tangle/issues/240 --- cloud_pipelines_backend/orchestrator_sql.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cloud_pipelines_backend/orchestrator_sql.py b/cloud_pipelines_backend/orchestrator_sql.py index 79017b6..c613a9e 100644 --- a/cloud_pipelines_backend/orchestrator_sql.py +++ b/cloud_pipelines_backend/orchestrator_sql.py @@ -707,13 +707,19 @@ def internal_process_one_running_execution( container_execution.status = bts.ContainerExecutionStatus.CANCELLED terminated = True + # Marking the execution nodes as cancelled only after the launched container is successfully terminated + for execution_node in votes_to_terminate: + _logger.info( + f"Marking the execution node {execution_node.id} as CANCELLED." + ) + execution_node.container_execution_status = ( + bts.ContainerExecutionStatus.CANCELLED + ) + # Mark the execution nodes as cancelled only after the launched container is successfully terminated (if needed) for execution_node in votes_to_terminate: _logger.info( - f"Cancelling execution {execution_node.id} and skipping all downstream executions." - ) - execution_node.container_execution_status = ( - bts.ContainerExecutionStatus.CANCELLED + f"Skipping all downstream executions of cancelled execution node {execution_node.id}." ) _mark_all_downstream_executions_as_skipped( session=session, execution=execution_node