diff --git a/src/alert-manager/src/alert-parser/node_alert_monitor.py b/src/alert-manager/src/alert-parser/node_alert_monitor.py index cb3f4435..6a2024e9 100644 --- a/src/alert-manager/src/alert-parser/node_alert_monitor.py +++ b/src/alert-manager/src/alert-parser/node_alert_monitor.py @@ -228,7 +228,7 @@ def handle_node_status_change(self, node: str, timestamp: float, status: float, elif period_alerts['alertname'].str.contains('CordonValidationFailedNodes').any(): validation_alerts = period_alerts[period_alerts['alertname'].str.contains('CordonValidationFailedNodes')] validation_time = validation_alerts['timestamp'].max() - to_status = NodeStatus.CORDONED.value + to_status = NodeStatus.TRIAGED_UNKNOWN.value reason, detail = self.alert_mapper.summary_events_into_reason_detail(shrinked_alerts) self.node_updater.update_status_action(node, from_status, to_status, validation_time, reason, detail) elif period_alerts['alertname'].str.contains('RecoverValidatedNodes').any(): diff --git a/src/alert-manager/src/alert-parser/tests/test_alert_monitor.py b/src/alert-manager/src/alert-parser/tests/test_alert_monitor.py index dc64bd0d..a98fefd8 100644 --- a/src/alert-manager/src/alert-parser/tests/test_alert_monitor.py +++ b/src/alert-manager/src/alert-parser/tests/test_alert_monitor.py @@ -178,7 +178,7 @@ def test_handle_node_status_change(monitor, mock_alert_fetcher, mock_alert_mappe mock_alert_fetcher.reset_mock() mock_alert_mapper.reset_mock() - # case 3: Node status change from validating to cordoned + # case 4: Validation failure moves the node to triaged_unknown status = -1 node_status = NodeStatusRecord( Timestamp=datetime.fromtimestamp(timestamp - 100, tz=timezone.utc), @@ -204,7 +204,31 @@ def test_handle_node_status_change(monitor, mock_alert_fetcher, mock_alert_mappe args = mock_node_updater.update_status_action.call_args.args assert args[0] == node assert args[1] == 'validating' - assert args[2] == 'cordoned' + assert args[2] == 'triaged_unknown' + + # Reset mocks for next test + mock_node_updater.reset_mock() + mock_alert_fetcher.reset_mock() + mock_alert_mapper.reset_mock() + + # case 5: Validation success moves the node to available_nodata + alerts = pd.DataFrame({ + 'alertname': ['RecoverValidatedNodes'], + 'timestamp': [datetime.fromtimestamp(timestamp, tz=timezone.utc)], + 'node_name': [node], + 'summary': [f'{node} passed validation'] + }) + mock_alert_fetcher.find_node_alerts.return_value = alerts + mock_alert_fetcher.shrink_alerts.return_value = alerts + mock_alert_mapper.summary_events_into_reason_detail.return_value = ("reason", "detail") + + monitor.handle_node_status_change(node, timestamp, status, alerts, node_status) + + assert mock_node_updater.update_status_action.called + args = mock_node_updater.update_status_action.call_args.args + assert args[0] == node + assert args[1] == 'validating' + assert args[2] == 'available_nodata' def test_handle_validating_node_with_empty_alerts(monitor, mock_alert_fetcher, mock_alert_mapper, mock_node_updater): diff --git a/src/alert-manager/src/job-status-change-notification/controllers/alert.js b/src/alert-manager/src/job-status-change-notification/controllers/alert.js index c4ba363b..48b6f840 100644 --- a/src/alert-manager/src/job-status-change-notification/controllers/alert.js +++ b/src/alert-manager/src/job-status-change-notification/controllers/alert.js @@ -87,7 +87,7 @@ const uncordonNodes = async (nodeList) => { status: "firing", labels: { alertname: "RecoverValidatedNodes", - severity: "info", + severity: "error", node_name: node, }, annotations: { @@ -109,7 +109,7 @@ const cordonNodes = async (nodeList) => { status: "firing", labels: { alertname: "CordonValidationFailedNodes", - severity: "info", + severity: "error", node_name: node.name, }, annotations: {