From 98e1f4918087dc36c400c289ea62cafadefd52f2 Mon Sep 17 00:00:00 2001 From: MJ Ponsonby Date: Mon, 13 Jul 2026 16:24:51 +0200 Subject: [PATCH 1/3] internal/rest/resources: Ensure node is fully removed instead of partially. Previously on removing the node failing, which happens if the node is not in the cluster, the function exits. Which means the trust store isn't cleaned. This stops an issue wherin a prejoin hook could fail and cause the trust store to remain in a broken state which is not good. Fixes: #784 Signed-off-by: MJ Ponsonby Assisted-by: Deepseek V4 Pro --- internal/rest/resources/cluster.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/rest/resources/cluster.go b/internal/rest/resources/cluster.go index f1bcc514..2748152c 100644 --- a/internal/rest/resources/cluster.go +++ b/internal/rest/resources/cluster.go @@ -709,9 +709,13 @@ func clusterMemberDelete(s types.State, r *http.Request) types.Response { // Remove the node from dqlite, if it has a record there. if index >= 0 { err = leader.Remove(ctx, info[index].ID) - if err != nil { + if err != nil && !force { return types.SmartError(err) } + + if err != nil { + logger.Error("Failed to remove cluster member from dqlite", slog.String("error", err.Error()), slog.Bool("force", force), slog.String("name", name)) + } } u := api.NewURL() From 9a66f36376123e5bdb0fc1a12b8a90c9e4f827be Mon Sep 17 00:00:00 2001 From: MJ Ponsonby Date: Mon, 13 Jul 2026 16:24:51 +0200 Subject: [PATCH 2/3] internal/rest/resources: Make join failure cleanup synchronous When PreJoin or StartAPI fails during a join, the reverts that clean up the cluster member state on the leader ran asynchronously in a goroutine. If the process exited or the network call failed, the leader could be left with stale core_cluster_members and truststore entries, causing a consistency error on the next join attempt (issue #784). Make the DeleteClusterMember call in the revert handler synchronous so the leader state is fully cleaned before the error is returned to the client. Signed-off-by: MJ Ponsonby Assisted-by: Deepseek V4 Pro --- internal/rest/resources/control.go | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/internal/rest/resources/control.go b/internal/rest/resources/control.go index 57082a99..951fa450 100644 --- a/internal/rest/resources/control.go +++ b/internal/rest/resources/control.go @@ -124,17 +124,12 @@ func controlPost(state types.State, r *http.Request) types.Response { return } - // The cluster member also gets re-executed as part of the deletion. - go func() { - <-r.Context().Done() - - // Use `force=1` to ensure the node is fully removed, in case its listener hasn't been set up. - // Use the background context as the original request context is already cancelled at this point. - err = internalClient.DeleteClusterMember(context.Background(), client, req.Name, "", true) - if err != nil { - logger.Error("Failed to clean up cluster state after join failure", slog.String("error", err.Error())) - } - }() + // Use `force=1` to ensure the node is fully removed, in case its listener hasn't been set up. + // Use the background context as the original request context is already cancelled at this point. + err = internalClient.DeleteClusterMember(context.Background(), client, req.Name, "", true) + if err != nil { + logger.Error("Failed to clean up cluster state after join failure", slog.String("error", err.Error())) + } }) // Replace the server keypair if the cluster member name has changed upon initialization. From 35005d05f7ffdcdb04023a78f5764f1d6bb709fa Mon Sep 17 00:00:00 2001 From: MJ Ponsonby Date: Mon, 13 Jul 2026 16:24:51 +0200 Subject: [PATCH 3/3] example: Add FAIL_PREJOIN env var and test for PreJoin failure cleanup Add support for FAIL_PREJOIN=1 environment variable in the example daemon's PreJoin hook, which causes it to return an error. Add a system test that verifies a node with a failing PreJoin hook can still rejoin the cluster successfully, proving that the synchronous cleanup after join failure properly cleans the cluster state. Signed-off-by: MJ Ponsonby Assisted-by: Deepseek V4 Pro --- example/cmd/microd/main.go | 4 +++ example/test/main.sh | 61 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/example/cmd/microd/main.go b/example/cmd/microd/main.go index a78c036b..83465528 100644 --- a/example/cmd/microd/main.go +++ b/example/cmd/microd/main.go @@ -157,6 +157,10 @@ func (c *cmdDaemon) run(cmd *cobra.Command, args []string) error { logger.Info("This is a hook that runs after the daemon is initialized and joins an existing cluster, before OnNewMember runs on all peers") logger.Info("Here are the extra configuration keys that were passed into the init --join command", slog.Any("config", initConfig)) + if os.Getenv("FAIL_PREJOIN") == "1" { + return fmt.Errorf("PreJoin failed: FAIL_PREJOIN is set") + } + return nil }, diff --git a/example/test/main.sh b/example/test/main.sh index a89a7a67..e1040282 100755 --- a/example/test/main.sh +++ b/example/test/main.sh @@ -1009,6 +1009,64 @@ test_extended_endpoints() { shutdown_systems } +test_prejoin_failure() { + echo "Testing PreJoin failure cleanup" + + # Start 3 systems (c4 will be started manually) + new_systems 3 --heartbeat 2s + + # Bootstrap c1 and join c2, c3 + microctl --state-dir "${test_dir}/c1" init "c1" 127.0.0.1:9001 --bootstrap + token_c2=$(microctl --state-dir "${test_dir}/c1" tokens add "c2") + token_c3=$(microctl --state-dir "${test_dir}/c1" tokens add "c3") + microctl --state-dir "${test_dir}/c2" init "c2" 127.0.0.1:9002 --token "${token_c2}" + microctl --state-dir "${test_dir}/c3" init "c3" 127.0.0.1:9003 --token "${token_c3}" + + # Wait for cluster to stabilize + while [[ -n "$(microctl --state-dir "${test_dir}/c1" cluster list -f yaml | yq '.[] | select(.role == "PENDING")')" ]]; do + sleep 2 + done + + echo " -> Starting c4 with FAIL_PREJOIN=1" + mkdir -p "${test_dir}/c4" + FAIL_PREJOIN=1 microd --state-dir "${test_dir}/c4" --heartbeat 2s & + microctl --state-dir "${test_dir}/c4" waitready + + # First join attempt should fail due to FAIL_PREJOIN + token_c4=$(microctl --state-dir "${test_dir}/c1" tokens add "c4") + ! microctl --state-dir "${test_dir}/c4" init "c4" 127.0.0.1:9004 --token "${token_c4}" || { + echo "ERROR: c4 join should have failed due to PreJoin hook failure" + return 1 + } + + echo " -> PreJoin failure triggered successfully, restarting c4 without FAIL_PREJOIN" + + # Kill c4 and restart without FAIL_PREJOIN + microctl --state-dir "${test_dir}/c4" shutdown || true + sleep 2 + + microd --state-dir "${test_dir}/c4" --heartbeat 2s & + microctl --state-dir "${test_dir}/c4" waitready + + # c4 should now be able to join successfully + # (this tests that the synchronous cleanup after the failed PreJoin left the cluster state clean) + token_c4=$(microctl --state-dir "${test_dir}/c1" tokens add "c4") + microctl --state-dir "${test_dir}/c4" init "c4" 127.0.0.1:9004 --token "${token_c4}" || { + echo "ERROR: c4 should be able to join after failed PreJoin cleanup" + return 1 + } + + # Wait for cluster to stabilize + while [[ -n "$(microctl --state-dir "${test_dir}/c1" cluster list -f yaml | yq '.[] | select(.role == "PENDING")')" ]]; do + sleep 2 + done + + # Verify c4 is a voter + [[ $(microctl --state-dir "${test_dir}/c1" cluster list -f yaml | yq '.[] | select(.clustermemberlocal.name == "c4").role') == "voter" ]] + + shutdown_systems +} + test_self_deletion() { echo "Testing self deletion" @@ -1060,6 +1118,7 @@ if [ "${1:-"all"}" = "all" ] || [ "${1}" = "" ]; then run_test membership_consistency run_test truststore_force_removal run_test parallel_joins + run_test prejoin_failure run_test self_deletion elif [ "${1}" = "recover" ]; then run_test recover @@ -1079,6 +1138,8 @@ elif [ "${1}" = "force-removal" ]; then run_test truststore_force_removal elif [ "${1}" = "parallel-join" ]; then run_test parallel_joins +elif [ "${1}" = "prejoin" ]; then + run_test prejoin_failure elif [ "${1}" = "self-deletion" ]; then run_test self_deletion elif [ "${1}" = "daemon-config" ]; then