diff --git a/example/cmd/microd/main.go b/example/cmd/microd/main.go index a78c036b..83465528 100644 --- a/example/cmd/microd/main.go +++ b/example/cmd/microd/main.go @@ -157,6 +157,10 @@ func (c *cmdDaemon) run(cmd *cobra.Command, args []string) error { logger.Info("This is a hook that runs after the daemon is initialized and joins an existing cluster, before OnNewMember runs on all peers") logger.Info("Here are the extra configuration keys that were passed into the init --join command", slog.Any("config", initConfig)) + if os.Getenv("FAIL_PREJOIN") == "1" { + return fmt.Errorf("PreJoin failed: FAIL_PREJOIN is set") + } + return nil }, diff --git a/example/test/main.sh b/example/test/main.sh index a89a7a67..e1040282 100755 --- a/example/test/main.sh +++ b/example/test/main.sh @@ -1009,6 +1009,64 @@ test_extended_endpoints() { shutdown_systems } +test_prejoin_failure() { + echo "Testing PreJoin failure cleanup" + + # Start 3 systems (c4 will be started manually) + new_systems 3 --heartbeat 2s + + # Bootstrap c1 and join c2, c3 + microctl --state-dir "${test_dir}/c1" init "c1" 127.0.0.1:9001 --bootstrap + token_c2=$(microctl --state-dir "${test_dir}/c1" tokens add "c2") + token_c3=$(microctl --state-dir "${test_dir}/c1" tokens add "c3") + microctl --state-dir "${test_dir}/c2" init "c2" 127.0.0.1:9002 --token "${token_c2}" + microctl --state-dir "${test_dir}/c3" init "c3" 127.0.0.1:9003 --token "${token_c3}" + + # Wait for cluster to stabilize + while [[ -n "$(microctl --state-dir "${test_dir}/c1" cluster list -f yaml | yq '.[] | select(.role == "PENDING")')" ]]; do + sleep 2 + done + + echo " -> Starting c4 with FAIL_PREJOIN=1" + mkdir -p "${test_dir}/c4" + FAIL_PREJOIN=1 microd --state-dir "${test_dir}/c4" --heartbeat 2s & + microctl --state-dir "${test_dir}/c4" waitready + + # First join attempt should fail due to FAIL_PREJOIN + token_c4=$(microctl --state-dir "${test_dir}/c1" tokens add "c4") + ! microctl --state-dir "${test_dir}/c4" init "c4" 127.0.0.1:9004 --token "${token_c4}" || { + echo "ERROR: c4 join should have failed due to PreJoin hook failure" + return 1 + } + + echo " -> PreJoin failure triggered successfully, restarting c4 without FAIL_PREJOIN" + + # Kill c4 and restart without FAIL_PREJOIN + microctl --state-dir "${test_dir}/c4" shutdown || true + sleep 2 + + microd --state-dir "${test_dir}/c4" --heartbeat 2s & + microctl --state-dir "${test_dir}/c4" waitready + + # c4 should now be able to join successfully + # (this tests that the synchronous cleanup after the failed PreJoin left the cluster state clean) + token_c4=$(microctl --state-dir "${test_dir}/c1" tokens add "c4") + microctl --state-dir "${test_dir}/c4" init "c4" 127.0.0.1:9004 --token "${token_c4}" || { + echo "ERROR: c4 should be able to join after failed PreJoin cleanup" + return 1 + } + + # Wait for cluster to stabilize + while [[ -n "$(microctl --state-dir "${test_dir}/c1" cluster list -f yaml | yq '.[] | select(.role == "PENDING")')" ]]; do + sleep 2 + done + + # Verify c4 is a voter + [[ $(microctl --state-dir "${test_dir}/c1" cluster list -f yaml | yq '.[] | select(.clustermemberlocal.name == "c4").role') == "voter" ]] + + shutdown_systems +} + test_self_deletion() { echo "Testing self deletion" @@ -1060,6 +1118,7 @@ if [ "${1:-"all"}" = "all" ] || [ "${1}" = "" ]; then run_test membership_consistency run_test truststore_force_removal run_test parallel_joins + run_test prejoin_failure run_test self_deletion elif [ "${1}" = "recover" ]; then run_test recover @@ -1079,6 +1138,8 @@ elif [ "${1}" = "force-removal" ]; then run_test truststore_force_removal elif [ "${1}" = "parallel-join" ]; then run_test parallel_joins +elif [ "${1}" = "prejoin" ]; then + run_test prejoin_failure elif [ "${1}" = "self-deletion" ]; then run_test self_deletion elif [ "${1}" = "daemon-config" ]; then diff --git a/internal/rest/resources/cluster.go b/internal/rest/resources/cluster.go index f1bcc514..2748152c 100644 --- a/internal/rest/resources/cluster.go +++ b/internal/rest/resources/cluster.go @@ -709,9 +709,13 @@ func clusterMemberDelete(s types.State, r *http.Request) types.Response { // Remove the node from dqlite, if it has a record there. if index >= 0 { err = leader.Remove(ctx, info[index].ID) - if err != nil { + if err != nil && !force { return types.SmartError(err) } + + if err != nil { + logger.Error("Failed to remove cluster member from dqlite", slog.String("error", err.Error()), slog.Bool("force", force), slog.String("name", name)) + } } u := api.NewURL() diff --git a/internal/rest/resources/control.go b/internal/rest/resources/control.go index 57082a99..951fa450 100644 --- a/internal/rest/resources/control.go +++ b/internal/rest/resources/control.go @@ -124,17 +124,12 @@ func controlPost(state types.State, r *http.Request) types.Response { return } - // The cluster member also gets re-executed as part of the deletion. - go func() { - <-r.Context().Done() - - // Use `force=1` to ensure the node is fully removed, in case its listener hasn't been set up. - // Use the background context as the original request context is already cancelled at this point. - err = internalClient.DeleteClusterMember(context.Background(), client, req.Name, "", true) - if err != nil { - logger.Error("Failed to clean up cluster state after join failure", slog.String("error", err.Error())) - } - }() + // Use `force=1` to ensure the node is fully removed, in case its listener hasn't been set up. + // Use the background context as the original request context is already cancelled at this point. + err = internalClient.DeleteClusterMember(context.Background(), client, req.Name, "", true) + if err != nil { + logger.Error("Failed to clean up cluster state after join failure", slog.String("error", err.Error())) + } }) // Replace the server keypair if the cluster member name has changed upon initialization.