Ensure node is fully removed instead of partially.

example/cmd/microd/main.go

@@ -157,6 +157,10 @@ func (c *cmdDaemon) run(cmd *cobra.Command, args []string) error {
 			logger.Info("This is a hook that runs after the daemon is initialized and joins an existing cluster, before OnNewMember runs on all peers")
 			logger.Info("Here are the extra configuration keys that were passed into the init --join command", slog.Any("config", initConfig))
 
+			if os.Getenv("FAIL_PREJOIN") == "1" {
+				return fmt.Errorf("PreJoin failed: FAIL_PREJOIN is set")
+			}
+
 			return nil
 		},
 

example/test/main.sh

@@ -1009,6 +1009,64 @@ test_extended_endpoints() {
   shutdown_systems
 }
 
+test_prejoin_failure() {
+  echo "Testing PreJoin failure cleanup"
+
+  # Start 3 systems (c4 will be started manually)
+  new_systems 3 --heartbeat 2s
+
+  # Bootstrap c1 and join c2, c3
+  microctl --state-dir "${test_dir}/c1" init "c1" 127.0.0.1:9001 --bootstrap
+  token_c2=$(microctl --state-dir "${test_dir}/c1" tokens add "c2")
+  token_c3=$(microctl --state-dir "${test_dir}/c1" tokens add "c3")
+  microctl --state-dir "${test_dir}/c2" init "c2" 127.0.0.1:9002 --token "${token_c2}"
+  microctl --state-dir "${test_dir}/c3" init "c3" 127.0.0.1:9003 --token "${token_c3}"
+
+  # Wait for cluster to stabilize
+  while [[ -n "$(microctl --state-dir "${test_dir}/c1" cluster list -f yaml | yq '.[] | select(.role == "PENDING")')" ]]; do
+    sleep 2
+  done
+
+  echo "  -> Starting c4 with FAIL_PREJOIN=1"
+  mkdir -p "${test_dir}/c4"
+  FAIL_PREJOIN=1 microd --state-dir "${test_dir}/c4" --heartbeat 2s &
+  microctl --state-dir "${test_dir}/c4" waitready
+
+  # First join attempt should fail due to FAIL_PREJOIN
+  token_c4=$(microctl --state-dir "${test_dir}/c1" tokens add "c4")
+  ! microctl --state-dir "${test_dir}/c4" init "c4" 127.0.0.1:9004 --token "${token_c4}" || {
+    echo "ERROR: c4 join should have failed due to PreJoin hook failure"
+    return 1
+  }
+
+  echo "  -> PreJoin failure triggered successfully, restarting c4 without FAIL_PREJOIN"
+
+  # Kill c4 and restart without FAIL_PREJOIN
+  microctl --state-dir "${test_dir}/c4" shutdown || true
+  sleep 2
+
+  microd --state-dir "${test_dir}/c4" --heartbeat 2s &
+  microctl --state-dir "${test_dir}/c4" waitready
+
+  # c4 should now be able to join successfully
+  # (this tests that the synchronous cleanup after the failed PreJoin left the cluster state clean)
+  token_c4=$(microctl --state-dir "${test_dir}/c1" tokens add "c4")
+  microctl --state-dir "${test_dir}/c4" init "c4" 127.0.0.1:9004 --token "${token_c4}" || {
+    echo "ERROR: c4 should be able to join after failed PreJoin cleanup"
+    return 1
+  }
+
+  # Wait for cluster to stabilize
+  while [[ -n "$(microctl --state-dir "${test_dir}/c1" cluster list -f yaml | yq '.[] | select(.role == "PENDING")')" ]]; do
+    sleep 2
+  done
+
+  # Verify c4 is a voter
+  [[ $(microctl --state-dir "${test_dir}/c1" cluster list -f yaml | yq '.[] | select(.clustermemberlocal.name == "c4").role') == "voter" ]]
+
+  shutdown_systems
+}
+
 test_self_deletion() {
   echo "Testing self deletion"
 
@@ -1060,6 +1118,7 @@ if [ "${1:-"all"}" = "all" ] || [ "${1}" = "" ]; then
   run_test membership_consistency
   run_test truststore_force_removal
   run_test parallel_joins
+  run_test prejoin_failure
   run_test self_deletion
 elif [ "${1}" = "recover" ]; then
   run_test recover
@@ -1079,6 +1138,8 @@ elif [ "${1}" = "force-removal" ]; then
   run_test truststore_force_removal
 elif [ "${1}" = "parallel-join" ]; then
   run_test parallel_joins
+elif [ "${1}" = "prejoin" ]; then
+  run_test prejoin_failure
 elif [ "${1}" = "self-deletion" ]; then
   run_test self_deletion
 elif [ "${1}" = "daemon-config" ]; then

internal/rest/resources/cluster.go

@@ -709,9 +709,13 @@ func clusterMemberDelete(s types.State, r *http.Request) types.Response {
 	// Remove the node from dqlite, if it has a record there.
 	if index >= 0 {
 		err = leader.Remove(ctx, info[index].ID)
-		if err != nil {
+		if err != nil && !force {
 			return types.SmartError(err)
 		}
+
+		if err != nil {
+			logger.Error("Failed to remove cluster member from dqlite", slog.String("error", err.Error()), slog.Bool("force", force), slog.String("name", name))
+		}
 	}
 
 	u := api.NewURL()

internal/rest/resources/control.go

@@ -124,17 +124,12 @@ func controlPost(state types.State, r *http.Request) types.Response {
 			return
 		}
 
-		// The cluster member also gets re-executed as part of the deletion.
-		go func() {
-			<-r.Context().Done()
-
-			// Use `force=1` to ensure the node is fully removed, in case its listener hasn't been set up.
-			// Use the background context as the original request context is already cancelled at this point.
-			err = internalClient.DeleteClusterMember(context.Background(), client, req.Name, "", true)
-			if err != nil {
-				logger.Error("Failed to clean up cluster state after join failure", slog.String("error", err.Error()))
-			}
-		}()
+		// Use `force=1` to ensure the node is fully removed, in case its listener hasn't been set up.
+		// Use the background context as the original request context is already cancelled at this point.
+		err = internalClient.DeleteClusterMember(context.Background(), client, req.Name, "", true)
+		if err != nil {
+			logger.Error("Failed to clean up cluster state after join failure", slog.String("error", err.Error()))
+		}
 	})
 
 	// Replace the server keypair if the cluster member name has changed upon initialization.