diff --git a/internal/stacks/ipfs_config.go b/internal/stacks/ipfs_config.go index e05f854..1777d65 100644 --- a/internal/stacks/ipfs_config.go +++ b/internal/stacks/ipfs_config.go @@ -34,6 +34,14 @@ func GenerateSwarmKey() (string, error) { // disables Kubo's AutoConf/public-network defaults, which are incompatible // with a private swarm (swarm.key / LIBP2P_FORCE_PNET) and otherwise prevent // the daemon from starting or from ever peering with other members. +// +// Routing.Type is set to "none" rather than "dht": a small private swarm hits +// a long-standing upstream bug (https://github.com/ipfs/kubo/issues/8346) +// where bitswap finds a provider via the DHT but never sends it a WANT +// message if the swarm connection to that peer already existed, so content +// fetches hang even though the peers are connected. With routing disabled, +// bitswap has no choice but to broadcast wants directly to its connected +// peers, which is all a 2+ node private swarm actually needs. func GenerateIPFSPrivateNetInitScript() string { return `#!/bin/sh ipfs config --json AutoConf.Enabled false @@ -41,7 +49,7 @@ ipfs config --json Bootstrap '[]' ipfs config --json DNS.Resolvers '{}' ipfs config --json Routing.DelegatedRouters '[]' ipfs config --json Ipns.DelegatedPublishers '[]' -ipfs config Routing.Type dht +ipfs config Routing.Type none ipfs config --json AutoTLS.Enabled false ipfs config --json Swarm.Transports.Network.Websocket false ` diff --git a/internal/stacks/stack_manager.go b/internal/stacks/stack_manager.go index d95fd8e..5c55c46 100644 --- a/internal/stacks/stack_manager.go +++ b/internal/stacks/stack_manager.go @@ -22,6 +22,7 @@ import ( "fmt" "net" "net/http" + "net/url" "os" "os/exec" "path" @@ -600,6 +601,8 @@ func (s *StackManager) peerIPFSNodes() error { return nil } + s.Log.Info("peering IPFS nodes") + type ipfsIDResponse struct { ID string `json:"ID"` } @@ -614,30 +617,55 @@ func (s *StackManager) peerIPFSNodes() error { peerIDs[member.ID] = idResp.ID } + type peeringPeer struct { + ID string `json:"ID"` + Addrs []string `json:"Addrs"` + } + for _, member := range s.Stack.Members { + peers := []peeringPeer{} for _, other := range s.Stack.Members { if member.ID == other.ID { continue } - addr := fmt.Sprintf("/dns4/ipfs_%s/tcp/4001/p2p/%s", other.ID, peerIDs[other.ID]) + peers = append(peers, peeringPeer{ + ID: peerIDs[other.ID], + Addrs: []string{fmt.Sprintf("/dns4/ipfs_%s/tcp/4001", other.ID)}, + }) + } + peersJSON, err := json.Marshal(peers) + if err != nil { + return err + } - // swarm/peering/add only registers the peer with Kubo's background - // reconnect service - it returns success even when the peer is - // unreachable, so it does not confirm a connection was made. - peeringURL := fmt.Sprintf("http://127.0.0.1:%d/api/v0/swarm/peering/add?arg=%s", member.ExposedIPFSApiPort, addr) - if err := core.RequestWithRetry(s.ctx, http.MethodPost, peeringURL, nil, nil); err != nil { - return fmt.Errorf("failed to peer IPFS node %s with %s: %w", member.ID, other.ID, err) - } + // swarm/peering/add only registers the peer with the *running* + // daemon's in-memory peering service and returns success even when + // the peer is unreachable - it does not persist to Peering.Peers in + // the on-disk config, so the pairing is lost on the next restart. + // Writing through /api/v0/config persists it for real, and Kubo + // picks up the change live (no restart needed). + configURL := fmt.Sprintf("http://127.0.0.1:%d/api/v0/config?arg=Peering.Peers&arg=%s&json=true", member.ExposedIPFSApiPort, url.QueryEscape(string(peersJSON))) + if err := core.RequestWithRetry(s.ctx, http.MethodPost, configURL, nil, nil); err != nil { + return fmt.Errorf("failed to persist IPFS peering config for member %s: %w", member.ID, err) + } + for _, other := range s.Stack.Members { + if member.ID == other.ID { + continue + } + addr := fmt.Sprintf("/dns4/ipfs_%s/tcp/4001/p2p/%s", other.ID, peerIDs[other.ID]) // swarm/connect dials synchronously and errors if the connection // fails, so retrying it confirms the nodes are actually connected - // rather than just registered to reconnect in the background. + // now rather than waiting on the peering service's own timing. connectURL := fmt.Sprintf("http://127.0.0.1:%d/api/v0/swarm/connect?arg=%s", member.ExposedIPFSApiPort, addr) if err := core.RequestWithRetry(s.ctx, http.MethodPost, connectURL, nil, nil); err != nil { return fmt.Errorf("failed to connect IPFS node %s to %s: %w", member.ID, other.ID, err) } + s.Log.Info(fmt.Sprintf("IPFS node %s peered with %s (%s)", member.ID, other.ID, peerIDs[other.ID])) } } + + s.Log.Info("IPFS nodes peered successfully") return nil }