diff --git a/README.md b/README.md index fd904096..b5b38214 100644 --- a/README.md +++ b/README.md @@ -482,6 +482,18 @@ A single controller instance can configure all server instances in your cluster. [Configuration example](examples/distributed.yaml#L21) +### Swarm + +Controller-only mode that renders a full Caddyfile and rolls it out to an existing Swarm service as an immutable Swarm **config**. + +This mode doesn't use (or expose) Caddy's Admin API. Instead, it updates the target service to mount the generated Caddyfile at `/etc/caddy/Caddyfile` (or `--swarm-caddyfile-target`). This triggers a Swarm service update (tasks get restarted according to the service's update/rollback settings). + +Each configuration change creates a new Swarm config object named `-` (see `--swarm-config-prefix`). Old configs are not garbage-collected automatically. + +This mode requires access to a Swarm **manager** Docker API. + +[Configuration example](examples/swarm.yaml#L1) + ### Standalone (default) This mode executes a controller and a server in the same instance and doesn't require additional configuration. @@ -496,7 +508,7 @@ Run `caddy help docker-proxy` to see the raw flag output. | CLI flag | Env var | Description | |---|---|---| -| `--mode` | `CADDY_DOCKER_MODE` | Which mode to run: `standalone` \| `controller` \| `server`.
**Default:** `standalone` | +| `--mode` | `CADDY_DOCKER_MODE` | Which mode to run: `standalone` \| `controller` \| `server` \| `swarm`.
**Default:** `standalone` | | `--docker-sockets` | `CADDY_DOCKER_SOCKETS` | Comma-separated Docker sockets.
**Default:** `DOCKER_HOST` or the default socket | | `--docker-certs-path` | `CADDY_DOCKER_CERTS_PATH` | Comma-separated cert paths (one per socket; leave entry empty for sockets without certs) | | `--docker-apis-version` | `CADDY_DOCKER_APIS_VERSION` | Comma-separated API versions (one per socket) | @@ -510,6 +522,10 @@ Run `caddy help docker-proxy` to see the raw flag output. | `--scan-stopped-containers` | `CADDY_DOCKER_SCAN_STOPPED_CONTAINERS` | Scan stopped containers and use their labels.
**Default:** `false` | | `--polling-interval` | `CADDY_DOCKER_POLLING_INTERVAL` | Interval to manually check Docker for a new Caddyfile.
**Default:** `30s` | | `--event-throttle-interval` | `CADDY_DOCKER_EVENT_THROTTLE_INTERVAL` | Interval to throttle Caddyfile updates triggered by Docker events.
**Default:** `100ms` | +| `--swarm-service` | `CADDY_DOCKER_SWARM_SERVICE` | Existing Swarm service name/ID to update when running in `swarm` mode | +| `--swarm-caddyfile-target` | `CADDY_DOCKER_SWARM_CADDYFILE_TARGET` | Target path inside the Swarm service task to mount the generated Caddyfile.
**Default:** `/etc/caddy/Caddyfile` | +| `--swarm-config-prefix` | `CADDY_DOCKER_SWARM_CONFIG_PREFIX` | Prefix for generated Swarm config objects.
**Default:** `caddyfile` | +| `--swarm-config-hash-len` | `CADDY_DOCKER_SWARM_CONFIG_HASH_LEN` | Length of sha256 hex used in generated Swarm config names.
**Default:** `32` | | _(env only)_ | `CADDY_ADMIN` | Override Caddy's admin listen address | | _(env only)_ | `CADDY_DOCKER_NO_SCOPE` | Disable Docker event scope filter (useful for Podman).
**Default:** `false` | diff --git a/cmd.go b/cmd.go index cf99aeb0..c056f445 100644 --- a/cmd.go +++ b/cmd.go @@ -2,9 +2,11 @@ package caddydockerproxy import ( "flag" + "fmt" "net" "os" "regexp" + "strconv" "strings" "time" @@ -28,7 +30,7 @@ func init() { fs := flag.NewFlagSet("docker-proxy", flag.ExitOnError) fs.String("mode", "standalone", - "Which mode this instance should run: standalone | controller | server") + "Which mode this instance should run: standalone | controller | server | swarm") fs.String("docker-sockets", "", "Docker sockets comma separate") @@ -70,6 +72,18 @@ func init() { fs.Duration("event-throttle-interval", 100*time.Millisecond, "Interval to throttle caddyfile updates triggered by docker events") + fs.String("swarm-service", "", + "Existing Swarm service name/ID to update (swarm mode only)") + + fs.String("swarm-caddyfile-target", "/etc/caddy/Caddyfile", + "Target path inside the Swarm service task to mount the generated Caddyfile (swarm mode only)") + + fs.String("swarm-config-prefix", "caddyfile", + "Prefix for generated Swarm config objects (swarm mode only)") + + fs.Int("swarm-config-hash-len", 32, + "Length of sha256 hex used in generated Swarm config name (swarm mode only)") + return fs }(), }) @@ -81,6 +95,12 @@ func cmdFunc(flags caddycmd.Flags) (int, error) { options := createOptions(flags) log := logger() + if options.SwarmMode { + if strings.TrimSpace(options.SwarmService) == "" { + return 1, fmt.Errorf("swarm mode requires --swarm-service (or CADDY_DOCKER_SWARM_SERVICE)") + } + } + if options.Mode&config.Server == config.Server { log.Info("Running caddy proxy server") @@ -172,6 +192,10 @@ func createOptions(flags caddycmd.Flags) *config.Options { dockerCertsPathFlag := flags.String("docker-certs-path") dockerAPIsVersionFlag := flags.String("docker-apis-version") ingressNetworksFlag := flags.String("ingress-networks") + swarmServiceFlag := flags.String("swarm-service") + swarmCaddyfileTargetFlag := flags.String("swarm-caddyfile-target") + swarmConfigPrefixFlag := flags.String("swarm-config-prefix") + swarmConfigHashLenFlag := flags.Int("swarm-config-hash-len") options := &config.Options{} @@ -186,6 +210,9 @@ func createOptions(flags caddycmd.Flags) *config.Options { options.Mode = config.Controller case "server": options.Mode = config.Server + case "swarm": + options.Mode = config.Controller + options.SwarmMode = true default: options.Mode = config.Standalone } @@ -297,5 +324,32 @@ func createOptions(flags caddycmd.Flags) *config.Options { options.EventThrottleInterval = eventThrottleIntervalFlag } + if swarmServiceEnv := os.Getenv("CADDY_DOCKER_SWARM_SERVICE"); swarmServiceEnv != "" { + options.SwarmService = swarmServiceEnv + } else { + options.SwarmService = swarmServiceFlag + } + + if swarmCaddyfileTargetEnv := os.Getenv("CADDY_DOCKER_SWARM_CADDYFILE_TARGET"); swarmCaddyfileTargetEnv != "" { + options.SwarmCaddyfileTarget = swarmCaddyfileTargetEnv + } else { + options.SwarmCaddyfileTarget = swarmCaddyfileTargetFlag + } + + if swarmConfigPrefixEnv := os.Getenv("CADDY_DOCKER_SWARM_CONFIG_PREFIX"); swarmConfigPrefixEnv != "" { + options.SwarmConfigPrefix = swarmConfigPrefixEnv + } else { + options.SwarmConfigPrefix = swarmConfigPrefixFlag + } + + options.SwarmConfigHashLen = swarmConfigHashLenFlag + if swarmConfigHashLenEnv := os.Getenv("CADDY_DOCKER_SWARM_CONFIG_HASH_LEN"); swarmConfigHashLenEnv != "" { + if v, err := strconv.Atoi(swarmConfigHashLenEnv); err != nil { + log.Error("Failed to parse CADDY_DOCKER_SWARM_CONFIG_HASH_LEN", zap.String("CADDY_DOCKER_SWARM_CONFIG_HASH_LEN", swarmConfigHashLenEnv), zap.Error(err)) + } else { + options.SwarmConfigHashLen = v + } + } + return options } diff --git a/config/options.go b/config/options.go index d885a71d..8caa799b 100644 --- a/config/options.go +++ b/config/options.go @@ -24,6 +24,15 @@ type Options struct { Secret string ControllerNetwork *net.IPNet IngressNetworks []string + + // SwarmMode enables atomic Caddyfile distribution via Swarm configs. + // When enabled, this instance updates an existing Swarm service by mounting + // a newly created, content-addressed Swarm config at SwarmCaddyfileTarget. + SwarmMode bool + SwarmService string + SwarmCaddyfileTarget string + SwarmConfigPrefix string + SwarmConfigHashLen int } // Mode represents how this instance should run diff --git a/docker/client.go b/docker/client.go index 3b3e92dd..16d5030a 100644 --- a/docker/client.go +++ b/docker/client.go @@ -16,6 +16,8 @@ import ( type Client interface { ContainerList(ctx context.Context, options container.ListOptions) ([]types.Container, error) ServiceList(ctx context.Context, options types.ServiceListOptions) ([]swarm.Service, error) + ServiceInspectWithRaw(ctx context.Context, serviceID string, opts swarm.ServiceInspectOptions) (swarm.Service, []byte, error) + ServiceUpdate(ctx context.Context, serviceID string, version swarm.Version, service swarm.ServiceSpec, options swarm.ServiceUpdateOptions) (swarm.ServiceUpdateResponse, error) TaskList(ctx context.Context, options types.TaskListOptions) ([]swarm.Task, error) Info(ctx context.Context) (system.Info, error) ContainerInspect(ctx context.Context, containerID string) (types.ContainerJSON, error) @@ -23,6 +25,7 @@ type Client interface { NetworkList(ctx context.Context, options network.ListOptions) ([]network.Summary, error) ConfigList(ctx context.Context, options types.ConfigListOptions) ([]swarm.Config, error) ConfigInspectWithRaw(ctx context.Context, id string) (swarm.Config, []byte, error) + ConfigCreate(ctx context.Context, config swarm.ConfigSpec) (swarm.ConfigCreateResponse, error) Events(ctx context.Context, options events.ListOptions) (<-chan events.Message, <-chan error) } @@ -45,6 +48,14 @@ func (wrapper *clientWrapper) ServiceList(ctx context.Context, options types.Ser return wrapper.client.ServiceList(ctx, options) } +func (wrapper *clientWrapper) ServiceInspectWithRaw(ctx context.Context, serviceID string, opts swarm.ServiceInspectOptions) (swarm.Service, []byte, error) { + return wrapper.client.ServiceInspectWithRaw(ctx, serviceID, opts) +} + +func (wrapper *clientWrapper) ServiceUpdate(ctx context.Context, serviceID string, version swarm.Version, service swarm.ServiceSpec, options swarm.ServiceUpdateOptions) (swarm.ServiceUpdateResponse, error) { + return wrapper.client.ServiceUpdate(ctx, serviceID, version, service, options) +} + func (wrapper *clientWrapper) TaskList(ctx context.Context, options types.TaskListOptions) ([]swarm.Task, error) { return wrapper.client.TaskList(ctx, options) } @@ -73,6 +84,10 @@ func (wrapper *clientWrapper) ConfigInspectWithRaw(ctx context.Context, id strin return wrapper.client.ConfigInspectWithRaw(ctx, id) } +func (wrapper *clientWrapper) ConfigCreate(ctx context.Context, config swarm.ConfigSpec) (swarm.ConfigCreateResponse, error) { + return wrapper.client.ConfigCreate(ctx, config) +} + func (wrapper *clientWrapper) Events(ctx context.Context, options events.ListOptions) (<-chan events.Message, <-chan error) { return wrapper.client.Events(ctx, options) } diff --git a/docker/client_mock.go b/docker/client_mock.go index 66fc0a6c..3785c864 100644 --- a/docker/client_mock.go +++ b/docker/client_mock.go @@ -2,6 +2,7 @@ package docker import ( "context" + "fmt" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" @@ -35,6 +36,28 @@ func (mock *ClientMock) ServiceList(ctx context.Context, options types.ServiceLi return mock.ServicesData, nil } +// ServiceInspectWithRaw returns information about a specific service +func (mock *ClientMock) ServiceInspectWithRaw(ctx context.Context, serviceID string, opts swarm.ServiceInspectOptions) (swarm.Service, []byte, error) { + for _, service := range mock.ServicesData { + if service.ID == serviceID || service.Spec.Name == serviceID { + return service, nil, nil + } + } + return swarm.Service{}, nil, fmt.Errorf("service not found: %s", serviceID) +} + +// ServiceUpdate updates a specific service +func (mock *ClientMock) ServiceUpdate(ctx context.Context, serviceID string, version swarm.Version, service swarm.ServiceSpec, options swarm.ServiceUpdateOptions) (swarm.ServiceUpdateResponse, error) { + for i := range mock.ServicesData { + if mock.ServicesData[i].ID == serviceID || mock.ServicesData[i].Spec.Name == serviceID { + mock.ServicesData[i].Spec = service + mock.ServicesData[i].Meta.Version.Index++ + return swarm.ServiceUpdateResponse{}, nil + } + } + return swarm.ServiceUpdateResponse{}, fmt.Errorf("service not found: %s", serviceID) +} + // TaskList list all tasks func (mock *ClientMock) TaskList(ctx context.Context, options types.TaskListOptions) ([]swarm.Task, error) { matchingTasks := []swarm.Task{} @@ -85,6 +108,23 @@ func (mock *ClientMock) ConfigInspectWithRaw(ctx context.Context, id string) (sw return swarm.Config{}, nil, nil } +// ConfigCreate creates a new swarm config +func (mock *ClientMock) ConfigCreate(ctx context.Context, config swarm.ConfigSpec) (swarm.ConfigCreateResponse, error) { + id := fmt.Sprintf("config-%d", len(mock.ConfigsData)+1) + mock.ConfigsData = append(mock.ConfigsData, swarm.Config{ + ID: id, + Spec: swarm.ConfigSpec{ + Annotations: swarm.Annotations{ + Name: config.Name, + Labels: config.Labels, + }, + Data: config.Data, + Templating: config.Templating, + }, + }) + return swarm.ConfigCreateResponse{ID: id}, nil +} + // Events listen for events in docker func (mock *ClientMock) Events(ctx context.Context, options events.ListOptions) (<-chan events.Message, <-chan error) { return mock.EventsChannel, mock.ErrorsChannel diff --git a/examples/swarm.yaml b/examples/swarm.yaml new file mode 100644 index 00000000..e8febb66 --- /dev/null +++ b/examples/swarm.yaml @@ -0,0 +1,65 @@ +version: '3.7' + +configs: + caddy_bootstrap: + file: ./Caddyfile + +services: + # Caddy runs on workers and does NOT need the Docker socket. + caddy: + # Vanilla Caddy image; no docker-proxy plugin required on workers. + image: caddy:2-alpine + ports: + - 80:80 + - 443:443 + configs: + - source: caddy_bootstrap + target: /etc/caddy/Caddyfile + mode: 0444 + volumes: + # Persist certificates and other Caddy state + - caddy_data:/data + deploy: + mode: global + placement: + constraints: + - node.role == worker + update_config: + parallelism: 1 + delay: 5s + monitor: 10s + failure_action: rollback + order: stop-first + rollback_config: + parallelism: 1 + delay: 5s + monitor: 10s + order: stop-first + + # Controller runs on a manager, generates a full Caddyfile from labels, + # and rolls it out by updating the existing Swarm service above. + controller: + image: lucaslorentz/caddy-docker-proxy:ci-alpine + environment: + - CADDY_DOCKER_MODE=swarm + # NOTE: Swarm service names are _. + # If you deploy this stack as `docker stack deploy -c examples/swarm.yaml caddy`, + # then the service name will be `caddy_caddy`. + - CADDY_DOCKER_SWARM_SERVICE=caddy_caddy + volumes: + - /var/run/docker.sock:/var/run/docker.sock + deploy: + placement: + constraints: + - node.role == manager + + whoami: + image: traefik/whoami + deploy: + labels: + caddy: whoami.example.com + caddy.reverse_proxy: "{{upstreams 80}}" + caddy.tls: internal + +volumes: + caddy_data: {} diff --git a/loader.go b/loader.go index cb8ecc80..f7a11916 100644 --- a/loader.go +++ b/loader.go @@ -42,14 +42,19 @@ type DockerLoader struct { lastVersion int64 serversVersions *utils.StringInt64CMap serversUpdating *utils.StringBoolCMap + + // swarmServiceClientIndex is the docker client index used to manage the + // configured SwarmService. Only used when options.SwarmMode is enabled. + swarmServiceClientIndex int } // CreateDockerLoader creates a docker loader func CreateDockerLoader(options *config.Options) *DockerLoader { return &DockerLoader{ - options: options, - serversVersions: utils.NewStringInt64CMap(), - serversUpdating: utils.NewStringBoolCMap(), + options: options, + serversVersions: utils.NewStringInt64CMap(), + serversUpdating: utils.NewStringBoolCMap(), + swarmServiceClientIndex: -1, } } @@ -159,6 +164,11 @@ func (dockerLoader *DockerLoader) Start() error { zap.Strings("DockerCertsPath", dockerLoader.options.DockerCertsPath), zap.Strings("DockerAPIsVersion", dockerLoader.options.DockerAPIsVersion), zap.String("CaddyfileAutosavePath", CaddyfileAutosavePath), + zap.Bool("SwarmMode", dockerLoader.options.SwarmMode), + zap.String("SwarmService", dockerLoader.options.SwarmService), + zap.String("SwarmCaddyfileTarget", dockerLoader.options.SwarmCaddyfileTarget), + zap.String("SwarmConfigPrefix", dockerLoader.options.SwarmConfigPrefix), + zap.Int("SwarmConfigHashLen", dockerLoader.options.SwarmConfigHashLen), ) ready := make(chan struct{}) @@ -255,30 +265,40 @@ func (dockerLoader *DockerLoader) update() bool { if caddyfileChanged { log.Info("New Caddyfile", zap.ByteString("caddyfile", caddyfile)) - tmpPath := CaddyfileAutosavePath + ".tmp" - if err := os.WriteFile(tmpPath, caddyfile, 0640); err != nil { - log.Warn("Failed to write temporary caddyfile", zap.Error(err), zap.String("path", tmpPath)) - } else if err := os.Rename(tmpPath, CaddyfileAutosavePath); err != nil { - log.Warn("Failed to autosave caddyfile", zap.Error(err), zap.String("path", CaddyfileAutosavePath)) - } + tmpPath := CaddyfileAutosavePath + ".tmp" + if err := os.WriteFile(tmpPath, caddyfile, 0640); err != nil { + log.Warn("Failed to write temporary caddyfile", zap.Error(err), zap.String("path", tmpPath)) + } else if err := os.Rename(tmpPath, CaddyfileAutosavePath); err != nil { + log.Warn("Failed to autosave caddyfile", zap.Error(err), zap.String("path", CaddyfileAutosavePath)) + } + + if !dockerLoader.options.SwarmMode { + adapter := caddyconfig.GetAdapter("caddyfile") - adapter := caddyconfig.GetAdapter("caddyfile") + configJSON, warn, err := adapter.Adapt(caddyfile, nil) - configJSON, warn, err := adapter.Adapt(caddyfile, nil) + if warn != nil { + log.Warn("Caddyfile to json warning", zap.String("warn", fmt.Sprintf("%v", warn))) + } + + if err != nil { + log.Error("Failed to convert caddyfile into json config", zap.Error(err)) + return false + } - if warn != nil { - log.Warn("Caddyfile to json warning", zap.String("warn", fmt.Sprintf("%v", warn))) + log.Info("New Config JSON", zap.ByteString("json", configJSON)) + + dockerLoader.lastJSONConfig = configJSON + dockerLoader.lastVersion++ } + } - if err != nil { - log.Error("Failed to convert caddyfile into json config", zap.Error(err)) + if dockerLoader.options.SwarmMode { + if err := dockerLoader.updateSwarmService(); err != nil { + log.Error("Failed to update swarm service", zap.Error(err)) return false } - - log.Info("New Config JSON", zap.ByteString("json", configJSON)) - - dockerLoader.lastJSONConfig = configJSON - dockerLoader.lastVersion++ + return true } var wg sync.WaitGroup diff --git a/swarm_mode.go b/swarm_mode.go new file mode 100644 index 00000000..bca2f57d --- /dev/null +++ b/swarm_mode.go @@ -0,0 +1,302 @@ +package caddydockerproxy + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "strings" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/swarm" + "github.com/docker/docker/errdefs" + "github.com/lucaslorentz/caddy-docker-proxy/v2/docker" + + "go.uber.org/zap" +) + +const ( + defaultSwarmCaddyfileTarget = "/etc/caddy/Caddyfile" + defaultSwarmConfigPrefix = "caddyfile" + defaultSwarmConfigHashLen = 32 + maxSwarmConfigSizeBytes = 1000 * 1024 +) + +func (dockerLoader *DockerLoader) updateSwarmService() error { + ctx := context.Background() + log := logger() + + svcName := strings.TrimSpace(dockerLoader.options.SwarmService) + if svcName == "" { + return fmt.Errorf("swarm service is not set") + } + + targetPath := strings.TrimSpace(dockerLoader.options.SwarmCaddyfileTarget) + if targetPath == "" { + targetPath = defaultSwarmCaddyfileTarget + } + + prefix := strings.TrimSpace(dockerLoader.options.SwarmConfigPrefix) + if prefix == "" { + prefix = defaultSwarmConfigPrefix + } + + hashLen := dockerLoader.options.SwarmConfigHashLen + if hashLen == 0 { + hashLen = defaultSwarmConfigHashLen + } + if hashLen < 8 || hashLen > 64 { + return fmt.Errorf("swarm-config-hash-len must be between 8 and 64 (got %d)", hashLen) + } + + caddyfile := dockerLoader.lastCaddyfile + if len(caddyfile) > maxSwarmConfigSizeBytes { + return fmt.Errorf("generated caddyfile is too large for Swarm config (%d bytes > %d bytes)", len(caddyfile), maxSwarmConfigSizeBytes) + } + + sum := sha256.Sum256(caddyfile) + fullHash := hex.EncodeToString(sum[:]) + configName := fmt.Sprintf("%s-%s", prefix, fullHash[:hashLen]) + + dockerClient, svc, err := dockerLoader.inspectSwarmService(ctx, svcName) + if err != nil { + return err + } + + configID, created, err := ensureSwarmConfig(ctx, dockerClient, configName, caddyfile, fullHash) + if err != nil { + return err + } + if created { + log.Info("Swarm config created", zap.String("name", configName), zap.String("id", configID), zap.String("service", svcName)) + } + + updated, err := dockerLoader.ensureServiceCaddyfileConfig(ctx, dockerClient, svcName, svc, configID, configName, targetPath) + if err != nil { + return err + } + if updated { + log.Info("Swarm service updated", zap.String("service", svc.Spec.Name), zap.String("config", configName), zap.String("target", targetPath)) + } + + return nil +} + +func (dockerLoader *DockerLoader) inspectSwarmService(ctx context.Context, service string) (docker.Client, swarm.Service, error) { + log := logger() + + // Try cached client first + if idx := dockerLoader.swarmServiceClientIndex; idx >= 0 && idx < len(dockerLoader.dockerClients) { + client := dockerLoader.dockerClients[idx] + if svc, ok := tryInspectSwarmService(ctx, client, service, log); ok { + return client, svc, nil + } + + // Invalidate cache and try all clients + dockerLoader.swarmServiceClientIndex = -1 + } + + for i, client := range dockerLoader.dockerClients { + if svc, ok := tryInspectSwarmService(ctx, client, service, log); ok { + dockerLoader.swarmServiceClientIndex = i + return client, svc, nil + } + } + + return nil, swarm.Service{}, fmt.Errorf("failed to inspect swarm service %q on any configured docker socket", service) +} + +func tryInspectSwarmService(ctx context.Context, client docker.Client, service string, log *zap.Logger) (swarm.Service, bool) { + info, err := client.Info(ctx) + if err != nil { + log.Debug("Swarm info check failed", zap.Error(err)) + return swarm.Service{}, false + } + if !info.Swarm.ControlAvailable { + return swarm.Service{}, false + } + + svc, _, err := client.ServiceInspectWithRaw(ctx, service, swarm.ServiceInspectOptions{}) + if err != nil { + return swarm.Service{}, false + } + + return svc, true +} + +func ensureSwarmConfig(ctx context.Context, dockerClient docker.Client, name string, data []byte, fullHash string) (string, bool, error) { + configs, err := dockerClient.ConfigList(ctx, types.ConfigListOptions{}) + if err != nil { + return "", false, err + } + + for _, cfg := range configs { + if cfg.Spec.Name != name { + continue + } + + fullCfg, _, err := dockerClient.ConfigInspectWithRaw(ctx, cfg.ID) + if err != nil { + return "", false, err + } + + sum := sha256.Sum256(fullCfg.Spec.Data) + existingHash := hex.EncodeToString(sum[:]) + if existingHash != fullHash { + return "", false, fmt.Errorf("swarm config name collision for %q (existing hash %s != desired hash %s); increase --swarm-config-hash-len", name, existingHash, fullHash) + } + + return cfg.ID, false, nil + } + + spec := swarm.ConfigSpec{ + Annotations: swarm.Annotations{ + Name: name, + Labels: map[string]string{}, + }, + Data: data, + } + + resp, err := dockerClient.ConfigCreate(ctx, spec) + if err != nil { + // If another instance created it concurrently, inspect again. + if errdefs.IsConflict(err) { + return ensureSwarmConfig(ctx, dockerClient, name, data, fullHash) + } + return "", false, err + } + + return resp.ID, true, nil +} + +func (dockerLoader *DockerLoader) ensureServiceCaddyfileConfig( + ctx context.Context, + dockerClient docker.Client, + svcName string, + svc swarm.Service, + configID string, + configName string, + targetPath string, +) (bool, error) { + if strings.TrimSpace(targetPath) == "" { + return false, fmt.Errorf("swarm caddyfile target is empty") + } + + const maxAttempts = 5 + var lastErr error + + for attempt := 1; attempt <= maxAttempts; attempt++ { + // Always inspect latest state before updating. + currentSvc, _, err := dockerClient.ServiceInspectWithRaw(ctx, svcName, swarm.ServiceInspectOptions{}) + if err != nil { + lastErr = err + break + } + + updated, err := updateServiceSpecConfigTarget(¤tSvc.Spec, targetPath, configID, configName) + if err != nil { + return false, err + } + if !updated { + return false, nil + } + + _, err = dockerClient.ServiceUpdate(ctx, currentSvc.ID, currentSvc.Version, currentSvc.Spec, swarm.ServiceUpdateOptions{}) + if err == nil { + return true, nil + } + + lastErr = err + if errdefs.IsConflict(err) { + time.Sleep(time.Duration(attempt*attempt) * 200 * time.Millisecond) + continue + } + return false, err + } + + if lastErr != nil { + return false, lastErr + } + return false, fmt.Errorf("failed to update swarm service") +} + +func updateServiceSpecConfigTarget(spec *swarm.ServiceSpec, targetPath, configID, configName string) (bool, error) { + if spec == nil { + return false, fmt.Errorf("service spec is nil") + } + if spec.TaskTemplate.ContainerSpec == nil { + return false, fmt.Errorf("service has no container spec") + } + + cs := spec.TaskTemplate.ContainerSpec + + for _, s := range cs.Secrets { + if s == nil || s.File == nil { + continue + } + if s.File.Name == targetPath { + return false, fmt.Errorf("service mounts a secret at %q; configs-only swarm mode cannot manage this path", targetPath) + } + } + + uid := "0" + gid := "0" + mode := os.FileMode(0444) + seenTarget := 0 + seenDesired := false + seenOther := false + + for _, cfg := range cs.Configs { + if cfg == nil || cfg.File == nil { + continue + } + if cfg.File.Name != targetPath { + continue + } + + seenTarget++ + if cfg.File.UID != "" { + uid = cfg.File.UID + } + if cfg.File.GID != "" { + gid = cfg.File.GID + } + if cfg.File.Mode != 0 { + mode = cfg.File.Mode + } + + if cfg.ConfigID == configID || cfg.ConfigName == configName { + seenDesired = true + } else { + seenOther = true + } + } + + if seenTarget == 1 && seenDesired && !seenOther { + return false, nil + } + + newConfigs := make([]*swarm.ConfigReference, 0, len(cs.Configs)+1) + for _, cfg := range cs.Configs { + if cfg == nil || cfg.File == nil || cfg.File.Name != targetPath { + newConfigs = append(newConfigs, cfg) + } + } + + newConfigs = append(newConfigs, &swarm.ConfigReference{ + ConfigID: configID, + ConfigName: configName, + File: &swarm.ConfigReferenceFileTarget{ + Name: targetPath, + UID: uid, + GID: gid, + Mode: mode, + }, + }) + + cs.Configs = newConfigs + spec.TaskTemplate.ContainerSpec = cs + return true, nil +} diff --git a/tests/swarm-mode/compose.yaml b/tests/swarm-mode/compose.yaml new file mode 100644 index 00000000..cea57652 --- /dev/null +++ b/tests/swarm-mode/compose.yaml @@ -0,0 +1,38 @@ +version: "3.7" + +services: + caddy: + image: caddy:2-alpine + ports: + - 80:80 + - 443:443 + networks: + - caddy + + controller: + image: caddy-docker-proxy:local + networks: + - caddy + environment: + - CADDY_DOCKER_MODE=swarm + - CADDY_DOCKER_SWARM_SERVICE=caddy_test_caddy + - CADDY_DOCKER_SWARM_CONFIG_PREFIX=cdp-test-caddy_test + volumes: + - source: "${DOCKER_SOCKET_PATH}" + target: "${DOCKER_SOCKET_PATH}" + type: ${DOCKER_SOCKET_TYPE} + + service: + image: traefik/whoami + networks: + - caddy + deploy: + labels: + caddy: service.local + caddy.reverse_proxy: "{{upstreams 80}}" + caddy.tls: "internal" + +networks: + caddy: + name: caddy_test + external: true diff --git a/tests/swarm-mode/run.sh b/tests/swarm-mode/run.sh new file mode 100644 index 00000000..eb5112ff --- /dev/null +++ b/tests/swarm-mode/run.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -e + +. ../functions.sh + +CONFIG_PREFIX="cdp-test-caddy_test" + +cleanup_configs() { + docker config ls --format "{{.Name}}" | grep "^${CONFIG_PREFIX}-" | xargs -r docker config rm >/dev/null || true +} + +cleanup_configs + +docker stack deploy -c compose.yaml --prune caddy_test + +retry bash -c "docker config ls --format '{{.Name}}' | grep '^${CONFIG_PREFIX}-'" + +retry curl --show-error -s -k -f --resolve service.local:443:127.0.0.1 https://service.local || +{ + echo "== Service errors ==" + docker service ps --no-trunc caddy_test_caddy --format "{{.Error}}" + echo "== Caddy logs ==" + docker service logs caddy_test_caddy + echo "== Controller logs ==" + docker service logs caddy_test_controller + exit 1 +}