From 8c3b0a82e0c5a4beda5cc86fd94c94f13b87716a Mon Sep 17 00:00:00 2001 From: Bianco95 Date: Mon, 29 Apr 2024 16:59:43 +0200 Subject: [PATCH 01/30] removed gpu bookkeeping and modified makefile target in order to disable cgo during build process --- Makefile | 2 +- cmd/main.go | 27 +-- pkg/docker/Create.go | 53 ----- pkg/docker/Delete.go | 4 - pkg/docker/Status.go | 10 +- pkg/docker/aux.go | 6 +- pkg/docker/gpustrategies/AmdHandler.go | 1 - pkg/docker/gpustrategies/NvidiaHandler.go | 270 ---------------------- 8 files changed, 9 insertions(+), 364 deletions(-) delete mode 100644 pkg/docker/gpustrategies/AmdHandler.go delete mode 100644 pkg/docker/gpustrategies/NvidiaHandler.go diff --git a/Makefile b/Makefile index 8ee2e34..2c2efb0 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ all: sidecars sidecars: - CGO_ENABLED=1 GOOS=linux go build -o bin/docker-sd cmd/main.go + CGO_ENABLED=0 GOOS=linux go build -o bin/docker-sd cmd/main.go diff --git a/cmd/main.go b/cmd/main.go index 6b110ca..3996557 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -10,7 +10,6 @@ import ( commonIL "github.com/intertwin-eu/interlink-docker-plugin/pkg/common" docker "github.com/intertwin-eu/interlink-docker-plugin/pkg/docker" - "github.com/intertwin-eu/interlink-docker-plugin/pkg/docker/gpustrategies" ) func main() { @@ -33,31 +32,9 @@ func main() { defer cancel() log.G(Ctx).Debug("Debug level: " + strconv.FormatBool(interLinkConfig.VerboseLogging)) - var gpuManager gpustrategies.GPUManagerInterface - gpuManager = &gpustrategies.GPUManager{ - GPUSpecsList: []gpustrategies.GPUSpecs{}, - Ctx: Ctx, - } - - err = gpuManager.Init() - if err != nil { - log.G(Ctx).Fatal(err) - } - - err = gpuManager.Discover() - if err != nil { - log.G(Ctx).Fatal(err) - } - - err = gpuManager.Check() - if err != nil { - log.G(Ctx).Fatal(err) - } - SidecarAPIs := docker.SidecarHandler{ - Config: interLinkConfig, - Ctx: Ctx, - GpuManager: gpuManager, + Config: interLinkConfig, + Ctx: Ctx, } mutex := http.NewServeMux() diff --git a/pkg/docker/Create.go b/pkg/docker/Create.go index 4b9f284..0fd6539 100644 --- a/pkg/docker/Create.go +++ b/pkg/docker/Create.go @@ -66,55 +66,6 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { containerName := podNamespace + "-" + podUID + "-" + container.Name - var isGpuRequested bool = false - var additionalGpuArgs []string - - if val, ok := container.Resources.Limits["nvidia.com/gpu"]; ok { - - numGpusRequested := val.Value() - - log.G(h.Ctx).Infof("Number of GPU requested: %d", numGpusRequested) - - // if the container is requesting 0 GPU, skip the GPU assignment - if numGpusRequested == 0 { - log.G(h.Ctx).Info("Container " + containerName + " is not requesting a GPU") - - } else { - - log.G(h.Ctx).Info("Container " + containerName + " is requesting " + val.String() + " GPU") - - isGpuRequested = true - - numGpusRequestedInt := int(numGpusRequested) - _, err := h.GpuManager.GetAvailableGPUs(numGpusRequestedInt) - - if err != nil { - HandleErrorAndRemoveData(h, w, statusCode, "Some errors occurred while creating container. Check Docker Sidecar's logs", err, &data) - return - } - - gpuSpecs, err := h.GpuManager.GetAndAssignAvailableGPUs(numGpusRequestedInt, containerName) - if err != nil { - HandleErrorAndRemoveData(h, w, statusCode, "Some errors occurred while creating container. Check Docker Sidecar's logs", err, &data) - return - } - - var gpuUUIDs string = "" - for _, gpuSpec := range gpuSpecs { - if gpuSpec.UUID == gpuSpecs[len(gpuSpecs)-1].UUID { - gpuUUIDs += strconv.Itoa(gpuSpec.Index) - } else { - gpuUUIDs += strconv.Itoa(gpuSpec.Index) + "," - } - } - - additionalGpuArgs = append(additionalGpuArgs, "--runtime=nvidia -e NVIDIA_VISIBLE_DEVICES="+gpuUUIDs) - } - - } else { - log.G(h.Ctx).Info("Container " + containerName + " is not requesting a GPU") - } - log.G(h.Ctx).Info("-- Preparing environment variables for " + containerName) var envVars string = "" @@ -156,10 +107,6 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { cmd = append(cmd, envVars) - if isGpuRequested { - cmd = append(cmd, additionalGpuArgs...) - } - var additionalPortArgs []string for _, port := range container.Ports { if port.HostPort != 0 { diff --git a/pkg/docker/Delete.go b/pkg/docker/Delete.go index 09feb83..6ff4237 100644 --- a/pkg/docker/Delete.go +++ b/pkg/docker/Delete.go @@ -40,7 +40,6 @@ func (h *SidecarHandler) DeleteHandler(w http.ResponseWriter, r *http.Request) { podUID := string(pod.UID) podNamespace := string(pod.Namespace) - for _, container := range pod.Spec.Containers { containerName := podNamespace + "-" + podUID + "-" + container.Name @@ -90,9 +89,6 @@ func (h *SidecarHandler) DeleteHandler(w http.ResponseWriter, r *http.Request) { } } - // check if the container has GPU devices attacched using the GpuManager and release them - h.GpuManager.Release(containerName) - os.RemoveAll(h.Config.DataRootFolder + pod.Namespace + "-" + string(pod.UID)) } diff --git a/pkg/docker/Status.go b/pkg/docker/Status.go index fd883a5..66582c3 100644 --- a/pkg/docker/Status.go +++ b/pkg/docker/Status.go @@ -46,7 +46,7 @@ func (h *SidecarHandler) StatusHandler(w http.ResponseWriter, r *http.Request) { resp = append(resp, commonIL.PodStatus{PodName: pod.Name, PodUID: podUID, PodNamespace: podNamespace}) for _, container := range pod.Spec.Containers { containerName := podNamespace + "-" + podUID + "-" + container.Name - + log.G(h.Ctx).Debug("- Getting status for container " + containerName) cmd := []string{"ps -af name=^" + containerName + "$ --format \"{{.Status}}\""} @@ -70,15 +70,13 @@ func (h *SidecarHandler) StatusHandler(w http.ResponseWriter, r *http.Request) { if execReturn.Stdout != "" { if containerstatus[0] == "Created" { log.G(h.Ctx).Info("-- Container " + containerName + " is going ready...") - resp[i].Containers = append(resp[i].Containers,v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Waiting: &v1.ContainerStateWaiting{}}, Ready: false}) + resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Waiting: &v1.ContainerStateWaiting{}}, Ready: false}) } else if containerstatus[0] == "Up" { log.G(h.Ctx).Info("-- Container " + containerName + " is running") - resp[i].Containers = append(resp[i].Containers,v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Running: &v1.ContainerStateRunning{}}, Ready: true}) + resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Running: &v1.ContainerStateRunning{}}, Ready: true}) } else if containerstatus[0] == "Exited" { log.G(h.Ctx).Info("-- Container " + containerName + " has been stopped") - resp[i].Containers = append(resp[i].Containers,v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Terminated: &v1.ContainerStateTerminated{}}, Ready: false}) - // release all the GPUs from the container - h.GpuManager.Release(containerName) + resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Terminated: &v1.ContainerStateTerminated{}}, Ready: false}) } } else { log.G(h.Ctx).Info("-- Container " + containerName + " doesn't exist") diff --git a/pkg/docker/aux.go b/pkg/docker/aux.go index d3ff871..721ea26 100644 --- a/pkg/docker/aux.go +++ b/pkg/docker/aux.go @@ -14,13 +14,11 @@ import ( "fmt" commonIL "github.com/intertwin-eu/interlink-docker-plugin/pkg/common" - "github.com/intertwin-eu/interlink-docker-plugin/pkg/docker/gpustrategies" ) type SidecarHandler struct { - Config commonIL.InterLinkConfig - Ctx context.Context - GpuManager gpustrategies.GPUManagerInterface + Config commonIL.InterLinkConfig + Ctx context.Context } // prepareMounts iterates along the struct provided in the data parameter and checks for ConfigMaps, Secrets and EmptyDirs to be mounted. diff --git a/pkg/docker/gpustrategies/AmdHandler.go b/pkg/docker/gpustrategies/AmdHandler.go deleted file mode 100644 index 45e09bd..0000000 --- a/pkg/docker/gpustrategies/AmdHandler.go +++ /dev/null @@ -1 +0,0 @@ -package gpustrategies diff --git a/pkg/docker/gpustrategies/NvidiaHandler.go b/pkg/docker/gpustrategies/NvidiaHandler.go deleted file mode 100644 index 5c454f0..0000000 --- a/pkg/docker/gpustrategies/NvidiaHandler.go +++ /dev/null @@ -1,270 +0,0 @@ -package gpustrategies - -import ( - "context" - "encoding/json" - "fmt" - "io/ioutil" - "strconv" - "strings" - - "github.com/NVIDIA/go-nvml/pkg/nvml" - "github.com/containerd/containerd/log" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/client" - - "sync" -) - -type GPUSpecs struct { - Name string - UUID string - Type string - ContainerID string - Available bool - Index int -} - -type GPUManager struct { - GPUSpecsList []GPUSpecs - GPUSpecsMutex sync.Mutex // Mutex to make GPUSpecsList access atomic - Vendor string - Ctx context.Context -} - -type GPUManagerInterface interface { - Init() error - Shutdown() error - GetGPUSpecsList() []GPUSpecs - Dump() error - Discover() error - Check() error - GetAvailableGPUs(numGPUs int) ([]GPUSpecs, error) - Assign(UUID string, containerID string) error - Release(UUID string) error - GetAndAssignAvailableGPUs(numGPUs int, containerID string) ([]GPUSpecs, error) -} - -func (a *GPUManager) Init() error { - - ret := nvml.Init() - if ret != nvml.SUCCESS { - return fmt.Errorf("Unable to initialize NVML") - } - - return nil -} - -// Discover implements the Discover function of the GPUManager interface -func (a *GPUManager) Discover() error { - - log.G(a.Ctx).Info("Discovering GPUs...") - - count, ret := nvml.DeviceGetCount() - if ret != nvml.SUCCESS { - return fmt.Errorf("Unable to get device count: %v", nvml.ErrorString(ret)) - } - - for i := 0; i < count; i++ { - device, ret := nvml.DeviceGetHandleByIndex(i) - if ret != nvml.SUCCESS { - return fmt.Errorf("Unable to get device at index %d: %v", i, nvml.ErrorString(ret)) - } - - uuid, ret := device.GetUUID() - if ret != nvml.SUCCESS { - return fmt.Errorf("Unable to get uuid of device at index %d: %v", i, nvml.ErrorString(ret)) - } - - name, ret := device.GetName() - if ret != nvml.SUCCESS { - return fmt.Errorf("Unable to get name of device at index %d: %v", i, nvml.ErrorString(ret)) - } - - index, ret := device.GetIndex() - if ret != nvml.SUCCESS { - return fmt.Errorf("Unable to get index of device at index %d: %v", i, nvml.ErrorString(ret)) - } - - // Add the GPU to the GPUSpecsList - a.GPUSpecsList = append(a.GPUSpecsList, GPUSpecs{Name: name, UUID: uuid, Type: "NVIDIA", ContainerID: "", Available: true, Index: index}) - } - - // print the GPUSpecsList if the length is greater than 0 - if len(a.GPUSpecsList) > 0 { - log.G(a.Ctx).Info("Discovered GPUs:") - for _, gpuSpec := range a.GPUSpecsList { - log.G(a.Ctx).Info(fmt.Sprintf("Name: %s, UUID: %s, Type: %s, Available: %t, Index: %d", gpuSpec.Name, gpuSpec.UUID, gpuSpec.Type, gpuSpec.Available, gpuSpec.Index)) - } - } else { - log.G(a.Ctx).Info("No GPUs discovered") - } - - return nil -} - -func (a *GPUManager) Check() error { - - log.G(a.Ctx).Info("Checking the availability of GPUs...") - - cli, err := client.NewEnvClient() - if err != nil { - return fmt.Errorf("unable to create a new Docker client: %v", err) - } - - containers, err := cli.ContainerList(context.Background(), container.ListOptions{All: false}) // With All set to false I get only the running containers, if I set All to true I get all the containers (running and stopped) - if err != nil { - return fmt.Errorf("unable to list containers: %v", err) - } - - for _, container := range containers { - containerInfo, err := cli.ContainerInspect(context.Background(), container.ID) - if err != nil { - return fmt.Errorf("unable to inspect container: %v", err) - } - - for _, env := range containerInfo.Config.Env { - if strings.Contains(env, "NVIDIA_VISIBLE_DEVICES=") { - indexOfEqualSign := strings.Index(env, "=") - gpuIDs := env[indexOfEqualSign+1:] - gpuIDsSplitted := strings.Split(gpuIDs, ",") - - for _, gpuID := range gpuIDsSplitted { - gpuIndex, err := strconv.Atoi(gpuID) - if err != nil { - return fmt.Errorf("unable to convert GPU ID to int: %v", err) - } - for i := range a.GPUSpecsList { - if a.GPUSpecsList[i].Index == gpuIndex { - a.GPUSpecsList[i].ContainerID = containerInfo.ID - a.GPUSpecsList[i].Available = false - } - } - } - } - } - } - - // print the GPUSpecsList that are not available - for _, gpuSpec := range a.GPUSpecsList { - if !gpuSpec.Available { - log.G(a.Ctx).Info(fmt.Sprintf("GPU with UUID %s is not available. It is in use by container %s", gpuSpec.UUID, gpuSpec.ContainerID)) - } else { - log.G(a.Ctx).Info(fmt.Sprintf("GPU with UUID %s is available", gpuSpec.UUID)) - } - } - - return nil -} - -func (a *GPUManager) Shutdown() error { - - log.G(a.Ctx).Info("Shutting down NVML...") - - ret := nvml.Shutdown() - if ret != nvml.SUCCESS { - return fmt.Errorf("Unable to shutdown NVML: %v", nvml.ErrorString(ret)) - } - - return nil -} - -func (a *GPUManager) GetGPUSpecsList() []GPUSpecs { - return a.GPUSpecsList -} - -func (a *GPUManager) Assign(UUID string, containerID string) error { - - for i := range a.GPUSpecsList { - if a.GPUSpecsList[i].UUID == UUID { - - if a.GPUSpecsList[i].Available == false { - return fmt.Errorf("GPU with UUID %s is already in use by container %s", UUID, a.GPUSpecsList[i].ContainerID) - } - - a.GPUSpecsList[i].ContainerID = containerID - a.GPUSpecsList[i].Available = false - break - } - } - return nil - -} - -func (a *GPUManager) Release(containerID string) error { - - log.G(a.Ctx).Info("Releasing GPU from container " + containerID) - - a.GPUSpecsMutex.Lock() - defer a.GPUSpecsMutex.Unlock() - - for i := range a.GPUSpecsList { - if a.GPUSpecsList[i].ContainerID == containerID { - - if a.GPUSpecsList[i].Available == true { - continue - } - - a.GPUSpecsList[i].ContainerID = "" - a.GPUSpecsList[i].Available = true - } - } - - log.G(a.Ctx).Info("Correctly released GPU from container " + containerID) - - return nil -} - -func (a *GPUManager) GetAvailableGPUs(numGPUs int) ([]GPUSpecs, error) { - - var availableGPUs []GPUSpecs - for _, gpuSpec := range a.GPUSpecsList { - if gpuSpec.Available == true { - availableGPUs = append(availableGPUs, gpuSpec) - if len(availableGPUs) == numGPUs { - return availableGPUs, nil - } - } - } - return nil, fmt.Errorf("Not enough available GPUs. Requested: %d, Available: %d", numGPUs, len(availableGPUs)) -} - -func (a *GPUManager) GetAndAssignAvailableGPUs(numGPUs int, containerID string) ([]GPUSpecs, error) { - - a.GPUSpecsMutex.Lock() - defer a.GPUSpecsMutex.Unlock() - - gpuSpecs, err := a.GetAvailableGPUs(numGPUs) - if err != nil { - return nil, err - } - - for _, gpuSpec := range gpuSpecs { - err = a.Assign(gpuSpec.UUID, containerID) - if err != nil { - return nil, err - } - } - - return gpuSpecs, nil -} - -// dump the GPUSpecsList into a JSON file -func (a *GPUManager) Dump() error { - - log.G(a.Ctx).Info("Dumping the GPUSpecsList into a JSON file...") - - // Convert the array to JSON format - jsonData, err := json.MarshalIndent(a.GPUSpecsList, "", " ") - if err != nil { - return fmt.Errorf("Error marshalling JSON: %v", err) - } - - // Write JSON data to a file - err = ioutil.WriteFile("gpu_specs.json", jsonData, 0644) - if err != nil { - return fmt.Errorf("Error writing to file: %v", err) - } - - return nil -} From 811f61ad15f8f8313ca9ba837824495db58899af Mon Sep 17 00:00:00 2001 From: Bianco95 Date: Mon, 29 Apr 2024 17:02:03 +0200 Subject: [PATCH 02/30] update go.mod and go.sum --- go.mod | 1 - go.sum | 2 -- 2 files changed, 3 deletions(-) diff --git a/go.mod b/go.mod index b5516fb..33e0a73 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.21 toolchain go1.21.3 require ( - github.com/NVIDIA/go-nvml v0.12.0-4 github.com/alexellis/go-execute v0.6.0 github.com/containerd/containerd v1.7.15 github.com/docker/docker v26.0.1+incompatible diff --git a/go.sum b/go.sum index dba9928..212f071 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,6 @@ github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOEl github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/NVIDIA/go-nvml v0.12.0-4 h1:BvPjnjJr6qje0zov57Md7TwEA8i/12kZeUQIpyWzTEE= -github.com/NVIDIA/go-nvml v0.12.0-4/go.mod h1:8Llmj+1Rr+9VGGwZuRer5N/aCjxGuR5nPb/9ebBiIEQ= github.com/alexellis/go-execute v0.6.0 h1:FVGoudJnWSObwf9qmehbvVuvhK6g1UpKOCBjS+OUXEA= github.com/alexellis/go-execute v0.6.0/go.mod h1:nlg2F6XdYydUm1xXQMMiuibQCV1mveybBkNWfdNznjk= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= From d02e37fe2a5af21923d0a2eb28150dcf5f817d9f Mon Sep 17 00:00:00 2001 From: Diego Ciangottini Date: Tue, 30 Apr 2024 09:31:23 +0200 Subject: [PATCH 03/30] cgo disabled in go releaser --- .goreleaser.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index f2018a4..c4ae02c 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -8,7 +8,7 @@ builds: - id: "docker-plugin" binary: docker-plugin env: - - CGO_ENABLED=1 + - CGO_ENABLED=0 goos: - linux #- darwin From fecb037c87dfeb8fc9ec1e1267da0f260bd03584 Mon Sep 17 00:00:00 2001 From: Bianco95 Date: Tue, 14 May 2024 16:23:18 +0200 Subject: [PATCH 04/30] updated no gpu light version --- pkg/docker/Status.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/docker/Status.go b/pkg/docker/Status.go index c93fd8a..e929849 100644 --- a/pkg/docker/Status.go +++ b/pkg/docker/Status.go @@ -85,8 +85,6 @@ func (h *SidecarHandler) StatusHandler(w http.ResponseWriter, r *http.Request) { } log.G(h.Ctx).Info("-- Container exit code is: " + strconv.Itoa(exitCode)) resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Terminated: &v1.ContainerStateTerminated{ExitCode: int32(exitCode)}}, Ready: false}) - // release all the GPUs from the container - h.GpuManager.Release(containerName) } } else { log.G(h.Ctx).Info("-- Container " + containerName + " doesn't exist") From 9a4af3544b0c87b94461cbd2b903ccb236f1daaf Mon Sep 17 00:00:00 2001 From: Diego Ciangottini Date: Thu, 30 May 2024 09:37:04 +0200 Subject: [PATCH 05/30] Update Dockerfile.sidecar-docker --- docker/Dockerfile.sidecar-docker | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/Dockerfile.sidecar-docker b/docker/Dockerfile.sidecar-docker index 4ccafeb..10ccff1 100644 --- a/docker/Dockerfile.sidecar-docker +++ b/docker/Dockerfile.sidecar-docker @@ -23,6 +23,7 @@ ENV PATH "$PATH:/bin" #creating a simple startup script to start both docker rootless and the sidecar RUN echo -e '#!/bin/bash\ndockerd-entrypoint.sh & /sidecar/docker-sidecar' > /sidecar/startup-docker.sh RUN chmod +x /sidecar/startup-docker.sh +RUN chmod +x /sidecar/docker-sidecar RUN chmod -R 777 /sidecar ENV INTERLINKCONFIGPATH=/InterLinkConfig.yaml From 039a6eec24eb9519d4c47d6d53c31d5663511700 Mon Sep 17 00:00:00 2001 From: Diego Ciangottini Date: Thu, 30 May 2024 09:51:41 +0200 Subject: [PATCH 06/30] Update Dockerfile.sidecar-docker --- docker/Dockerfile.sidecar-docker | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docker/Dockerfile.sidecar-docker b/docker/Dockerfile.sidecar-docker index 10ccff1..fc013dc 100644 --- a/docker/Dockerfile.sidecar-docker +++ b/docker/Dockerfile.sidecar-docker @@ -2,7 +2,7 @@ FROM golang:1.21 as build-stage WORKDIR /app COPY .. . -RUN CGO_ENABLED=1 GOOS=linux go build -o bin/docker-sidecar cmd/main.go +RUN CGO_ENABLED=0 GOOS=linux go build -o bin/docker-sidecar cmd/main.go FROM bash:latest as bash-stage @@ -11,12 +11,19 @@ FROM docker:24.0-dind-rootless AS build-release-stage WORKDIR / +USER root:root + +RUN mkdir -p /sidecar && chown -R 1000:1000 /sidecar + + +USER 1000:1000 + COPY --from=build-stage /app/bin/docker-sidecar /sidecar/docker-sidecar # adding bash binary to be able to perform commands within the sidecar binary COPY --from=bash-stage /usr/local/bin/bash /bin -USER root:root +ENV INTERLINKCONFIGPATH=/InterLinkConfig.yaml ENV PATH "$PATH:/bin" @@ -24,11 +31,6 @@ ENV PATH "$PATH:/bin" RUN echo -e '#!/bin/bash\ndockerd-entrypoint.sh & /sidecar/docker-sidecar' > /sidecar/startup-docker.sh RUN chmod +x /sidecar/startup-docker.sh RUN chmod +x /sidecar/docker-sidecar -RUN chmod -R 777 /sidecar - -ENV INTERLINKCONFIGPATH=/InterLinkConfig.yaml - -USER 1000:1000 #setting up the path for the docker daemon ENV DOCKER_HOST=unix:///run/user/1000/docker.sock From ca441710166cc1dc1d4ace1a462598c076f29c86 Mon Sep 17 00:00:00 2001 From: Diego Ciangottini Date: Thu, 30 May 2024 10:06:28 +0200 Subject: [PATCH 07/30] Update Dockerfile.sidecar-docker --- docker/Dockerfile.sidecar-docker | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/Dockerfile.sidecar-docker b/docker/Dockerfile.sidecar-docker index fc013dc..414e732 100644 --- a/docker/Dockerfile.sidecar-docker +++ b/docker/Dockerfile.sidecar-docker @@ -30,7 +30,6 @@ ENV PATH "$PATH:/bin" #creating a simple startup script to start both docker rootless and the sidecar RUN echo -e '#!/bin/bash\ndockerd-entrypoint.sh & /sidecar/docker-sidecar' > /sidecar/startup-docker.sh RUN chmod +x /sidecar/startup-docker.sh -RUN chmod +x /sidecar/docker-sidecar #setting up the path for the docker daemon ENV DOCKER_HOST=unix:///run/user/1000/docker.sock From 7ef0e363e75c7548a23e55ae46e71dffed471b7d Mon Sep 17 00:00:00 2001 From: Bianco95 Date: Fri, 31 May 2024 09:10:04 +0200 Subject: [PATCH 08/30] updated light version branch --- pkg/docker/Create.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/docker/Create.go b/pkg/docker/Create.go index 5069c0b..d1ddb94 100644 --- a/pkg/docker/Create.go +++ b/pkg/docker/Create.go @@ -146,10 +146,6 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { //cmd = append(cmd, "--security-opt=apparmor:unconfined") } - if isGpuRequested { - cmd = append(cmd, additionalGpuArgs...) - } - var additionalPortArgs []string for _, port := range container.Ports { From 76f99e08772f3868a4e52fb4bbf276b1b0171527 Mon Sep 17 00:00:00 2001 From: Diego Ciangottini Date: Fri, 31 May 2024 11:59:48 +0200 Subject: [PATCH 09/30] remove rootless dind use just dind --- docker/Dockerfile.sidecar-docker | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/docker/Dockerfile.sidecar-docker b/docker/Dockerfile.sidecar-docker index 414e732..5f3a5e7 100644 --- a/docker/Dockerfile.sidecar-docker +++ b/docker/Dockerfile.sidecar-docker @@ -7,16 +7,13 @@ RUN CGO_ENABLED=0 GOOS=linux go build -o bin/docker-sidecar cmd/main.go FROM bash:latest as bash-stage # Deploy the application binary into a lean image -FROM docker:24.0-dind-rootless AS build-release-stage +FROM docker:26.1.3-dind AS build-release-stage WORKDIR / USER root:root -RUN mkdir -p /sidecar && chown -R 1000:1000 /sidecar - - -USER 1000:1000 +RUN mkdir -p /sidecar COPY --from=build-stage /app/bin/docker-sidecar /sidecar/docker-sidecar @@ -28,12 +25,9 @@ ENV INTERLINKCONFIGPATH=/InterLinkConfig.yaml ENV PATH "$PATH:/bin" #creating a simple startup script to start both docker rootless and the sidecar -RUN echo -e '#!/bin/bash\ndockerd-entrypoint.sh & /sidecar/docker-sidecar' > /sidecar/startup-docker.sh +RUN echo -e '#!dockerd & /sidecar/docker-sidecar' > /sidecar/startup-docker.sh RUN chmod +x /sidecar/startup-docker.sh -#setting up the path for the docker daemon -ENV DOCKER_HOST=unix:///run/user/1000/docker.sock - WORKDIR /sidecar ENTRYPOINT ["/sidecar/startup-docker.sh"] From b6bfcc9a6c713a134c68aac25c5f85f123836e5d Mon Sep 17 00:00:00 2001 From: Bianco95 Date: Fri, 31 May 2024 12:33:24 +0200 Subject: [PATCH 10/30] updated memory limits by setting ulimits and memlock --- pkg/docker/Create.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/docker/Create.go b/pkg/docker/Create.go index d1ddb94..d6ec62d 100644 --- a/pkg/docker/Create.go +++ b/pkg/docker/Create.go @@ -169,16 +169,19 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { //} memoryLimitsArray := []string{} + uLimitMemoryArray := []string{} cpuLimitsArray := []string{} if container.Resources.Limits.Memory().Value() != 0 { memoryLimitsArray = append(memoryLimitsArray, "--memory", strconv.Itoa(int(container.Resources.Limits.Memory().Value()))+"b") + uLimitMemoryArray = append(uLimitMemoryArray, "--ulimits", "\"memlock="+strconv.Itoa(int(container.Resources.Limits.Memory().Value()))+"\"") } if container.Resources.Limits.Cpu().Value() != 0 { cpuLimitsArray = append(cpuLimitsArray, "--cpus", strconv.FormatFloat(float64(container.Resources.Limits.Cpu().Value()), 'f', -1, 64)) } cmd = append(cmd, memoryLimitsArray...) + cmd = append(cmd, uLimitMemoryArray...) cmd = append(cmd, cpuLimitsArray...) containerCommands := []string{} From f5bbe394935d256f179deb7ef48624ff8b284f60 Mon Sep 17 00:00:00 2001 From: Diego Ciangottini Date: Fri, 31 May 2024 13:42:58 +0200 Subject: [PATCH 11/30] Update Dockerfile.sidecar-docker --- docker/Dockerfile.sidecar-docker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.sidecar-docker b/docker/Dockerfile.sidecar-docker index 5f3a5e7..2d4eb15 100644 --- a/docker/Dockerfile.sidecar-docker +++ b/docker/Dockerfile.sidecar-docker @@ -25,7 +25,7 @@ ENV INTERLINKCONFIGPATH=/InterLinkConfig.yaml ENV PATH "$PATH:/bin" #creating a simple startup script to start both docker rootless and the sidecar -RUN echo -e '#!dockerd & /sidecar/docker-sidecar' > /sidecar/startup-docker.sh +RUN echo -e '#!/bin/bash\ndockerd & /sidecar/docker-sidecar' > /sidecar/startup-docker.sh RUN chmod +x /sidecar/startup-docker.sh WORKDIR /sidecar From d214cadac624ba267cf2188d0a44f9fd774c429b Mon Sep 17 00:00:00 2001 From: Diego Ciangottini Date: Fri, 31 May 2024 15:03:00 +0200 Subject: [PATCH 12/30] revert ulimits --- pkg/docker/Create.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/docker/Create.go b/pkg/docker/Create.go index d6ec62d..069ab35 100644 --- a/pkg/docker/Create.go +++ b/pkg/docker/Create.go @@ -174,7 +174,6 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { if container.Resources.Limits.Memory().Value() != 0 { memoryLimitsArray = append(memoryLimitsArray, "--memory", strconv.Itoa(int(container.Resources.Limits.Memory().Value()))+"b") - uLimitMemoryArray = append(uLimitMemoryArray, "--ulimits", "\"memlock="+strconv.Itoa(int(container.Resources.Limits.Memory().Value()))+"\"") } if container.Resources.Limits.Cpu().Value() != 0 { cpuLimitsArray = append(cpuLimitsArray, "--cpus", strconv.FormatFloat(float64(container.Resources.Limits.Cpu().Value()), 'f', -1, 64)) From e68a39e5b7730fefeb140e41539b7478c21be8c0 Mon Sep 17 00:00:00 2001 From: Diego Ciangottini Date: Sun, 2 Jun 2024 12:16:16 +0200 Subject: [PATCH 13/30] Update Dockerfile.sidecar-docker --- docker/Dockerfile.sidecar-docker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.sidecar-docker b/docker/Dockerfile.sidecar-docker index 2d4eb15..42d1ae7 100644 --- a/docker/Dockerfile.sidecar-docker +++ b/docker/Dockerfile.sidecar-docker @@ -25,7 +25,7 @@ ENV INTERLINKCONFIGPATH=/InterLinkConfig.yaml ENV PATH "$PATH:/bin" #creating a simple startup script to start both docker rootless and the sidecar -RUN echo -e '#!/bin/bash\ndockerd & /sidecar/docker-sidecar' > /sidecar/startup-docker.sh +RUN echo -e '#!/bin/bash\ndockerd --mtu=1350 & /sidecar/docker-sidecar' > /sidecar/startup-docker.sh RUN chmod +x /sidecar/startup-docker.sh WORKDIR /sidecar From a03c3c1dbceaa157945fe00bdaf758e5d38f79df Mon Sep 17 00:00:00 2001 From: Diego Ciangottini Date: Wed, 17 Jul 2024 21:20:28 +0200 Subject: [PATCH 14/30] increase dind timeout Signed-off-by: Diego Ciangottini --- pkg/docker/Create.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/docker/Create.go b/pkg/docker/Create.go index 9552aa0..b24f66f 100644 --- a/pkg/docker/Create.go +++ b/pkg/docker/Create.go @@ -317,7 +317,7 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { log.G(h.Ctx).Info("\u2705 [POD FLOW] DIND container created successfully with ID: " + dindContainerID) // create a variable of maximum number of retries - maxRetries := 10 + maxRetries := 20 // wait until the dind container is up and running by check that the command docker ps inside of it does not return an error for { @@ -337,7 +337,7 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { if strings.Contains(string(output), "API listen on /var/run/docker.sock") { break } else { - time.Sleep(1 * time.Second) + time.Sleep(60 * time.Second) } maxRetries -= 1 From c0090655a663acb7fb64bd9d0f927ece48fe2cf3 Mon Sep 17 00:00:00 2001 From: Bianco95 Date: Tue, 27 Aug 2024 08:38:57 +0200 Subject: [PATCH 15/30] sync with main --- cmd/main.go | 22 ++- pkg/docker/Create.go | 212 +++++++++++++++--------- pkg/docker/Delete.go | 115 +++++-------- pkg/docker/aux.go | 6 +- pkg/docker/dindmanager/DindHandler.go | 223 ++++++++++++++++++++++++++ 5 files changed, 420 insertions(+), 158 deletions(-) create mode 100644 pkg/docker/dindmanager/DindHandler.go diff --git a/cmd/main.go b/cmd/main.go index 3996557..91e0e41 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -3,6 +3,7 @@ package main import ( "context" "net/http" + "os" "strconv" "github.com/sirupsen/logrus" @@ -10,6 +11,7 @@ import ( commonIL "github.com/intertwin-eu/interlink-docker-plugin/pkg/common" docker "github.com/intertwin-eu/interlink-docker-plugin/pkg/docker" + "github.com/intertwin-eu/interlink-docker-plugin/pkg/docker/dindmanager" ) func main() { @@ -32,9 +34,25 @@ func main() { defer cancel() log.G(Ctx).Debug("Debug level: " + strconv.FormatBool(interLinkConfig.VerboseLogging)) + availableDinds := os.Getenv("AVAILABLEDINDS") + if availableDinds == "" { + availableDinds = "2" + } + var dindHandler dindmanager.DindManagerInterface + dindHandler = &dindmanager.DindManager{ + DindList: []dindmanager.DindSpecs{}, + Ctx: Ctx, + } + availableDindsInt, err := strconv.ParseInt(availableDinds, 10, 8) + if err != nil { + log.G(Ctx).Fatal(err) + } + dindHandler.BuildDindContainers(int8(availableDindsInt)) + SidecarAPIs := docker.SidecarHandler{ - Config: interLinkConfig, - Ctx: Ctx, + Config: interLinkConfig, + Ctx: Ctx, + DindManager: dindHandler, } mutex := http.NewServeMux() diff --git a/pkg/docker/Create.go b/pkg/docker/Create.go index b24f66f..76da4b5 100644 --- a/pkg/docker/Create.go +++ b/pkg/docker/Create.go @@ -17,7 +17,6 @@ import ( commonIL "github.com/intertwin-eu/interlink-docker-plugin/pkg/common" - OSexec "os/exec" "path/filepath" ) @@ -198,7 +197,37 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { log.G(h.Ctx).Info("\u23F3 [CREATE CALL] Received create call from InterLink ") - var execReturn exec.ExecResult + // create bool variable to set if a new dind container has to be created + newDindContainerCreated := false + + // get a dind container ID from dind manager of the sidecard handler + dindContainerID, err := h.DindManager.GetAvailableDind() + if err != nil { + + log.G(h.Ctx).Info("\u2705 [POD FLOW] No available DIND container found, creating a new one") + + h.DindManager.BuildDindContainers(1) + dindContainerID, err = h.DindManager.GetAvailableDind() + if err != nil { + HandleErrorAndRemoveData(h, w, "During creation of new DIND container, an error occurred during the request of get available DIND container", err, "", "") + return + } + newDindContainerCreated = true + } + + // remove the dind container from the list of available dind containers + err = h.DindManager.SetDindUnavailable(dindContainerID) + if err != nil { + HandleErrorAndRemoveData(h, w, "An error occurred during the removal of the DIND container from the list of available DIND containers", err, "", "") + return + } + + if !newDindContainerCreated { + // create a new dind container in background + go h.DindManager.BuildDindContainers(1) + } + + //var execReturn exec.ExecResult statusCode := http.StatusOK bodyBytes, err := io.ReadAll(r.Body) @@ -251,7 +280,7 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { // from dockerRunStructs, create two arrays: one for initContainers and one for containers var initContainers []DockerRunStruct var containers []DockerRunStruct - var gpuArgs string + //var gpuArgs string for _, dockerRunStruct := range dockerRunStructs { if dockerRunStruct.IsInitContainer { @@ -261,90 +290,117 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { } } - // check if between the containers there is a container that requires a GPU - for _, container := range containers { - if container.GpuArgs != "" { - gpuArgs = container.GpuArgs - } - } - - gpuArgsAsArray := []string{} - if gpuArgs != "" { - gpuArgsAsArray = strings.Split(gpuArgs, " ") - } - - dindImage := "ghcr.io/extrality/nvidia-dind" - if gpuArgs == "" { - dindImage = "docker:dind" - } - - // create a dedicated docker network for the dind container - shell := exec.ExecTask{ - Command: "docker", - Args: []string{"network", "create", "--driver", "bridge", string(data.Pod.UID) + "_dind_network"}, - Shell: true, - } - execReturn, err = shell.Execute() + // // check if between the containers there is a container that requires a GPU + // for _, container := range containers { + // if container.GpuArgs != "" { + // gpuArgs = container.GpuArgs + // } + // } + + // gpuArgsAsArray := []string{} + // if gpuArgs != "" { + // gpuArgsAsArray = strings.Split(gpuArgs, " ") + // } + + // dindImage := "ghcr.io/extrality/nvidia-dind" + // if gpuArgs == "" { + // dindImage = "docker:dind" + // } + + // // create a dedicated docker network for the dind container + // shell := exec.ExecTask{ + // Command: "docker", + // Args: []string{"network", "create", "--driver", "bridge", string(data.Pod.UID) + "_dind_network"}, + // Shell: true, + // } + // execReturnNetworkCommand, err := shell.Execute() + // if err != nil { + // HandleErrorAndRemoveData(h, w, "An error occurred during the creation of the network for the DIND container", err, "", "") + // return + // } + + // // log the docker network creation command + // log.G(h.Ctx).Info("\u2705 [POD FLOW] Docker network created successfully with command: " + "docker " + strings.Join(shell.Args, " ")) + + // dindContainerArgs := []string{"run"} + // dindContainerArgs = append(dindContainerArgs, gpuArgsAsArray...) + // if _, err := os.Stat("/cvmfs"); err == nil { + // dindContainerArgs = append(dindContainerArgs, "-v", "/cvmfs:/cvmfs") + // } + + // // add the network to the dind container + // dindContainerArgs = append(dindContainerArgs, "--network", string(data.Pod.UID)+"_dind_network") + // dindContainerArgs = append(dindContainerArgs, "--privileged", "-v", wd+":/"+wd, "-v", "/home:/home", "-v", "/var/lib/docker/overlay2:/var/lib/docker/overlay2", "-v", "/var/lib/docker/image:/var/lib/docker/image", "-d", "--name", string(data.Pod.UID)+"_dind", dindImage) + + // var dindContainerID string + // shell = exec.ExecTask{ + // Command: "docker", + // Args: dindContainerArgs, + // Shell: true, + // } + + // execReturn, err = shell.Execute() + // if err != nil { + // HandleErrorAndRemoveData(h, w, "An error occurred during the execution of DIND container command", err, "", "") + // return + // } + // dindContainerID = execReturn.Stdout + + // // log also the command executed to create the DIND container + // log.G(h.Ctx).Info("\u2705 [POD FLOW] DIND container command executed successfully: " + "docker " + strings.Join(shell.Args, " ")) + + // log.G(h.Ctx).Info("\u2705 [POD FLOW] DIND container created successfully with ID: " + dindContainerID) + + // // create a variable of maximum number of retries + // maxRetries := 20 + // output := []byte{} + + // // wait until the dind container is up and running by check that the command docker ps inside of it does not return an error + // for { + + // if maxRetries == 0 { + // HandleErrorAndRemoveData(h, w, "The number of attempts to check if the DIND container is running is 0. This means that an error occurred during the creation of the DIND container UID. "+dindContainerID+" output: "+string(output)+" Network creation output "+string(execReturnNetworkCommand.Stdout), err, "", "") + // return + // } + + // cmd := OSexec.Command("docker", "logs", string(data.Pod.UID)+"_dind") + // output, err = cmd.CombinedOutput() + + // if err != nil { + // time.Sleep(1 * time.Second) + // } + + // if strings.Contains(string(output), "API listen on /var/run/docker.sock") { + // break + // } else { + // time.Sleep(1 * time.Second) + // } + + // maxRetries -= 1 + + // } + + // log.G(h.Ctx).Info("\u2705 [POD FLOW] DIND container is up and running, ready to create the containers inside of it") + + // set the podUID to the dind container + err = h.DindManager.SetPodUIDToDind(dindContainerID, podUID) if err != nil { - HandleErrorAndRemoveData(h, w, "An error occurred during the creation of the network for the DIND container", err, "", "") + HandleErrorAndRemoveData(h, w, "An error occurred during the setting of the pod UID to the DIND container", err, "", "") return } - dindContainerArgs := []string{"run"} - dindContainerArgs = append(dindContainerArgs, gpuArgsAsArray...) - if _, err := os.Stat("/cvmfs"); err == nil { - dindContainerArgs = append(dindContainerArgs, "-v", "/cvmfs:/cvmfs") - } - - // add the network to the dind container - dindContainerArgs = append(dindContainerArgs, "--network", string(data.Pod.UID)+"_dind_network") - dindContainerArgs = append(dindContainerArgs, "--privileged", "-v", wd+":/"+wd, "-v", "/home:/home", "-v", "/var/lib/docker/overlay2:/var/lib/docker/overlay2", "-v", "/var/lib/docker/image:/var/lib/docker/image", "-d", "--name", string(data.Pod.UID)+"_dind", dindImage) - - var dindContainerID string - shell = exec.ExecTask{ + // run the docker command to rename the container to the pod UID + shell := exec.ExecTask{ Command: "docker", - Args: dindContainerArgs, + Args: []string{"rename", dindContainerID, string(data.Pod.UID) + "_dind"}, Shell: true, } - execReturn, err = shell.Execute() + _, err = shell.Execute() if err != nil { - HandleErrorAndRemoveData(h, w, "An error occurred during the execution of DIND container command", err, "", "") + HandleErrorAndRemoveData(h, w, "An error occurred during the rename of the DIND container", err, "", "") return } - dindContainerID = execReturn.Stdout - - log.G(h.Ctx).Info("\u2705 [POD FLOW] DIND container created successfully with ID: " + dindContainerID) - - // create a variable of maximum number of retries - maxRetries := 20 - - // wait until the dind container is up and running by check that the command docker ps inside of it does not return an error - for { - - if maxRetries == 0 { - HandleErrorAndRemoveData(h, w, "The number of attempts to check if the DIND container is running is 0. This means that an error occurred during the creation of the DIND container", err, "", "") - return - } - - cmd := OSexec.Command("docker", "logs", string(data.Pod.UID)+"_dind") - output, err := cmd.CombinedOutput() - - if err != nil { - time.Sleep(1 * time.Second) - } - - if strings.Contains(string(output), "API listen on /var/run/docker.sock") { - break - } else { - time.Sleep(60 * time.Second) - } - - maxRetries -= 1 - - } - - log.G(h.Ctx).Info("\u2705 [POD FLOW] DIND container is up and running, ready to create the containers inside of it") if len(initContainers) > 0 { @@ -360,7 +416,7 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { return } - shell = exec.ExecTask{ + shell := exec.ExecTask{ Command: "docker", Args: []string{"exec", string(data.Pod.UID) + "_dind", "/bin/sh", podDirectoryPath + "/init_containers_command.sh"}, } @@ -426,7 +482,7 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { Args: []string{"exec", string(data.Pod.UID) + "_dind", "/bin/sh", podDirectoryPath + "/containers_command.sh"}, } - execReturn, err = shell.Execute() + _, err = shell.Execute() if err != nil { HandleErrorAndRemoveData(h, w, "An error occurred during the execution of the container command script", err, "", "") return diff --git a/pkg/docker/Delete.go b/pkg/docker/Delete.go index a24d19c..fdf271d 100644 --- a/pkg/docker/Delete.go +++ b/pkg/docker/Delete.go @@ -9,6 +9,7 @@ import ( exec "github.com/alexellis/go-execute/pkg/v1" "github.com/containerd/containerd/log" + "github.com/intertwin-eu/interlink-docker-plugin/pkg/docker/dindmanager" v1 "k8s.io/api/core/v1" "path/filepath" @@ -42,76 +43,36 @@ func (h *SidecarHandler) DeleteHandler(w http.ResponseWriter, r *http.Request) { podUID := string(pod.UID) podNamespace := string(pod.Namespace) - for _, container := range pod.Spec.Containers { + log.G(h.Ctx).Debug("\u2705 [DELETE CALL] Deleting POD " + podUID + "_dind") - containerName := podNamespace + "-" + podUID + "-" + container.Name - - log.G(h.Ctx).Debug("\u2705 [DELETE CALL] Deleting container " + containerName) - - // added a timeout to the stop container command - cmd := []string{"exec", podUID + "_dind", "docker", "stop", "-t", "10", containerName} - shell := exec.ExecTask{ - Command: "docker", - Args: cmd, - Shell: true, - } - execReturn, _ = shell.Execute() + cmd := []string{"rm", "-f", podUID + "_dind"} + shell := exec.ExecTask{ + Command: "docker", + Args: cmd, + Shell: true, + } + execReturn, _ = shell.Execute() + execReturn.Stdout = strings.ReplaceAll(execReturn.Stdout, "\n", "") - if execReturn.Stderr != "" { - if strings.Contains(execReturn.Stderr, "No such container") { - log.G(h.Ctx).Debug("\u26A0 [DELETE CALL] Unable to find container " + containerName + ". Probably already removed? Skipping its removal") - } else { - log.G(h.Ctx).Error("\u274C [DELETE CALL] Error stopping container " + containerName + ". Skipping its removal") - statusCode = http.StatusInternalServerError - w.WriteHeader(statusCode) - w.Write([]byte("Some errors occurred while deleting container. Check Docker Sidecar's logs")) - return - } - continue - } + if execReturn.Stderr != "" { + log.G(h.Ctx).Error("\u274C [DELETE CALL] Error deleting container " + podUID + "_dind") + statusCode = http.StatusInternalServerError + } else { + log.G(h.Ctx).Info("\u2705 [DELETE CALL] Deleted container " + podUID + "_dind") + } - if execReturn.Stdout != "" { - cmd = []string{"exec", podUID + "_dind", "docker", "rm", execReturn.Stdout} - shell = exec.ExecTask{ - Command: "docker", - Args: cmd, - Shell: true, - } - execReturn, _ = shell.Execute() - execReturn.Stdout = strings.ReplaceAll(execReturn.Stdout, "\n", "") - - if execReturn.Stderr != "" { - log.G(h.Ctx).Error("\u274C [DELETE CALL] Error deleting container " + containerName) - statusCode = http.StatusInternalServerError - w.WriteHeader(statusCode) - w.Write([]byte("Some errors occurred while deleting container. Check Docker Sidecar's logs")) - return - } else { - log.G(h.Ctx).Info("\u2705 [DELETE CALL] Deleted container " + containerName) - } - } + dindSpec := dindmanager.DindSpecs{} + dindSpec, err = h.DindManager.GetDindFromPodUID(podUID) - cmd = []string{"rm", "-f", podUID + "_dind"} - shell = exec.ExecTask{ - Command: "docker", - Args: cmd, - Shell: true, - } - execReturn, _ = shell.Execute() - execReturn.Stdout = strings.ReplaceAll(execReturn.Stdout, "\n", "") + if err != nil { + log.G(h.Ctx).Error("\u274C [DELETE CALL] Error retrieving DindSpecs, maybe the Dind container has already been deleted") + } else { + log.G(h.Ctx).Info("\u2705 [DELETE CALL] Retrieved DindSpecs: " + dindSpec.DindID + " " + dindSpec.PodUID + " " + dindSpec.DindNetworkID + " ") - if execReturn.Stderr != "" { - log.G(h.Ctx).Error("\u274C [DELETE CALL] Error deleting container " + podUID + "_dind") - statusCode = http.StatusInternalServerError - w.WriteHeader(statusCode) - w.Write([]byte("Some errors occurred while deleting container. Check Docker Sidecar's logs")) - return - } else { - log.G(h.Ctx).Info("\u2705 [DELETE CALL] Deleted container " + podUID + "_dind") - } + // log the retrieved dindSpec + log.G(h.Ctx).Info("\u2705 [DELETE CALL] Retrieved DindSpecs: " + dindSpec.DindID + " " + dindSpec.PodUID + " " + dindSpec.DindNetworkID + " ") - // delete also the network of the docker dind container that is called string(data.Pod.UID) + "_dind_network" - cmd = []string{"network", "rm", podUID + "_dind_network"} + cmd = []string{"network", "rm", dindSpec.DindNetworkID} shell = exec.ExecTask{ Command: "docker", Args: cmd, @@ -120,23 +81,25 @@ func (h *SidecarHandler) DeleteHandler(w http.ResponseWriter, r *http.Request) { execReturn, _ = shell.Execute() execReturn.Stdout = strings.ReplaceAll(execReturn.Stdout, "\n", "") if execReturn.Stderr != "" { - log.G(h.Ctx).Error("\u274C [DELETE CALL] Error deleting network " + podUID + "_dind_network") + log.G(h.Ctx).Error("\u274C [DELETE CALL] Error deleting network " + dindSpec.DindNetworkID) } else { - log.G(h.Ctx).Info("\u2705 [DELETE CALL] Deleted network " + podUID + "_dind_network") + log.G(h.Ctx).Info("\u2705 [DELETE CALL] Deleted network " + dindSpec.DindNetworkID) } - - wd, err := os.Getwd() + // set the dind available again + err = h.DindManager.RemoveDindFromList(dindSpec.PodUID) if err != nil { - HandleErrorAndRemoveData(h, w, "Unable to get current working directory", err, "", "") - return + log.G(h.Ctx).Error("\u274C [DELETE CALL] Error setting DIND container available") } - podDirectoryPathToDelete := filepath.Join(wd, h.Config.DataRootFolder+"/"+podNamespace+"-"+podUID) - log.G(h.Ctx).Info("\u2705 [DELETE CALL] Deleting directory " + podDirectoryPathToDelete) - - err = os.RemoveAll(podDirectoryPathToDelete) - - //os.RemoveAll(h.Config.DataRootFolder + pod.Namespace + "-" + string(pod.UID)) } + wd, err := os.Getwd() + if err != nil { + HandleErrorAndRemoveData(h, w, "Unable to get current working directory", err, "", "") + return + } + podDirectoryPathToDelete := filepath.Join(wd, h.Config.DataRootFolder+"/"+podNamespace+"-"+podUID) + log.G(h.Ctx).Info("\u2705 [DELETE CALL] Deleting directory " + podDirectoryPathToDelete) + + err = os.RemoveAll(podDirectoryPathToDelete) w.WriteHeader(statusCode) if statusCode != http.StatusOK { diff --git a/pkg/docker/aux.go b/pkg/docker/aux.go index 86e37f2..f190434 100644 --- a/pkg/docker/aux.go +++ b/pkg/docker/aux.go @@ -12,11 +12,13 @@ import ( v1 "k8s.io/api/core/v1" commonIL "github.com/intertwin-eu/interlink-docker-plugin/pkg/common" + "github.com/intertwin-eu/interlink-docker-plugin/pkg/docker/dindmanager" ) type SidecarHandler struct { - Config commonIL.InterLinkConfig - Ctx context.Context + Config commonIL.InterLinkConfig + Ctx context.Context + DindManager dindmanager.DindManagerInterface } func parseContainerCommandAndReturnArgs(Ctx context.Context, config commonIL.InterLinkConfig, podUID string, podNamespace string, container v1.Container) ([]string, []string, []string, error) { diff --git a/pkg/docker/dindmanager/DindHandler.go b/pkg/docker/dindmanager/DindHandler.go new file mode 100644 index 0000000..2c2e7c9 --- /dev/null +++ b/pkg/docker/dindmanager/DindHandler.go @@ -0,0 +1,223 @@ +package dindmanager + +import ( + "context" + "crypto/rand" + "fmt" + "os" + "strings" + "time" + + exec "github.com/alexellis/go-execute/pkg/v1" + "github.com/containerd/containerd/log" + + OSexec "os/exec" +) + +type DindManagerInterface interface { + BuildDindContainers(nDindContainer int8) error + PrintDindList() error + GetAvailableDind() (string, error) + SetDindUnavailable(dindID string) error + RemoveDindFromList(PodUID string) error + SetPodUIDToDind(dindID string, podUID string) error + GetDindFromPodUID(podUID string) (DindSpecs, error) + SetDindAvailable(PodUID string) error +} + +type DindSpecs struct { + DindID string + PodUID string + DindNetworkID string + Available bool +} + +type DindManager struct { + DindList []DindSpecs + Ctx context.Context +} + +// GenerateUUIDv4 generates a random UUIDv4 +func GenerateUUIDv4() (string, error) { + uuid := make([]byte, 16) + _, err := rand.Read(uuid) + if err != nil { + return "", err + } + + uuid[6] = (uuid[6] & 0x0f) | 0x40 + uuid[8] = (uuid[8] & 0x3f) | 0x80 + + return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:16]), nil +} + +func (a *DindManager) BuildDindContainers(nDindContainer int8) error { + + // print the number of DIND containers to be created + log.G(a.Ctx).Info(fmt.Sprintf("\u2705 Creating %d DIND containers", nDindContainer)) + + // get the working dir + wd, err := os.Getwd() + if err != nil { + return err + } + + // get the env variable GPUENABLED, if 1 then the DIND container will have GPU support, otherwise it will not + + gpuEnabled := os.Getenv("GPUENABLED") + dindImage := "docker:dind" + if gpuEnabled == "1" { + dindImage = "ghcr.io/extrality/nvidia-dind" + } + + for i := int8(0); i < nDindContainer; i++ { + + // generate a random UID for the DIND container + randUID, err := GenerateUUIDv4() + if err != nil { + return err + } + + // create the networks + shell := exec.ExecTask{ + Command: "docker", + Args: []string{"network", "create", "--driver", "bridge", randUID + "_dind_network"}, + Shell: true, + } + _, err = shell.Execute() + + log.G(a.Ctx).Info(fmt.Sprintf("\u2705 DIND network %s created", randUID+"_dind_network")) + + if err != nil { + return err + } + + dindContainerArgs := []string{"run"} + //dindContainerArgs = append(dindContainerArgs, gpuArgsAsArray...) + if _, err := os.Stat("/cvmfs"); err == nil { + dindContainerArgs = append(dindContainerArgs, "-v", "/cvmfs:/cvmfs") + } + + // add the network to the dind container + dindContainerArgs = append(dindContainerArgs, "--network", randUID+"_dind_network") + // "--runtime=nvidia" is added to the dind container if the GPUENABLED env variable is set to 1 + + if gpuEnabled == "1" { + dindContainerArgs = append(dindContainerArgs, "--runtime=nvidia") + } + dindContainerArgs = append(dindContainerArgs, "--privileged", "-v", wd+":/"+wd, "-v", "/home:/home", "-v", "/var/lib/docker/overlay2:/var/lib/docker/overlay2", "-v", "/var/lib/docker/image:/var/lib/docker/image", "-d", "--name", randUID+"_dind", dindImage) + + var dindContainerID string + shell = exec.ExecTask{ + Command: "docker", + Args: dindContainerArgs, + Shell: true, + } + + execReturn, err := shell.Execute() + if err != nil { + return err + } + dindContainerID = execReturn.Stdout + + // create a variable of maximum number of retries + maxRetries := 20 + output := []byte{} + + // wait until the dind container is up and running by check that the command docker ps inside of it does not return an error + for { + + if maxRetries == 0 { + return fmt.Errorf("DIND container %s not up and running", dindContainerID) + } + + cmd := OSexec.Command("docker", "logs", randUID+"_dind") + output, err = cmd.CombinedOutput() + + if err != nil { + time.Sleep(1 * time.Second) + } + + if strings.Contains(string(output), "API listen on /var/run/docker.sock") { + break + } else { + time.Sleep(1 * time.Second) + } + + maxRetries -= 1 + + } + + log.G(a.Ctx).Info(fmt.Sprintf("\u2705 DIND container %s is up and running", dindContainerID)) + + // add the dind container to the list of DIND containers + a.DindList = append(a.DindList, DindSpecs{DindID: randUID + "_dind", PodUID: "", DindNetworkID: randUID + "_dind_network", Available: true}) + } + + return nil +} + +func (a *DindManager) PrintDindList() error { + for _, dindSpec := range a.DindList { + log.G(a.Ctx).Info(fmt.Sprintf("DindID: %s, PodUID: %s, DindNetworkID: %s, Available: %t", dindSpec.DindID, dindSpec.PodUID, dindSpec.DindNetworkID, dindSpec.Available)) + } + return nil +} + +func (a *DindManager) GetDindFromPodUID(podUID string) (DindSpecs, error) { + for _, dindSpec := range a.DindList { + if dindSpec.PodUID == podUID { + return dindSpec, nil + } + } + return DindSpecs{}, fmt.Errorf("DIND container with PodUID %s not found", podUID) +} + +func (a *DindManager) GetAvailableDind() (string, error) { + for _, dindSpec := range a.DindList { + if dindSpec.Available { + return dindSpec.DindID, nil + } + } + return "", fmt.Errorf("No available DIND container") +} + +func (a *DindManager) SetDindUnavailable(dindID string) error { + for i, dindSpec := range a.DindList { + if dindSpec.DindID == dindID { + a.DindList[i].Available = false + return nil + } + } + return fmt.Errorf("DIND container %s not found", dindID) +} + +func (a *DindManager) SetDindAvailable(PodUI string) error { + for i, dindSpec := range a.DindList { + if dindSpec.PodUID == PodUI { + a.DindList[i].Available = true + return nil + } + } + return fmt.Errorf("DIND container %s not found", PodUI) +} + +func (a *DindManager) SetPodUIDToDind(dindID string, podUID string) error { + for i, dindSpec := range a.DindList { + if dindSpec.DindID == dindID { + a.DindList[i].PodUID = podUID + return nil + } + } + return fmt.Errorf("DIND container %s not found", dindID) +} + +func (a *DindManager) RemoveDindFromList(PodUID string) error { + for i, dindSpec := range a.DindList { + if dindSpec.PodUID == PodUID { + a.DindList = append(a.DindList[:i], a.DindList[i+1:]...) + return nil + } + } + return fmt.Errorf("DIND container with PodUID %s not found", PodUID) +} From c62cabc9f698c865e0c0cbaf15668624ff34f47e Mon Sep 17 00:00:00 2001 From: Bianco95 Date: Tue, 27 Aug 2024 10:53:39 +0200 Subject: [PATCH 16/30] sync with main --- cmd/main.go | 1 + pkg/docker/dindmanager/DindHandler.go | 47 +++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/cmd/main.go b/cmd/main.go index 91e0e41..857f038 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -47,6 +47,7 @@ func main() { if err != nil { log.G(Ctx).Fatal(err) } + dindHandler.CleanDindContainers() dindHandler.BuildDindContainers(int8(availableDindsInt)) SidecarAPIs := docker.SidecarHandler{ diff --git a/pkg/docker/dindmanager/DindHandler.go b/pkg/docker/dindmanager/DindHandler.go index 2c2e7c9..a63aa8f 100644 --- a/pkg/docker/dindmanager/DindHandler.go +++ b/pkg/docker/dindmanager/DindHandler.go @@ -15,6 +15,7 @@ import ( ) type DindManagerInterface interface { + CleanDindContainers() error BuildDindContainers(nDindContainer int8) error PrintDindList() error GetAvailableDind() (string, error) @@ -51,6 +52,52 @@ func GenerateUUIDv4() (string, error) { return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:16]), nil } +func (a *DindManager) CleanDindContainers() error { + + // print the number of DIND containers to be created + log.G(a.Ctx).Info(fmt.Sprintf("\u2705 Start cleaning zombie DIND containers")) + + // exec this command docker ps -a --format "{{.Names}}" | grep '_dind$' | wc -l + shell := exec.ExecTask{ + Command: "docker", + Args: []string{"ps", "-a", "--format", "{{.Names}}", "|", "grep", "_dind$", "|", "wc", "-l"}, + Shell: true, + } + execReturn, err := shell.Execute() + if err != nil { + return err + } + + // log the number of zombie DIND containers (remove the \n at the end of the string) + log.G(a.Ctx).Info(fmt.Sprintf("\u2705 %s zombie DIND containers found", strings.ReplaceAll(execReturn.Stdout, "\n", ""))) + + shell = exec.ExecTask{ + Command: "docker", + Args: []string{"ps", "-a", "--format", "{{.Names}}", "|", "grep", "_dind$", "|", "xargs", "-I", "{}", "docker", "rm", "-f", "{}"}, + Shell: true, + } + _, err = shell.Execute() + if err != nil { + return err + } + + // exec this command docker network ls --filter name=_dind_network$ --format "{{.ID}}" | xargs -r docker network rm + + shell = exec.ExecTask{ + Command: "docker", + Args: []string{"network", "ls", "--filter", "name=_dind_network$", "--format", "{{.ID}}", "|", "xargs", "-r", "docker", "network", "rm"}, + Shell: true, + } + _, err = shell.Execute() + if err != nil { + return err + } + + log.G(a.Ctx).Info(fmt.Sprintf("\u2705 DIND zombie containers cleaned")) + + return nil +} + func (a *DindManager) BuildDindContainers(nDindContainer int8) error { // print the number of DIND containers to be created From 7d6864614ddb8b489f808d51011107e9f64a019a Mon Sep 17 00:00:00 2001 From: Bianco95 Date: Fri, 13 Sep 2024 11:53:45 +0200 Subject: [PATCH 17/30] updated handling of delete danglind data --- pkg/docker/Create.go | 52 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/pkg/docker/Create.go b/pkg/docker/Create.go index 76da4b5..a1b7899 100644 --- a/pkg/docker/Create.go +++ b/pkg/docker/Create.go @@ -16,10 +16,25 @@ import ( "errors" commonIL "github.com/intertwin-eu/interlink-docker-plugin/pkg/common" + "github.com/intertwin-eu/interlink-docker-plugin/pkg/docker/dindmanager" "path/filepath" ) +func (h *SidecarHandler) CheckGpuRequestFromContainer(containerData v1.Container) error { + + numGpusRequested := 0 + if val, ok := containerData.Resources.Limits["nvidia.com/gpu"]; ok { + numGpusRequested = int(val.Value()) + } + + if numGpusRequested > 0 { + return errors.New("GPU requests are not supported") + } + + return nil +} + func (h *SidecarHandler) prepareDockerRuns(podData commonIL.RetrievedPodData, w http.ResponseWriter) ([]DockerRunStruct, error) { var dockerRunStructs []DockerRunStruct @@ -77,6 +92,12 @@ func (h *SidecarHandler) prepareDockerRuns(podData commonIL.RetrievedPodData, w containerName := podNamespace + "-" + podUID + "-" + container.Name + // check if the container has GPU requests + err := h.CheckGpuRequestFromContainer(container) + if err != nil { + return dockerRunStructs, errors.New("GPU requests are not supported") + } + var envVars string = "" for _, envVar := range container.Env { if envVar.Value != "" { @@ -518,4 +539,35 @@ func HandleErrorAndRemoveData(h *SidecarHandler, w http.ResponseWriter, s string if podNamespace != "" && podUID != "" { os.RemoveAll(h.Config.DataRootFolder + podNamespace + "-" + podUID) } + + dindSpec := dindmanager.DindSpecs{} + dindSpec, err = h.DindManager.GetDindFromPodUID(podUID) + + if err != nil { + log.G(h.Ctx).Error("\u274C [CREATE CALL] Error retrieving DindSpecs, maybe the Dind container has already been deleted") + } else { + log.G(h.Ctx).Info("\u2705 [CREATE CALL] Retrieved DindSpecs: " + dindSpec.DindID + " " + dindSpec.PodUID + " " + dindSpec.DindNetworkID + " ") + + // log the retrieved dindSpec + log.G(h.Ctx).Info("\u2705 [CREATE CALL] Retrieved DindSpecs: " + dindSpec.DindID + " " + dindSpec.PodUID + " " + dindSpec.DindNetworkID + " ") + + cmd := []string{"network", "rm", dindSpec.DindNetworkID} + shell := exec.ExecTask{ + Command: "docker", + Args: cmd, + Shell: true, + } + execReturn, _ := shell.Execute() + execReturn.Stdout = strings.ReplaceAll(execReturn.Stdout, "\n", "") + if execReturn.Stderr != "" { + log.G(h.Ctx).Error("\u274C [CREATE CALL] Error deleting network " + dindSpec.DindNetworkID) + } else { + log.G(h.Ctx).Info("\u2705 [CREATE CALL] Deleted network " + dindSpec.DindNetworkID) + } + // set the dind available again + err = h.DindManager.RemoveDindFromList(dindSpec.PodUID) + if err != nil { + log.G(h.Ctx).Error("\u274C [CREATE CALL] Error setting DIND container available") + } + } } From 7b92fa2f9b7f5d48d994df6785f8229a32b951b8 Mon Sep 17 00:00:00 2001 From: Diego Ciangottini Date: Sun, 22 Sep 2024 09:21:01 +0200 Subject: [PATCH 18/30] listen on socket Signed-off-by: Diego Ciangottini --- .DS_Store | Bin 0 -> 6148 bytes cmd/main.go | 36 +++++++++++++++++++++++++++++++++--- pkg/.DS_Store | Bin 0 -> 6148 bytes pkg/common/types.go | 1 + 4 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 .DS_Store create mode 100644 pkg/.DS_Store diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..38989cfa6df641bd7e5faf47c0c975149e603d6b GIT binary patch literal 6148 zcmeHKze~eF6nq+ByMqqS;^HE@ z2!c52B>26%tGQ=dM-{mTcVF^;B2sZTqU zP?K7+AH_;Blat~0^RzAc5m{M3le1($74D~iPP){TeFP6##N~f@UmV>NTh~o8-h6Zn z-!xX8fOC&pbRby^uV58%m%CbA8gYNg$vT*nGp3IM_0U6Gc5r&G*R3Nicm29{yjaw~ zMO){tJ0F#Ncep4OX@C{?G4~~B?Mg;n-t*hf`|Z@b+~YGOf1UY%%dls&mC6SF>=p0| zcm+Nc;P*p_!5CRg4eD11CjAKj4B*y=W4(6)Lv{coi>X0)V9JyNO{ub<7|N8xAKJXg zVrtNolhVw%j%8N%3q@&m_(PpeDl+J2uYgxTDo`+|72f~XCx8FTB7f!;@Cy7Z1ym4k z#%oxT-CJ{uKH#l*4nrGaAs+xEi>X0)VE&JQmcdV6fge@i E14TLEX8-^I literal 0 HcmV?d00001 diff --git a/cmd/main.go b/cmd/main.go index 857f038..f873668 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -2,9 +2,13 @@ package main import ( "context" + "net" "net/http" "os" + "os/signal" "strconv" + "strings" + "syscall" "github.com/sirupsen/logrus" "github.com/virtual-kubelet/virtual-kubelet/log" @@ -61,9 +65,35 @@ func main() { mutex.HandleFunc("/create", SidecarAPIs.CreateHandler) mutex.HandleFunc("/delete", SidecarAPIs.DeleteHandler) mutex.HandleFunc("/getLogs", SidecarAPIs.GetLogsHandler) - err = http.ListenAndServe(":"+interLinkConfig.Sidecarport, mutex) - if err != nil { - log.G(Ctx).Fatal(err) + if strings.HasPrefix(interLinkConfig.Socket, "unix://") { + // Create a Unix domain socket and listen for incoming connections. + socket, err := net.Listen("unix", strings.ReplaceAll(interLinkConfig.Socket, "unix://", "")) + if err != nil { + panic(err) + } + + // Cleanup the sockfile. + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + go func() { + <-c + os.Remove(strings.ReplaceAll(interLinkConfig.Socket, "unix://", "")) + os.Exit(1) + }() + server := http.Server{ + Handler: mutex, + } + + log.G(Ctx).Info(socket) + + if err := server.Serve(socket); err != nil { + log.G(Ctx).Fatal(err) + } + } else { + err = http.ListenAndServe(":"+interLinkConfig.Sidecarport, mutex) + if err != nil { + log.G(Ctx).Fatal(err) + } } } diff --git a/pkg/.DS_Store b/pkg/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..0c8885141335ec4fe931b31e71960ae38997f8fd GIT binary patch literal 6148 zcmeHKK~BR!475vxg1GdEIPC`z^ar5|AD|yVO%O;lY*nczE_nb#{D>P*-~qgZ@!Bd$ zL&XiD%8tCV@!A__62&nQ@pN5Jh{i;eLj^}i7=9307ahpREV9UHk4TS;yUDDI&2r%F zhX2TbJi9}xC{lxC@chEd_IeY`a=wUj*z?opv(Jz0{c`A2|LU{6@4hD&*aCUf$e^MJ zx}{bBJ$gFdUhn2{{np8&vquxZ^VsFn&*N3{aB&8l0cT*}89>bz$&VF%bOxLOXJEsC zd>;Z-urO>D)29PNY5{;5%tp9{eM5m&zu2gV6Pb9tello+>+JS!Ocmn s4bU5?i1@XNn-EM=DTc3<;uB~R*n>=fg<-1*3&ei}5)D2$1AofE7b?U|P5=M^ literal 0 HcmV?d00001 diff --git a/pkg/common/types.go b/pkg/common/types.go index 0810681..a0b0ff2 100644 --- a/pkg/common/types.go +++ b/pkg/common/types.go @@ -48,6 +48,7 @@ type InterLinkConfig struct { Scancelpath string `yaml:"ScancelPath"` Squeuepath string `yaml:"SqueuePath"` Interlinkport string `yaml:"InterlinkPort"` + Socket string `yaml:"Socket"` Sidecarport string `yaml:"SidecarPort"` Commandprefix string `yaml:"CommandPrefix"` ExportPodData bool `yaml:"ExportPodData"` From cc6f9b2626e0f2efe957276494ca04455e2bb013 Mon Sep 17 00:00:00 2001 From: Bianco95 Date: Thu, 16 Jan 2025 16:12:00 +0100 Subject: [PATCH 19/30] updated fpga integration --- cmd/main.go | 2 + pkg/docker/Create.go | 40 +++++- pkg/docker/Delete.go | 5 + pkg/docker/fpgastrategies/AMDHandler.go | 167 ++++++++++++++++-------- pkg/docker/func.go | 2 + pkg/docker/types.go | 1 + 6 files changed, 163 insertions(+), 54 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index f511061..5abdf9d 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -229,6 +229,8 @@ func main() { if err != nil { log.G(ctx).Info("\u274C Error during fpga discover: %w", err) } + + SidecarAPIs.FPGAManager = fpgaManager } log.G(ctx).Info(fmt.Sprintf("\u2705 Going to start the sidecar on port %s", interLinkConfig.Sidecarport)) diff --git a/pkg/docker/Create.go b/pkg/docker/Create.go index a4db876..f9171db 100644 --- a/pkg/docker/Create.go +++ b/pkg/docker/Create.go @@ -2,6 +2,7 @@ package docker import ( "encoding/json" + "fmt" "io" "net/http" "os" @@ -28,6 +29,7 @@ import ( func (h *SidecarHandler) prepareDockerRuns(podData commonIL.RetrievedPodData, w http.ResponseWriter) ([]DockerRunStruct, error) { var dockerRunStructs []DockerRunStruct + var fpgaArgs string = "" podUID := string(podData.Pod.UID) podNamespace := string(podData.Pod.Namespace) @@ -77,11 +79,40 @@ func (h *SidecarHandler) prepareDockerRuns(podData commonIL.RetrievedPodData, w for containerType, containers := range allContainers { isInitContainer := containerType == "initContainers" + var envVars string = "" + for _, container := range containers { containerName := podNamespace + "-" + podUID + "-" + container.Name - var envVars string = "" + var isFPGARequested bool = false + + if val, ok := container.Resources.Limits["xilinx.com/fpga"]; ok { + numFPGAsRequested := val.Value() + if numFPGAsRequested == 0 { + log.G(h.Ctx).Info("\u2705 Container " + containerName + " is not requesting a FPGA") + } else { + + isFPGARequested = true + log.G(h.Ctx).Info("\u2705 Container " + containerName + " is requesting " + strconv.Itoa(int(numFPGAsRequested)) + " FPGA(s)") + + numFPGAsRequestedInt := int(numFPGAsRequested) + _, err := h.FPGAManager.GetAvailableFPGAs(numFPGAsRequestedInt) + if err != nil { + HandleErrorAndRemoveData(h, w, "An error occurred during the request of available FPGAs", err, podNamespace, podUID) + return dockerRunStructs, errors.New("An error occurred during the request of available FPGAs") + } + assignedFPGAs, err := h.FPGAManager.GetAndAssignAvailableFPGAs(numFPGAsRequestedInt, containerName) + if err != nil { + HandleErrorAndRemoveData(h, w, "An error occurred during request of get and assign of an available GPU", err, podNamespace, podUID) + return dockerRunStructs, errors.New("An error occurred during request of get and assign of an available GPU") + } + for _, fpgaSpec := range assignedFPGAs { + fpgaArgs += " --device=" + fpgaSpec.DeviceToMount + ":" + fpgaSpec.DeviceToMount + } + } + } + for _, envVar := range container.Env { if envVar.Value != "" { if strings.Contains(envVar.Value, "[") { @@ -121,6 +152,12 @@ func (h *SidecarHandler) prepareDockerRuns(podData commonIL.RetrievedPodData, w cmd = append(cmd, "--privileged") } + if isFPGARequested { + cmd = append(cmd, fpgaArgs) + } + + fmt.Printf("Container fpga args: %s\n", fpgaArgs) + var additionalPortArgs []string for _, port := range container.Ports { @@ -189,6 +226,7 @@ func (h *SidecarHandler) prepareDockerRuns(podData commonIL.RetrievedPodData, w Name: containerName, Command: "docker " + strings.Join(shell.Args, " "), IsInitContainer: isInitContainer, + FpgaArgs: fpgaArgs, }) } } diff --git a/pkg/docker/Delete.go b/pkg/docker/Delete.go index cb148f6..bcf903a 100644 --- a/pkg/docker/Delete.go +++ b/pkg/docker/Delete.go @@ -55,6 +55,11 @@ func (h *SidecarHandler) DeleteHandler(w http.ResponseWriter, r *http.Request) { podUID := string(pod.UID) podNamespace := string(pod.Namespace) + for _, container := range pod.Spec.Containers { + containerName := podNamespace + "-" + podUID + "-" + container.Name + h.FPGAManager.Release(containerName) + } + log.G(h.Ctx).Debug("\u2705 [DELETE CALL] Deleting POD " + podUID + "_dind") cmd := []string{"rm", "-f", podUID + "_dind"} diff --git a/pkg/docker/fpgastrategies/AMDHandler.go b/pkg/docker/fpgastrategies/AMDHandler.go index ec48768..b8d4c8f 100644 --- a/pkg/docker/fpgastrategies/AMDHandler.go +++ b/pkg/docker/fpgastrategies/AMDHandler.go @@ -6,22 +6,26 @@ import ( "fmt" "io/ioutil" "os" - "os/exec" + "regexp" "strings" + exec "github.com/alexellis/go-execute/pkg/v1" + "sync" "github.com/containerd/containerd/log" ) type FPGASpecs struct { - BDF string - Shell string - LogicUUID string - ContainerID string - DeviceReady string - Available bool - Index int + BDF string + Shell string + LogicUUID string + deviceID string + ContainerID string + DeviceReady string + DeviceToMount string + Available bool + Index int } type FPGAManager struct { @@ -34,14 +38,14 @@ type FPGAManager struct { type FPGAManagerInterface interface { Init() error Shutdown() error - GetGPUSpecsList() []FPGASpecs + GetFPGASpecsList() []FPGASpecs Dump() error Discover() error Check() error - GetAvailableGPUs(numGPUs int) ([]FPGASpecs, error) + GetAvailableFPGAs(numFPGAs int) ([]FPGASpecs, error) Assign(UUID string, containerID string) error Release(UUID string) error - GetAndAssignAvailableGPUs(numGPUs int, containerID string) ([]FPGASpecs, error) + GetAndAssignAvailableFPGAs(numFPGAs int, containerID string) ([]FPGASpecs, error) } func (a *FPGAManager) Init() error { @@ -56,58 +60,115 @@ func (a *FPGAManager) Init() error { return fmt.Errorf("/tools/Xilinx/Vitis/2023.2 does not exist: %v", err) } - // Source the setup.sh to initialize Xilinx tools - cmd := exec.Command("bash", "-c", "source /opt/xilinx/xrt/setup.sh") - err := cmd.Run() + // Source the setup.sh to initialize Xilinx tools using shell + shellArgs := []string{"source", "/opt/xilinx/xrt/setup.sh"} + shell := exec.ExecTask{ + Command: "/bin/bash", + Args: shellArgs, + Shell: true, + } + + _, err := shell.Execute() if err != nil { - return fmt.Errorf("Error sourcing setup.sh: %v", err) + return fmt.Errorf("Error running source setup.sh command: %v", err) } + return nil } // Discover implements the Discover function of the FPGAManager interface func (a *FPGAManager) Discover() error { - cmd := exec.Command("lspci", "|", "grep", "Xilinx") - output, err := cmd.CombinedOutput() + shellArgs := []string{"|", "grep", "Xilinx"} + + shell := exec.ExecTask{ + Command: "/usr/bin/lspci", + Args: shellArgs, + Shell: true, + } + + regexPattern := `^\[[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-9a-fA-F]\].*` + re := regexp.MustCompile(regexPattern) + + shell.Execute() + output, err := shell.Execute() if err != nil { return fmt.Errorf("Error running lspci command: %v", err) } - // Extract BDF and other information from lspci output - lines := strings.Split(string(output), "\n") + lines := strings.Split(string(output.Stdout), "\n") for _, line := range lines { if strings.Contains(line, "Xilinx") { - // Parse the BDF and other necessary information from the lspci line - // Example: "01:00.0 Processing accelerators: Xilinx Corporation Device 505c" parts := strings.Fields(line) if len(parts) < 3 { continue } - bdf := parts[0] + shellArgs := []string{"/opt/xilinx/xrt/setup.sh"} + shell := exec.ExecTask{ + Command: "source", + Args: shellArgs, + Shell: true, + } - // Now, source the setup.sh and run `xbutil examine` to gather FPGA information - cmd = exec.Command("bash", "-c", "source /opt/xilinx/xrt/setup.sh && xbutil examine") - examineOutput, err := cmd.CombinedOutput() + _, err := shell.Execute() + if err != nil { + return fmt.Errorf("Error running source setup.sh command: %v", err) + } + + cmd := exec.ExecTask{ + Command: "/opt/xilinx/xrt/bin/xbutil", + Args: []string{"examine"}, // "--device", bdf + Shell: false, + } + outputXbutil, err := cmd.Execute() if err != nil { return fmt.Errorf("Error running xbutil examine: %v", err) } - // Parse the xbutil examine output to extract FPGA details - examineLines := strings.Split(string(examineOutput), "\n") + examineLines := strings.Split(string(outputXbutil.Stdout), "\n") for _, examineLine := range examineLines { - // Find the line with the device details (look for "Devices present") - if strings.Contains(examineLine, "Devices present") { - // Parse the detailed information from the xbutil examine output - // For example: "[0000:01:00.1] : xilinx_u55c_gen3x16_xdma_base_3 97088961-FEAE-DA91-52A2-1D9DFD63CCEF user(inst=129) Yes" + + if re.MatchString(examineLine) { fpgas := strings.Split(examineLine, " : ") - if len(fpgas) > 1 { + fmt.Printf("FPGAs: %v\n", fpgas) + found := false + + deviceID := "" + reDeviceID := regexp.MustCompile(`user\(inst=(\d+)\)`) + matches := reDeviceID.FindStringSubmatch(fpgas[1]) + if len(matches) > 1 { + deviceID = matches[1] + } + + logicUUID := "" + reLogicUUID := regexp.MustCompile(`[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}`) + matches = reLogicUUID.FindStringSubmatch(fpgas[1]) + if len(matches) > 0 { + logicUUID = matches[0] + } + + shell := "" + reLogiShell := regexp.MustCompile(`xilinx_.*_base_`) + matches = reLogiShell.FindStringSubmatch(fpgas[1]) + if len(matches) > 0 { + shell = matches[0] + } + + for _, fpgaSpec := range a.FPGASpecsList { + if fpgaSpec.LogicUUID == fpgas[1] { + found = true + break + } + } + if len(fpgas) > 1 && !found { spec := FPGASpecs{ - BDF: bdf, - Shell: fpgas[0], - LogicUUID: fpgas[1], - Available: true, // Assuming it's available for now, can update based on further output parsing + BDF: bdf, + Shell: shell, + LogicUUID: logicUUID, + deviceID: deviceID, + DeviceToMount: "/dev/dri/renderD" + deviceID, + Available: true, // Assuming it's available for now, can update based on further output parsing } a.FPGASpecsList = append(a.FPGASpecsList, spec) } @@ -119,7 +180,7 @@ func (a *FPGAManager) Discover() error { if len(a.FPGASpecsList) > 0 { log.G(a.Ctx).Info("\u2705 Discovered FPGAs:") for _, fpgaSpec := range a.FPGASpecsList { - log.G(a.Ctx).Info(fmt.Sprintf("\u2705 BDF: %s, Shell: %s, LogicUUID: %s, Available: %t", fpgaSpec.BDF, fpgaSpec.Shell, fpgaSpec.LogicUUID, fpgaSpec.Available)) + log.G(a.Ctx).Info(fmt.Sprintf("\u2705 BDF: %s, Shell: %s, LogicUUID: %s, DeviceID %s, Available: %t", fpgaSpec.BDF, fpgaSpec.Shell, fpgaSpec.LogicUUID, fpgaSpec.deviceID, fpgaSpec.Available)) } } else { log.G(a.Ctx).Info(" \u2705 No FPGAs discovered") @@ -137,7 +198,7 @@ func (a *FPGAManager) Shutdown() error { return nil } -func (a *FPGAManager) GetGPUSpecsList() []FPGASpecs { +func (a *FPGAManager) GetFPGASpecsList() []FPGASpecs { return a.FPGASpecsList } @@ -146,8 +207,8 @@ func (a *FPGAManager) Assign(UUID string, containerID string) error { for i := range a.FPGASpecsList { if a.FPGASpecsList[i].LogicUUID == UUID { - if a.FPGASpecsList[i].Available == false { - return fmt.Errorf("GPU with UUID %s is already in use by container %s", UUID, a.FPGASpecsList[i].ContainerID) + if !a.FPGASpecsList[i].Available { + return fmt.Errorf("FPGA with UUID %s is already in use by container %s", UUID, a.FPGASpecsList[i].ContainerID) } a.FPGASpecsList[i].ContainerID = containerID @@ -167,7 +228,7 @@ func (a *FPGAManager) Release(containerID string) error { for i := range a.FPGASpecsList { if a.FPGASpecsList[i].ContainerID == containerID { - if a.FPGASpecsList[i].Available == true { + if a.FPGASpecsList[i].Available { continue } @@ -179,26 +240,26 @@ func (a *FPGAManager) Release(containerID string) error { return nil } -func (a *FPGAManager) GetAvailableGPUs(numGPUs int) ([]FPGASpecs, error) { +func (a *FPGAManager) GetAvailableFPGAs(numFPGAs int) ([]FPGASpecs, error) { - var availableGPUs []FPGASpecs - for _, gpuSpec := range a.FPGASpecsList { - if gpuSpec.Available == true { - availableGPUs = append(availableGPUs, gpuSpec) - if len(availableGPUs) == numGPUs { - return availableGPUs, nil + var availableFPGAs []FPGASpecs + for _, fpgaSpec := range a.FPGASpecsList { + if fpgaSpec.Available { + availableFPGAs = append(availableFPGAs, fpgaSpec) + if len(availableFPGAs) == numFPGAs { + return availableFPGAs, nil } } } - return nil, fmt.Errorf("Not enough available GPUs. Requested: %d, Available: %d", numGPUs, len(availableGPUs)) + return nil, fmt.Errorf("Not enough available FPGAs. Requested: %d, Available: %d", numFPGAs, len(availableFPGAs)) } -func (a *FPGAManager) GetAndAssignAvailableGPUs(numGPUs int, containerID string) ([]FPGASpecs, error) { +func (a *FPGAManager) GetAndAssignAvailableFPGAs(numFPGAs int, containerID string) ([]FPGASpecs, error) { a.FPGASpecsMutex.Lock() defer a.FPGASpecsMutex.Unlock() - fpgaSpecs, err := a.GetAvailableGPUs(numGPUs) + fpgaSpecs, err := a.GetAvailableFPGAs(numFPGAs) if err != nil { return nil, err } @@ -213,7 +274,7 @@ func (a *FPGAManager) GetAndAssignAvailableGPUs(numGPUs int, containerID string) return fpgaSpecs, nil } -// dump the GPUSpecsList into a JSON file +// dump the FPGASpecsList into a JSON file func (a *FPGAManager) Dump() error { // Convert the array to JSON format @@ -223,7 +284,7 @@ func (a *FPGAManager) Dump() error { } // Write JSON data to a file - err = ioutil.WriteFile("gpu_specs.json", jsonData, 0644) + err = ioutil.WriteFile("fpga_specs.json", jsonData, 0644) if err != nil { return fmt.Errorf("Error writing to file: %v", err) } diff --git a/pkg/docker/func.go b/pkg/docker/func.go index f190434..e2d5431 100644 --- a/pkg/docker/func.go +++ b/pkg/docker/func.go @@ -13,12 +13,14 @@ import ( commonIL "github.com/intertwin-eu/interlink-docker-plugin/pkg/common" "github.com/intertwin-eu/interlink-docker-plugin/pkg/docker/dindmanager" + "github.com/intertwin-eu/interlink-docker-plugin/pkg/docker/fpgastrategies" ) type SidecarHandler struct { Config commonIL.InterLinkConfig Ctx context.Context DindManager dindmanager.DindManagerInterface + FPGAManager fpgastrategies.FPGAManagerInterface } func parseContainerCommandAndReturnArgs(Ctx context.Context, config commonIL.InterLinkConfig, podUID string, podNamespace string, container v1.Container) ([]string, []string, []string, error) { diff --git a/pkg/docker/types.go b/pkg/docker/types.go index bbe7694..d485e07 100644 --- a/pkg/docker/types.go +++ b/pkg/docker/types.go @@ -5,6 +5,7 @@ type DockerRunStruct struct { Command string `json:"command"` IsInitContainer bool `json:"isInitContainer"` GpuArgs string `json:"gpuArgs"` + FpgaArgs string `json:"fpgaArgs"` } type CreateStruct struct { PodUID string `json:"PodUID"` From 64127dfd7f4c763d6af54e7d57acdd75f398c568 Mon Sep 17 00:00:00 2001 From: Bianco95 Date: Mon, 17 Feb 2025 12:31:36 +0100 Subject: [PATCH 20/30] started implementing the possibility of disable the fpga bookkeping --- pkg/docker/dindmanager/DindHandler.go | 2 +- pkg/docker/fpgastrategies/AMDHandler.go | 42 +++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/pkg/docker/dindmanager/DindHandler.go b/pkg/docker/dindmanager/DindHandler.go index 342c5b9..a63aa8f 100644 --- a/pkg/docker/dindmanager/DindHandler.go +++ b/pkg/docker/dindmanager/DindHandler.go @@ -152,7 +152,7 @@ func (a *DindManager) BuildDindContainers(nDindContainer int8) error { if gpuEnabled == "1" { dindContainerArgs = append(dindContainerArgs, "--runtime=nvidia") } - dindContainerArgs = append(dindContainerArgs, "--privileged", "-v", wd+":/"+wd, "-v", "/home:/home:ro", "-v", "/var/lib/docker/overlay2:/var/lib/docker/overlay2", "-v", "/var/lib/docker/image:/var/lib/docker/image", "-d", "--name", randUID+"_dind", dindImage) + dindContainerArgs = append(dindContainerArgs, "--privileged", "-v", wd+":/"+wd, "-v", "/home:/home", "-v", "/var/lib/docker/overlay2:/var/lib/docker/overlay2", "-v", "/var/lib/docker/image:/var/lib/docker/image", "-d", "--name", randUID+"_dind", dindImage) var dindContainerID string shell = exec.ExecTask{ diff --git a/pkg/docker/fpgastrategies/AMDHandler.go b/pkg/docker/fpgastrategies/AMDHandler.go index b8d4c8f..57ef08b 100644 --- a/pkg/docker/fpgastrategies/AMDHandler.go +++ b/pkg/docker/fpgastrategies/AMDHandler.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "os" "regexp" + "sort" "strings" exec "github.com/alexellis/go-execute/pkg/v1" @@ -241,8 +242,48 @@ func (a *FPGAManager) Release(containerID string) error { } func (a *FPGAManager) GetAvailableFPGAs(numFPGAs int) ([]FPGASpecs, error) { + a.FPGASpecsMutex.Lock() + defer a.FPGASpecsMutex.Unlock() var availableFPGAs []FPGASpecs + disableBookkeeping := os.Getenv("FPGA_DISABLE_BOOKKEEPING") == "1" + + if disableBookkeeping { + fpgaUsage := make(map[string]int) + for _, fpga := range a.FPGASpecsList { + if fpga.ContainerID != "" { + fpgaUsage[fpga.BDF]++ + } else { + // If an unassigned FPGA is found, prioritize it + availableFPGAs = append(availableFPGAs, fpga) + } + } + + if len(availableFPGAs) >= numFPGAs { + return availableFPGAs[:numFPGAs], nil + } + + // If not enough unassigned FPGAs, find the least assigned ones + sort.Slice(a.FPGASpecsList, func(i, j int) bool { + return fpgaUsage[a.FPGASpecsList[i].BDF] < fpgaUsage[a.FPGASpecsList[j].BDF] + }) + + for _, fpga := range a.FPGASpecsList { + if len(availableFPGAs) < numFPGAs { + availableFPGAs = append(availableFPGAs, fpga) + } else { + break + } + } + + if len(availableFPGAs) >= numFPGAs { + return availableFPGAs[:numFPGAs], nil + } + + return nil, fmt.Errorf("Not enough FPGAs available. Requested: %d, Found: %d", numFPGAs, len(availableFPGAs)) + } + + // Default behavior: return only available FPGAs for _, fpgaSpec := range a.FPGASpecsList { if fpgaSpec.Available { availableFPGAs = append(availableFPGAs, fpgaSpec) @@ -251,6 +292,7 @@ func (a *FPGAManager) GetAvailableFPGAs(numFPGAs int) ([]FPGASpecs, error) { } } } + return nil, fmt.Errorf("Not enough available FPGAs. Requested: %d, Available: %d", numFPGAs, len(availableFPGAs)) } From 7135ad8995ee52d03f1ffee69e4efc1930fd694f Mon Sep 17 00:00:00 2001 From: Bianco95 Date: Wed, 19 Feb 2025 11:30:36 +0100 Subject: [PATCH 21/30] wip --- pkg/common/types.go | 1 + pkg/docker/Create.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/pkg/common/types.go b/pkg/common/types.go index 7a4d899..9f8a765 100644 --- a/pkg/common/types.go +++ b/pkg/common/types.go @@ -36,6 +36,7 @@ type RetrievedPodData struct { Pod v1.Pod `json:"pod"` Containers []RetrievedContainer `json:"container"` InitContainers []RetrievedContainer `json:"initContainer"` + JobScript string `json:"jobScript"` } // InterLinkConfig holds the whole configuration diff --git a/pkg/docker/Create.go b/pkg/docker/Create.go index f9171db..93bee24 100644 --- a/pkg/docker/Create.go +++ b/pkg/docker/Create.go @@ -283,6 +283,8 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { return } + log.G(h.Ctx).Info("\u2705 [POD FLOW] Body request for pod creation read successfully", string(bodyBytes)) + var req []commonIL.RetrievedPodData err = json.Unmarshal(bodyBytes, &req) From 2286b84e26167102ac85098671a48e0f68cff1c5 Mon Sep 17 00:00:00 2001 From: Bianco95 Date: Wed, 12 Mar 2025 09:48:09 +0100 Subject: [PATCH 22/30] handled mount of xilinx directories; fixed fpga over bookking logic --- pkg/docker/Create.go | 55 +++++++++++-------- pkg/docker/dindmanager/DindHandler.go | 6 +++ pkg/docker/fpgastrategies/AMDHandler.go | 72 ++++++++++++++++++++++++- 3 files changed, 108 insertions(+), 25 deletions(-) diff --git a/pkg/docker/Create.go b/pkg/docker/Create.go index dd92dc5..082cff6 100644 --- a/pkg/docker/Create.go +++ b/pkg/docker/Create.go @@ -97,14 +97,28 @@ func (h *SidecarHandler) prepareDockerRuns(podData commonIL.RetrievedPodData, w numFPGAsRequestedInt := int(numFPGAsRequested) _, err := h.FPGAManager.GetAvailableFPGAs(numFPGAsRequestedInt) + log.G(h.Ctx).Info("\u2705 [CREATE CALL] Retrieved available FPGAs") if err != nil { + // log the err + log.G(h.Ctx).Error("\u274C [CREATE CALL] Error retrieving available FPGAs") HandleErrorAndRemoveData(h, w, "An error occurred during the request of available FPGAs", err, podNamespace, podUID) return dockerRunStructs, errors.New("An error occurred during the request of available FPGAs") } + + log.G(h.Ctx).Info("\u2705 [CREATE CALL] ********* BEFORE Requested FPGAs are available") + assignedFPGAs, err := h.FPGAManager.GetAndAssignAvailableFPGAs(numFPGAsRequestedInt, containerName) + log.G(h.Ctx).Info("\u2705 [CREATE CALL] ********* AFTER Requested FPGAs are available") + + // log the assigned FPGAs + log.G(h.Ctx).Info("\u2705 [CREATE CALL] Assigned FPGAs: ") + for _, fpgaSpec := range assignedFPGAs { + log.G(h.Ctx).Info("\u2705 [CREATE CALL] " + fpgaSpec.DeviceToMount) + } if err != nil { - HandleErrorAndRemoveData(h, w, "An error occurred during request of get and assign of an available GPU", err, podNamespace, podUID) - return dockerRunStructs, errors.New("An error occurred during request of get and assign of an available GPU") + log.G(h.Ctx).Error("\u274C [CREATE CALL] Error during request of get and assign of an available FPGA") + HandleErrorAndRemoveData(h, w, "An error occurred during request of get and assign of an available FPGA", err, podNamespace, podUID) + return dockerRunStructs, errors.New("An error occurred during request of get and assign of an available FPGA") } for _, fpgaSpec := range assignedFPGAs { fpgaArgs += " --device=" + fpgaSpec.DeviceToMount + ":" + fpgaSpec.DeviceToMount @@ -155,6 +169,13 @@ func (h *SidecarHandler) prepareDockerRuns(podData commonIL.RetrievedPodData, w } } + // if FPGA is requested, mount in read mode the /tools/Xilinx/ path in the container + if isFPGARequested { + envVars += " -v /tools/Xilinx/:/tools/Xilinx/:ro" + } + + log.G(h.Ctx).Info("\u2705 [POD FLOW] Before creating run command") + //envVars += " --network=host" cmd := []string{"run", "--user", "root", "-d", "--name", containerName} @@ -309,8 +330,6 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { return } - log.G(h.Ctx).Info("\u2705 [POD FLOW] Body request for pod creation read successfully", string(bodyBytes)) - var req []commonIL.RetrievedPodData err = json.Unmarshal(bodyBytes, &req) @@ -362,10 +381,16 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { return } + routeIP := strings.Split(podIpAddress, ".") + routeIP[3] = "251" + route := strings.Join(routeIP, ".") + + log.G(h.Ctx).Info("\u2705 [POD FLOW] Route IP is: " + route) + // inside the dind container, add the route to the pod IP ip route add 10.0.0.0/8 via 10.244.12.251 shell = exec.ExecTask{ Command: "docker", - Args: []string{"exec", dindContainerID, "ip", "route", "add", "10.0.0.0/8", "via", "10.244.12.251"}, + Args: []string{"exec", dindContainerID, "ip", "route", "add", "10.0.0.0/8", "via", route}, Shell: true, } @@ -375,24 +400,6 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { return } - // exec the command echo "nameserver 10.96.0.10" > /etc/resolv.conf - // shell = exec.ExecTask{ - // Command: "docker", - // Args: []string{"exec", dindContainerID, "-u", "0", "sh", "-c", "echo 'nameserver 10.96.0.10' > /etc/resolv.conf"}, - // Shell: true, - // } - - // log.G(h.Ctx).Info("\u2705 [POD FLOW] Executing command to add nameserver to resolv.conf file") - // log.G(h.Ctx).Info("\u2705 [POD FLOW] Command: " + "docker " + strings.Join(shell.Args, " ")) - - // _, err = shell.Execute() - // if err != nil { - // HandleErrorAndRemoveData(h, w, "An error occurred during the addition of the nameserver to the resolv.conf file", err, "", "") - // return - // } - - // log.G(h.Ctx).Info("\u2705 [POD FLOW] Nameserver added to resolv.conf file") - } // if the podDirectoryPath does not exist, create it @@ -532,6 +539,8 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { log.G(h.Ctx).Info("\u2705 [POD FLOW] All init containers created and executed successfully") } + log.G(h.Ctx).Info("\u2705 [POD FLOW] Start creating containers") + // create a file called containers_command.sh and write the containers commands to it, use WriteFile function containersCommand := "#!/bin/sh\n" diff --git a/pkg/docker/dindmanager/DindHandler.go b/pkg/docker/dindmanager/DindHandler.go index 08d39ad..676d5fd 100644 --- a/pkg/docker/dindmanager/DindHandler.go +++ b/pkg/docker/dindmanager/DindHandler.go @@ -145,6 +145,12 @@ func (a *DindManager) BuildDindContainers(nDindContainer int8) error { dindContainerArgs = append(dindContainerArgs, "-v", "/cvmfs:/cvmfs") } + if os.Getenv("FPGAENABLED") == "1" { + if _, err := os.Stat("/tools/Xilinx/"); err == nil { + dindContainerArgs = append(dindContainerArgs, "-v", "/tools/Xilinx/:/tools/Xilinx/:ro") + } + } + // add the network to the dind container dindContainerArgs = append(dindContainerArgs, "--network", randUID+"_dind_network") diff --git a/pkg/docker/fpgastrategies/AMDHandler.go b/pkg/docker/fpgastrategies/AMDHandler.go index 57ef08b..bbb65cb 100644 --- a/pkg/docker/fpgastrategies/AMDHandler.go +++ b/pkg/docker/fpgastrategies/AMDHandler.go @@ -157,7 +157,7 @@ func (a *FPGAManager) Discover() error { } for _, fpgaSpec := range a.FPGASpecsList { - if fpgaSpec.LogicUUID == fpgas[1] { + if fpgaSpec.LogicUUID == logicUUID { found = true break } @@ -208,6 +208,14 @@ func (a *FPGAManager) Assign(UUID string, containerID string) error { for i := range a.FPGASpecsList { if a.FPGASpecsList[i].LogicUUID == UUID { + // check if the BOOKKEEPING is disabled + disableBookkeeping := os.Getenv("FPGA_DISABLE_BOOKKEEPING") == "1" + if disableBookkeeping { + a.FPGASpecsList[i].ContainerID = containerID + a.FPGASpecsList[i].Available = false + break + } + if !a.FPGASpecsList[i].Available { return fmt.Errorf("FPGA with UUID %s is already in use by container %s", UUID, a.FPGASpecsList[i].ContainerID) } @@ -241,7 +249,7 @@ func (a *FPGAManager) Release(containerID string) error { return nil } -func (a *FPGAManager) GetAvailableFPGAs(numFPGAs int) ([]FPGASpecs, error) { +/* func (a *FPGAManager) GetAvailableFPGAs(numFPGAs int) ([]FPGASpecs, error) { a.FPGASpecsMutex.Lock() defer a.FPGASpecsMutex.Unlock() @@ -293,6 +301,66 @@ func (a *FPGAManager) GetAvailableFPGAs(numFPGAs int) ([]FPGASpecs, error) { } } + return nil, fmt.Errorf("Not enough available FPGAs. Requested: %d, Available: %d", numFPGAs, len(availableFPGAs)) +} */ + +func (a *FPGAManager) GetAvailableFPGAs(numFPGAs int) ([]FPGASpecs, error) { + + var availableFPGAs []FPGASpecs + + fmt.Println("Checking for available FPGAs") + + disableBookkeeping := os.Getenv("FPGA_DISABLE_BOOKKEEPING") == "1" + + fmt.Println(fmt.Sprintf("FPGA_DISABLE_BOOKKEEPING: %v", disableBookkeeping)) + + if disableBookkeeping { + fmt.Println("FPGA_DISABLE_BOOKKEEPING is set to 1. Disabling bookkeeping") + + fpgaUsage := make(map[string]int) + for _, fpga := range a.FPGASpecsList { + if fpga.ContainerID != "" { + fpgaUsage[fpga.BDF]++ + } else { + // If an unassigned FPGA is found, prioritize it + availableFPGAs = append(availableFPGAs, fpga) + } + } + + fmt.Println(fmt.Sprintf("FPGA Usage: %v", fpgaUsage)) + + if len(availableFPGAs) >= numFPGAs { + return availableFPGAs[:numFPGAs], nil + } + + // If not enough unassigned FPGAs, find the least assigned ones + sort.Slice(a.FPGASpecsList, func(i, j int) bool { + return fpgaUsage[a.FPGASpecsList[i].BDF] < fpgaUsage[a.FPGASpecsList[j].BDF] + }) + + for _, fpga := range a.FPGASpecsList { + if len(availableFPGAs) < numFPGAs { + availableFPGAs = append(availableFPGAs, fpga) + } else { + break + } + } + + if len(availableFPGAs) >= numFPGAs { + return availableFPGAs[:numFPGAs], nil + } + + return nil, fmt.Errorf("Not enough FPGAs available. Requested: %d, Found: %d", numFPGAs, len(availableFPGAs)) + } + + for _, fpgaSpec := range a.FPGASpecsList { + if fpgaSpec.Available { + availableFPGAs = append(availableFPGAs, fpgaSpec) + if len(availableFPGAs) == numFPGAs { + return availableFPGAs, nil + } + } + } return nil, fmt.Errorf("Not enough available FPGAs. Requested: %d, Available: %d", numFPGAs, len(availableFPGAs)) } From 4a63965d1de85ade1cabb6d58a8765faaa35e7f8 Mon Sep 17 00:00:00 2001 From: Giulio Bianchini Date: Thu, 22 Jan 2026 14:27:59 +0100 Subject: [PATCH 23/30] removed code related to handle old mesh overlay network --- pkg/docker/Create.go | 62 +++++++++----------------------------------- pkg/docker/Delete.go | 9 ++++++- 2 files changed, 20 insertions(+), 51 deletions(-) diff --git a/pkg/docker/Create.go b/pkg/docker/Create.go index 082cff6..a02a52b 100644 --- a/pkg/docker/Create.go +++ b/pkg/docker/Create.go @@ -2,6 +2,7 @@ package docker import ( "encoding/json" + "fmt" "io" "net/http" "os" @@ -25,7 +26,7 @@ import ( trace "go.opentelemetry.io/otel/trace" ) -func (h *SidecarHandler) prepareDockerRuns(podData commonIL.RetrievedPodData, w http.ResponseWriter, podIp string) ([]DockerRunStruct, error) { +func (h *SidecarHandler) prepareDockerRuns(podData commonIL.RetrievedPodData, w http.ResponseWriter) ([]DockerRunStruct, error) { var dockerRunStructs []DockerRunStruct var fpgaArgs string = "" @@ -92,6 +93,12 @@ func (h *SidecarHandler) prepareDockerRuns(podData commonIL.RetrievedPodData, w log.G(h.Ctx).Info("\u2705 Container " + containerName + " is not requesting a FPGA") } else { + if h.FPGAManager == nil { + log.G(h.Ctx).Error("\u274C [CREATE CALL] FPGA Manager is not initialized") + HandleErrorAndRemoveData(h, w, "FPGA Manager is not initialized", errors.New("FPGA Manager is not initialized"), podNamespace, podUID) + return dockerRunStructs, errors.New("FPGA Manager is not initialized") + } + isFPGARequested = true log.G(h.Ctx).Info("\u2705 Container " + containerName + " is requesting " + strconv.Itoa(int(numFPGAsRequested)) + " FPGA(s)") @@ -353,55 +360,15 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { podDirectoryPath := filepath.Join(wd, h.Config.DataRootFolder+"/"+podNamespace+"-"+podUID) - podIpAddress := "" + // log the pod specifics + log.G(h.Ctx).Info(fmt.Sprintf("\u2705 [POD FLOW] Pod specs: %+v", data.Pod)) + annotations := make([]string, 0, len(data.Pod.Annotations)) for key, value := range data.Pod.Annotations { annotations = append(annotations, key+"="+value) - - // if the key is interlink.eu/pod-ip and the value is not empty, set the pod IP to the DIND container - if key == "interlink.eu/pod-ip" && value != "" { - podIpAddress = value - } } log.G(h.Ctx).Info("\u2705 [POD FLOW] Pod Annotations are: " + strings.Join(annotations, ", ")) - log.G(h.Ctx).Info("\u2705 [POD FLOW] Pod IP Address is: " + podIpAddress) - - // if podIpAddress is != "" then exec the command docker network connect vk0 --ip podIpAddress - if podIpAddress != "" { - shell := exec.ExecTask{ - Command: "docker", - Args: []string{"network", "connect", "vk0", "--ip", podIpAddress, dindContainerID}, - Shell: true, - } - - _, err = shell.Execute() - if err != nil { - HandleErrorAndRemoveData(h, w, "An error occurred during the connection of the DIND container to the vk0 network", err, "", "") - return - } - - routeIP := strings.Split(podIpAddress, ".") - routeIP[3] = "251" - route := strings.Join(routeIP, ".") - - log.G(h.Ctx).Info("\u2705 [POD FLOW] Route IP is: " + route) - - // inside the dind container, add the route to the pod IP ip route add 10.0.0.0/8 via 10.244.12.251 - shell = exec.ExecTask{ - Command: "docker", - Args: []string{"exec", dindContainerID, "ip", "route", "add", "10.0.0.0/8", "via", route}, - Shell: true, - } - - _, err = shell.Execute() - if err != nil { - HandleErrorAndRemoveData(h, w, "An error occurred during the addition of the route to the pod IP", err, "", "") - return - } - - } - // if the podDirectoryPath does not exist, create it if _, err := os.Stat(podDirectoryPath); os.IsNotExist(err) { err = os.MkdirAll(podDirectoryPath, os.ModePerm) @@ -412,7 +379,7 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { } // call prepareDockerRuns to get the DockerRunStruct array - dockerRunStructs, err := h.prepareDockerRuns(data, w, podIpAddress) + dockerRunStructs, err := h.prepareDockerRuns(data, w) if err != nil { HandleErrorAndRemoveData(h, w, "An error occurred during preparing of docker run commmands", err, "", "") return @@ -544,11 +511,6 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { // create a file called containers_command.sh and write the containers commands to it, use WriteFile function containersCommand := "#!/bin/sh\n" - // if podIpAddress is != "" , add the echo "nameserver 10.0 " > /etc/resolv.conf command to the containers_command.sh - if podIpAddress != "" { - containersCommand += "echo 'nameserver 10.96.0.10' > /etc/resolv.conf" + "\n" - } - for _, container := range containers { containersCommand += container.Command + "\n" } diff --git a/pkg/docker/Delete.go b/pkg/docker/Delete.go index bcf903a..bed1c86 100644 --- a/pkg/docker/Delete.go +++ b/pkg/docker/Delete.go @@ -57,7 +57,14 @@ func (h *SidecarHandler) DeleteHandler(w http.ResponseWriter, r *http.Request) { for _, container := range pod.Spec.Containers { containerName := podNamespace + "-" + podUID + "-" + container.Name - h.FPGAManager.Release(containerName) + // if the FPGA manager is nil we don't need to release the container + if h.FPGAManager != nil { + // release the container from the FPGA manager + err = h.FPGAManager.Release(containerName) + if err != nil { + log.G(h.Ctx).Error("\u274C [DELETE CALL] Error releasing container " + containerName) + } + } } log.G(h.Ctx).Debug("\u2705 [DELETE CALL] Deleting POD " + podUID + "_dind") From f46d2c26b3d0ee9a711def1e132f1d8e3bc77972 Mon Sep 17 00:00:00 2001 From: Giulio Bianchini Date: Mon, 2 Mar 2026 15:13:48 +0100 Subject: [PATCH 24/30] updated Create to handle mesh network overlay --- cmd/main.go | 3 +- go.mod | 98 +++++----- go.sum | 223 ++++++++++------------ pkg/common/func.go | 95 +--------- pkg/common/types.go | 29 ++- pkg/docker/Create.go | 225 +++++++++++++++++++++- pkg/docker/func.go | 121 +++++++++++- pkg/docker/meshutils.go | 401 ++++++++++++++++++++++++++++++++++++++++ pkg/docker/types.go | 109 +++++++++++ 9 files changed, 1015 insertions(+), 289 deletions(-) create mode 100644 pkg/docker/meshutils.go diff --git a/cmd/main.go b/cmd/main.go index 5abdf9d..63cb735 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -18,7 +18,6 @@ import ( //"github.com/containerd/log" "github.com/containerd/log" "github.com/google/uuid" - commonIL "github.com/intertwin-eu/interlink-docker-plugin/pkg/common" docker "github.com/intertwin-eu/interlink-docker-plugin/pkg/docker" "github.com/intertwin-eu/interlink-docker-plugin/pkg/docker/dindmanager" "github.com/intertwin-eu/interlink-docker-plugin/pkg/docker/fpgastrategies" @@ -158,7 +157,7 @@ func initProvider(ctx context.Context) (func(context.Context) error, error) { func main() { logger := logrus.StandardLogger() - interLinkConfig, err := commonIL.NewInterLinkConfig() + interLinkConfig, err := docker.NewInterLinkConfig() if err != nil { log.G(context.Background()).Fatal(err) } diff --git a/go.mod b/go.mod index 6e9a349..dbdef64 100644 --- a/go.mod +++ b/go.mod @@ -1,84 +1,70 @@ module github.com/intertwin-eu/interlink-docker-plugin -go 1.22 - -toolchain go1.22.2 +go 1.24.0 require ( github.com/alexellis/go-execute v0.6.0 github.com/containerd/containerd v1.7.15 - github.com/docker/docker v26.0.1+incompatible + github.com/containerd/log v0.1.0 github.com/google/uuid v1.6.0 + github.com/interlink-hq/interlink v0.0.0-20251216132847-75253b484a35 github.com/sirupsen/logrus v1.9.3 github.com/virtual-kubelet/virtual-kubelet v1.11.0 - go.opentelemetry.io/otel v1.27.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 - go.opentelemetry.io/otel/sdk v1.27.0 - go.opentelemetry.io/otel/trace v1.27.0 - google.golang.org/grpc v1.64.0 + go.opentelemetry.io/otel v1.36.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0 + go.opentelemetry.io/otel/sdk v1.36.0 + go.opentelemetry.io/otel/trace v1.36.0 + google.golang.org/grpc v1.72.2 gopkg.in/yaml.v2 v2.4.0 - k8s.io/api v0.29.3 - k8s.io/apimachinery v0.29.3 - k8s.io/client-go v0.29.3 - sigs.k8s.io/yaml v1.3.0 + k8s.io/api v0.33.1 + k8s.io/apimachinery v0.33.1 + k8s.io/client-go v0.33.1 + sigs.k8s.io/yaml v1.4.0 ) require ( - github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/cenkalti/backoff/v4 v4.3.0 // indirect - github.com/containerd/log v0.1.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/distribution/reference v0.6.0 // indirect - github.com/docker/go-connections v0.5.0 // indirect - github.com/docker/go-units v0.5.0 // indirect + github.com/cenkalti/backoff/v5 v5.0.2 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect - github.com/felixge/httpsnoop v1.0.3 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-openapi/jsonpointer v0.20.0 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/swag v0.22.4 // indirect + github.com/go-openapi/swag v0.23.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.4 // indirect - github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/gofuzz v1.2.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect - github.com/imdario/mergo v0.3.16 // indirect + github.com/google/gnostic-models v0.6.9 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/moby/docker-image-spec v1.3.1 // indirect - github.com/moby/term v0.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/morikuni/aec v1.0.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b // indirect github.com/pkg/errors v0.9.1 // indirect github.com/spf13/pflag v1.0.5 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect - go.opentelemetry.io/otel/metric v1.27.0 // indirect - go.opentelemetry.io/proto/otlp v1.3.1 // indirect - golang.org/x/mod v0.15.0 // indirect - golang.org/x/net v0.25.0 // indirect - golang.org/x/oauth2 v0.20.0 // indirect - golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/term v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect - golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.18.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect - google.golang.org/protobuf v1.34.1 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 // indirect + go.opentelemetry.io/otel/metric v1.36.0 // indirect + go.opentelemetry.io/proto/otlp v1.7.0 // indirect + golang.org/x/net v0.40.0 // indirect + golang.org/x/oauth2 v0.27.0 // indirect + golang.org/x/sys v0.33.0 // indirect + golang.org/x/term v0.32.0 // indirect + golang.org/x/text v0.25.0 // indirect + golang.org/x/time v0.9.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250528174236-200df99c418a // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a // indirect + google.golang.org/protobuf v1.36.6 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - gotest.tools/v3 v3.5.1 // indirect - k8s.io/klog/v2 v2.110.1 // indirect - k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect - k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect - sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect + k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect ) diff --git a/go.sum b/go.sum index f3b0f36..cb07f45 100644 --- a/go.sum +++ b/go.sum @@ -1,67 +1,53 @@ -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= -github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/alexellis/go-execute v0.6.0 h1:FVGoudJnWSObwf9qmehbvVuvhK6g1UpKOCBjS+OUXEA= github.com/alexellis/go-execute v0.6.0/go.mod h1:nlg2F6XdYydUm1xXQMMiuibQCV1mveybBkNWfdNznjk= -github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= -github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8= +github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/containerd/containerd v1.7.15 h1:afEHXdil9iAm03BmhjzKyXnnEBtjaLJefdU7DV0IFes= github.com/containerd/containerd v1.7.15/go.mod h1:ISzRRTMF8EXNpJlTzyr2XMhN+j9K302C21/+cr3kUnY= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= -github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v26.0.1+incompatible h1:t39Hm6lpXuXtgkF0dm1t9a5HkbUfdGy6XbWexmGr+hA= -github.com/docker/docker v26.0.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= -github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= -github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ= -github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= -github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= -github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= +github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= -github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= -github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= +github.com/interlink-hq/interlink v0.0.0-20251216132847-75253b484a35 h1:RDRGKw1FCkWEtdRK3orbdtiSzmajB4vLtzVr01YUSec= +github.com/interlink-hq/interlink v0.0.0-20251216132847-75253b484a35/go.mod h1:iRxbA0yamfzqDsEqqJw+svTnwD+gWh6hM6kjxRFTMl8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -77,33 +63,24 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= -github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= -github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= -github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= -github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= -github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= -github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b h1:YWuSjZCQAPM8UUBLkYUk1e+rZcvWHJmFb6i6rM44Xs8= -github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= +github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= +github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= +github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -111,35 +88,39 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/virtual-kubelet/virtual-kubelet v1.11.0 h1:LOMcZQfP083xmYH9mYtyHAR+ybFbK1uMaRA+EtDcd1I= github.com/virtual-kubelet/virtual-kubelet v1.11.0/go.mod h1:WQfPHbIlzfhMNYkh6hFXF1ctGfNM8UJCYLYpLa/trxc= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 h1:x8Z78aZx8cOF0+Kkazoc7lwUNMGy0LrzEMxTm4BbTxg= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0/go.mod h1:62CPTSry9QZtOaSsE3tOzhx6LzDhHnXJ6xHeMNNiM6Q= -go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= -go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 h1:qFffATk0X+HD+f1Z8lswGiOQYKHRlzfmdJm0wEaVrFA= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0/go.mod h1:MOiCmryaYtc+V0Ei+Tx9o5S1ZjA7kzLucuVuyzBZloQ= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= -go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= -go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= -go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= -go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= -go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= -go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= -go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= -go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= +go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 h1:dNzwXjZKpMpE2JhmO+9HsPl42NIXFIFSUSSs0fiqra0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0/go.mod h1:90PoxvaEB5n6AOdZvi+yWJQoE95U8Dhhw2bSyRqnTD0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0 h1:JgtbA0xkWHnTmYk7YusopJFX6uleBmAuZ8n05NEh8nQ= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0/go.mod h1:179AK5aar5R3eS9FucPy6rggvU0g52cvKId8pv4+v0c= +go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= +go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= +go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= +go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= +go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= +go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= +go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= +go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= +go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os= +go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -147,59 +128,56 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.15.0 h1:SernR4v+D55NyBH2QiEQrlBAnj1ECL6AGrA5+dPaMY8= -golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= -golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= +golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= -golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= +golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= +golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ= -golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 h1:P8OJ/WCl/Xo4E4zoe4/bifHpSmmKwARqyqE4nW6J2GQ= -google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5/go.mod h1:RGnPtTG7r4i8sPlNyDeikXF99hMM+hN6QMm4ooG9g2g= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 h1:AgADTJarZTBqgjiUzRgfaBchgYB3/WFTC80GPwsMcRI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= -google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= -google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/genproto/googleapis/api v0.0.0-20250528174236-200df99c418a h1:SGktgSolFCo75dnHJF2yMvnns6jCmHFJ0vE4Vn2JKvQ= +google.golang.org/genproto/googleapis/api v0.0.0-20250528174236-200df99c418a/go.mod h1:a77HrdMjoeKbnd2jmgcWdaS++ZLZAEq3orIOAEIKiVw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a h1:v2PbRU4K3llS09c7zodFpNePeamkAwG3mPrAery9VeE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.72.2 h1:TdbGzwb82ty4OusHWepvFWGLgIbNo1/SUynEN0ssqv8= +google.golang.org/grpc v1.72.2/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -207,23 +185,24 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= -gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= -k8s.io/api v0.29.3 h1:2ORfZ7+bGC3YJqGpV0KSDDEVf8hdGQ6A03/50vj8pmw= -k8s.io/api v0.29.3/go.mod h1:y2yg2NTyHUUkIoTC+phinTnEa3KFM6RZ3szxt014a80= -k8s.io/apimachinery v0.29.3 h1:2tbx+5L7RNvqJjn7RIuIKu9XTsIZ9Z5wX2G22XAa5EU= -k8s.io/apimachinery v0.29.3/go.mod h1:hx/S4V2PNW4OMg3WizRrHutyB5la0iCUbZym+W0EQIU= -k8s.io/client-go v0.29.3 h1:R/zaZbEAxqComZ9FHeQwOh3Y1ZUs7FaHKZdQtIc2WZg= -k8s.io/client-go v0.29.3/go.mod h1:tkDisCvgPfiRpxGnOORfkljmS+UrW+WtXAy2fTvXJB0= -k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= -k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= -sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +k8s.io/api v0.33.1 h1:tA6Cf3bHnLIrUK4IqEgb2v++/GYUtqiu9sRVk3iBXyw= +k8s.io/api v0.33.1/go.mod h1:87esjTn9DRSRTD4fWMXamiXxJhpOIREjWOSjsW1kEHw= +k8s.io/apimachinery v0.33.1 h1:mzqXWV8tW9Rw4VeW9rEkqvnxj59k1ezDUl20tFK/oM4= +k8s.io/apimachinery v0.33.1/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= +k8s.io/client-go v0.33.1 h1:ZZV/Ks2g92cyxWkRRnfUDsnhNn28eFpt26aGc8KbXF4= +k8s.io/client-go v0.33.1/go.mod h1:JAsUrl1ArO7uRVFWfcj6kOomSlCv+JpvIsp6usAGefA= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= +sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/pkg/common/func.go b/pkg/common/func.go index 2aeda2b..c4abff4 100644 --- a/pkg/common/func.go +++ b/pkg/common/func.go @@ -4,10 +4,7 @@ import ( "context" "flag" "fmt" - "io" - "net/http" "os" - "strconv" "time" "go.opentelemetry.io/otel/attribute" @@ -19,13 +16,13 @@ import ( trace "go.opentelemetry.io/otel/trace" ) -var InterLinkConfigInst InterLinkConfig +var InterLinkConfigInst DockerConfig var Clientset *kubernetes.Clientset // TODO: implement factory design // NewInterLinkConfig returns a variable of type InterLinkConfig, used in many other functions and the first encountered error. -func NewInterLinkConfig() (InterLinkConfig, error) { +func NewInterLinkConfig() (DockerConfig, error) { if !InterLinkConfigInst.set { var path string verbose := flag.Bool("verbose", false, "Enable or disable Debug level logging") @@ -51,49 +48,21 @@ func NewInterLinkConfig() (InterLinkConfig, error) { if _, err := os.Stat(path); err != nil { log.G(context.Background()).Error("File " + path + " doesn't exist. You can set a custom path by exporting INTERLINKCONFIGPATH. Exiting...") - return InterLinkConfig{}, err + return DockerConfig{}, err } log.G(context.Background()).Info("\u2705 Loading InterLink config from " + path) yfile, err := os.ReadFile(path) if err != nil { log.G(context.Background()).Error("\u274C Error opening config file, exiting...") - return InterLinkConfig{}, err + return DockerConfig{}, err } yaml.Unmarshal(yfile, &InterLinkConfigInst) - if os.Getenv("INTERLINKURL") != "" { - InterLinkConfigInst.Interlinkurl = os.Getenv("INTERLINKURL") - } - - if os.Getenv("SIDECARURL") != "" { - InterLinkConfigInst.Sidecarurl = os.Getenv("SIDECARURL") - } - - if os.Getenv("INTERLINKPORT") != "" { - InterLinkConfigInst.Interlinkport = os.Getenv("INTERLINKPORT") - } - - if os.Getenv("SIDECARPORT") != "" { - InterLinkConfigInst.Sidecarport = os.Getenv("SIDECARPORT") - } - - if os.Getenv("SBATCHPATH") != "" { - InterLinkConfigInst.Sbatchpath = os.Getenv("SBATCHPATH") - } - - if os.Getenv("SCANCELPATH") != "" { - InterLinkConfigInst.Scancelpath = os.Getenv("SCANCELPATH") - } - - if os.Getenv("POD_IP") != "" { - InterLinkConfigInst.PodIP = os.Getenv("POD_IP") - } - if os.Getenv("TSOCKS") != "" { if os.Getenv("TSOCKS") != "true" && os.Getenv("TSOCKS") != "false" { fmt.Println("export TSOCKS as true or false") - return InterLinkConfig{}, err + return DockerConfig{}, err } if os.Getenv("TSOCKS") == "true" { InterLinkConfigInst.Tsocks = true @@ -106,69 +75,17 @@ func NewInterLinkConfig() (InterLinkConfig, error) { path = os.Getenv("TSOCKSPATH") if _, err := os.Stat(path); err != nil { log.G(context.Background()).Error("File " + path + " doesn't exist. You can set a custom path by exporting TSOCKSPATH. Exiting...") - return InterLinkConfig{}, err + return DockerConfig{}, err } InterLinkConfigInst.Tsockspath = path } - if os.Getenv("VKTOKENFILE") != "" { - path = os.Getenv("VKTOKENFILE") - if _, err := os.Stat(path); err != nil { - log.G(context.Background()).Error("File " + path + " doesn't exist. You can set a custom path by exporting VKTOKENFILE. Exiting...") - return InterLinkConfig{}, err - } - - InterLinkConfigInst.VKTokenFile = path - } else { - path = InterLinkConfigInst.DataRootFolder + "token" - InterLinkConfigInst.VKTokenFile = path - } - InterLinkConfigInst.set = true } return InterLinkConfigInst, nil } -// PingInterLink pings the InterLink API and returns true if there's an answer. The second return value is given by the answer provided by the API. -func PingInterLink(ctx context.Context) (bool, int, error) { - log.G(ctx).Info("Pinging: " + InterLinkConfigInst.Interlinkurl + ":" + InterLinkConfigInst.Interlinkport + "/pinglink") - retVal := -1 - req, err := http.NewRequest(http.MethodPost, InterLinkConfigInst.Interlinkurl+":"+InterLinkConfigInst.Interlinkport+"/pinglink", nil) - - if err != nil { - log.G(ctx).Error(err) - } - - token, err := os.ReadFile(InterLinkConfigInst.VKTokenFile) // just pass the file name - if err != nil { - log.G(ctx).Error(err) - return false, retVal, err - } - req.Header.Add("Authorization", "Bearer "+string(token)) - resp, err := http.DefaultClient.Do(req) - if err != nil { - return false, retVal, err - } - - if resp.StatusCode == http.StatusOK { - retBytes, err := io.ReadAll(resp.Body) - if err != nil { - log.G(ctx).Error(err) - return false, retVal, err - } - retVal, err = strconv.Atoi(string(retBytes)) - if err != nil { - log.G(ctx).Error(err) - return false, retVal, err - } - return true, retVal, nil - } else { - log.G(ctx).Error("server error: " + fmt.Sprint(resp.StatusCode)) - return false, retVal, nil - } -} - func WithHTTPReturnCode(code int) SpanOption { return func(cfg *SpanConfig) { cfg.HTTPReturnCode = code diff --git a/pkg/common/types.go b/pkg/common/types.go index 9f8a765..49b4fef 100644 --- a/pkg/common/types.go +++ b/pkg/common/types.go @@ -6,6 +6,27 @@ import ( v1 "k8s.io/api/core/v1" ) +type DockerConfig struct { + VKConfigPath string `yaml:"VKConfigPath"` + Socket string `yaml:"Socket"` + ExportPodData bool `yaml:"ExportPodData"` + Commandprefix string `yaml:"CommandPrefix"` + ImagePrefix string `yaml:"ImagePrefix"` + DataRootFolder string `yaml:"DataRootFolder"` + Namespace string `yaml:"Namespace"` + Tsocks bool `yaml:"Tsocks"` + Tsockspath string `yaml:"TsocksPath"` + Tsockslogin string `yaml:"TsocksLoginNode"` + BashPath string `yaml:"BashPath"` + VerboseLogging bool `yaml:"VerboseLogging"` + ErrorsOnlyLogging bool `yaml:"ErrorsOnlyLogging"` + SingularityDefaultOptions []string `yaml:"SingularityDefaultOptions"` + SingularityPrefix string `yaml:"SingularityPrefix"` + SingularityPath string `yaml:"SingularityPath"` + EnableProbes bool `yaml:"EnableProbes"` + set bool +} + // PodCreateRequests is a struct holding data for a create request. Retrieved ConfigMaps and Secrets are held along the Pod description itself. type PodCreateRequests struct { Pod v1.Pod `json:"pod"` @@ -31,14 +52,6 @@ type RetrievedContainer struct { EmptyDirs []string `json:"emptyDirs"` } -// RetrievedPoData is used in InterLink to rearrange data structure in a suitable way for the sidecar -type RetrievedPodData struct { - Pod v1.Pod `json:"pod"` - Containers []RetrievedContainer `json:"container"` - InitContainers []RetrievedContainer `json:"initContainer"` - JobScript string `json:"jobScript"` -} - // InterLinkConfig holds the whole configuration type InterLinkConfig struct { VKConfigPath string `yaml:"VKConfigPath"` diff --git a/pkg/docker/Create.go b/pkg/docker/Create.go index a02a52b..77eba4f 100644 --- a/pkg/docker/Create.go +++ b/pkg/docker/Create.go @@ -16,7 +16,7 @@ import ( "errors" - commonIL "github.com/intertwin-eu/interlink-docker-plugin/pkg/common" + commonIL "github.com/interlink-hq/interlink/pkg/interlink" "github.com/intertwin-eu/interlink-docker-plugin/pkg/docker/dindmanager" "path/filepath" @@ -337,7 +337,7 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { return } - var req []commonIL.RetrievedPodData + var req commonIL.RetrievedPodData err = json.Unmarshal(bodyBytes, &req) if err != nil { @@ -353,7 +353,10 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { log.G(h.Ctx).Info("\u2705 [POD FLOW] Request data unmarshalled successfully and current working directory detected") - for _, data := range req { + var newReq []commonIL.RetrievedPodData + newReq = []commonIL.RetrievedPodData{req} + + for _, data := range newReq { podUID := string(data.Pod.UID) podNamespace := string(data.Pod.Namespace) @@ -385,6 +388,124 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { return } + if preExecAnnotations, ok := data.Pod.Annotations["slurm-job.vk.io/pre-exec"]; ok { + if strings.Contains(preExecAnnotations, "cat <<'EOFMESH' > $TMPDIR/mesh.sh") { + meshScript, err := extractHeredoc(preExecAnnotations, "EOFMESH") + if err == nil && meshScript != "" { + log.G(h.Ctx).Info("โœ… [POD FLOW] Mesh.sh script extracted from annotation") + + // Extract the binary download section (before EOFSLIRP) + downloadSection := extractDownloadSection(meshScript) + + // Extract the WG_IFACE variable definition from outer script + wgIfaceDefinition := extractWGIfaceDefinition(meshScript) + + // Extract the WireGuard config section + wgConfigSection := extractWGConfigSection(meshScript) + + // Extract just the inner script (EOFSLIRP content) + innerScript, err := extractHeredoc(meshScript, "EOFSLIRP") + if err == nil && innerScript != "" { + finalDNSNameserver, finalDNSSearch := extractFinalDNSConfig(innerScript) + + // Store these for DIND configuration + h.FinalDNSNameserver = finalDNSNameserver + h.FinalDNSSearch = finalDNSSearch + + // Clean the header from inner script (remove duplicate shebang, set commands, etc.) + innerScript = cleanInnerScriptHeader(innerScript) + + // Remove the final DNS config from inner script (we'll apply it to DIND) + innerScript = removeFinalDNSConfig(innerScript) + + // Remove WG_IFACE definition from inner script (it's already extracted) + innerScript = removeWGIfaceDefinition(innerScript) + + // Remove the slirp4netns execution at the end of inner script + innerScript = removeSlirp4netnsExecution(innerScript) + + // Replace command execution with sleep infinity + innerScript = strings.Replace(innerScript, "$@", "sleep infinity", -1) + + // Build the complete script with correct order + meshScript = `#!/bin/bash +set -e +set -m + +export PATH=$PATH:$PWD:/usr/sbin:/sbin + +# Set up temporary directory +TMPDIR=${SLIRP_TMPDIR:-/tmp/.slirp.$RANDOM$RANDOM} +mkdir -p $TMPDIR +cd $TMPDIR + +` + downloadSection + ` + +` + wgIfaceDefinition + ` + +` + wgConfigSection + ` + +` + innerScript + } else { + // Fallback: just clean up the outer script + meshScript = removeSlirp4netnsDownload(meshScript) + meshScript = removeUnshareWrapper(meshScript) + meshScript = strings.Replace(meshScript, "$@", "sleep infinity", -1) + meshScript = removeSlirp4netnsExecution(meshScript) + } + + log.G(h.Ctx).Info("โœ… [POD FLOW] Mesh.sh script cleaned and simplified") + + // Create a special network overlay container name + networkContainerName := podNamespace + "-" + podUID + "-network-overlay" + + // Save the modified mesh script to the pod directory + meshScriptPath := filepath.Join(podDirectoryPath, "mesh.sh") + err = os.WriteFile(meshScriptPath, []byte(meshScript), 0755) + if err != nil { + HandleErrorAndRemoveData(h, w, "An error occurred during the creation of mesh.sh script", err, podNamespace, podUID) + return + } + + log.G(h.Ctx).Info("โœ… [POD FLOW] Mesh.sh script saved to " + meshScriptPath) + + // Prepare docker run command for the network overlay container + networkCmd := []string{ + "run", + "--user", "root", + "-d", + "--name", networkContainerName, + "--privileged", + "--cap-add", "NET_ADMIN", + "--cap-add", "SYS_ADMIN", + "--cap-add", "NET_RAW", + "-v", meshScriptPath + ":/mesh.sh:ro", + "-v", podDirectoryPath + ":" + podDirectoryPath, + "-v", "/tmp:/tmp", + "--network", "host", + "nicolaka/netshoot", + "/bin/bash", "/mesh.sh", + } + + // Prepend this as the first init container + dockerRunStructs = append([]DockerRunStruct{{ + Name: networkContainerName, + Command: "docker " + strings.Join(networkCmd, " "), + IsInitContainer: false, + FpgaArgs: "", + }}, dockerRunStructs...) + + log.G(h.Ctx).Info("โœ… [POD FLOW] Network overlay container prepared: " + networkContainerName) + } else { + log.G(h.Ctx).Error("โŒ [POD FLOW] Failed to extract mesh.sh script from annotation") + if err != nil { + HandleErrorAndRemoveData(h, w, "Failed to extract mesh.sh heredoc", err, podNamespace, podUID) + return + } + } + } + } + log.G(h.Ctx).Info("\u2705 [POD FLOW] Docker run commands prepared successfully") // from dockerRunStructs, create two arrays: one for initContainers and one for containers @@ -420,6 +541,64 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { return } + isMeshScriptPresent := false + + // Configure DNS on the DIND container if mesh script is present + if preExecAnnotations, ok := data.Pod.Annotations["slurm-job.vk.io/pre-exec"]; ok { + if strings.Contains(preExecAnnotations, "cat <<'EOFMESH'") { + log.G(h.Ctx).Info("โœ… [POD FLOW] Configuring DNS on DIND container for cluster connectivity") + + isMeshScriptPresent = true + + // Use the extracted final DNS configuration + dnsNameserver := h.FinalDNSNameserver + dnsSearch := h.FinalDNSSearch + + if dnsNameserver == "" { + dnsNameserver = "10.96.0.10" // Fallback + } + if dnsSearch == "" { + dnsSearch = "default.svc.cluster.local svc.cluster.local cluster.local" // Fallback + } + + // Create DNS configuration script for DIND + dnsConfigScript := `#!/bin/sh +set -e + +# Backup original resolv.conf +cp /etc/resolv.conf /etc/resolv.conf.backup 2>/dev/null || true + +# Create new resolv.conf with cluster DNS +cat > /etc/resolv.conf << EOF +nameserver 8.8.8.8 +search ` + dnsSearch + ` +EOF + +echo "DNS configured for cluster connectivity" +` + + // Write DNS config script to pod directory + dnsScriptPath := filepath.Join(podDirectoryPath, "configure-dns.sh") + err = os.WriteFile(dnsScriptPath, []byte(dnsConfigScript), 0755) + if err != nil { + log.G(h.Ctx).Warning("โš ๏ธ Failed to create DNS config script: " + err.Error()) + } else { + // Execute DNS configuration on DIND container + dnsExecCmd := exec.ExecTask{ + Command: "docker", + Args: []string{"exec", string(data.Pod.UID) + "_dind", "sh", dnsScriptPath}, + Shell: true, + } + _, err = dnsExecCmd.Execute() + if err != nil { + log.G(h.Ctx).Warning("โš ๏ธ Failed to configure DNS on DIND container: " + err.Error()) + } else { + log.G(h.Ctx).Info("โœ… [POD FLOW] DNS configured on DIND container successfully with NS: " + dnsNameserver + ", Search: " + dnsSearch) + } + } + } + } + createResponse := CreateStruct{PodUID: string(data.Pod.UID), PodJID: dindContainerID} createResponseBytes, err := json.Marshal(createResponse) if err != nil { @@ -511,11 +690,48 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { // create a file called containers_command.sh and write the containers commands to it, use WriteFile function containersCommand := "#!/bin/sh\n" + if isMeshScriptPresent { + + dnsNameserver := h.FinalDNSNameserver + dnsSearch := h.FinalDNSSearch + + if dnsNameserver == "" { + dnsNameserver = "10.96.0.10" // Fallback + } + if dnsSearch == "" { + dnsSearch = "default.svc.cluster.local svc.cluster.local cluster.local" // Fallback + } + + // Add a delay to ensure network container is ready + containersCommand += "echo 'Waiting for network overlay to be ready...'\n" + containersCommand += "sleep 10\n" + containersCommand += "echo 'Starting containers...'\n\n" + + for _, container := range containers { + containersCommand += "# Start container: " + container.Name + "\n" + containersCommand += container.Command + "\n" + containersCommand += "sleep 2\n" + + // Configure DNS inside the container + containersCommand += "# Configure DNS for container: " + container.Name + "\n" + containersCommand += "docker exec " + container.Name + " sh -c '\n" + containersCommand += "cp /etc/resolv.conf /etc/resolv.conf.backup 2>/dev/null || true\n" + containersCommand += "cat > /etc/resolv.conf << EOF\n" + containersCommand += "nameserver " + dnsNameserver + "\n" + containersCommand += "search " + dnsSearch + "\n" + containersCommand += "EOF\n" + containersCommand += "' || echo 'Warning: Could not configure DNS for " + container.Name + "'\n" + containersCommand += "echo 'DNS configured for container: " + container.Name + "'\n\n" + } + } + for _, container := range containers { containersCommand += container.Command + "\n" + containersCommand += "sleep 30\n" } err = os.WriteFile(podDirectoryPath+"/containers_command.sh", []byte(containersCommand), 0644) if err != nil { + log.G(h.Ctx).Error("\u274C [POD FLOW] Error writing containers command script: " + err.Error()) HandleErrorAndRemoveData(h, w, "An error occurred during the creation of the container commands script.", err, "", "") return } @@ -527,8 +743,11 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { Args: []string{"exec", string(data.Pod.UID) + "_dind", "/bin/sh", podDirectoryPath + "/containers_command.sh"}, } + log.G(h.Ctx).Info("\u2705 [POD FLOW] Executing containers creation script inside DIND container; command to execute: docker " + strings.Join(shell.Args, " ")) + _, err = shell.Execute() if err != nil { + log.G(h.Ctx).Error("\u274C [POD FLOW] Error executing containers command script: " + err.Error()) HandleErrorAndRemoveData(h, w, "An error occurred during the execution of the container command script", err, "", "") return } diff --git a/pkg/docker/func.go b/pkg/docker/func.go index e2d5431..52909a8 100644 --- a/pkg/docker/func.go +++ b/pkg/docker/func.go @@ -3,27 +3,130 @@ package docker import ( "context" "errors" + "flag" + "fmt" "os" "path/filepath" "strings" + "time" exec2 "github.com/alexellis/go-execute/pkg/v1" "github.com/containerd/containerd/log" + "go.opentelemetry.io/otel/attribute" + trace "go.opentelemetry.io/otel/trace" + "gopkg.in/yaml.v2" v1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" - commonIL "github.com/intertwin-eu/interlink-docker-plugin/pkg/common" + commonIL "github.com/interlink-hq/interlink/pkg/interlink" "github.com/intertwin-eu/interlink-docker-plugin/pkg/docker/dindmanager" "github.com/intertwin-eu/interlink-docker-plugin/pkg/docker/fpgastrategies" ) +var InterLinkConfigInst DockerConfig +var Clientset *kubernetes.Clientset + +// TODO: implement factory design + +// NewInterLinkConfig returns a variable of type InterLinkConfig, used in many other functions and the first encountered error. +func NewInterLinkConfig() (DockerConfig, error) { + if !InterLinkConfigInst.set { + var path string + verbose := flag.Bool("verbose", false, "Enable or disable Debug level logging") + errorsOnly := flag.Bool("errorsonly", false, "Prints only errors if enabled") + InterLinkConfigPath := flag.String("interlinkconfigpath", "", "Path to InterLink config") + flag.Parse() + + if *verbose { + InterLinkConfigInst.VerboseLogging = true + InterLinkConfigInst.ErrorsOnlyLogging = false + } else if *errorsOnly { + InterLinkConfigInst.VerboseLogging = false + InterLinkConfigInst.ErrorsOnlyLogging = true + } + + if *InterLinkConfigPath != "" { + path = *InterLinkConfigPath + } else if os.Getenv("INTERLINKCONFIGPATH") != "" { + path = os.Getenv("INTERLINKCONFIGPATH") + } else { + path = "/etc/interlink/InterLinkConfig.yaml" + } + + if _, err := os.Stat(path); err != nil { + log.G(context.Background()).Error("File " + path + " doesn't exist. You can set a custom path by exporting INTERLINKCONFIGPATH. Exiting...") + return DockerConfig{}, err + } + + log.G(context.Background()).Info("\u2705 Loading InterLink config from " + path) + yfile, err := os.ReadFile(path) + if err != nil { + log.G(context.Background()).Error("\u274C Error opening config file, exiting...") + return DockerConfig{}, err + } + yaml.Unmarshal(yfile, &InterLinkConfigInst) + + if os.Getenv("TSOCKS") != "" { + if os.Getenv("TSOCKS") != "true" && os.Getenv("TSOCKS") != "false" { + fmt.Println("export TSOCKS as true or false") + return DockerConfig{}, err + } + if os.Getenv("TSOCKS") == "true" { + InterLinkConfigInst.Tsocks = true + } else { + InterLinkConfigInst.Tsocks = false + } + } + + if os.Getenv("TSOCKSPATH") != "" { + path = os.Getenv("TSOCKSPATH") + if _, err := os.Stat(path); err != nil { + log.G(context.Background()).Error("File " + path + " doesn't exist. You can set a custom path by exporting TSOCKSPATH. Exiting...") + return DockerConfig{}, err + } + + InterLinkConfigInst.Tsockspath = path + } + + InterLinkConfigInst.set = true + } + return InterLinkConfigInst, nil +} + +func WithHTTPReturnCode(code int) SpanOption { + return func(cfg *SpanConfig) { + cfg.HTTPReturnCode = code + cfg.SetHTTPCode = true + } +} + +func SetDurationSpan(startTime int64, span trace.Span, opts ...SpanOption) { + endTime := time.Now().UnixMicro() + config := &SpanConfig{} + + for _, opt := range opts { + opt(config) + } + + duration := endTime - startTime + span.SetAttributes(attribute.Int64("end.timestamp", endTime), + attribute.Int64("duration", duration)) + + if config.SetHTTPCode { + span.SetAttributes(attribute.Int("exit.code", config.HTTPReturnCode)) + } +} + type SidecarHandler struct { - Config commonIL.InterLinkConfig - Ctx context.Context - DindManager dindmanager.DindManagerInterface - FPGAManager fpgastrategies.FPGAManagerInterface + Config DockerConfig + Ctx context.Context + DindManager dindmanager.DindManagerInterface + FPGAManager fpgastrategies.FPGAManagerInterface + FinalDNSNameserver string + FinalDNSSearch string } -func parseContainerCommandAndReturnArgs(Ctx context.Context, config commonIL.InterLinkConfig, podUID string, podNamespace string, container v1.Container) ([]string, []string, []string, error) { +func parseContainerCommandAndReturnArgs(Ctx context.Context, config DockerConfig, podUID string, podNamespace string, container v1.Container) ([]string, []string, []string, error) { dirPath := config.DataRootFolder + podNamespace + "-" + podUID if _, err := os.Stat(dirPath); os.IsNotExist(err) { @@ -83,7 +186,7 @@ func parseContainerCommandAndReturnArgs(Ctx context.Context, config commonIL.Int } } -func prepareMounts(Ctx context.Context, config commonIL.InterLinkConfig, data commonIL.RetrievedPodData, container v1.Container) (string, error) { +func prepareMounts(Ctx context.Context, config DockerConfig, data commonIL.RetrievedPodData, container v1.Container) (string, error) { mountedData := "" podUID := string(data.Pod.UID) @@ -94,7 +197,7 @@ func prepareMounts(Ctx context.Context, config commonIL.InterLinkConfig, data co return "", err } - allContainers := append(data.Containers, data.InitContainers...) + allContainers := append(data.Containers) for _, cont := range allContainers { @@ -148,7 +251,7 @@ func prepareMounts(Ctx context.Context, config commonIL.InterLinkConfig, data co return mountedData, nil } -func mountData(Ctx context.Context, config commonIL.InterLinkConfig, pod v1.Pod, data interface{}, container v1.Container) ([]string, error) { +func mountData(Ctx context.Context, config DockerConfig, pod v1.Pod, data interface{}, container v1.Container) ([]string, error) { wd, err := os.Getwd() if err != nil { log.G(Ctx).Error(err) diff --git a/pkg/docker/meshutils.go b/pkg/docker/meshutils.go new file mode 100644 index 0000000..13c943c --- /dev/null +++ b/pkg/docker/meshutils.go @@ -0,0 +1,401 @@ +package docker + +import ( + "fmt" + "strings" +) + +func cleanInnerScriptHeader(content string) string { + lines := strings.Split(content, "\n") + var cleanedLines []string + skipHeader := true + + for _, line := range lines { + trimmed := strings.TrimSpace(line) + + // Skip the header section of the inner script + if skipHeader { + // Skip shebang + if strings.HasPrefix(trimmed, "#!") { + continue + } + // Skip set commands + if trimmed == "set -e" || trimmed == "set -euo pipefail" { + continue + } + // Skip PATH export that duplicates outer script + if strings.HasPrefix(trimmed, "export PATH=$TMPDIR") || + strings.Contains(trimmed, "Ensure PATH includes tmpdir") { + continue + } + // Skip empty lines at start + if trimmed == "" { + continue + } + // Once we hit actual content, stop skipping + if trimmed != "" && !strings.HasPrefix(trimmed, "#") { + skipHeader = false + } + } + + if !skipHeader { + cleanedLines = append(cleanedLines, line) + } + } + + return strings.Join(cleanedLines, "\n") +} + +func extractWGIfaceDefinition(content string) string { + // Look for WG_IFACE definition in the outer script first + lines := strings.Split(content, "\n") + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "WG_IFACE=") || strings.HasPrefix(trimmed, "export WG_IFACE=") { + return trimmed + } + // Also check in comments or before EOFSLIRP + if strings.Contains(trimmed, "# Set WireGuard interface name") { + // Next non-empty line should have the definition + for i, l := range lines { + if strings.TrimSpace(l) == trimmed { + if i+1 < len(lines) { + nextLine := strings.TrimSpace(lines[i+1]) + if strings.HasPrefix(nextLine, "WG_IFACE=") { + return nextLine + } + } + } + } + } + } + + // If not found in outer script, try to find in EOFSLIRP + innerScript, err := extractHeredoc(content, "EOFSLIRP") + if err == nil { + lines = strings.Split(innerScript, "\n") + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "WG_IFACE=") { + return trimmed + } + } + } + + return "" +} + +func removeWGIfaceDefinition(content string) string { + lines := strings.Split(content, "\n") + var cleanedLines []string + + for _, line := range lines { + trimmed := strings.TrimSpace(line) + // Skip WG_IFACE definition lines + if strings.HasPrefix(trimmed, "WG_IFACE=") || + strings.HasPrefix(trimmed, "export WG_IFACE=") || + trimmed == "# Get WireGuard interface name from parent" { + continue + } + cleanedLines = append(cleanedLines, line) + } + + return strings.Join(cleanedLines, "\n") +} + +func extractWGConfigSection(content string) string { + // Find the WireGuard config creation section + wgStart := strings.Index(content, "# Create WireGuard config") + if wgStart == -1 { + wgStart = strings.Index(content, "cat <<'EOFWG'") + } + + if wgStart == -1 { + return "" + } + + // Find where this section ends (before "Generate the execution script") + wgEnd := strings.Index(content[wgStart:], "# Generate the execution script") + if wgEnd == -1 { + wgEnd = strings.Index(content[wgStart:], "cat <<'EOFSLIRP'") + } + + if wgEnd == -1 { + return "" + } + + wgSection := strings.TrimSpace(content[wgStart : wgStart+wgEnd]) + + // Ensure the config file is created in TMPDIR with explicit path + wgSection = strings.Replace(wgSection, + "> $WG_IFACE.conf", + "> $TMPDIR/$WG_IFACE.conf", + 1) + + return wgSection +} + +func extractDownloadSection(content string) string { + // Find the start of downloads + downloadStart := strings.Index(content, "echo \"=== Downloading binaries") + if downloadStart == -1 { + downloadStart = strings.Index(content, "# Download wstunnel") + } + + if downloadStart == -1 { + return "" + } + + // Find the end of downloads (before the WireGuard config creation) + downloadEnd := strings.Index(content[downloadStart:], "# Create WireGuard config") + if downloadEnd == -1 { + downloadEnd = strings.Index(content[downloadStart:], "cat <<'EOFWG'") + } + + if downloadEnd == -1 { + return "" + } + + downloadSection := content[downloadStart : downloadStart+downloadEnd] + + // Remove slirp4netns download from the section + lines := strings.Split(downloadSection, "\n") + var cleanedLines []string + skipSlirp := false + + for _, line := range lines { + trimmed := strings.TrimSpace(line) + + // Start skipping at slirp4netns download + if strings.Contains(trimmed, "# Download slirp4netns") || + strings.Contains(trimmed, "echo \"Downloading slirp4netns...\"") { + skipSlirp = true + continue + } + + // Stop skipping after the slirp4netns section + if skipSlirp && (strings.HasPrefix(trimmed, "# ") && !strings.Contains(trimmed, "slirp")) { + skipSlirp = false + } + + // Skip lines related to slirp4netns + if strings.Contains(line, "slirp4netns") { + continue + } + + if !skipSlirp { + cleanedLines = append(cleanedLines, line) + } + } + + return strings.Join(cleanedLines, "\n") +} + +func extractHeredoc(content, marker string) (string, error) { + // Find the start of the heredoc + startPattern := fmt.Sprintf("cat <<'%s'", marker) + startIdx := strings.Index(content, startPattern) + if startIdx == -1 { + return "", fmt.Errorf("heredoc start marker not found") + } + + // Find the line after the cat command (start of actual content) + contentStart := strings.Index(content[startIdx:], "\n") + if contentStart == -1 { + return "", fmt.Errorf("invalid heredoc format") + } + contentStart += startIdx + 1 + + // Find the end marker + endMarker := "\n" + marker + endIdx := strings.Index(content[contentStart:], endMarker) + if endIdx == -1 { + return "", fmt.Errorf("heredoc end marker not found") + } + + // Extract the content between start and end markers + return content[contentStart : contentStart+endIdx], nil +} + +func removeHeredoc(content, marker string) string { + // Find the start of the heredoc + startPattern := fmt.Sprintf("cat <<'%s'", marker) + startIdx := strings.Index(content, startPattern) + if startIdx == -1 { + return content // No heredoc found, return as-is + } + + // Find the line after the cat command (start of actual content) + contentStart := strings.Index(content[startIdx:], "\n") + if contentStart == -1 { + return content // Invalid heredoc format + } + contentStart += startIdx + 1 + + // Find the end marker + endMarker := "\n" + marker + endIdx := strings.Index(content[contentStart:], endMarker) + if endIdx == -1 { + return content // Heredoc end marker not found + } + + // Calculate the actual end position (after the end marker) + heredocEnd := contentStart + endIdx + len(endMarker) + + // Skip trailing newline if present + if heredocEnd < len(content) && content[heredocEnd] == '\n' { + heredocEnd++ + } + + // Remove the heredoc block and return + return content[:startIdx] + content[heredocEnd:] +} + +func removeSlirp4netnsDownload(content string) string { + // Remove the slirp4netns download section + slirpDownloadStart := strings.Index(content, "# Download slirp4netns") + if slirpDownloadStart == -1 { + return content + } + + // Find the end of this download block (next echo or next section) + slirpDownloadEnd := strings.Index(content[slirpDownloadStart:], "# Check if iproute2") + if slirpDownloadEnd == -1 { + slirpDownloadEnd = strings.Index(content[slirpDownloadStart:], "\n\n") + } + + if slirpDownloadEnd != -1 { + return content[:slirpDownloadStart] + content[slirpDownloadStart+slirpDownloadEnd:] + } + + return content +} + +func removeUnshareWrapper(content string) string { + // Remove the entire unshare mode detection and execution section + unshareStart := strings.Index(content, "# Detect best unshare strategy") + if unshareStart == -1 { + unshareStart = strings.Index(content, "echo \"=== Starting network namespace ===\"") + } + + if unshareStart == -1 { + return content + } + + // Remove everything from the unshare section to the end + return content[:unshareStart] +} + +func removeSlirp4netnsExecution(content string) string { + // Remove slirp4netns execution lines + lines := strings.Split(content, "\n") + var cleanedLines []string + + skipUntilBlank := false + for _, line := range lines { + trimmed := strings.TrimSpace(line) + + // Skip slirp4netns related lines + if strings.Contains(trimmed, "./slirp4netns") || + strings.Contains(trimmed, "SLIRPPID") || + strings.Contains(trimmed, "slirp4netns_") || + strings.Contains(trimmed, "Bring the main job to foreground") || + strings.Contains(trimmed, "fg 1") { + skipUntilBlank = true + continue + } + + // Skip comments about slirp4netns + if strings.Contains(trimmed, "Create the tap0 device with slirp4netns") || + strings.Contains(trimmed, "Starting slirp4netns") || + strings.Contains(trimmed, "Wait a bit for slirp4netns") { + continue + } + + if skipUntilBlank && trimmed == "" { + skipUntilBlank = false + continue + } + + if !skipUntilBlank { + cleanedLines = append(cleanedLines, line) + } + } + + return strings.Join(cleanedLines, "\n") +} + +func extractFinalDNSConfig(content string) (nameserver string, searchDomains string) { + // Look for the final DNS configuration (the simple echo commands) + // Pattern: echo "nameserver X.X.X.X" > /etc/resolv.conf + // echo "search ..." >> /etc/resolv.conf + + lines := strings.Split(content, "\n") + for i, line := range lines { + trimmed := strings.TrimSpace(line) + + // Find the nameserver line + if strings.Contains(trimmed, "echo") && strings.Contains(trimmed, "nameserver") && strings.Contains(trimmed, "> /etc/resolv.conf") { + // Extract nameserver IP + // Pattern: echo "nameserver 10.43.0.10" > /etc/resolv.conf + start := strings.Index(trimmed, "nameserver") + if start != -1 { + afterNameserver := trimmed[start+len("nameserver"):] + fields := strings.Fields(afterNameserver) + if len(fields) > 0 { + nameserver = strings.Trim(fields[0], `"'`) + } + } + + // Look for the next line with search domains + if i+1 < len(lines) { + nextLine := strings.TrimSpace(lines[i+1]) + if strings.Contains(nextLine, "echo") && strings.Contains(nextLine, "search") && strings.Contains(nextLine, ">> /etc/resolv.conf") { + // Extract search domains + // Pattern: echo "search mlaas.svc.cluster.local svc.cluster.local cluster.local" >> /etc/resolv.conf + start := strings.Index(nextLine, `"search`) + end := strings.LastIndex(nextLine, `"`) + if start != -1 && end != -1 && end > start { + searchContent := nextLine[start+1 : end] + // Remove the "search " prefix + searchDomains = strings.TrimPrefix(searchContent, "search ") + } + } + } + break + } + } + + return nameserver, searchDomains +} + +func removeFinalDNSConfig(content string) string { + // Remove the final DNS configuration lines + lines := strings.Split(content, "\n") + var cleanedLines []string + skipNext := false + + for _, line := range lines { + trimmed := strings.TrimSpace(line) + + if skipNext { + // Skip this line (the search domains line) + skipNext = false + continue + } + + // Check if this is the nameserver echo line + if strings.Contains(trimmed, "echo") && + strings.Contains(trimmed, "nameserver") && + strings.Contains(trimmed, "> /etc/resolv.conf") && + !strings.Contains(trimmed, ">>") { + // Skip this line and set flag to skip next line + skipNext = true + continue + } + + cleanedLines = append(cleanedLines, line) + } + + return strings.Join(cleanedLines, "\n") +} diff --git a/pkg/docker/types.go b/pkg/docker/types.go index d485e07..a6e8190 100644 --- a/pkg/docker/types.go +++ b/pkg/docker/types.go @@ -1,5 +1,33 @@ package docker +import ( + "time" + + v1 "k8s.io/api/core/v1" +) + +type DockerConfig struct { + VKConfigPath string `yaml:"VKConfigPath"` + Sidecarport string `yaml:"SidecarPort"` + Socket string `yaml:"Socket"` + ExportPodData bool `yaml:"ExportPodData"` + Commandprefix string `yaml:"CommandPrefix"` + ImagePrefix string `yaml:"ImagePrefix"` + DataRootFolder string `yaml:"DataRootFolder"` + Namespace string `yaml:"Namespace"` + Tsocks bool `yaml:"Tsocks"` + Tsockspath string `yaml:"TsocksPath"` + Tsockslogin string `yaml:"TsocksLoginNode"` + BashPath string `yaml:"BashPath"` + VerboseLogging bool `yaml:"VerboseLogging"` + ErrorsOnlyLogging bool `yaml:"ErrorsOnlyLogging"` + SingularityDefaultOptions []string `yaml:"SingularityDefaultOptions"` + SingularityPrefix string `yaml:"SingularityPrefix"` + SingularityPath string `yaml:"SingularityPath"` + EnableProbes bool `yaml:"EnableProbes"` + set bool +} + type DockerRunStruct struct { Name string `json:"name"` Command string `json:"command"` @@ -11,3 +39,84 @@ type CreateStruct struct { PodUID string `json:"PodUID"` PodJID string `json:"PodJID"` } + +// PodCreateRequests is a struct holding data for a create request. Retrieved ConfigMaps and Secrets are held along the Pod description itself. +type PodCreateRequests struct { + Pod v1.Pod `json:"pod"` + ConfigMaps []v1.ConfigMap `json:"configmaps"` + Secrets []v1.Secret `json:"secrets"` +} + +// PodStatus is a simplified v1.Pod struct, holding only necessary variables to uniquely identify a job/service in the sidecar. It is used to request +type PodStatus struct { + PodName string `json:"name"` + PodUID string `json:"UID"` + PodNamespace string `json:"namespace"` + JobID string `json:"JID"` + Containers []v1.ContainerStatus `json:"containers"` + InitContainers []v1.ContainerStatus `json:"initContainers"` +} + +// RetrievedContainer is used in InterLink to rearrange data structure in a suitable way for the sidecar +type RetrievedContainer struct { + Name string `json:"name"` + ConfigMaps []v1.ConfigMap `json:"configMaps"` + Secrets []v1.Secret `json:"secrets"` + EmptyDirs []string `json:"emptyDirs"` +} + +// InterLinkConfig holds the whole configuration +type InterLinkConfig struct { + VKConfigPath string `yaml:"VKConfigPath"` + VKTokenFile string `yaml:"VKTokenFile"` + Interlinkurl string `yaml:"InterlinkURL"` + Sidecarurl string `yaml:"SidecarURL"` + Sbatchpath string `yaml:"SbatchPath"` + Scancelpath string `yaml:"ScancelPath"` + Squeuepath string `yaml:"SqueuePath"` + Interlinkport string `yaml:"InterlinkPort"` + Socket string `yaml:"Socket"` + Sidecarport string `yaml:"SidecarPort"` + Commandprefix string `yaml:"CommandPrefix"` + ExportPodData bool `yaml:"ExportPodData"` + DataRootFolder string `yaml:"DataRootFolder"` + ServiceAccount string `yaml:"ServiceAccount"` + Namespace string `yaml:"Namespace"` + Tsocks bool `yaml:"Tsocks"` + Tsockspath string `yaml:"TsocksPath"` + Tsocksconfig string `yaml:"TsocksConfig"` + Tsockslogin string `yaml:"TsocksLoginNode"` + BashPath string `yaml:"BashPath"` + VerboseLogging bool `yaml:"VerboseLogging"` + ErrorsOnlyLogging bool `yaml:"ErrorsOnlyLogging"` + PodIP string `yaml:"PodIP"` + SingularityPrefix string `yaml:"SingularityPrefix"` + set bool +} + +// ContainerLogOpts is a struct in which it is possible to specify options to retrieve logs from the sidecar +type ContainerLogOpts struct { + Tail int `json:"Tail"` + LimitBytes int `json:"Bytes"` + Timestamps bool `json:"Timestamps"` + Follow bool `json:"Follow"` + Previous bool `json:"Previous"` + SinceSeconds int `json:"SinceSeconds"` + SinceTime time.Time `json:"SinceTime"` +} + +// LogStruct is needed to identify the job/container running on the sidecar to retrieve the logs from. Using ContainerLogOpts struct allows to specify more options on how to collect logs +type LogStruct struct { + Namespace string `json:"Namespace"` + PodUID string `json:"PodUID"` + PodName string `json:"PodName"` + ContainerName string `json:"ContainerName"` + Opts ContainerLogOpts `json:"Opts"` +} + +type SpanConfig struct { + HTTPReturnCode int + SetHTTPCode bool +} + +type SpanOption func(*SpanConfig) From 486367ae25d4593bcdafb41e06609746bc28aa1f Mon Sep 17 00:00:00 2001 From: Giulio Bianchini Date: Wed, 4 Mar 2026 08:40:11 +0100 Subject: [PATCH 25/30] wip --- pkg/docker/Create.go | 68 +++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 32 deletions(-) diff --git a/pkg/docker/Create.go b/pkg/docker/Create.go index 77eba4f..58eeedf 100644 --- a/pkg/docker/Create.go +++ b/pkg/docker/Create.go @@ -196,30 +196,15 @@ func (h *SidecarHandler) prepareDockerRuns(podData commonIL.RetrievedPodData, w cmd = append(cmd, fpgaArgs) } - cmd = append(cmd, "-p", "8888:8888") - - // if podIp != "" { - // // add --ip flag to the docker run command - // cmd = append(cmd, "--ip", podIp) - - // // add --net vk0 - // cmd = append(cmd, "--net", "vk0") - - // // --dns 10.96.0.10 - // cmd = append(cmd, "--dns", "10.96.0.10") - - // // add NET_ADMIN capability - // cmd = append(cmd, "--cap-add", "NET_ADMIN") - // } - var additionalPortArgs []string for _, port := range container.Ports { - if port.HostPort != 0 { - additionalPortArgs = append(additionalPortArgs, "-p", strconv.Itoa(int(port.HostPort))+":"+strconv.Itoa(int(port.ContainerPort))) - } + log.G(h.Ctx).Info("\u2705 [POD FLOW] Container port: " + strconv.Itoa(int(port.ContainerPort)) + " Protocol: " + string(port.Protocol) + " HostPort: " + strconv.Itoa(int(port.HostPort))) + additionalPortArgs = append(additionalPortArgs, "-p", strconv.Itoa(int(port.ContainerPort))+":"+strconv.Itoa(int(port.ContainerPort))) } + log.G(h.Ctx).Info("\u2705 [POD FLOW] Additional port arguments for container " + containerName + ": " + strings.Join(additionalPortArgs, " ")) + cmd = append(cmd, additionalPortArgs...) mounts, err := prepareMounts(h.Ctx, h.Config, podData, container) @@ -363,6 +348,10 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { podDirectoryPath := filepath.Join(wd, h.Config.DataRootFolder+"/"+podNamespace+"-"+podUID) + // Sentinel file written by mesh.sh once network setup is complete. + // containers_command.sh polls for this file before starting workload containers. + meshReadyFile := filepath.Join(podDirectoryPath, "mesh_ready") + // log the pod specifics log.G(h.Ctx).Info(fmt.Sprintf("\u2705 [POD FLOW] Pod specs: %+v", data.Pod)) @@ -424,14 +413,17 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { // Remove the slirp4netns execution at the end of inner script innerScript = removeSlirp4netnsExecution(innerScript) - // Replace command execution with sleep infinity - innerScript = strings.Replace(innerScript, "$@", "sleep infinity", -1) + // Replace command execution with a sentinel touch followed by sleep infinity. + // The sentinel file signals to containers_command.sh that network setup is done. + innerScript = strings.Replace(innerScript, "$@", "touch "+meshReadyFile+" && sleep infinity", -1) // Build the complete script with correct order meshScript = `#!/bin/bash set -e set -m +sleep 20s + export PATH=$PATH:$PWD:/usr/sbin:/sbin # Set up temporary directory @@ -447,10 +439,10 @@ cd $TMPDIR ` + innerScript } else { - // Fallback: just clean up the outer script + // Fallback: just clean up the outer script, still touch the sentinel before sleeping. meshScript = removeSlirp4netnsDownload(meshScript) meshScript = removeUnshareWrapper(meshScript) - meshScript = strings.Replace(meshScript, "$@", "sleep infinity", -1) + meshScript = strings.Replace(meshScript, "$@", "touch "+meshReadyFile+" && sleep infinity", -1) meshScript = removeSlirp4netnsExecution(meshScript) } @@ -702,12 +694,23 @@ echo "DNS configured for cluster connectivity" dnsSearch = "default.svc.cluster.local svc.cluster.local cluster.local" // Fallback } - // Add a delay to ensure network container is ready + // The first container in the list is the network-overlay; start it immediately. + // All subsequent containers are workload containers and must wait for the sentinel. + networkOverlay := containers[0] + workloadContainers := containers[1:] + + containersCommand += "# Start network overlay container first\n" + containersCommand += networkOverlay.Command + "\n\n" + + // Poll for the sentinel file written by mesh.sh once network setup is complete. containersCommand += "echo 'Waiting for network overlay to be ready...'\n" - containersCommand += "sleep 10\n" - containersCommand += "echo 'Starting containers...'\n\n" + containersCommand += "while [ ! -f " + meshReadyFile + " ]; do\n" + containersCommand += " echo 'Network not ready yet, waiting 2s...'\n" + containersCommand += " sleep 2\n" + containersCommand += "done\n" + containersCommand += "echo 'Network overlay is ready (sentinel file found), starting containers...'\n\n" - for _, container := range containers { + for _, container := range workloadContainers { containersCommand += "# Start container: " + container.Name + "\n" containersCommand += container.Command + "\n" containersCommand += "sleep 2\n" @@ -718,16 +721,17 @@ echo "DNS configured for cluster connectivity" containersCommand += "cp /etc/resolv.conf /etc/resolv.conf.backup 2>/dev/null || true\n" containersCommand += "cat > /etc/resolv.conf << EOF\n" containersCommand += "nameserver " + dnsNameserver + "\n" + containersCommand += "nameserver 8.8.8.8 \n" containersCommand += "search " + dnsSearch + "\n" containersCommand += "EOF\n" containersCommand += "' || echo 'Warning: Could not configure DNS for " + container.Name + "'\n" containersCommand += "echo 'DNS configured for container: " + container.Name + "'\n\n" } - } - - for _, container := range containers { - containersCommand += container.Command + "\n" - containersCommand += "sleep 30\n" + } else { + for _, container := range containers { + containersCommand += container.Command + "\n" + containersCommand += "sleep 1\n" + } } err = os.WriteFile(podDirectoryPath+"/containers_command.sh", []byte(containersCommand), 0644) if err != nil { From a20b6d95050e061eb9ad11b96deed322d79a4690 Mon Sep 17 00:00:00 2001 From: Bianco95 Date: Wed, 4 Mar 2026 09:09:34 +0100 Subject: [PATCH 26/30] updated DockerConfig to handle FPGA setup --- cmd/main.go | 10 +++++++--- pkg/docker/Create.go | 4 ++-- pkg/docker/dindmanager/DindHandler.go | 12 +++++++----- pkg/docker/fpgastrategies/AMDHandler.go | 16 +++++++++------- pkg/docker/func.go | 10 ++++++++++ pkg/docker/types.go | 4 ++++ 6 files changed, 39 insertions(+), 17 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 63cb735..e38d02b 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -178,8 +178,10 @@ func main() { availableDinds = "2" } var dindHandler dindmanager.DindManagerInterface = &dindmanager.DindManager{ - DindList: []dindmanager.DindSpecs{}, - Ctx: ctx, + DindList: []dindmanager.DindSpecs{}, + Ctx: ctx, + FPGAEnabled: interLinkConfig.FPGAEnabled, + XilinxToolsPath: interLinkConfig.XilinxToolsPath, } availableDindsInt, err := strconv.ParseInt(availableDinds, 10, 8) if err != nil { @@ -214,10 +216,12 @@ func main() { log.G(ctx).Info("\u2705 Tracing is disabled") } - if os.Getenv("FPGAENABLED") == "1" { + if interLinkConfig.FPGAEnabled { fpgaManager := &fpgastrategies.FPGAManager{ FPGASpecsList: []fpgastrategies.FPGASpecs{}, Ctx: ctx, + VitisPath: interLinkConfig.VitisPath, + XRTPath: interLinkConfig.XRTPath, } err = fpgaManager.Init() if err != nil { diff --git a/pkg/docker/Create.go b/pkg/docker/Create.go index 58eeedf..b28e65a 100644 --- a/pkg/docker/Create.go +++ b/pkg/docker/Create.go @@ -176,9 +176,9 @@ func (h *SidecarHandler) prepareDockerRuns(podData commonIL.RetrievedPodData, w } } - // if FPGA is requested, mount in read mode the /tools/Xilinx/ path in the container + // if FPGA is requested, mount in read mode the Xilinx tools path in the container if isFPGARequested { - envVars += " -v /tools/Xilinx/:/tools/Xilinx/:ro" + envVars += " -v " + h.Config.XilinxToolsPath + ":" + h.Config.XilinxToolsPath + ":ro" } log.G(h.Ctx).Info("\u2705 [POD FLOW] Before creating run command") diff --git a/pkg/docker/dindmanager/DindHandler.go b/pkg/docker/dindmanager/DindHandler.go index 676d5fd..b97c190 100644 --- a/pkg/docker/dindmanager/DindHandler.go +++ b/pkg/docker/dindmanager/DindHandler.go @@ -34,8 +34,10 @@ type DindSpecs struct { } type DindManager struct { - DindList []DindSpecs - Ctx context.Context + DindList []DindSpecs + Ctx context.Context + FPGAEnabled bool + XilinxToolsPath string } // GenerateUUIDv4 generates a random UUIDv4 @@ -145,9 +147,9 @@ func (a *DindManager) BuildDindContainers(nDindContainer int8) error { dindContainerArgs = append(dindContainerArgs, "-v", "/cvmfs:/cvmfs") } - if os.Getenv("FPGAENABLED") == "1" { - if _, err := os.Stat("/tools/Xilinx/"); err == nil { - dindContainerArgs = append(dindContainerArgs, "-v", "/tools/Xilinx/:/tools/Xilinx/:ro") + if a.FPGAEnabled { + if _, err := os.Stat(a.XilinxToolsPath); err == nil { + dindContainerArgs = append(dindContainerArgs, "-v", a.XilinxToolsPath+":"+a.XilinxToolsPath+":ro") } } diff --git a/pkg/docker/fpgastrategies/AMDHandler.go b/pkg/docker/fpgastrategies/AMDHandler.go index bbb65cb..53e8cbf 100644 --- a/pkg/docker/fpgastrategies/AMDHandler.go +++ b/pkg/docker/fpgastrategies/AMDHandler.go @@ -34,6 +34,8 @@ type FPGAManager struct { FPGASpecsMutex sync.Mutex Vendor string Ctx context.Context + VitisPath string + XRTPath string } type FPGAManagerInterface interface { @@ -52,17 +54,17 @@ type FPGAManagerInterface interface { func (a *FPGAManager) Init() error { // Check if the Xilinx setup.sh file exists - if _, err := os.Stat("/opt/xilinx/xrt/setup.sh"); os.IsNotExist(err) { - return fmt.Errorf("/opt/xilinx/xrt/setup.sh does not exist: %v", err) + if _, err := os.Stat(a.XRTPath + "/setup.sh"); os.IsNotExist(err) { + return fmt.Errorf("%s/setup.sh does not exist: %v", a.XRTPath, err) } // Check if the path to Vitis exists - if _, err := os.Stat("/tools/Xilinx/Vitis/2023.2"); os.IsNotExist(err) { - return fmt.Errorf("/tools/Xilinx/Vitis/2023.2 does not exist: %v", err) + if _, err := os.Stat(a.VitisPath); os.IsNotExist(err) { + return fmt.Errorf("%s does not exist: %v", a.VitisPath, err) } // Source the setup.sh to initialize Xilinx tools using shell - shellArgs := []string{"source", "/opt/xilinx/xrt/setup.sh"} + shellArgs := []string{"source", a.XRTPath + "/setup.sh"} shell := exec.ExecTask{ Command: "/bin/bash", Args: shellArgs, @@ -105,7 +107,7 @@ func (a *FPGAManager) Discover() error { continue } bdf := parts[0] - shellArgs := []string{"/opt/xilinx/xrt/setup.sh"} + shellArgs := []string{a.XRTPath + "/setup.sh"} shell := exec.ExecTask{ Command: "source", Args: shellArgs, @@ -118,7 +120,7 @@ func (a *FPGAManager) Discover() error { } cmd := exec.ExecTask{ - Command: "/opt/xilinx/xrt/bin/xbutil", + Command: a.XRTPath + "/bin/xbutil", Args: []string{"examine"}, // "--device", bdf Shell: false, } diff --git a/pkg/docker/func.go b/pkg/docker/func.go index 52909a8..fb55ad1 100644 --- a/pkg/docker/func.go +++ b/pkg/docker/func.go @@ -88,6 +88,16 @@ func NewInterLinkConfig() (DockerConfig, error) { InterLinkConfigInst.Tsockspath = path } + if InterLinkConfigInst.XilinxToolsPath == "" { + InterLinkConfigInst.XilinxToolsPath = "/tools/Xilinx/" + } + if InterLinkConfigInst.VitisPath == "" { + InterLinkConfigInst.VitisPath = "/tools/Xilinx/Vitis/2023.2" + } + if InterLinkConfigInst.XRTPath == "" { + InterLinkConfigInst.XRTPath = "/opt/xilinx/xrt" + } + InterLinkConfigInst.set = true } return InterLinkConfigInst, nil diff --git a/pkg/docker/types.go b/pkg/docker/types.go index a6e8190..685a709 100644 --- a/pkg/docker/types.go +++ b/pkg/docker/types.go @@ -25,6 +25,10 @@ type DockerConfig struct { SingularityPrefix string `yaml:"SingularityPrefix"` SingularityPath string `yaml:"SingularityPath"` EnableProbes bool `yaml:"EnableProbes"` + FPGAEnabled bool `yaml:"FPGAEnabled"` + XilinxToolsPath string `yaml:"XilinxToolsPath"` + VitisPath string `yaml:"VitisPath"` + XRTPath string `yaml:"XRTPath"` set bool } From 2ed6360da0bd310b255667d884288c212fe90a4f Mon Sep 17 00:00:00 2001 From: Giulio Bianchini Date: Tue, 17 Mar 2026 16:26:20 +0100 Subject: [PATCH 27/30] updated docker configuration to handle manual setup of subnet to use --- cmd/main.go | 30 +- pkg/common/types.go | 51 +-- pkg/docker/Create.go | 68 ++-- pkg/docker/Status.go | 48 +++ pkg/docker/dindmanager/DindHandler.go | 421 +++++++++++++++--------- pkg/docker/fpgastrategies/AMDHandler.go | 202 +++++++----- pkg/docker/types.go | 52 +-- 7 files changed, 539 insertions(+), 333 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index e38d02b..bc026fa 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -162,6 +162,9 @@ func main() { log.G(context.Background()).Fatal(err) } + // log the interlinkconfig + log.G(context.Background()).Info("\u2705 InterLinkConfig: ", interLinkConfig) + if interLinkConfig.VerboseLogging { logger.SetLevel(logrus.DebugLevel) } else if interLinkConfig.ErrorsOnlyLogging { @@ -177,16 +180,35 @@ func main() { if availableDinds == "" { availableDinds = "2" } + availableDindsInt, err := strconv.ParseInt(availableDinds, 10, 8) + if err != nil { + log.G(ctx).Fatal("Error parsing AVAILABLEDINDS: ", err) + } + + // Build the subnet pool from the config (empty slice = no pool). + subnetPool, err := dindmanager.InitSubnetPool(interLinkConfig.DockerNetworkSubnet) + if err != nil { + log.G(ctx).Fatal("Error initialising subnet pool: ", err) + } + if len(subnetPool) > 0 { + log.G(ctx).Info(fmt.Sprintf( + "\u2705 Subnet pool initialised: %d /24 subnets available from %v", + len(subnetPool), interLinkConfig.DockerNetworkSubnet, + )) + } else { + log.G(ctx).Info("\u2705 No DockerNetworkSubnet configured โ€” Docker will assign subnets automatically") + } + var dindHandler dindmanager.DindManagerInterface = &dindmanager.DindManager{ DindList: []dindmanager.DindSpecs{}, Ctx: ctx, FPGAEnabled: interLinkConfig.FPGAEnabled, XilinxToolsPath: interLinkConfig.XilinxToolsPath, + SubnetPool: subnetPool, } - availableDindsInt, err := strconv.ParseInt(availableDinds, 10, 8) - if err != nil { - log.G(ctx).Info("\u2705 Error parsing availableDinds") - } + + dindHandler.(*dindmanager.DindManager).InitialPoolSz = len(subnetPool) + dindHandler.CleanDindContainers() dindHandler.BuildDindContainers(int8(availableDindsInt)) diff --git a/pkg/common/types.go b/pkg/common/types.go index 49b4fef..4912358 100644 --- a/pkg/common/types.go +++ b/pkg/common/types.go @@ -54,31 +54,32 @@ type RetrievedContainer struct { // InterLinkConfig holds the whole configuration type InterLinkConfig struct { - VKConfigPath string `yaml:"VKConfigPath"` - VKTokenFile string `yaml:"VKTokenFile"` - Interlinkurl string `yaml:"InterlinkURL"` - Sidecarurl string `yaml:"SidecarURL"` - Sbatchpath string `yaml:"SbatchPath"` - Scancelpath string `yaml:"ScancelPath"` - Squeuepath string `yaml:"SqueuePath"` - Interlinkport string `yaml:"InterlinkPort"` - Socket string `yaml:"Socket"` - Sidecarport string `yaml:"SidecarPort"` - Commandprefix string `yaml:"CommandPrefix"` - ExportPodData bool `yaml:"ExportPodData"` - DataRootFolder string `yaml:"DataRootFolder"` - ServiceAccount string `yaml:"ServiceAccount"` - Namespace string `yaml:"Namespace"` - Tsocks bool `yaml:"Tsocks"` - Tsockspath string `yaml:"TsocksPath"` - Tsocksconfig string `yaml:"TsocksConfig"` - Tsockslogin string `yaml:"TsocksLoginNode"` - BashPath string `yaml:"BashPath"` - VerboseLogging bool `yaml:"VerboseLogging"` - ErrorsOnlyLogging bool `yaml:"ErrorsOnlyLogging"` - PodIP string `yaml:"PodIP"` - SingularityPrefix string `yaml:"SingularityPrefix"` - set bool + VKConfigPath string `yaml:"VKConfigPath"` + VKTokenFile string `yaml:"VKTokenFile"` + Interlinkurl string `yaml:"InterlinkURL"` + Sidecarurl string `yaml:"SidecarURL"` + Sbatchpath string `yaml:"SbatchPath"` + Scancelpath string `yaml:"ScancelPath"` + Squeuepath string `yaml:"SqueuePath"` + Interlinkport string `yaml:"InterlinkPort"` + Socket string `yaml:"Socket"` + Sidecarport string `yaml:"SidecarPort"` + Commandprefix string `yaml:"CommandPrefix"` + ExportPodData bool `yaml:"ExportPodData"` + DataRootFolder string `yaml:"DataRootFolder"` + ServiceAccount string `yaml:"ServiceAccount"` + Namespace string `yaml:"Namespace"` + Tsocks bool `yaml:"Tsocks"` + Tsockspath string `yaml:"TsocksPath"` + Tsocksconfig string `yaml:"TsocksConfig"` + Tsockslogin string `yaml:"TsocksLoginNode"` + BashPath string `yaml:"BashPath"` + VerboseLogging bool `yaml:"VerboseLogging"` + ErrorsOnlyLogging bool `yaml:"ErrorsOnlyLogging"` + PodIP string `yaml:"PodIP"` + SingularityPrefix string `yaml:"SingularityPrefix"` + DockerNetworkSubnet []string `yaml:"DockerNetworkSubnet"` + set bool } // ContainerLogOpts is a struct in which it is possible to specify options to retrieve logs from the sidecar diff --git a/pkg/docker/Create.go b/pkg/docker/Create.go index b28e65a..814f3ba 100644 --- a/pkg/docker/Create.go +++ b/pkg/docker/Create.go @@ -554,40 +554,40 @@ cd $TMPDIR } // Create DNS configuration script for DIND - dnsConfigScript := `#!/bin/sh -set -e - -# Backup original resolv.conf -cp /etc/resolv.conf /etc/resolv.conf.backup 2>/dev/null || true - -# Create new resolv.conf with cluster DNS -cat > /etc/resolv.conf << EOF -nameserver 8.8.8.8 -search ` + dnsSearch + ` -EOF - -echo "DNS configured for cluster connectivity" -` - - // Write DNS config script to pod directory - dnsScriptPath := filepath.Join(podDirectoryPath, "configure-dns.sh") - err = os.WriteFile(dnsScriptPath, []byte(dnsConfigScript), 0755) - if err != nil { - log.G(h.Ctx).Warning("โš ๏ธ Failed to create DNS config script: " + err.Error()) - } else { - // Execute DNS configuration on DIND container - dnsExecCmd := exec.ExecTask{ - Command: "docker", - Args: []string{"exec", string(data.Pod.UID) + "_dind", "sh", dnsScriptPath}, - Shell: true, - } - _, err = dnsExecCmd.Execute() - if err != nil { - log.G(h.Ctx).Warning("โš ๏ธ Failed to configure DNS on DIND container: " + err.Error()) - } else { - log.G(h.Ctx).Info("โœ… [POD FLOW] DNS configured on DIND container successfully with NS: " + dnsNameserver + ", Search: " + dnsSearch) - } - } + /* dnsConfigScript := `#!/bin/sh + set -e + + # Backup original resolv.conf + cp /etc/resolv.conf /etc/resolv.conf.backup 2>/dev/null || true + + # Create new resolv.conf with cluster DNS + cat > /etc/resolv.conf << EOF + nameserver 8.8.8.8 + search ` + dnsSearch + ` + EOF + + echo "DNS configured for cluster connectivity" + ` + + // Write DNS config script to pod directory + dnsScriptPath := filepath.Join(podDirectoryPath, "configure-dns.sh") + err = os.WriteFile(dnsScriptPath, []byte(dnsConfigScript), 0755) + if err != nil { + log.G(h.Ctx).Warning("โš ๏ธ Failed to create DNS config script: " + err.Error()) + } else { + // Execute DNS configuration on DIND container + dnsExecCmd := exec.ExecTask{ + Command: "docker", + Args: []string{"exec", string(data.Pod.UID) + "_dind", "sh", dnsScriptPath}, + Shell: true, + } + _, err = dnsExecCmd.Execute() + if err != nil { + log.G(h.Ctx).Warning("โš ๏ธ Failed to configure DNS on DIND container: " + err.Error()) + } else { + log.G(h.Ctx).Info("โœ… [POD FLOW] DNS configured on DIND container successfully with NS: " + dnsNameserver + ", Search: " + dnsSearch) + } + } */ } } diff --git a/pkg/docker/Status.go b/pkg/docker/Status.go index 6e742c4..7ef51ba 100644 --- a/pkg/docker/Status.go +++ b/pkg/docker/Status.go @@ -87,8 +87,34 @@ func (h *SidecarHandler) StatusHandler(w http.ResponseWriter, r *http.Request) { resp = append(resp, commonIL.PodStatus{PodName: pod.Name, PodUID: podUID, PodNamespace: podNamespace, JobID: dindUUID}) + disabledInitContainers := make(map[string]bool) + if ann, ok := pod.Annotations["interlink.eu/disable-offload-init-containers"]; ok { + for _, name := range strings.Split(ann, ",") { + name = strings.TrimSpace(name) + if name != "" { + disabledInitContainers[name] = true + } + } + } + // check if the pod has initContainers and get their status for _, container := range pod.Spec.InitContainers { + + if disabledInitContainers[container.Name] { + log.G(h.Ctx).Infof("โœ… [STATUS CALL] init container %s is marked as non-offloaded, reporting as Completed", container.Name) + resp[i].InitContainers = append(resp[i].InitContainers, v1.ContainerStatus{ + Name: container.Name, + Ready: false, + State: v1.ContainerState{ + Terminated: &v1.ContainerStateTerminated{ + ExitCode: 0, + Reason: "Completed", + }, + }, + }) + continue + } + containerName := podNamespace + "-" + podUID + "-" + container.Name cmd := []string{"exec " + podUID + "_dind" + " docker ps -af name=^" + containerName + "$ --format \"{{.Status}}\""} shell := exec.ExecTask{ @@ -130,8 +156,30 @@ func (h *SidecarHandler) StatusHandler(w http.ResponseWriter, r *http.Request) { } } + disabledContainers := make(map[string]bool) + if ann, ok := pod.Annotations["interlink.eu/disable-offload-containers"]; ok { + for _, name := range strings.Split(ann, ",") { + name = strings.TrimSpace(name) + if name != "" { + disabledContainers[name] = true + } + } + } + for _, container := range pod.Spec.Containers { + if disabledContainers[container.Name] { + log.G(h.Ctx).Infof("โœ… [STATUS CALL] container %s is marked as non-offloaded, reporting as Running", container.Name) + resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{ + Name: container.Name, + Ready: true, + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }) + continue + } + containerName := podNamespace + "-" + podUID + "-" + container.Name cmd := []string{"exec " + podUID + "_dind" + " docker ps -af name=^" + containerName + "$ --format \"{{.Status}}\""} diff --git a/pkg/docker/dindmanager/DindHandler.go b/pkg/docker/dindmanager/DindHandler.go index b97c190..7da55db 100644 --- a/pkg/docker/dindmanager/DindHandler.go +++ b/pkg/docker/dindmanager/DindHandler.go @@ -4,8 +4,10 @@ import ( "context" "crypto/rand" "fmt" + "net" "os" "strings" + "sync" "time" exec "github.com/alexellis/go-execute/pkg/v1" @@ -27,10 +29,11 @@ type DindManagerInterface interface { } type DindSpecs struct { - DindID string - PodUID string - DindNetworkID string - Available bool + DindID string + PodUID string + DindNetworkID string + AllocatedSubnet string // empty if no subnet pool was configured + Available bool } type DindManager struct { @@ -38,28 +41,127 @@ type DindManager struct { Ctx context.Context FPGAEnabled bool XilinxToolsPath string + + // Subnet pool: pre-generated /24 subnets from the parent CIDRs in config. + // Protected by mu because container creation/deletion can happen concurrently. + SubnetPool []string + mu sync.Mutex + InitialPoolSz int // total subnets at startup, used for logging +} + +// --------------------------------------------------------------------------- +// Subnet pool helpers +// --------------------------------------------------------------------------- + +// InitSubnetPool expands a list of parent CIDRs (e.g. ["192.168.0.0/16", +// "10.12.0.0/16"]) into an ordered slice of allocatable /24 subnets. +// Supported parent prefix lengths: /8, /16, /24. +// Returns an empty slice (no error) when parentCIDRs is nil/empty. +func InitSubnetPool(parentCIDRs []string) ([]string, error) { + var pool []string + for _, cidr := range parentCIDRs { + subnets, err := generateSubnetsFromCIDR(cidr) + if err != nil { + return nil, fmt.Errorf("failed to expand CIDR %s: %w", cidr, err) + } + pool = append(pool, subnets...) + } + return pool, nil +} + +// generateSubnetsFromCIDR returns all usable /24 subnets within parentCIDR. +// +// - /8 โ†’ iterates second octet [1-254] ร— third octet [1-254] (~64 k subnets) +// - /16 โ†’ iterates third octet [1-254] (254 subnets) +// - /24 โ†’ returns the CIDR itself (1 subnet) +func generateSubnetsFromCIDR(cidr string) ([]string, error) { + _, ipNet, err := net.ParseCIDR(cidr) + if err != nil { + return nil, fmt.Errorf("invalid CIDR %q: %w", cidr, err) + } + + ones, bits := ipNet.Mask.Size() + if bits != 32 { + return nil, fmt.Errorf("only IPv4 CIDRs are supported (got %s)", cidr) + } + + base := ipNet.IP.To4() + var subnets []string + + switch ones { + case 8: + for second := 1; second <= 254; second++ { + for third := 1; third <= 254; third++ { + subnets = append(subnets, fmt.Sprintf("%d.%d.%d.0/24", base[0], second, third)) + } + } + case 16: + for third := 1; third <= 254; third++ { + subnets = append(subnets, fmt.Sprintf("%d.%d.%d.0/24", base[0], base[1], third)) + } + case 24: + subnets = append(subnets, cidr) + default: + return nil, fmt.Errorf("unsupported prefix length /%d in %s: only /8, /16 and /24 are supported", ones, cidr) + } + + return subnets, nil +} + +// allocateSubnet pops the next available subnet from the pool. +// Returns ("", false) when the pool is empty or was never configured. +func (a *DindManager) allocateSubnet() (string, bool) { + a.mu.Lock() + defer a.mu.Unlock() + if len(a.SubnetPool) == 0 { + return "", false + } + subnet := a.SubnetPool[0] + a.SubnetPool = a.SubnetPool[1:] + return subnet, true +} + +// freeSubnet returns a subnet to the pool (appended at the end so the pool +// stays ordered and previously-used ranges are re-used last). +func (a *DindManager) freeSubnet(subnet string) { + if subnet == "" { + return + } + a.mu.Lock() + defer a.mu.Unlock() + a.SubnetPool = append(a.SubnetPool, subnet) } -// GenerateUUIDv4 generates a random UUIDv4 +// remainingSubnets returns the current pool size (thread-safe). +func (a *DindManager) remainingSubnets() int { + a.mu.Lock() + defer a.mu.Unlock() + return len(a.SubnetPool) +} + +// --------------------------------------------------------------------------- +// UUIDv4 generator (unchanged) +// --------------------------------------------------------------------------- + func GenerateUUIDv4() (string, error) { uuid := make([]byte, 16) _, err := rand.Read(uuid) if err != nil { return "", err } - uuid[6] = (uuid[6] & 0x0f) | 0x40 uuid[8] = (uuid[8] & 0x3f) | 0x80 - - return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:16]), nil + return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", + uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:16]), nil } -func (a *DindManager) CleanDindContainers() error { +// --------------------------------------------------------------------------- +// DindManager methods +// --------------------------------------------------------------------------- - // print the number of DIND containers to be created - log.G(a.Ctx).Info(fmt.Sprintf("\u2705 Start cleaning zombie DIND containers")) +func (a *DindManager) CleanDindContainers() error { + log.G(a.Ctx).Info("\u2705 Start cleaning zombie DIND containers") - // exec this command docker ps -a --format "{{.Names}}" | grep '_dind$' | wc -l shell := exec.ExecTask{ Command: "docker", Args: []string{"ps", "-a", "--format", "{{.Names}}", "|", "grep", "_dind$", "|", "wc", "-l"}, @@ -69,50 +171,49 @@ func (a *DindManager) CleanDindContainers() error { if err != nil { return err } - - // log the number of zombie DIND containers (remove the \n at the end of the string) - log.G(a.Ctx).Info(fmt.Sprintf("\u2705 %s zombie DIND containers found", strings.ReplaceAll(execReturn.Stdout, "\n", ""))) + log.G(a.Ctx).Info(fmt.Sprintf("\u2705 %s zombie DIND containers found", + strings.ReplaceAll(execReturn.Stdout, "\n", ""))) shell = exec.ExecTask{ Command: "docker", Args: []string{"ps", "-a", "--format", "{{.Names}}", "|", "grep", "_dind$", "|", "xargs", "-I", "{}", "docker", "rm", "-f", "{}"}, Shell: true, } - _, err = shell.Execute() - if err != nil { + if _, err = shell.Execute(); err != nil { return err } - // exec this command docker network ls --filter name=_dind_network$ --format "{{.ID}}" | xargs -r docker network rm - shell = exec.ExecTask{ Command: "docker", Args: []string{"network", "ls", "--filter", "name=_dind_network$", "--format", "{{.ID}}", "|", "xargs", "-r", "docker", "network", "rm"}, Shell: true, } - _, err = shell.Execute() - if err != nil { + if _, err = shell.Execute(); err != nil { return err } - log.G(a.Ctx).Info(fmt.Sprintf("\u2705 DIND zombie containers cleaned")) - + log.G(a.Ctx).Info("\u2705 DIND zombie containers cleaned") return nil } func (a *DindManager) BuildDindContainers(nDindContainer int8) error { - - // print the number of DIND containers to be created log.G(a.Ctx).Info(fmt.Sprintf("\u2705 Creating %d DIND containers", nDindContainer)) - // get the working dir + // Log subnet pool status before we start allocating. + if a.InitialPoolSz > 0 { + log.G(a.Ctx).Info(fmt.Sprintf( + "\u2705 Subnet pool: %d/%d subnets available before container creation", + a.remainingSubnets(), a.InitialPoolSz, + )) + } else { + log.G(a.Ctx).Info("\u2705 No subnet pool configured โ€” networks will use Docker's automatic addressing") + } + wd, err := os.Getwd() if err != nil { return err } - // get the env variable GPUENABLED, if 1 then the DIND container will have GPU support, otherwise it will not - gpuEnabled := os.Getenv("GPUENABLED") dindImage := "docker:dind" if gpuEnabled == "1" { @@ -121,28 +222,52 @@ func (a *DindManager) BuildDindContainers(nDindContainer int8) error { for i := int8(0); i < nDindContainer; i++ { - // generate a random UID for the DIND container randUID, err := GenerateUUIDv4() if err != nil { return err } - // create the networks + // ---------------------------------------------------------------- + // Allocate a /24 subnet for this container's bridge network (if a + // pool is configured). + // ---------------------------------------------------------------- + allocatedSubnet, hasSubnet := a.allocateSubnet() + + networkArgs := []string{"network", "create", "--driver", "bridge"} + if hasSubnet { + networkArgs = append(networkArgs, "--subnet", allocatedSubnet) + } + networkArgs = append(networkArgs, randUID+"_dind_network") + shell := exec.ExecTask{ Command: "docker", - Args: []string{"network", "create", "--driver", "bridge", randUID + "_dind_network"}, + Args: networkArgs, Shell: true, } - _, err = shell.Execute() - - log.G(a.Ctx).Info(fmt.Sprintf("\u2705 DIND network %s created", randUID+"_dind_network")) - - if err != nil { + if _, err = shell.Execute(); err != nil { + // Return the subnet to the pool so it is not lost on error. + a.freeSubnet(allocatedSubnet) return err } + if hasSubnet { + log.G(a.Ctx).Info(fmt.Sprintf( + "\u2705 DIND network %s created with subnet %s (%d/%d subnets remaining in pool)", + randUID+"_dind_network", allocatedSubnet, + a.remainingSubnets(), a.InitialPoolSz, + )) + } else { + log.G(a.Ctx).Info(fmt.Sprintf( + "\u2705 DIND network %s created (no subnet pool configured)", + randUID+"_dind_network", + )) + } + + // ---------------------------------------------------------------- + // Build the docker-run argument list. + // ---------------------------------------------------------------- dindContainerArgs := []string{"run"} - //dindContainerArgs = append(dindContainerArgs, gpuArgsAsArray...) + if _, err := os.Stat("/cvmfs"); err == nil { dindContainerArgs = append(dindContainerArgs, "-v", "/cvmfs:/cvmfs") } @@ -153,173 +278,120 @@ func (a *DindManager) BuildDindContainers(nDindContainer int8) error { } } - // add the network to the dind container dindContainerArgs = append(dindContainerArgs, "--network", randUID+"_dind_network") - - // append also --net vk0 - //dindContainerArgs = append(dindContainerArgs, "--net", "vk0") dindContainerArgs = append(dindContainerArgs, "--cap-add", "NET_ADMIN") - // "--runtime=nvidia" is added to the dind container if the GPUENABLED env variable is set to 1 if gpuEnabled == "1" { dindContainerArgs = append(dindContainerArgs, "--runtime=nvidia") } - dindContainerArgs = append(dindContainerArgs, "--privileged", "-v", wd+":/"+wd, "-v", "/home:/home", "-v", "/var/lib/docker/overlay2:/var/lib/docker/overlay2", "-v", "/var/lib/docker/image:/var/lib/docker/image", "-d", "--name", randUID+"_dind", dindImage) + dindContainerArgs = append(dindContainerArgs, + "--privileged", + "-v", wd+":"+"/"+wd, + "-v", "/home:/home", + "-v", "/var/lib/docker/overlay2:/var/lib/docker/overlay2", + "-v", "/var/lib/docker/image:/var/lib/docker/image", + "-d", "--name", randUID+"_dind", dindImage, + ) - var dindContainerID string shell = exec.ExecTask{ Command: "docker", Args: dindContainerArgs, Shell: true, } - - // log the command to be executed with docker and the args log.G(a.Ctx).Info(fmt.Sprintf("\u2705 Command is %s", shell.Command+" "+strings.Join(shell.Args, " "))) execReturn, err := shell.Execute() if err != nil { log.G(a.Ctx).Error(fmt.Sprintf("\u274c Error creating DIND container %s", randUID+"_dind")) log.G(a.Ctx).Error(fmt.Sprintf("\u274c %s", execReturn.Stderr)) + // Free the subnet so it can be reused. + a.freeSubnet(allocatedSubnet) return err } - dindContainerID = execReturn.Stdout + dindContainerID := execReturn.Stdout - // create a variable of maximum number of retries + // ---------------------------------------------------------------- + // Wait for the daemon inside the DinD container to be ready. + // ---------------------------------------------------------------- maxRetries := 20 - output := []byte{} - - // wait until the dind container is up and running by check that the command docker ps inside of it does not return an error for { - if maxRetries == 0 { - return fmt.Errorf("DIND container %s not up and running", dindContainerID) + a.freeSubnet(allocatedSubnet) + return fmt.Errorf("DIND container %s did not become ready in time", dindContainerID) } cmd := OSexec.Command("docker", "logs", randUID+"_dind") - output, err = cmd.CombinedOutput() - - if err != nil { - time.Sleep(1 * time.Second) - } - - if strings.Contains(string(output), "API listen on /var/run/docker.sock") { + output, err := cmd.CombinedOutput() + if err == nil && strings.Contains(string(output), "API listen on /var/run/docker.sock") { break - } else { - time.Sleep(1 * time.Second) } - - maxRetries -= 1 - + time.Sleep(1 * time.Second) + maxRetries-- } - log.G(a.Ctx).Info(fmt.Sprintf("\u2705 DIND container %s is up and running", dindContainerID)) - shell = exec.ExecTask{ - Command: "docker", - Args: []string{"exec", randUID + "_dind", "apt-get", "update"}, - Shell: true, - } - _, err = shell.Execute() - if err != nil { - return err - } - - log.G(a.Ctx).Info(fmt.Sprintf("\u2705 Executed apt-get update")) - - shell = exec.ExecTask{ - Command: "docker", - Args: []string{"exec", randUID + "_dind", "apt-get", "install", "-y", "net-tools"}, - Shell: true, - } - _, err = shell.Execute() - if err != nil { - return err - } - - log.G(a.Ctx).Info(fmt.Sprintf("\u2705 Installed net-tools")) - - shell = exec.ExecTask{ - Command: "docker", - Args: []string{"exec", randUID + "_dind", "apt-get", "install", "-y", "iproute2"}, - Shell: true, - } - _, err = shell.Execute() - if err != nil { - return err + // ---------------------------------------------------------------- + // Install required tools inside the DinD container. + // ---------------------------------------------------------------- + for _, pkg := range []string{"net-tools", "iproute2"} { + shell = exec.ExecTask{ + Command: "docker", + Args: []string{"exec", randUID + "_dind", "apt-get", "install", "-y", pkg}, + Shell: true, + } + if _, err = shell.Execute(); err != nil { + a.freeSubnet(allocatedSubnet) + return err + } + log.G(a.Ctx).Info(fmt.Sprintf("\u2705 Installed %s", pkg)) } - // log.G(a.Ctx).Info(fmt.Sprintf("\u2705 Installed iproute2")) - - // // run docker network connet vk0 container-id - // shell = exec.ExecTask{ - // Command: "docker", - // Args: []string{"network", "connect", "vk0", randUID + "_dind"}, - // Shell: true, - // } - // _, err = shell.Execute() - // if err != nil { - // return err - // } - - // shell = exec.ExecTask{ - // Command: "docker", - // Args: []string{"exec", randUID + "_dind", "docker", "network", "create", "--subnet", "10.244.12.0/24", "-o", "com.docker.network.bridge.name=vk0", "vk0", "-o", "com.docker.network.driver.mtu=1500"}, - // Shell: true, - // } - // _, err = shell.Execute() - // if err != nil { - // return err - // } - - // log.G(a.Ctx).Info(fmt.Sprintf("\u2705 Created network vk0")) - - // shell = exec.ExecTask{ - // Command: "docker", - // Args: []string{"exec", randUID + "_dind", "ip", "link", "set", "to-cluster", "master", "vk0"}, - // Shell: true, - // } - // _, err = shell.Execute() - // if err != nil { - // return err - // } - - // log.G(a.Ctx).Info(fmt.Sprintf("\u2705 Set vk0 as master")) - - // add the dind container to the list of DIND containers - a.DindList = append(a.DindList, DindSpecs{DindID: randUID + "_dind", PodUID: "", DindNetworkID: randUID + "_dind_network", Available: true}) + // ---------------------------------------------------------------- + // Register the new DinD in the list. + // ---------------------------------------------------------------- + a.DindList = append(a.DindList, DindSpecs{ + DindID: randUID + "_dind", + PodUID: "", + DindNetworkID: randUID + "_dind_network", + AllocatedSubnet: allocatedSubnet, // empty string when no pool + Available: true, + }) } return nil } func (a *DindManager) PrintDindList() error { - for _, dindSpec := range a.DindList { - log.G(a.Ctx).Info(fmt.Sprintf("DindID: %s, PodUID: %s, DindNetworkID: %s, Available: %t", dindSpec.DindID, dindSpec.PodUID, dindSpec.DindNetworkID, dindSpec.Available)) + for _, d := range a.DindList { + log.G(a.Ctx).Info(fmt.Sprintf( + "DindID: %s, PodUID: %s, DindNetworkID: %s, AllocatedSubnet: %q, Available: %t", + d.DindID, d.PodUID, d.DindNetworkID, d.AllocatedSubnet, d.Available, + )) } return nil } func (a *DindManager) GetDindFromPodUID(podUID string) (DindSpecs, error) { - for _, dindSpec := range a.DindList { - if dindSpec.PodUID == podUID { - return dindSpec, nil + for _, d := range a.DindList { + if d.PodUID == podUID { + return d, nil } } return DindSpecs{}, fmt.Errorf("DIND container with PodUID %s not found", podUID) } func (a *DindManager) GetAvailableDind() (string, error) { - for _, dindSpec := range a.DindList { - if dindSpec.Available { - return dindSpec.DindID, nil + for _, d := range a.DindList { + if d.Available { + return d.DindID, nil } } - return "", fmt.Errorf("No available DIND container") + return "", fmt.Errorf("no available DIND container") } func (a *DindManager) SetDindUnavailable(dindID string) error { - for i, dindSpec := range a.DindList { - if dindSpec.DindID == dindID { + for i, d := range a.DindList { + if d.DindID == dindID { a.DindList[i].Available = false return nil } @@ -327,19 +399,19 @@ func (a *DindManager) SetDindUnavailable(dindID string) error { return fmt.Errorf("DIND container %s not found", dindID) } -func (a *DindManager) SetDindAvailable(PodUI string) error { - for i, dindSpec := range a.DindList { - if dindSpec.PodUID == PodUI { +func (a *DindManager) SetDindAvailable(PodUID string) error { + for i, d := range a.DindList { + if d.PodUID == PodUID { a.DindList[i].Available = true return nil } } - return fmt.Errorf("DIND container %s not found", PodUI) + return fmt.Errorf("DIND container with PodUID %s not found", PodUID) } func (a *DindManager) SetPodUIDToDind(dindID string, podUID string) error { - for i, dindSpec := range a.DindList { - if dindSpec.DindID == dindID { + for i, d := range a.DindList { + if d.DindID == dindID { a.DindList[i].PodUID = podUID return nil } @@ -347,12 +419,45 @@ func (a *DindManager) SetPodUIDToDind(dindID string, podUID string) error { return fmt.Errorf("DIND container %s not found", dindID) } +// RemoveDindFromList removes the DinD entry associated with PodUID from the +// in-memory list, tears down its Docker network, and returns its subnet to +// the pool so it can be reused by future containers. func (a *DindManager) RemoveDindFromList(PodUID string) error { - for i, dindSpec := range a.DindList { - if dindSpec.PodUID == PodUID { - a.DindList = append(a.DindList[:i], a.DindList[i+1:]...) - return nil + for i, d := range a.DindList { + if d.PodUID != PodUID { + continue + } + + // Remove the dedicated bridge network from Docker. + if d.DindNetworkID != "" { + shell := exec.ExecTask{ + Command: "docker", + Args: []string{"network", "rm", d.DindNetworkID}, + Shell: true, + } + if execReturn, err := shell.Execute(); err != nil { + // Log but don't abort โ€” the container may already be gone. + log.G(a.Ctx).Warn(fmt.Sprintf( + "\u26a0\ufe0f Could not remove network %s: %s (stderr: %s)", + d.DindNetworkID, err, execReturn.Stderr, + )) + } else { + log.G(a.Ctx).Info(fmt.Sprintf("\u2705 Removed Docker network %s", d.DindNetworkID)) + } } + + // Return the subnet to the pool. + a.freeSubnet(d.AllocatedSubnet) + if d.AllocatedSubnet != "" { + log.G(a.Ctx).Info(fmt.Sprintf( + "\u2705 Subnet %s returned to pool (%d/%d now available)", + d.AllocatedSubnet, a.remainingSubnets(), a.InitialPoolSz, + )) + } + + // Splice the entry out of the list. + a.DindList = append(a.DindList[:i], a.DindList[i+1:]...) + return nil } return fmt.Errorf("DIND container with PodUID %s not found", PodUID) } diff --git a/pkg/docker/fpgastrategies/AMDHandler.go b/pkg/docker/fpgastrategies/AMDHandler.go index 53e8cbf..604b906 100644 --- a/pkg/docker/fpgastrategies/AMDHandler.go +++ b/pkg/docker/fpgastrategies/AMDHandler.go @@ -46,7 +46,7 @@ type FPGAManagerInterface interface { Discover() error Check() error GetAvailableFPGAs(numFPGAs int) ([]FPGASpecs, error) - Assign(UUID string, containerID string) error + Assign(BDF string, containerID string) error Release(UUID string) error GetAndAssignAvailableFPGAs(numFPGAs int, containerID string) ([]FPGASpecs, error) } @@ -90,103 +90,136 @@ func (a *FPGAManager) Discover() error { Shell: true, } - regexPattern := `^\[[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-9a-fA-F]\].*` - re := regexp.MustCompile(regexPattern) - - shell.Execute() output, err := shell.Execute() if err != nil { return fmt.Errorf("Error running lspci command: %v", err) } lines := strings.Split(string(output.Stdout), "\n") + xilinxFound := false for _, line := range lines { if strings.Contains(line, "Xilinx") { - parts := strings.Fields(line) - if len(parts) < 3 { - continue - } - bdf := parts[0] - shellArgs := []string{a.XRTPath + "/setup.sh"} - shell := exec.ExecTask{ - Command: "source", - Args: shellArgs, - Shell: true, - } + xilinxFound = true + log.G(a.Ctx).Info(fmt.Sprintf("\u2705 Found potential FPGA: %s", line)) + break + } + } - _, err := shell.Execute() - if err != nil { - return fmt.Errorf("Error running source setup.sh command: %v", err) - } + if !xilinxFound { + log.G(a.Ctx).Info("\u2705 No FPGAs discovered") + return nil + } - cmd := exec.ExecTask{ - Command: a.XRTPath + "/bin/xbutil", - Args: []string{"examine"}, // "--device", bdf - Shell: false, - } - outputXbutil, err := cmd.Execute() - if err != nil { - return fmt.Errorf("Error running xbutil examine: %v", err) - } + // Source XRT setup.sh before running xbutil + sourceShell := exec.ExecTask{ + Command: "source", + Args: []string{a.XRTPath + "/setup.sh"}, + Shell: true, + } + _, err = sourceShell.Execute() + if err != nil { + return fmt.Errorf("Error running source setup.sh command: %v", err) + } + + // Get a temp path but don't create the file โ€” xbutil will create it + tmpFile, err := os.CreateTemp("", "xbutil-examine-*.json") + if err != nil { + return fmt.Errorf("Error creating temp file for xbutil output: %v", err) + } + tmpFilePath := tmpFile.Name() + tmpFile.Close() + os.Remove(tmpFilePath) // Remove it so xbutil can write to it freely + defer os.Remove(tmpFilePath) + + log.G(a.Ctx).Info(fmt.Sprintf("\u2705 Running xbutil examine -f json -o %s", tmpFilePath)) + + cmd := exec.ExecTask{ + Command: a.XRTPath + "/bin/xbutil", + Args: []string{"examine", "-f", "json", "-o", tmpFilePath}, + Shell: false, + } + result, err := cmd.Execute() + if err != nil { + return fmt.Errorf("Error running xbutil examine: %v", err) + } - examineLines := strings.Split(string(outputXbutil.Stdout), "\n") - for _, examineLine := range examineLines { - - if re.MatchString(examineLine) { - fpgas := strings.Split(examineLine, " : ") - fmt.Printf("FPGAs: %v\n", fpgas) - found := false - - deviceID := "" - reDeviceID := regexp.MustCompile(`user\(inst=(\d+)\)`) - matches := reDeviceID.FindStringSubmatch(fpgas[1]) - if len(matches) > 1 { - deviceID = matches[1] - } - - logicUUID := "" - reLogicUUID := regexp.MustCompile(`[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}`) - matches = reLogicUUID.FindStringSubmatch(fpgas[1]) - if len(matches) > 0 { - logicUUID = matches[0] - } - - shell := "" - reLogiShell := regexp.MustCompile(`xilinx_.*_base_`) - matches = reLogiShell.FindStringSubmatch(fpgas[1]) - if len(matches) > 0 { - shell = matches[0] - } - - for _, fpgaSpec := range a.FPGASpecsList { - if fpgaSpec.LogicUUID == logicUUID { - found = true - break - } - } - if len(fpgas) > 1 && !found { - spec := FPGASpecs{ - BDF: bdf, - Shell: shell, - LogicUUID: logicUUID, - deviceID: deviceID, - DeviceToMount: "/dev/dri/renderD" + deviceID, - Available: true, // Assuming it's available for now, can update based on further output parsing - } - a.FPGASpecsList = append(a.FPGASpecsList, spec) - } - } + // Log stderr to help debug if JSON is still empty + if result.Stderr != "" { + log.G(a.Ctx).Info(fmt.Sprintf("xbutil stderr: %s", result.Stderr)) + } + + // Verify the file was actually written + info, err := os.Stat(tmpFilePath) + if err != nil || info.Size() == 0 { + return fmt.Errorf("xbutil did not write output to %s (stderr: %s)", tmpFilePath, result.Stderr) + } + + // Parse the JSON output + jsonData, err := os.ReadFile(tmpFilePath) + if err != nil { + return fmt.Errorf("Error reading xbutil JSON output: %v", err) + } + + var examineOutput struct { + System struct { + Host struct { + Devices []struct { + BDF string `json:"bdf"` + VBNV string `json:"vbnv"` + ID string `json:"id"` + Instance string `json:"instance"` + IsReady string `json:"is_ready"` + } `json:"devices"` + } `json:"host"` + } `json:"system"` + } + + if err := json.Unmarshal(jsonData, &examineOutput); err != nil { + return fmt.Errorf("Error parsing xbutil JSON output: %v", err) + } + + reDeviceID := regexp.MustCompile(`user\(inst=(\d+)\)`) + + for _, device := range examineOutput.System.Host.Devices { + log.G(a.Ctx).Info(fmt.Sprintf("\u2705 Processing device BDF: %s, VBNV: %s, ID: %s", device.BDF, device.VBNV, device.ID)) + + // Check if already in the list + alreadyFound := false + for _, fpgaSpec := range a.FPGASpecsList { + if fpgaSpec.BDF == device.BDF { + alreadyFound = true + break } } + if alreadyFound { + continue + } + + deviceID := "" + if matches := reDeviceID.FindStringSubmatch(device.Instance); len(matches) > 1 { + deviceID = matches[1] + } + + spec := FPGASpecs{ + BDF: device.BDF, + Shell: device.VBNV, + LogicUUID: device.ID, + deviceID: deviceID, + DeviceToMount: "/dev/dri/renderD" + deviceID, + Available: device.IsReady == "true", + } + a.FPGASpecsList = append(a.FPGASpecsList, spec) } if len(a.FPGASpecsList) > 0 { log.G(a.Ctx).Info("\u2705 Discovered FPGAs:") for _, fpgaSpec := range a.FPGASpecsList { - log.G(a.Ctx).Info(fmt.Sprintf("\u2705 BDF: %s, Shell: %s, LogicUUID: %s, DeviceID %s, Available: %t", fpgaSpec.BDF, fpgaSpec.Shell, fpgaSpec.LogicUUID, fpgaSpec.deviceID, fpgaSpec.Available)) + log.G(a.Ctx).Info(fmt.Sprintf("\u2705 BDF: %s, Shell: %s, LogicUUID: %s, DeviceID: %s, Available: %t", + fpgaSpec.BDF, fpgaSpec.Shell, fpgaSpec.LogicUUID, fpgaSpec.deviceID, fpgaSpec.Available)) } + log.G(a.Ctx).Info(fmt.Sprintf("\u2705 Total FPGAs discovered: %d", len(a.FPGASpecsList))) } else { - log.G(a.Ctx).Info(" \u2705 No FPGAs discovered") + log.G(a.Ctx).Info("\u2705 No FPGAs discovered") } return nil @@ -205,12 +238,9 @@ func (a *FPGAManager) GetFPGASpecsList() []FPGASpecs { return a.FPGASpecsList } -func (a *FPGAManager) Assign(UUID string, containerID string) error { - +func (a *FPGAManager) Assign(BDF string, containerID string) error { for i := range a.FPGASpecsList { - if a.FPGASpecsList[i].LogicUUID == UUID { - - // check if the BOOKKEEPING is disabled + if a.FPGASpecsList[i].BDF == BDF { disableBookkeeping := os.Getenv("FPGA_DISABLE_BOOKKEEPING") == "1" if disableBookkeeping { a.FPGASpecsList[i].ContainerID = containerID @@ -219,7 +249,7 @@ func (a *FPGAManager) Assign(UUID string, containerID string) error { } if !a.FPGASpecsList[i].Available { - return fmt.Errorf("FPGA with UUID %s is already in use by container %s", UUID, a.FPGASpecsList[i].ContainerID) + return fmt.Errorf("FPGA with BDF %s is already in use by container %s", BDF, a.FPGASpecsList[i].ContainerID) } a.FPGASpecsList[i].ContainerID = containerID @@ -228,7 +258,6 @@ func (a *FPGAManager) Assign(UUID string, containerID string) error { } } return nil - } func (a *FPGAManager) Release(containerID string) error { @@ -367,7 +396,6 @@ func (a *FPGAManager) GetAvailableFPGAs(numFPGAs int) ([]FPGASpecs, error) { } func (a *FPGAManager) GetAndAssignAvailableFPGAs(numFPGAs int, containerID string) ([]FPGASpecs, error) { - a.FPGASpecsMutex.Lock() defer a.FPGASpecsMutex.Unlock() @@ -377,7 +405,7 @@ func (a *FPGAManager) GetAndAssignAvailableFPGAs(numFPGAs int, containerID strin } for _, fpgaSpec := range fpgaSpecs { - err = a.Assign(fpgaSpec.LogicUUID, containerID) + err = a.Assign(fpgaSpec.BDF, containerID) // โ† was fpgaSpec.LogicUUID if err != nil { return nil, err } diff --git a/pkg/docker/types.go b/pkg/docker/types.go index 685a709..b1265b8 100644 --- a/pkg/docker/types.go +++ b/pkg/docker/types.go @@ -29,6 +29,7 @@ type DockerConfig struct { XilinxToolsPath string `yaml:"XilinxToolsPath"` VitisPath string `yaml:"VitisPath"` XRTPath string `yaml:"XRTPath"` + DockerNetworkSubnet []string `yaml:"DockerNetworkSubnet"` set bool } @@ -71,31 +72,32 @@ type RetrievedContainer struct { // InterLinkConfig holds the whole configuration type InterLinkConfig struct { - VKConfigPath string `yaml:"VKConfigPath"` - VKTokenFile string `yaml:"VKTokenFile"` - Interlinkurl string `yaml:"InterlinkURL"` - Sidecarurl string `yaml:"SidecarURL"` - Sbatchpath string `yaml:"SbatchPath"` - Scancelpath string `yaml:"ScancelPath"` - Squeuepath string `yaml:"SqueuePath"` - Interlinkport string `yaml:"InterlinkPort"` - Socket string `yaml:"Socket"` - Sidecarport string `yaml:"SidecarPort"` - Commandprefix string `yaml:"CommandPrefix"` - ExportPodData bool `yaml:"ExportPodData"` - DataRootFolder string `yaml:"DataRootFolder"` - ServiceAccount string `yaml:"ServiceAccount"` - Namespace string `yaml:"Namespace"` - Tsocks bool `yaml:"Tsocks"` - Tsockspath string `yaml:"TsocksPath"` - Tsocksconfig string `yaml:"TsocksConfig"` - Tsockslogin string `yaml:"TsocksLoginNode"` - BashPath string `yaml:"BashPath"` - VerboseLogging bool `yaml:"VerboseLogging"` - ErrorsOnlyLogging bool `yaml:"ErrorsOnlyLogging"` - PodIP string `yaml:"PodIP"` - SingularityPrefix string `yaml:"SingularityPrefix"` - set bool + VKConfigPath string `yaml:"VKConfigPath"` + VKTokenFile string `yaml:"VKTokenFile"` + Interlinkurl string `yaml:"InterlinkURL"` + Sidecarurl string `yaml:"SidecarURL"` + Sbatchpath string `yaml:"SbatchPath"` + Scancelpath string `yaml:"ScancelPath"` + Squeuepath string `yaml:"SqueuePath"` + Interlinkport string `yaml:"InterlinkPort"` + Socket string `yaml:"Socket"` + Sidecarport string `yaml:"SidecarPort"` + Commandprefix string `yaml:"CommandPrefix"` + ExportPodData bool `yaml:"ExportPodData"` + DataRootFolder string `yaml:"DataRootFolder"` + ServiceAccount string `yaml:"ServiceAccount"` + Namespace string `yaml:"Namespace"` + Tsocks bool `yaml:"Tsocks"` + Tsockspath string `yaml:"TsocksPath"` + Tsocksconfig string `yaml:"TsocksConfig"` + Tsockslogin string `yaml:"TsocksLoginNode"` + BashPath string `yaml:"BashPath"` + VerboseLogging bool `yaml:"VerboseLogging"` + ErrorsOnlyLogging bool `yaml:"ErrorsOnlyLogging"` + PodIP string `yaml:"PodIP"` + SingularityPrefix string `yaml:"SingularityPrefix"` + DockerNetworkSubnet []string `yaml:"DockerNetworkSubnet"` + set bool } // ContainerLogOpts is a struct in which it is possible to specify options to retrieve logs from the sidecar From 6515754dc7295c98e8bfb8fc4e1820974e95dabe Mon Sep 17 00:00:00 2001 From: Giulio Bianchini Date: Tue, 28 Jul 2026 09:44:05 +0200 Subject: [PATCH 28/30] prevent DIND subnet pool exhaustion from orphaned containers/networks Signed-off-by: Giulio Bianchini --- cmd/main.go | 13 ++- pkg/docker/Create.go | 151 ++++++++++++-------------- pkg/docker/Delete.go | 63 +++++------ pkg/docker/dindmanager/DindHandler.go | 112 ++++++++++++++++++- pkg/docker/func.go | 29 ++--- 5 files changed, 229 insertions(+), 139 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index bc026fa..588db12 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -212,6 +212,17 @@ func main() { dindHandler.CleanDindContainers() dindHandler.BuildDindContainers(int8(availableDindsInt)) + // Periodically reclaim orphan DIND networks left behind by failed creates, so + // a single failure cannot permanently exhaust the subnet pool. Only networks + // with no attached container are removed, and never while a build is running. + go func() { + ticker := time.NewTicker(5 * time.Minute) + defer ticker.Stop() + for range ticker.C { + dindHandler.ReapOrphanNetworks() + } + }() + SidecarAPIs := docker.SidecarHandler{ Config: interLinkConfig, Ctx: ctx, @@ -275,7 +286,7 @@ func main() { // Cleanup the sockfile. c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt, syscall.SIGTERM) + signal.Notify(c, os.Interrupt, syscall.SIGTERM, syscall.SIGHUP) go func() { <-c os.Remove(strings.ReplaceAll(interLinkConfig.Socket, "unix://", "")) diff --git a/pkg/docker/Create.go b/pkg/docker/Create.go index 814f3ba..d318182 100644 --- a/pkg/docker/Create.go +++ b/pkg/docker/Create.go @@ -17,7 +17,6 @@ import ( "errors" commonIL "github.com/interlink-hq/interlink/pkg/interlink" - "github.com/intertwin-eu/interlink-docker-plugin/pkg/docker/dindmanager" "path/filepath" @@ -283,39 +282,12 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { attribute.Int64("start.timestamp", start), )) - // create bool variable to set if a new dind container has to be created - newDindContainerCreated := false - - // get a dind container ID from dind manager of the sidecard handler - dindContainerID, err := h.DindManager.GetAvailableDind() - if err != nil { - - log.G(h.Ctx).Info("\u2705 [POD FLOW] No available DIND container found, creating a new one") - - h.DindManager.BuildDindContainers(1) - dindContainerID, err = h.DindManager.GetAvailableDind() - if err != nil { - HandleErrorAndRemoveData(h, w, "During creation of new DIND container, an error occurred during the request of get available DIND container", err, "", "") - return - } - newDindContainerCreated = true - } - - // remove the dind container from the list of available dind containers - err = h.DindManager.SetDindUnavailable(dindContainerID) - if err != nil { - HandleErrorAndRemoveData(h, w, "An error occurred during the removal of the DIND container from the list of available DIND containers", err, "", "") - return - } - - if !newDindContainerCreated { - // create a new dind container in background - go h.DindManager.BuildDindContainers(1) - } - - //var execReturn exec.ExecResult statusCode := http.StatusOK + // Read and parse the request body FIRST, so the pod UID is known before a DIND + // container is claimed. Claiming with the pod UID (ClaimAvailableDind) makes + // the assignment atomic \u2014 Available=false and PodUID are set together \u2014 so a + // failure at any later point can always find and clean up the right DIND. bodyBytes, err := io.ReadAll(r.Body) if err != nil { HandleErrorAndRemoveData(h, w, "An error occurred during read of body request for pod creation", err, "", "") @@ -323,20 +295,41 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { } var req commonIL.RetrievedPodData - err = json.Unmarshal(bodyBytes, &req) - - if err != nil { + if err = json.Unmarshal(bodyBytes, &req); err != nil { HandleErrorAndRemoveData(h, w, "An error occurred during json unmarshal of data from pod creation request", err, "", "") return } - wd, err := os.Getwd() + log.G(h.Ctx).Info("\u2705 [POD FLOW] Request data unmarshalled successfully") + + podNamespace := string(req.Pod.Namespace) + podUID := string(req.Pod.UID) + + // Atomically claim a DIND container for this pod. + newDindContainerCreated := false + dindContainerID, err := h.DindManager.ClaimAvailableDind(podUID) if err != nil { - HandleErrorAndRemoveData(h, w, "Unable to get current working directory", err, "", "") - return + + log.G(h.Ctx).Info("\u2705 [POD FLOW] No available DIND container found, creating a new one") + + // The build error must be reported: without it the only thing reaching the + // caller is the generic "no available DIND container" from the claim below. + if buildErr := h.DindManager.BuildDindContainers(1); buildErr != nil { + HandleErrorAndRemoveData(h, w, "An error occurred during the creation of a new DIND container", buildErr, podNamespace, podUID) + return + } + dindContainerID, err = h.DindManager.ClaimAvailableDind(podUID) + if err != nil { + HandleErrorAndRemoveData(h, w, "During creation of new DIND container, an error occurred during the request of get available DIND container", err, podNamespace, podUID) + return + } + newDindContainerCreated = true } - log.G(h.Ctx).Info("\u2705 [POD FLOW] Request data unmarshalled successfully and current working directory detected") + if !newDindContainerCreated { + // replenish the pool in the background since we consumed a pre-warmed one + go h.DindManager.BuildDindContainers(1) + } var newReq []commonIL.RetrievedPodData newReq = []commonIL.RetrievedPodData{req} @@ -346,7 +339,7 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { podUID := string(data.Pod.UID) podNamespace := string(data.Pod.Namespace) - podDirectoryPath := filepath.Join(wd, h.Config.DataRootFolder+"/"+podNamespace+"-"+podUID) + podDirectoryPath := filepath.Join(h.Config.DataRootFolder, podNamespace+"-"+podUID) // Sentinel file written by mesh.sh once network setup is complete. // containers_command.sh polls for this file before starting workload containers. @@ -365,7 +358,7 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { if _, err := os.Stat(podDirectoryPath); os.IsNotExist(err) { err = os.MkdirAll(podDirectoryPath, os.ModePerm) if err != nil { - HandleErrorAndRemoveData(h, w, "An error occurred during the creation of the pod directory", err, "", "") + HandleErrorAndRemoveData(h, w, "An error occurred during the creation of the pod directory", err, podNamespace, podUID) return } } @@ -373,7 +366,7 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { // call prepareDockerRuns to get the DockerRunStruct array dockerRunStructs, err := h.prepareDockerRuns(data, w) if err != nil { - HandleErrorAndRemoveData(h, w, "An error occurred during preparing of docker run commmands", err, "", "") + HandleErrorAndRemoveData(h, w, "An error occurred during preparing of docker run commmands", err, podNamespace, podUID) return } @@ -513,13 +506,6 @@ cd $TMPDIR } } - // set the podUID to the dind container - err = h.DindManager.SetPodUIDToDind(dindContainerID, podUID) - if err != nil { - HandleErrorAndRemoveData(h, w, "An error occurred during the setting of the pod UID to the DIND container", err, "", "") - return - } - // run the docker command to rename the container to the pod UID shell := exec.ExecTask{ Command: "docker", @@ -529,7 +515,7 @@ cd $TMPDIR _, err = shell.Execute() if err != nil { - HandleErrorAndRemoveData(h, w, "An error occurred during the rename of the DIND container", err, "", "") + HandleErrorAndRemoveData(h, w, "An error occurred during the rename of the DIND container", err, podNamespace, podUID) return } @@ -595,7 +581,7 @@ cd $TMPDIR createResponseBytes, err := json.Marshal(createResponse) if err != nil { statusCode = http.StatusInternalServerError - HandleErrorAndRemoveData(h, w, "An error occurred during the json marshal of the returned JID", err, "", "") + HandleErrorAndRemoveData(h, w, "An error occurred during the json marshal of the returned JID", err, podNamespace, podUID) return } @@ -647,7 +633,7 @@ cd $TMPDIR _, err := shell.Execute() if err != nil { - HandleErrorAndRemoveData(h, w, "An error occurred during the exec of the init container command", err, "", "") + HandleErrorAndRemoveData(h, w, "An error occurred during the exec of the init container command", err, podNamespace, podUID) return } @@ -660,7 +646,7 @@ cd $TMPDIR statusReturn, err := shell.Execute() if err != nil { - HandleErrorAndRemoveData(h, w, "An error occurred during inspect of init container", err, "", "") + HandleErrorAndRemoveData(h, w, "An error occurred during inspect of init container", err, podNamespace, podUID) return } @@ -736,7 +722,7 @@ cd $TMPDIR err = os.WriteFile(podDirectoryPath+"/containers_command.sh", []byte(containersCommand), 0644) if err != nil { log.G(h.Ctx).Error("\u274C [POD FLOW] Error writing containers command script: " + err.Error()) - HandleErrorAndRemoveData(h, w, "An error occurred during the creation of the container commands script.", err, "", "") + HandleErrorAndRemoveData(h, w, "An error occurred during the creation of the container commands script.", err, podNamespace, podUID) return } @@ -752,7 +738,7 @@ cd $TMPDIR _, err = shell.Execute() if err != nil { log.G(h.Ctx).Error("\u274C [POD FLOW] Error executing containers command script: " + err.Error()) - HandleErrorAndRemoveData(h, w, "An error occurred during the execution of the container command script", err, "", "") + HandleErrorAndRemoveData(h, w, "An error occurred during the execution of the container command script", err, podNamespace, podUID) return } @@ -772,34 +758,41 @@ func HandleErrorAndRemoveData(h *SidecarHandler, w http.ResponseWriter, s string if podNamespace != "" && podUID != "" { os.RemoveAll(h.Config.DataRootFolder + podNamespace + "-" + podUID) } - dindSpec := dindmanager.DindSpecs{} - dindSpec, err = h.DindManager.GetDindFromPodUID(podUID) - if err != nil { - log.G(h.Ctx).Error("\u274C [CREATE CALL] Error retrieving DindSpecs, maybe the Dind container has already been deleted") - } else { - log.G(h.Ctx).Info("\u2705 [CREATE CALL] Retrieved DindSpecs: " + dindSpec.DindID + " " + dindSpec.PodUID + " " + dindSpec.DindNetworkID + " ") + // Without a pod UID there is no DIND to reconcile (e.g. the body was never + // parsed). Bail out here rather than matching an unrelated DIND by "". + if podUID == "" { + return + } - // log the retrieved dindSpec - log.G(h.Ctx).Info("\u2705 [CREATE CALL] Retrieved DindSpecs: " + dindSpec.DindID + " " + dindSpec.PodUID + " " + dindSpec.DindNetworkID + " ") + dindSpec, lookupErr := h.DindManager.GetDindFromPodUID(podUID) + if lookupErr != nil { + log.G(h.Ctx).Info("\u2139\uFE0F [CREATE CALL] No DIND container assigned to pod " + podUID + " to clean up") + return + } - cmd := []string{"network", "rm", dindSpec.DindNetworkID} - shell := exec.ExecTask{ - Command: "docker", - Args: cmd, - Shell: true, - } - execReturn, _ := shell.Execute() - execReturn.Stdout = strings.ReplaceAll(execReturn.Stdout, "\n", "") - if execReturn.Stderr != "" { - log.G(h.Ctx).Error("\u274C [CREATE CALL] Error deleting network " + dindSpec.DindNetworkID) - } else { - log.G(h.Ctx).Info("\u2705 [CREATE CALL] Deleted network " + dindSpec.DindNetworkID) + log.G(h.Ctx).Info("\u2705 [CREATE CALL] Cleaning up DIND for pod " + podUID + ": " + dindSpec.DindID + " (network " + dindSpec.DindNetworkID + ")") + + // Force-remove the DIND container FIRST. While it is running it holds an + // endpoint on its bridge network, so the network cannot be removed and its + // /24 subnet stays effectively allocated \u2014 which then causes the next create + // drawing that subnet from the pool to fail with an address overlap. The + // container may be named after the pod UID (after the rename step) or after + // its original build UID (before it); remove both, ignoring "no such + // container" errors. + for _, name := range []string{podUID + "_dind", dindSpec.DindID} { + if name == "" { + continue } - // set the dind available again - err = h.DindManager.RemoveDindFromList(dindSpec.PodUID) - if err != nil { - log.G(h.Ctx).Error("\u274C [CREATE CALL] Error setting DIND container available") + rm := exec.ExecTask{Command: "docker", Args: []string{"rm", "-f", name}, Shell: true} + if execReturn, rmErr := rm.Execute(); rmErr != nil || execReturn.ExitCode != 0 { + log.G(h.Ctx).Warning("\u26A0\uFE0F [CREATE CALL] Could not remove DIND container " + name + ": " + strings.TrimSpace(execReturn.Stderr)) } } + + // Now that the container is gone, RemoveDindFromList can actually delete the + // bridge network and return the subnet to the pool. + if remErr := h.DindManager.RemoveDindFromList(dindSpec.PodUID); remErr != nil { + log.G(h.Ctx).Error("\u274C [CREATE CALL] Error removing DIND from list: " + remErr.Error()) + } } diff --git a/pkg/docker/Delete.go b/pkg/docker/Delete.go index bed1c86..ee266bf 100644 --- a/pkg/docker/Delete.go +++ b/pkg/docker/Delete.go @@ -5,13 +5,13 @@ import ( "io" "net/http" "os" + "strconv" "strings" "time" exec "github.com/alexellis/go-execute/pkg/v1" "github.com/containerd/containerd/log" commonIL "github.com/intertwin-eu/interlink-docker-plugin/pkg/common" - "github.com/intertwin-eu/interlink-docker-plugin/pkg/docker/dindmanager" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" trace "go.opentelemetry.io/otel/trace" @@ -78,49 +78,40 @@ func (h *SidecarHandler) DeleteHandler(w http.ResponseWriter, r *http.Request) { execReturn, _ = shell.Execute() execReturn.Stdout = strings.ReplaceAll(execReturn.Stdout, "\n", "") - if execReturn.Stderr != "" { - log.G(h.Ctx).Error("\u274C [DELETE CALL] Error deleting container " + podUID + "_dind") - statusCode = http.StatusInternalServerError + // go-execute reports a nil error for non-zero exits, so inspect ExitCode + // rather than Stderr (docker prints benign warnings to stderr on success). + // Removing a missing container is NOT treated as fatal: a pod can fail before + // its DIND is ever created, and delete must stay idempotent \u2014 returning 500 + // here would make interLink retry the delete forever. + if execReturn.ExitCode != 0 { + log.G(h.Ctx).Warning("\u26A0\uFE0F [DELETE CALL] docker rm -f " + podUID + "_dind exited " + + strconv.Itoa(execReturn.ExitCode) + ": " + strings.TrimSpace(execReturn.Stderr)) } else { log.G(h.Ctx).Info("\u2705 [DELETE CALL] Deleted container " + podUID + "_dind") } - dindSpec := dindmanager.DindSpecs{} - dindSpec, err = h.DindManager.GetDindFromPodUID(podUID) - - if err != nil { - log.G(h.Ctx).Error("\u274C [DELETE CALL] Error retrieving DindSpecs, maybe the Dind container has already been deleted") + dindSpec, lookupErr := h.DindManager.GetDindFromPodUID(podUID) + if lookupErr != nil { + // The in-memory entry may already be gone (a failed create cleaned it, or + // it never got a pod UID). We cannot name its network here, but the + // orphan-network reaper below reclaims any unattached _dind_network. + log.G(h.Ctx).Info("\u2139\uFE0F [DELETE CALL] No DIND entry for pod " + podUID + " (already removed?)") } else { - log.G(h.Ctx).Info("\u2705 [DELETE CALL] Retrieved DindSpecs: " + dindSpec.DindID + " " + dindSpec.PodUID + " " + dindSpec.DindNetworkID + " ") + log.G(h.Ctx).Info("\u2705 [DELETE CALL] Retrieved DindSpecs: " + dindSpec.DindID + " " + dindSpec.PodUID + " " + dindSpec.DindNetworkID) - // log the retrieved dindSpec - log.G(h.Ctx).Info("\u2705 [DELETE CALL] Retrieved DindSpecs: " + dindSpec.DindID + " " + dindSpec.PodUID + " " + dindSpec.DindNetworkID + " ") - - cmd = []string{"network", "rm", dindSpec.DindNetworkID} - shell = exec.ExecTask{ - Command: "docker", - Args: cmd, - Shell: true, - } - execReturn, _ = shell.Execute() - execReturn.Stdout = strings.ReplaceAll(execReturn.Stdout, "\n", "") - if execReturn.Stderr != "" { - log.G(h.Ctx).Error("\u274C [DELETE CALL] Error deleting network " + dindSpec.DindNetworkID) - } else { - log.G(h.Ctx).Info("\u2705 [DELETE CALL] Deleted network " + dindSpec.DindNetworkID) - } - // set the dind available again - err = h.DindManager.RemoveDindFromList(dindSpec.PodUID) - if err != nil { - log.G(h.Ctx).Error("\u274C [DELETE CALL] Error setting DIND container available") + // RemoveDindFromList removes the bridge network and returns the subnet to + // the pool. The DIND container was already force-removed above, so its + // network no longer has an attached endpoint and can now be deleted. + if remErr := h.DindManager.RemoveDindFromList(dindSpec.PodUID); remErr != nil { + log.G(h.Ctx).Error("\u274C [DELETE CALL] Error removing DIND from list: " + remErr.Error()) } } - wd, err := os.Getwd() - if err != nil { - HandleErrorAndRemoveData(h, w, "Unable to get current working directory", err, "", "") - return - } - podDirectoryPathToDelete := filepath.Join(wd, h.Config.DataRootFolder+"/"+podNamespace+"-"+podUID) + + // Best-effort: reclaim any orphan DIND networks left behind by a create that + // failed before its entry was fully registered. + h.DindManager.ReapOrphanNetworks() + + podDirectoryPathToDelete := filepath.Join(h.Config.DataRootFolder, podNamespace+"-"+podUID) log.G(h.Ctx).Info("\u2705 [DELETE CALL] Deleting directory " + podDirectoryPathToDelete) err = os.RemoveAll(podDirectoryPathToDelete) diff --git a/pkg/docker/dindmanager/DindHandler.go b/pkg/docker/dindmanager/DindHandler.go index 7da55db..e88766b 100644 --- a/pkg/docker/dindmanager/DindHandler.go +++ b/pkg/docker/dindmanager/DindHandler.go @@ -14,6 +14,7 @@ import ( "github.com/containerd/containerd/log" OSexec "os/exec" + "sync/atomic" ) type DindManagerInterface interface { @@ -21,11 +22,13 @@ type DindManagerInterface interface { BuildDindContainers(nDindContainer int8) error PrintDindList() error GetAvailableDind() (string, error) + ClaimAvailableDind(podUID string) (string, error) SetDindUnavailable(dindID string) error RemoveDindFromList(PodUID string) error SetPodUIDToDind(dindID string, podUID string) error GetDindFromPodUID(podUID string) (DindSpecs, error) SetDindAvailable(PodUID string) error + ReapOrphanNetworks() } type DindSpecs struct { @@ -47,6 +50,16 @@ type DindManager struct { SubnetPool []string mu sync.Mutex InitialPoolSz int // total subnets at startup, used for logging + + // listMu protects DindList. Kept separate from mu (which protects SubnetPool) + // so that RemoveDindFromList can hold listMu while calling freeSubnet (mu) + // without self-deadlock. Lock ordering is always listMu -> mu, never reverse. + listMu sync.Mutex + + // buildsInFlight counts in-progress BuildDindContainers calls. The orphan + // network reaper checks it so it never removes a bridge network in the brief + // window after it is created but before its DIND container attaches to it. + buildsInFlight int32 } // --------------------------------------------------------------------------- @@ -197,6 +210,11 @@ func (a *DindManager) CleanDindContainers() error { } func (a *DindManager) BuildDindContainers(nDindContainer int8) error { + // Signal to the reaper that a build is running: freshly created networks are + // briefly container-less and must not be reaped as "orphans". + atomic.AddInt32(&a.buildsInFlight, 1) + defer atomic.AddInt32(&a.buildsInFlight, -1) + log.G(a.Ctx).Info(fmt.Sprintf("\u2705 Creating %d DIND containers", nDindContainer)) // Log subnet pool status before we start allocating. @@ -215,7 +233,7 @@ func (a *DindManager) BuildDindContainers(nDindContainer int8) error { } gpuEnabled := os.Getenv("GPUENABLED") - dindImage := "docker:dind" + dindImage := "docker:29.3.0-dind" if gpuEnabled == "1" { dindImage = "ghcr.io/extrality/nvidia-dind" } @@ -301,27 +319,45 @@ func (a *DindManager) BuildDindContainers(nDindContainer int8) error { log.G(a.Ctx).Info(fmt.Sprintf("\u2705 Command is %s", shell.Command+" "+strings.Join(shell.Args, " "))) execReturn, err := shell.Execute() + // go-execute returns a nil error for a non-zero exit, so the exit code has + // to be checked explicitly or a failed "docker run" is silently treated as + // a success (empty container ID, then a readiness timeout 20s later). + if err == nil && execReturn.ExitCode != 0 { + err = fmt.Errorf("docker run exited with code %d: %s", + execReturn.ExitCode, strings.TrimSpace(execReturn.Stderr)) + } if err != nil { - log.G(a.Ctx).Error(fmt.Sprintf("\u274c Error creating DIND container %s", randUID+"_dind")) + log.G(a.Ctx).Error(fmt.Sprintf("\u274c Error creating DIND container %s: %v", randUID+"_dind", err)) log.G(a.Ctx).Error(fmt.Sprintf("\u274c %s", execReturn.Stderr)) // Free the subnet so it can be reused. a.freeSubnet(allocatedSubnet) return err } - dindContainerID := execReturn.Stdout + dindContainerID := strings.TrimSpace(execReturn.Stdout) + if dindContainerID == "" { + a.freeSubnet(allocatedSubnet) + return fmt.Errorf("docker run returned an empty container ID for %s (stderr: %s)", + randUID+"_dind", strings.TrimSpace(execReturn.Stderr)) + } // ---------------------------------------------------------------- // Wait for the daemon inside the DinD container to be ready. // ---------------------------------------------------------------- maxRetries := 20 + lastOutput := "" for { if maxRetries == 0 { a.freeSubnet(allocatedSubnet) + // Surface what the daemon actually logged, otherwise the timeout + // is indistinguishable from a container that never started. + log.G(a.Ctx).Error(fmt.Sprintf("โŒ Last output of \"docker logs %s\": %s", + randUID+"_dind", strings.TrimSpace(lastOutput))) return fmt.Errorf("DIND container %s did not become ready in time", dindContainerID) } cmd := OSexec.Command("docker", "logs", randUID+"_dind") output, err := cmd.CombinedOutput() + lastOutput = string(output) if err == nil && strings.Contains(string(output), "API listen on /var/run/docker.sock") { break } @@ -349,6 +385,7 @@ func (a *DindManager) BuildDindContainers(nDindContainer int8) error { // ---------------------------------------------------------------- // Register the new DinD in the list. // ---------------------------------------------------------------- + a.listMu.Lock() a.DindList = append(a.DindList, DindSpecs{ DindID: randUID + "_dind", PodUID: "", @@ -356,12 +393,15 @@ func (a *DindManager) BuildDindContainers(nDindContainer int8) error { AllocatedSubnet: allocatedSubnet, // empty string when no pool Available: true, }) + a.listMu.Unlock() } return nil } func (a *DindManager) PrintDindList() error { + a.listMu.Lock() + defer a.listMu.Unlock() for _, d := range a.DindList { log.G(a.Ctx).Info(fmt.Sprintf( "DindID: %s, PodUID: %s, DindNetworkID: %s, AllocatedSubnet: %q, Available: %t", @@ -372,6 +412,8 @@ func (a *DindManager) PrintDindList() error { } func (a *DindManager) GetDindFromPodUID(podUID string) (DindSpecs, error) { + a.listMu.Lock() + defer a.listMu.Unlock() for _, d := range a.DindList { if d.PodUID == podUID { return d, nil @@ -381,6 +423,8 @@ func (a *DindManager) GetDindFromPodUID(podUID string) (DindSpecs, error) { } func (a *DindManager) GetAvailableDind() (string, error) { + a.listMu.Lock() + defer a.listMu.Unlock() for _, d := range a.DindList { if d.Available { return d.DindID, nil @@ -389,7 +433,28 @@ func (a *DindManager) GetAvailableDind() (string, error) { return "", fmt.Errorf("no available DIND container") } +// ClaimAvailableDind atomically finds an available DIND container, marks it +// unavailable, and records the pod UID against it โ€” all under a single lock. +// Doing this in one step (instead of GetAvailableDind + SetDindUnavailable + +// SetPodUIDToDind) prevents two concurrent creates from grabbing the same +// container, and guarantees the DIND is findable by pod UID for cleanup the +// moment it is claimed, closing the window where a failure would orphan it. +func (a *DindManager) ClaimAvailableDind(podUID string) (string, error) { + a.listMu.Lock() + defer a.listMu.Unlock() + for i, d := range a.DindList { + if d.Available { + a.DindList[i].Available = false + a.DindList[i].PodUID = podUID + return a.DindList[i].DindID, nil + } + } + return "", fmt.Errorf("no available DIND container") +} + func (a *DindManager) SetDindUnavailable(dindID string) error { + a.listMu.Lock() + defer a.listMu.Unlock() for i, d := range a.DindList { if d.DindID == dindID { a.DindList[i].Available = false @@ -400,6 +465,8 @@ func (a *DindManager) SetDindUnavailable(dindID string) error { } func (a *DindManager) SetDindAvailable(PodUID string) error { + a.listMu.Lock() + defer a.listMu.Unlock() for i, d := range a.DindList { if d.PodUID == PodUID { a.DindList[i].Available = true @@ -410,6 +477,8 @@ func (a *DindManager) SetDindAvailable(PodUID string) error { } func (a *DindManager) SetPodUIDToDind(dindID string, podUID string) error { + a.listMu.Lock() + defer a.listMu.Unlock() for i, d := range a.DindList { if d.DindID == dindID { a.DindList[i].PodUID = podUID @@ -419,10 +488,47 @@ func (a *DindManager) SetPodUIDToDind(dindID string, podUID string) error { return fmt.Errorf("DIND container %s not found", dindID) } +// ReapOrphanNetworks removes DIND bridge networks that have no attached +// container. A create that fails after its network is created leaves such a +// network behind; because it still holds its /24 subnet, a later create drawing +// the same subnet from the pool fails with an address-overlap error. Removing +// only unattached networks is safe: Docker refuses to remove a network still in +// use by a live DIND container, so those are skipped. The build-in-flight guard +// avoids racing a network that was just created but whose container has not yet +// attached. +func (a *DindManager) ReapOrphanNetworks() { + if atomic.LoadInt32(&a.buildsInFlight) > 0 { + return + } + list := exec.ExecTask{ + Command: "docker", + Args: []string{"network", "ls", "--filter", "name=_dind_network", "--format", "{{.Name}}"}, + Shell: true, + } + execReturn, err := list.Execute() + if err != nil || execReturn.ExitCode != 0 { + return + } + for _, name := range strings.Fields(execReturn.Stdout) { + rm := exec.ExecTask{ + Command: "docker", + Args: []string{"network", "rm", name}, + Shell: true, + } + // Fails harmlessly (non-zero exit) if the network still has a live + // container attached; only truly orphaned networks are removed. + if r, _ := rm.Execute(); r.ExitCode == 0 { + log.G(a.Ctx).Info("๐Ÿงน Reaped orphan DIND network " + name) + } + } +} + // RemoveDindFromList removes the DinD entry associated with PodUID from the // in-memory list, tears down its Docker network, and returns its subnet to // the pool so it can be reused by future containers. func (a *DindManager) RemoveDindFromList(PodUID string) error { + a.listMu.Lock() + defer a.listMu.Unlock() for i, d := range a.DindList { if d.PodUID != PodUID { continue diff --git a/pkg/docker/func.go b/pkg/docker/func.go index fb55ad1..5d6811f 100644 --- a/pkg/docker/func.go +++ b/pkg/docker/func.go @@ -154,18 +154,13 @@ func parseContainerCommandAndReturnArgs(Ctx context.Context, config DockerConfig prefileName := container.Name + "_" + podUID + "_" + podNamespace - wd, err := os.Getwd() - if err != nil { - return nil, nil, nil, err - } - if len(container.Command) > 0 { fileName := prefileName + "_script.sh" if len(container.Args) == 0 { - fileNamePath := filepath.Join(wd, config.DataRootFolder+podNamespace+"-"+podUID, fileName) - err = os.WriteFile(fileNamePath, []byte(strings.Join(container.Command, " ")), 0644) + fileNamePath := filepath.Join(config.DataRootFolder+podNamespace+"-"+podUID, fileName) + err := os.WriteFile(fileNamePath, []byte(strings.Join(container.Command, " ")), 0644) if err != nil { log.G(Ctx).Error(err) return nil, nil, nil, err @@ -174,15 +169,15 @@ func parseContainerCommandAndReturnArgs(Ctx context.Context, config DockerConfig } argsFileName := container.Name + "_args" - argsFileNamePath := filepath.Join(wd, config.DataRootFolder+podNamespace+"-"+podUID, argsFileName) - err = os.WriteFile(argsFileNamePath, []byte(strings.Join(container.Args, " ")), 0644) + argsFileNamePath := filepath.Join(config.DataRootFolder+podNamespace+"-"+podUID, argsFileName) + err := os.WriteFile(argsFileNamePath, []byte(strings.Join(container.Args, " ")), 0644) if err != nil { log.G(Ctx).Error(err) return nil, nil, nil, err } fullFileContent := strings.Join(container.Command, " ") + " \"$(cat " + argsFileName + ")\"" - fullFileNamePath := filepath.Join(wd, config.DataRootFolder+podNamespace+"-"+podUID, fileName) + fullFileNamePath := filepath.Join(config.DataRootFolder+podNamespace+"-"+podUID, fileName) err = os.WriteFile(fullFileNamePath, []byte(fullFileContent), 0644) if err != nil { log.G(Ctx).Error(err) @@ -262,12 +257,6 @@ func prepareMounts(Ctx context.Context, config DockerConfig, data commonIL.Retri } func mountData(Ctx context.Context, config DockerConfig, pod v1.Pod, data interface{}, container v1.Container) ([]string, error) { - wd, err := os.Getwd() - if err != nil { - log.G(Ctx).Error(err) - return nil, err - } - for _, mountSpec := range container.VolumeMounts { var podVolumeSpec *v1.VolumeSource @@ -287,7 +276,7 @@ func mountData(Ctx context.Context, config DockerConfig, pod v1.Pod, data interf } if podVolumeSpec != nil && podVolumeSpec.ConfigMap != nil { - podConfigMapDir := filepath.Join(wd+"/"+config.DataRootFolder+pod.Namespace+"-"+string(pod.UID)+"/", "configMaps/", vol.Name) + podConfigMapDir := filepath.Join(config.DataRootFolder+pod.Namespace+"-"+string(pod.UID), "configMaps", vol.Name) mode := os.FileMode(*podVolumeSpec.ConfigMap.DefaultMode) correctMountPath := "" @@ -338,7 +327,7 @@ func mountData(Ctx context.Context, config DockerConfig, pod v1.Pod, data interf } if podVolumeSpec != nil && podVolumeSpec.Secret != nil { mode := os.FileMode(*podVolumeSpec.Secret.DefaultMode) - podSecretDir := filepath.Join(wd+"/"+config.DataRootFolder+pod.Namespace+"-"+string(pod.UID)+"/", "secrets/", vol.Name) + podSecretDir := filepath.Join(config.DataRootFolder+pod.Namespace+"-"+string(pod.UID), "secrets", vol.Name) if mount.Data != nil { for key := range mount.Data { @@ -403,7 +392,7 @@ func mountData(Ctx context.Context, config DockerConfig, pod v1.Pod, data interf } } - edPath = filepath.Join(wd + "/" + config.DataRootFolder + pod.Namespace + "-" + string(pod.UID) + "/" + "emptyDirs/" + vol.Name) + edPath = filepath.Join(config.DataRootFolder+pod.Namespace+"-"+string(pod.UID), "emptyDirs", vol.Name) cmd := []string{"-p " + edPath} shell := exec2.ExecTask{ Command: "mkdir", @@ -432,5 +421,5 @@ func mountData(Ctx context.Context, config DockerConfig, pod v1.Pod, data interf } } - return nil, err + return nil, errors.New("Volume " + container.Name + " not found in pod spec") } From b8f7af506337fd1ec3242f39cb12f4e0fa0160d4 Mon Sep 17 00:00:00 2001 From: Giulio Bianchini Date: Thu, 20 Aug 2026 10:03:27 +0200 Subject: [PATCH 29/30] fix crash and resource-leak paths in pod creation, status and DIND rollback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Create.go - Do not touch the ResponseWriter after the create handler has returned. The asynchronous container-creation goroutine called HandleErrorAndRemoveData, which writes to w long after the response was sent; that races with the http server and can corrupt the next response on a keep-alive connection. Split the cleanup out into cleanupPodData, which never touches w, and use it from the goroutine. HandleErrorAndRemoveData stays for the request goroutine. - Recover panics in that goroutine. net/http does not recover panics in bare goroutines spawned by a handler, so any nil deref there took the whole plugin down along with the bookkeeping of every other running pod. - Bound the mesh_ready wait. When mesh.sh fails the sentinel is never written, and the poll loop in containers_command.sh spun forever: the pod stayed in Waiting with no diagnostics and its DIND stayed claimed until a human deleted it. The loop now times out after 300s, dumps the overlay container logs and exits non-zero, and the docker exec itself runs under a 10m context so it cannot leak the goroutine for any other reason. Script output is logged on failure. - Select the network-overlay container by name instead of assuming it is containers[0]. The startup path was gated on the mesh annotation, a weaker condition than the one under which the overlay is actually built; when the two disagreed a workload container was started as the overlay and every other container waited on a sentinel nothing would write. It also panicked on an empty container list. The mismatch is now logged as a warning and the containers start directly. - Scope envVars and fpgaArgs to a single container. Declared outside the loop, every container inherited the -e/-v flags and --device entries of the containers processed before it. Status.go - Do not fail a whole status batch because one pod has no DIND container. interLink asks for every pod on the node in one call, so a 404 for a single missing DIND left all the other pods unreconciled. That pod is now skipped with a warning, which is also safer than inventing a terminal status: the DIND is legitimately absent between the claim and the rename. An all-skipped response marshals to [] rather than null. DindHandler.go - Tear the DIND down before returning its subnet to the pool. The failure paths freed the subnet while the container and its network still existed, so the next network create for that range failed with "Pool overlaps with other one on this address space" โ€” permanently, since the reaper cannot remove a network whose container is alive. rollbackDind removes container, then network, then frees the subnet, and deliberately leaks the subnet with a loud error if the network cannot be removed. - Check the exit code of docker network create. go-execute returns a nil error for a non-zero exit, so a rejected create was treated as a success and only surfaced later as an unrelated docker run failure. AMDHandler.go - Track FPGA assignees as a list instead of a single ContainerID, so sharing under FPGA_DISABLE_BOOKKEEPING keeps every assignee and Release can find each one instead of only the last. Assign/GetAvailableFPGAs no longer double-lock FPGASpecsMutex (added assignLocked/getAvailableFPGAsLocked), the fallback sort runs on a copy rather than reordering the shared list under no lock, and the same physical device is no longer returned twice in one call. Create_test.go - Unit tests for selectNetworkOverlay, including the no-overlay regression and the empty-list panic. --- pkg/docker/Create.go | 151 ++++++++++++++--- pkg/docker/Create_test.go | 123 ++++++++++++++ pkg/docker/Status.go | 50 +++--- pkg/docker/dindmanager/DindHandler.go | 75 ++++++++- pkg/docker/fpgastrategies/AMDHandler.go | 214 ++++++++++-------------- 5 files changed, 439 insertions(+), 174 deletions(-) create mode 100644 pkg/docker/Create_test.go diff --git a/pkg/docker/Create.go b/pkg/docker/Create.go index d318182..ac53fc4 100644 --- a/pkg/docker/Create.go +++ b/pkg/docker/Create.go @@ -1,6 +1,7 @@ package docker import ( + "context" "encoding/json" "fmt" "io" @@ -10,6 +11,8 @@ import ( "strings" "time" + OSexec "os/exec" + exec "github.com/alexellis/go-execute/pkg/v1" "github.com/containerd/containerd/log" v1 "k8s.io/api/core/v1" @@ -25,10 +28,49 @@ import ( trace "go.opentelemetry.io/otel/trace" ) +const ( + // meshReadyTimeoutSeconds bounds how long containers_command.sh waits for the + // mesh_ready sentinel written by the network-overlay container. + meshReadyTimeoutSeconds = 300 + + // containersCommandTimeout bounds the whole containers_command.sh execution + // inside the DIND container. It must be larger than meshReadyTimeoutSeconds so + // the script gets the chance to report the mesh timeout itself. + containersCommandTimeout = 10 * time.Minute +) + +// selectNetworkOverlay splits containers into the network-overlay container and +// the workload containers that have to wait for the mesh sentinel it writes. +// ok is false when overlayName is empty (no overlay was built for this pod) or +// when no container carries that name, in which case overlay is the zero value +// and workload holds every container unchanged. +// +// The overlay is prepended to the list, so it is normally at index 0. Taking it +// from there unconditionally was the bug: the condition that selected the mesh +// startup path (the pod carries a mesh annotation) is weaker than the conditions +// under which the overlay is actually built, so when the two disagreed an +// ordinary workload container was started as the overlay and every other +// container waited on a sentinel nothing would ever write. It also panicked on +// an empty container list. Matching by name makes the two impossible to confuse. +func selectNetworkOverlay(containers []DockerRunStruct, overlayName string) (overlay DockerRunStruct, workload []DockerRunStruct, ok bool) { + if overlayName == "" { + return DockerRunStruct{}, containers, false + } + for idx, c := range containers { + if c.Name != overlayName { + continue + } + workload = make([]DockerRunStruct, 0, len(containers)-1) + workload = append(workload, containers[:idx]...) + workload = append(workload, containers[idx+1:]...) + return c, workload, true + } + return DockerRunStruct{}, containers, false +} + func (h *SidecarHandler) prepareDockerRuns(podData commonIL.RetrievedPodData, w http.ResponseWriter) ([]DockerRunStruct, error) { var dockerRunStructs []DockerRunStruct - var fpgaArgs string = "" podUID := string(podData.Pod.UID) podNamespace := string(podData.Pod.Namespace) @@ -78,10 +120,14 @@ func (h *SidecarHandler) prepareDockerRuns(podData commonIL.RetrievedPodData, w for containerType, containers := range allContainers { isInitContainer := containerType == "initContainers" - var envVars string = "" - for _, container := range containers { + // envVars and fpgaArgs MUST be scoped to the single container: when they + // were declared outside this loop every container inherited the -e/-v + // flags and --device entries of the containers processed before it. + var envVars string = "" + var fpgaArgs string = "" + containerName := podNamespace + "-" + podUID + "-" + container.Name var isFPGARequested bool = false @@ -370,6 +416,14 @@ func (h *SidecarHandler) CreateHandler(w http.ResponseWriter, r *http.Request) { return } + // Name of the network-overlay container, set below only if one is actually + // built and prepended to dockerRunStructs. Everything downstream keys off + // this instead of re-testing the annotation: the annotation check and the + // overlay creation do not have the same preconditions, and assuming they + // agree is what made the startup script treat an ordinary workload container + // as the overlay. + networkOverlayName := "" + if preExecAnnotations, ok := data.Pod.Annotations["slurm-job.vk.io/pre-exec"]; ok { if strings.Contains(preExecAnnotations, "cat <<'EOFMESH' > $TMPDIR/mesh.sh") { meshScript, err := extractHeredoc(preExecAnnotations, "EOFMESH") @@ -480,6 +534,8 @@ cd $TMPDIR FpgaArgs: "", }}, dockerRunStructs...) + networkOverlayName = networkContainerName + log.G(h.Ctx).Info("โœ… [POD FLOW] Network overlay container prepared: " + networkContainerName) } else { log.G(h.Ctx).Error("โŒ [POD FLOW] Failed to extract mesh.sh script from annotation") @@ -605,6 +661,17 @@ cd $TMPDIR span.End() go func() { + // The HTTP response has already been sent at this point, so nothing below + // may touch w. A panic here would also not be recovered by net/http (it is + // a bare goroutine, not the handler), and would take the whole plugin down + // together with the bookkeeping of every other running pod. + defer func() { + if r := recover(); r != nil { + log.G(h.Ctx).Errorf("โŒ [POD FLOW] panic while creating containers for pod %s: %v", podUID, r) + cleanupPodData(h, "Panic during asynchronous container creation", + fmt.Errorf("panic: %v", r), podNamespace, podUID) + } + }() if len(initContainers) > 0 { @@ -633,7 +700,7 @@ cd $TMPDIR _, err := shell.Execute() if err != nil { - HandleErrorAndRemoveData(h, w, "An error occurred during the exec of the init container command", err, podNamespace, podUID) + cleanupPodData(h, "An error occurred during the exec of the init container command", err, podNamespace, podUID) return } @@ -646,7 +713,7 @@ cd $TMPDIR statusReturn, err := shell.Execute() if err != nil { - HandleErrorAndRemoveData(h, w, "An error occurred during inspect of init container", err, podNamespace, podUID) + cleanupPodData(h, "An error occurred during inspect of init container", err, podNamespace, podUID) return } @@ -668,7 +735,14 @@ cd $TMPDIR // create a file called containers_command.sh and write the containers commands to it, use WriteFile function containersCommand := "#!/bin/sh\n" - if isMeshScriptPresent { + networkOverlay, workloadContainers, hasOverlay := selectNetworkOverlay(containers, networkOverlayName) + + if isMeshScriptPresent && !hasOverlay { + log.G(h.Ctx).Warning("โš ๏ธ [POD FLOW] Pod " + podUID + " carries a mesh pre-exec annotation but no network-overlay " + + "container was created; starting its containers directly, without cluster network setup or cluster DNS") + } + + if hasOverlay { dnsNameserver := h.FinalDNSNameserver dnsSearch := h.FinalDNSSearch @@ -680,19 +754,27 @@ cd $TMPDIR dnsSearch = "default.svc.cluster.local svc.cluster.local cluster.local" // Fallback } - // The first container in the list is the network-overlay; start it immediately. - // All subsequent containers are workload containers and must wait for the sentinel. - networkOverlay := containers[0] - workloadContainers := containers[1:] - containersCommand += "# Start network overlay container first\n" containersCommand += networkOverlay.Command + "\n\n" - // Poll for the sentinel file written by mesh.sh once network setup is complete. - containersCommand += "echo 'Waiting for network overlay to be ready...'\n" + // Poll for the sentinel file written by mesh.sh once network setup is + // complete. The wait MUST be bounded: mesh.sh downloads binaries and + // brings up WireGuard, and when any of that fails the sentinel is never + // written. An unbounded loop left the docker exec below blocked with no + // diagnostics, the pod stuck in Waiting and its DIND claimed until a + // human deleted the pod. Timing out instead fails the pod loudly and + // lets the deferred cleanup reclaim the DIND. + containersCommand += "echo 'Waiting for network overlay to be ready (timeout " + strconv.Itoa(meshReadyTimeoutSeconds) + "s)...'\n" + containersCommand += "meshWaited=0\n" containersCommand += "while [ ! -f " + meshReadyFile + " ]; do\n" - containersCommand += " echo 'Network not ready yet, waiting 2s...'\n" + containersCommand += " if [ \"$meshWaited\" -ge " + strconv.Itoa(meshReadyTimeoutSeconds) + " ]; then\n" + containersCommand += " echo 'ERROR: network overlay did not become ready after " + strconv.Itoa(meshReadyTimeoutSeconds) + "s; aborting container startup'\n" + containersCommand += " docker logs " + networkOverlay.Name + " 2>&1 | tail -n 50\n" + containersCommand += " exit 1\n" + containersCommand += " fi\n" + containersCommand += " echo \"Network not ready yet, waiting 2s (${meshWaited}s elapsed)...\"\n" containersCommand += " sleep 2\n" + containersCommand += " meshWaited=$((meshWaited+2))\n" containersCommand += "done\n" containersCommand += "echo 'Network overlay is ready (sentinel file found), starting containers...'\n\n" @@ -722,23 +804,29 @@ cd $TMPDIR err = os.WriteFile(podDirectoryPath+"/containers_command.sh", []byte(containersCommand), 0644) if err != nil { log.G(h.Ctx).Error("\u274C [POD FLOW] Error writing containers command script: " + err.Error()) - HandleErrorAndRemoveData(h, w, "An error occurred during the creation of the container commands script.", err, podNamespace, podUID) + cleanupPodData(h, "An error occurred during the creation of the container commands script.", err, podNamespace, podUID) return } log.G(h.Ctx).Info("\u2705 [POD FLOW] Containers commands written to the script file") - shell = exec.ExecTask{ - Command: "docker", - Args: []string{"exec", string(data.Pod.UID) + "_dind", "/bin/sh", podDirectoryPath + "/containers_command.sh"}, - } + // Bound the exec as well, so a script that blocks for any other reason + // cannot leak this goroutine and keep the DIND claimed forever. Unlike + // go-execute, OSexec.CommandContext can be cancelled. + execCtx, cancelExec := context.WithTimeout(h.Ctx, containersCommandTimeout) + defer cancelExec() - log.G(h.Ctx).Info("\u2705 [POD FLOW] Executing containers creation script inside DIND container; command to execute: docker " + strings.Join(shell.Args, " ")) + execArgs := []string{"exec", string(data.Pod.UID) + "_dind", "/bin/sh", podDirectoryPath + "/containers_command.sh"} + log.G(h.Ctx).Info("\u2705 [POD FLOW] Executing containers creation script inside DIND container; command to execute: docker " + strings.Join(execArgs, " ")) - _, err = shell.Execute() + scriptOutput, err := OSexec.CommandContext(execCtx, "docker", execArgs...).CombinedOutput() if err != nil { + if execCtx.Err() == context.DeadlineExceeded { + err = fmt.Errorf("containers command script timed out after %s: %w", containersCommandTimeout, err) + } log.G(h.Ctx).Error("\u274C [POD FLOW] Error executing containers command script: " + err.Error()) - HandleErrorAndRemoveData(h, w, "An error occurred during the execution of the container command script", err, podNamespace, podUID) + log.G(h.Ctx).Error("\u274C [POD FLOW] Script output: " + strings.TrimSpace(string(scriptOutput))) + cleanupPodData(h, "An error occurred during the execution of the container command script", err, podNamespace, podUID) return } @@ -749,11 +837,26 @@ cd $TMPDIR } +// HandleErrorAndRemoveData reports the failure on the HTTP response and then +// reclaims everything provisioned for the pod. It may only be called from the +// request goroutine, before the response has been written; anything running +// after the handler returned must call cleanupPodData directly, because writing +// to a ResponseWriter after its handler returned races with the http server and +// can corrupt the next response on a keep-alive connection. func HandleErrorAndRemoveData(h *SidecarHandler, w http.ResponseWriter, s string, err error, podNamespace string, podUID string) { - log.G(h.Ctx).Error(err) - log.G(h.Ctx).Info("\u274C Error description: " + s) w.WriteHeader(http.StatusInternalServerError) w.Write([]byte("Some errors occurred while creating container. Check Docker Sidecar's logs")) + cleanupPodData(h, s, err, podNamespace, podUID) +} + +// cleanupPodData tears down the pod data directory and the DIND container, +// network and subnet claimed for the pod. It never touches the HTTP response, so +// it is safe to call from the asynchronous container-creation goroutine. +func cleanupPodData(h *SidecarHandler, s string, err error, podNamespace string, podUID string) { + if err != nil { + log.G(h.Ctx).Error(err) + } + log.G(h.Ctx).Info("\u274C Error description: " + s) if podNamespace != "" && podUID != "" { os.RemoveAll(h.Config.DataRootFolder + podNamespace + "-" + podUID) diff --git a/pkg/docker/Create_test.go b/pkg/docker/Create_test.go new file mode 100644 index 0000000..dbd7d9c --- /dev/null +++ b/pkg/docker/Create_test.go @@ -0,0 +1,123 @@ +package docker + +import "testing" + +func names(structs []DockerRunStruct) []string { + out := make([]string, 0, len(structs)) + for _, s := range structs { + out = append(out, s.Name) + } + return out +} + +func equal(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func TestSelectNetworkOverlay(t *testing.T) { + overlay := DockerRunStruct{Name: "ns-uid-network-overlay"} + appC := DockerRunStruct{Name: "ns-uid-app"} + sidecar := DockerRunStruct{Name: "ns-uid-sidecar"} + + tests := []struct { + name string + containers []DockerRunStruct + overlayName string + wantOK bool + wantOverlay string + wantWorkload []string + }{ + { + name: "overlay first, as prepended", + containers: []DockerRunStruct{overlay, appC, sidecar}, + overlayName: overlay.Name, + wantOK: true, + wantOverlay: overlay.Name, + wantWorkload: []string{appC.Name, sidecar.Name}, + }, + { + name: "overlay not at index 0 is still found, order preserved", + containers: []DockerRunStruct{appC, overlay, sidecar}, + overlayName: overlay.Name, + wantOK: true, + wantOverlay: overlay.Name, + wantWorkload: []string{appC.Name, sidecar.Name}, + }, + { + name: "overlay is the only container", + containers: []DockerRunStruct{overlay}, + overlayName: overlay.Name, + wantOK: true, + wantOverlay: overlay.Name, + wantWorkload: []string{}, + }, + { + // The regression: a mesh annotation was present but no overlay was built. + // containers[0] used to be started as the overlay, and every remaining + // container then waited forever on a sentinel nothing would write. + name: "no overlay built: no workload container is mistaken for it", + containers: []DockerRunStruct{appC, sidecar}, + overlayName: "", + wantOK: false, + wantWorkload: []string{appC.Name, sidecar.Name}, + }, + { + name: "overlay name set but absent from the list", + containers: []DockerRunStruct{appC, sidecar}, + overlayName: overlay.Name, + wantOK: false, + wantWorkload: []string{appC.Name, sidecar.Name}, + }, + { + // Used to panic on containers[0]. + name: "empty container list", + containers: nil, + overlayName: overlay.Name, + wantOK: false, + wantWorkload: []string{}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + gotOverlay, gotWorkload, gotOK := selectNetworkOverlay(tc.containers, tc.overlayName) + + if gotOK != tc.wantOK { + t.Fatalf("ok = %v, want %v", gotOK, tc.wantOK) + } + if gotOverlay.Name != tc.wantOverlay { + t.Errorf("overlay = %q, want %q", gotOverlay.Name, tc.wantOverlay) + } + if got := names(gotWorkload); !equal(got, tc.wantWorkload) { + t.Errorf("workload = %v, want %v", got, tc.wantWorkload) + } + }) + } +} + +// The original list must not be modified: it is still used by the non-mesh +// startup path. +func TestSelectNetworkOverlayDoesNotMutateInput(t *testing.T) { + containers := []DockerRunStruct{ + {Name: "ns-uid-network-overlay"}, + {Name: "ns-uid-app"}, + {Name: "ns-uid-sidecar"}, + } + before := names(containers) + + if _, _, ok := selectNetworkOverlay(containers, "ns-uid-network-overlay"); !ok { + t.Fatal("expected the overlay to be found") + } + + if after := names(containers); !equal(before, after) { + t.Errorf("input list was mutated: %v -> %v", before, after) + } +} diff --git a/pkg/docker/Status.go b/pkg/docker/Status.go index 7ef51ba..fd3a467 100644 --- a/pkg/docker/Status.go +++ b/pkg/docker/Status.go @@ -28,7 +28,9 @@ func (h *SidecarHandler) StatusHandler(w http.ResponseWriter, r *http.Request) { attribute.Int64("start.timestamp", start), )) - var resp []commonIL.PodStatus + // Initialised (not just declared) so a response in which every pod was skipped + // marshals to [] rather than null. + resp := []commonIL.PodStatus{} var req []*v1.Pod statusCode := http.StatusOK @@ -50,7 +52,7 @@ func (h *SidecarHandler) StatusHandler(w http.ResponseWriter, r *http.Request) { return } - for i, pod := range req { + for _, pod := range req { podUID := string(pod.UID) podNamespace := string(pod.Namespace) @@ -74,18 +76,22 @@ func (h *SidecarHandler) StatusHandler(w http.ResponseWriter, r *http.Request) { //dindUUID := strings.ReplaceAll(execReturn.Stdout, "\n", "") dindUUID := strings.Join(strings.Fields(execReturn.Stdout), "") - log.G(h.Ctx).Info("\u2705 [STATUS CALL] UUID of the dind container retrieved successfully: ", dindUUID) - // if the string is empty or the length of the string is 0, return an error and 404 status code\ - if len(dindUUID) == 0 || dindUUID == "" { - log.G(h.Ctx).Error("\u274C [STATUS CALL] Error retrieving UUID of the dind container") - statusCode = http.StatusNotFound - w.WriteHeader(statusCode) - w.Write([]byte("DIND container with UUID " + dindUUID + " not found. Maybe it was deleted or never existed.")) - return + // go-execute reports a nil error for a non-zero exit, so an unknown container + // shows up here as empty stdout. Skip that pod instead of failing the whole + // request: interLink asks for the status of every pod on the node in one call, + // and answering 404 for the batch left every other pod unreconciled. Reporting + // no status for this pod is also safer than inventing a terminal one \u2014 the DIND + // is legitimately absent for a short window between the claim and the rename. + if dindUUID == "" { + log.G(h.Ctx).Warning("\u26A0\uFE0F [STATUS CALL] No DIND container found for pod " + podUID + + " (not created yet, or already deleted); skipping it in this status response") + continue } - resp = append(resp, commonIL.PodStatus{PodName: pod.Name, PodUID: podUID, PodNamespace: podNamespace, JobID: dindUUID}) + log.G(h.Ctx).Info("\u2705 [STATUS CALL] UUID of the dind container retrieved successfully: ", dindUUID) + + podStatus := commonIL.PodStatus{PodName: pod.Name, PodUID: podUID, PodNamespace: podNamespace, JobID: dindUUID} disabledInitContainers := make(map[string]bool) if ann, ok := pod.Annotations["interlink.eu/disable-offload-init-containers"]; ok { @@ -102,7 +108,7 @@ func (h *SidecarHandler) StatusHandler(w http.ResponseWriter, r *http.Request) { if disabledInitContainers[container.Name] { log.G(h.Ctx).Infof("โœ… [STATUS CALL] init container %s is marked as non-offloaded, reporting as Completed", container.Name) - resp[i].InitContainers = append(resp[i].InitContainers, v1.ContainerStatus{ + podStatus.InitContainers = append(podStatus.InitContainers, v1.ContainerStatus{ Name: container.Name, Ready: false, State: v1.ContainerState{ @@ -139,9 +145,9 @@ func (h *SidecarHandler) StatusHandler(w http.ResponseWriter, r *http.Request) { log.G(h.Ctx).Info("\u2705 [STATUS CALL] The container " + container.Name + " is in the state: " + initContainerStatus[0]) if initContainerStatus[0] == "Created" { - resp[i].InitContainers = append(resp[i].InitContainers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Waiting: &v1.ContainerStateWaiting{}}, Ready: false}) + podStatus.InitContainers = append(podStatus.InitContainers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Waiting: &v1.ContainerStateWaiting{}}, Ready: false}) } else if initContainerStatus[0] == "Up" { - resp[i].InitContainers = append(resp[i].InitContainers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Running: &v1.ContainerStateRunning{}}, Ready: true}) + podStatus.InitContainers = append(podStatus.InitContainers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Running: &v1.ContainerStateRunning{}}, Ready: true}) } else if initContainerStatus[0] == "Exited" { containerExitCode := strings.Split(initContainerStatus[1], "(") exitCode, err := strconv.Atoi(strings.Trim(containerExitCode[1], ")")) @@ -149,10 +155,10 @@ func (h *SidecarHandler) StatusHandler(w http.ResponseWriter, r *http.Request) { log.G(h.Ctx).Error(err) exitCode = 0 } - resp[i].InitContainers = append(resp[i].InitContainers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Terminated: &v1.ContainerStateTerminated{ExitCode: int32(exitCode)}}, Ready: false}) + podStatus.InitContainers = append(podStatus.InitContainers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Terminated: &v1.ContainerStateTerminated{ExitCode: int32(exitCode)}}, Ready: false}) } } else { - resp[i].InitContainers = append(resp[i].InitContainers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Waiting: &v1.ContainerStateWaiting{}}, Ready: false}) + podStatus.InitContainers = append(podStatus.InitContainers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Waiting: &v1.ContainerStateWaiting{}}, Ready: false}) } } @@ -170,7 +176,7 @@ func (h *SidecarHandler) StatusHandler(w http.ResponseWriter, r *http.Request) { if disabledContainers[container.Name] { log.G(h.Ctx).Infof("โœ… [STATUS CALL] container %s is marked as non-offloaded, reporting as Running", container.Name) - resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{ + podStatus.Containers = append(podStatus.Containers, v1.ContainerStatus{ Name: container.Name, Ready: true, State: v1.ContainerState{ @@ -205,9 +211,9 @@ func (h *SidecarHandler) StatusHandler(w http.ResponseWriter, r *http.Request) { log.G(h.Ctx).Info("\u2705 [STATUS CALL] The container " + container.Name + " is in the state: " + containerstatus[0]) if containerstatus[0] == "Created" { - resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Waiting: &v1.ContainerStateWaiting{}}, Ready: false}) + podStatus.Containers = append(podStatus.Containers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Waiting: &v1.ContainerStateWaiting{}}, Ready: false}) } else if containerstatus[0] == "Up" { - resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Running: &v1.ContainerStateRunning{}}, Ready: true}) + podStatus.Containers = append(podStatus.Containers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Running: &v1.ContainerStateRunning{}}, Ready: true}) } else if containerstatus[0] == "Exited" { containerExitCode := strings.Split(containerstatus[1], "(") exitCode, err := strconv.Atoi(strings.Trim(containerExitCode[1], ")")) @@ -215,12 +221,14 @@ func (h *SidecarHandler) StatusHandler(w http.ResponseWriter, r *http.Request) { log.G(h.Ctx).Error(err) exitCode = 0 } - resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Terminated: &v1.ContainerStateTerminated{ExitCode: int32(exitCode)}}, Ready: false}) + podStatus.Containers = append(podStatus.Containers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Terminated: &v1.ContainerStateTerminated{ExitCode: int32(exitCode)}}, Ready: false}) } } else { - resp[i].Containers = append(resp[i].Containers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Waiting: &v1.ContainerStateWaiting{}}, Ready: false}) + podStatus.Containers = append(podStatus.Containers, v1.ContainerStatus{Name: container.Name, State: v1.ContainerState{Waiting: &v1.ContainerStateWaiting{}}, Ready: false}) } } + + resp = append(resp, podStatus) } w.WriteHeader(statusCode) diff --git a/pkg/docker/dindmanager/DindHandler.go b/pkg/docker/dindmanager/DindHandler.go index e88766b..d4c2691 100644 --- a/pkg/docker/dindmanager/DindHandler.go +++ b/pkg/docker/dindmanager/DindHandler.go @@ -152,6 +152,54 @@ func (a *DindManager) remainingSubnets() int { return len(a.SubnetPool) } +// rollbackDind undoes a partially created DIND: it force-removes the container, +// then its bridge network, and only then returns the subnet to the pool. +// +// The order matters. A running container holds an endpoint on its network, so +// Docker refuses to remove the network while it exists, and a network that still +// exists keeps owning its /24. Returning the subnet to the pool without tearing +// those down (as the failure paths used to do) hands out an address range Docker +// still considers allocated, so the next "docker network create --subnet" for it +// fails with "Pool overlaps with other one on this address space" โ€” permanently, +// since ReapOrphanNetworks cannot remove a network whose container is alive. +func (a *DindManager) rollbackDind(randUID string, allocatedSubnet string) { + rm := exec.ExecTask{ + Command: "docker", + Args: []string{"rm", "-f", randUID + "_dind"}, + Shell: true, + } + if execReturn, err := rm.Execute(); err != nil || execReturn.ExitCode != 0 { + log.G(a.Ctx).Warn(fmt.Sprintf("โš ๏ธ Rollback: could not remove container %s_dind: %s", + randUID, strings.TrimSpace(execReturn.Stderr))) + } + + rmNet := exec.ExecTask{ + Command: "docker", + Args: []string{"network", "rm", randUID + "_dind_network"}, + Shell: true, + } + execReturn, err := rmNet.Execute() + // "not found" means the network is already gone, so the range is free and the + // subnet can be recycled: docker network rm is not idempotent and reports it + // as a failure. + alreadyGone := strings.Contains(execReturn.Stderr, "not found") + if (err != nil || execReturn.ExitCode != 0) && !alreadyGone { + // The subnet is deliberately NOT returned to the pool in this case: the + // network still exists and still owns the range, so reusing it would fail. + log.G(a.Ctx).Error(fmt.Sprintf( + "โŒ Rollback: could not remove network %s_dind_network (%s); subnet %s is leaked rather than returned to the pool to avoid an overlap on reuse", + randUID, strings.TrimSpace(execReturn.Stderr), allocatedSubnet)) + return + } + + a.freeSubnet(allocatedSubnet) + if allocatedSubnet != "" { + log.G(a.Ctx).Info(fmt.Sprintf( + "โœ… Rollback: subnet %s returned to pool (%d/%d now available)", + allocatedSubnet, a.remainingSubnets(), a.InitialPoolSz)) + } +} + // --------------------------------------------------------------------------- // UUIDv4 generator (unchanged) // --------------------------------------------------------------------------- @@ -262,8 +310,20 @@ func (a *DindManager) BuildDindContainers(nDindContainer int8) error { Args: networkArgs, Shell: true, } - if _, err = shell.Execute(); err != nil { - // Return the subnet to the pool so it is not lost on error. + // go-execute returns a nil error for a non-zero exit, so the exit code has to + // be checked explicitly. Otherwise a rejected "network create" (typically + // "Pool overlaps with other one on this address space") is treated as a + // success and the failure only surfaces later, as an unrelated "docker run" + // error, with the subnet already considered in use. + netReturn, err := shell.Execute() + if err == nil && netReturn.ExitCode != 0 { + err = fmt.Errorf("docker network create exited with code %d: %s", + netReturn.ExitCode, strings.TrimSpace(netReturn.Stderr)) + } + if err != nil { + log.G(a.Ctx).Error(fmt.Sprintf("โŒ Error creating DIND network %s_dind_network: %v", randUID, err)) + // The network was not created, so nothing to tear down: just return the + // subnet to the pool so it is not lost. a.freeSubnet(allocatedSubnet) return err } @@ -329,13 +389,14 @@ func (a *DindManager) BuildDindContainers(nDindContainer int8) error { if err != nil { log.G(a.Ctx).Error(fmt.Sprintf("\u274c Error creating DIND container %s: %v", randUID+"_dind", err)) log.G(a.Ctx).Error(fmt.Sprintf("\u274c %s", execReturn.Stderr)) - // Free the subnet so it can be reused. - a.freeSubnet(allocatedSubnet) + // Tear down the network (and any half-started container) before the subnet + // goes back into the pool. + a.rollbackDind(randUID, allocatedSubnet) return err } dindContainerID := strings.TrimSpace(execReturn.Stdout) if dindContainerID == "" { - a.freeSubnet(allocatedSubnet) + a.rollbackDind(randUID, allocatedSubnet) return fmt.Errorf("docker run returned an empty container ID for %s (stderr: %s)", randUID+"_dind", strings.TrimSpace(execReturn.Stderr)) } @@ -347,11 +408,11 @@ func (a *DindManager) BuildDindContainers(nDindContainer int8) error { lastOutput := "" for { if maxRetries == 0 { - a.freeSubnet(allocatedSubnet) // Surface what the daemon actually logged, otherwise the timeout // is indistinguishable from a container that never started. log.G(a.Ctx).Error(fmt.Sprintf("โŒ Last output of \"docker logs %s\": %s", randUID+"_dind", strings.TrimSpace(lastOutput))) + a.rollbackDind(randUID, allocatedSubnet) return fmt.Errorf("DIND container %s did not become ready in time", dindContainerID) } @@ -376,7 +437,7 @@ func (a *DindManager) BuildDindContainers(nDindContainer int8) error { Shell: true, } if _, err = shell.Execute(); err != nil { - a.freeSubnet(allocatedSubnet) + a.rollbackDind(randUID, allocatedSubnet) return err } log.G(a.Ctx).Info(fmt.Sprintf("\u2705 Installed %s", pkg)) diff --git a/pkg/docker/fpgastrategies/AMDHandler.go b/pkg/docker/fpgastrategies/AMDHandler.go index 604b906..cd510d1 100644 --- a/pkg/docker/fpgastrategies/AMDHandler.go +++ b/pkg/docker/fpgastrategies/AMDHandler.go @@ -18,11 +18,14 @@ import ( ) type FPGASpecs struct { - BDF string - Shell string - LogicUUID string - deviceID string - ContainerID string + BDF string + Shell string + LogicUUID string + deviceID string + // ContainerIDs holds every container currently assigned this FPGA. It is + // normally at most one entry; it can hold more than one only when + // FPGA_DISABLE_BOOKKEEPING=1 lets containers share a device. + ContainerIDs []string DeviceReady string DeviceToMount string Available bool @@ -238,26 +241,35 @@ func (a *FPGAManager) GetFPGASpecsList() []FPGASpecs { return a.FPGASpecsList } +// Assign locks FPGASpecsMutex itself, so it must not be called by a method +// that already holds it โ€” use assignLocked in that case. func (a *FPGAManager) Assign(BDF string, containerID string) error { - for i := range a.FPGASpecsList { - if a.FPGASpecsList[i].BDF == BDF { - disableBookkeeping := os.Getenv("FPGA_DISABLE_BOOKKEEPING") == "1" - if disableBookkeeping { - a.FPGASpecsList[i].ContainerID = containerID - a.FPGASpecsList[i].Available = false - break - } + a.FPGASpecsMutex.Lock() + defer a.FPGASpecsMutex.Unlock() + return a.assignLocked(BDF, containerID) +} - if !a.FPGASpecsList[i].Available { - return fmt.Errorf("FPGA with BDF %s is already in use by container %s", BDF, a.FPGASpecsList[i].ContainerID) - } +// assignLocked adds containerID to the FPGA identified by BDF. Caller must +// hold FPGASpecsMutex. +func (a *FPGAManager) assignLocked(BDF string, containerID string) error { + for i := range a.FPGASpecsList { + if a.FPGASpecsList[i].BDF != BDF { + continue + } - a.FPGASpecsList[i].ContainerID = containerID - a.FPGASpecsList[i].Available = false - break + disableBookkeeping := os.Getenv("FPGA_DISABLE_BOOKKEEPING") == "1" + if !disableBookkeeping && len(a.FPGASpecsList[i].ContainerIDs) > 0 { + return fmt.Errorf("FPGA with BDF %s is already in use by container(s) %v", BDF, a.FPGASpecsList[i].ContainerIDs) } + + // Appending (instead of overwriting) is what makes sharing under + // FPGA_DISABLE_BOOKKEEPING actually work: every assignee stays tracked, + // so Release can later find each one instead of only the last assignee. + a.FPGASpecsList[i].ContainerIDs = append(a.FPGASpecsList[i].ContainerIDs, containerID) + a.FPGASpecsList[i].Available = false + return nil } - return nil + return fmt.Errorf("FPGA with BDF %s not found", BDF) } func (a *FPGAManager) Release(containerID string) error { @@ -266,147 +278,105 @@ func (a *FPGAManager) Release(containerID string) error { defer a.FPGASpecsMutex.Unlock() for i := range a.FPGASpecsList { - if a.FPGASpecsList[i].ContainerID == containerID { - - if a.FPGASpecsList[i].Available { + ids := a.FPGASpecsList[i].ContainerIDs + for j, id := range ids { + if id != containerID { continue } - - a.FPGASpecsList[i].ContainerID = "" - a.FPGASpecsList[i].Available = true + a.FPGASpecsList[i].ContainerIDs = append(ids[:j], ids[j+1:]...) + if len(a.FPGASpecsList[i].ContainerIDs) == 0 { + a.FPGASpecsList[i].Available = true + } + break } } return nil } -/* func (a *FPGAManager) GetAvailableFPGAs(numFPGAs int) ([]FPGASpecs, error) { +// GetAvailableFPGAs locks FPGASpecsMutex itself, so it must not be called by +// a method that already holds it โ€” use getAvailableFPGAsLocked in that case. +func (a *FPGAManager) GetAvailableFPGAs(numFPGAs int) ([]FPGASpecs, error) { a.FPGASpecsMutex.Lock() defer a.FPGASpecsMutex.Unlock() + return a.getAvailableFPGAsLocked(numFPGAs) +} +// getAvailableFPGAsLocked picks up to numFPGAs FPGAs. Caller must hold +// FPGASpecsMutex. +// +// Default behavior: only ever return fully-unassigned FPGAs. +// +// FPGA_DISABLE_BOOKKEEPING=1 relaxes that so callers can still get an answer +// once every device is in use: unassigned FPGAs are still preferred, but if +// there aren't enough, devices already in use are added back, least-shared +// first (fpgaUsage counts current assignees per BDF via len(ContainerIDs), so +// it reflects real concurrent sharing, not just "used at all"). +func (a *FPGAManager) getAvailableFPGAsLocked(numFPGAs int) ([]FPGASpecs, error) { var availableFPGAs []FPGASpecs - disableBookkeeping := os.Getenv("FPGA_DISABLE_BOOKKEEPING") == "1" - - if disableBookkeeping { - fpgaUsage := make(map[string]int) - for _, fpga := range a.FPGASpecsList { - if fpga.ContainerID != "" { - fpgaUsage[fpga.BDF]++ - } else { - // If an unassigned FPGA is found, prioritize it - availableFPGAs = append(availableFPGAs, fpga) - } - } - - if len(availableFPGAs) >= numFPGAs { - return availableFPGAs[:numFPGAs], nil - } + seen := make(map[string]bool) - // If not enough unassigned FPGAs, find the least assigned ones - sort.Slice(a.FPGASpecsList, func(i, j int) bool { - return fpgaUsage[a.FPGASpecsList[i].BDF] < fpgaUsage[a.FPGASpecsList[j].BDF] - }) - - for _, fpga := range a.FPGASpecsList { - if len(availableFPGAs) < numFPGAs { - availableFPGAs = append(availableFPGAs, fpga) - } else { - break - } - } + disableBookkeeping := os.Getenv("FPGA_DISABLE_BOOKKEEPING") == "1" - if len(availableFPGAs) >= numFPGAs { - return availableFPGAs[:numFPGAs], nil + for _, fpga := range a.FPGASpecsList { + if len(fpga.ContainerIDs) == 0 { + availableFPGAs = append(availableFPGAs, fpga) + seen[fpga.BDF] = true } - - return nil, fmt.Errorf("Not enough FPGAs available. Requested: %d, Found: %d", numFPGAs, len(availableFPGAs)) } - // Default behavior: return only available FPGAs - for _, fpgaSpec := range a.FPGASpecsList { - if fpgaSpec.Available { - availableFPGAs = append(availableFPGAs, fpgaSpec) - if len(availableFPGAs) == numFPGAs { - return availableFPGAs, nil - } - } + if len(availableFPGAs) >= numFPGAs { + return availableFPGAs[:numFPGAs], nil } - return nil, fmt.Errorf("Not enough available FPGAs. Requested: %d, Available: %d", numFPGAs, len(availableFPGAs)) -} */ - -func (a *FPGAManager) GetAvailableFPGAs(numFPGAs int) ([]FPGASpecs, error) { - - var availableFPGAs []FPGASpecs - - fmt.Println("Checking for available FPGAs") - - disableBookkeeping := os.Getenv("FPGA_DISABLE_BOOKKEEPING") == "1" - - fmt.Println(fmt.Sprintf("FPGA_DISABLE_BOOKKEEPING: %v", disableBookkeeping)) - - if disableBookkeeping { - fmt.Println("FPGA_DISABLE_BOOKKEEPING is set to 1. Disabling bookkeeping") - - fpgaUsage := make(map[string]int) - for _, fpga := range a.FPGASpecsList { - if fpga.ContainerID != "" { - fpgaUsage[fpga.BDF]++ - } else { - // If an unassigned FPGA is found, prioritize it - availableFPGAs = append(availableFPGAs, fpga) - } - } + if !disableBookkeeping { + return nil, fmt.Errorf("Not enough available FPGAs. Requested: %d, Available: %d", numFPGAs, len(availableFPGAs)) + } - fmt.Println(fmt.Sprintf("FPGA Usage: %v", fpgaUsage)) + log.G(a.Ctx).Info(fmt.Sprintf("โœ… FPGA_DISABLE_BOOKKEEPING is set: only %d/%d unassigned FPGAs found, falling back to sharing least-used devices", len(availableFPGAs), numFPGAs)) - if len(availableFPGAs) >= numFPGAs { - return availableFPGAs[:numFPGAs], nil - } + fpgaUsage := make(map[string]int, len(a.FPGASpecsList)) + for _, fpga := range a.FPGASpecsList { + fpgaUsage[fpga.BDF] = len(fpga.ContainerIDs) + } - // If not enough unassigned FPGAs, find the least assigned ones - sort.Slice(a.FPGASpecsList, func(i, j int) bool { - return fpgaUsage[a.FPGASpecsList[i].BDF] < fpgaUsage[a.FPGASpecsList[j].BDF] - }) + shared := make([]FPGASpecs, len(a.FPGASpecsList)) + copy(shared, a.FPGASpecsList) + sort.Slice(shared, func(i, j int) bool { + return fpgaUsage[shared[i].BDF] < fpgaUsage[shared[j].BDF] + }) - for _, fpga := range a.FPGASpecsList { - if len(availableFPGAs) < numFPGAs { - availableFPGAs = append(availableFPGAs, fpga) - } else { - break - } + for _, fpga := range shared { + if seen[fpga.BDF] { + // Already included from the unassigned pass above โ€” including it + // again would hand the caller the same physical device twice. + continue } - - if len(availableFPGAs) >= numFPGAs { - return availableFPGAs[:numFPGAs], nil + availableFPGAs = append(availableFPGAs, fpga) + seen[fpga.BDF] = true + if len(availableFPGAs) == numFPGAs { + break } - - return nil, fmt.Errorf("Not enough FPGAs available. Requested: %d, Found: %d", numFPGAs, len(availableFPGAs)) } - for _, fpgaSpec := range a.FPGASpecsList { - if fpgaSpec.Available { - availableFPGAs = append(availableFPGAs, fpgaSpec) - if len(availableFPGAs) == numFPGAs { - return availableFPGAs, nil - } - } + if len(availableFPGAs) >= numFPGAs { + return availableFPGAs[:numFPGAs], nil } - return nil, fmt.Errorf("Not enough available FPGAs. Requested: %d, Available: %d", numFPGAs, len(availableFPGAs)) + + return nil, fmt.Errorf("Not enough FPGAs available. Requested: %d, Found: %d", numFPGAs, len(availableFPGAs)) } func (a *FPGAManager) GetAndAssignAvailableFPGAs(numFPGAs int, containerID string) ([]FPGASpecs, error) { a.FPGASpecsMutex.Lock() defer a.FPGASpecsMutex.Unlock() - fpgaSpecs, err := a.GetAvailableFPGAs(numFPGAs) + fpgaSpecs, err := a.getAvailableFPGAsLocked(numFPGAs) if err != nil { return nil, err } for _, fpgaSpec := range fpgaSpecs { - err = a.Assign(fpgaSpec.BDF, containerID) // โ† was fpgaSpec.LogicUUID - if err != nil { + if err := a.assignLocked(fpgaSpec.BDF, containerID); err != nil { return nil, err } } From 5335458aa6e7738f24fcd83f3105066d4b8bab6c Mon Sep 17 00:00:00 2001 From: Giulio Bianchini Date: Thu, 20 Aug 2026 10:09:42 +0200 Subject: [PATCH 30/30] stop tracking .DS_Store --- .DS_Store | Bin 6148 -> 0 bytes .gitignore | 3 ++- pkg/.DS_Store | Bin 6148 -> 0 bytes 3 files changed, 2 insertions(+), 1 deletion(-) delete mode 100644 .DS_Store delete mode 100644 pkg/.DS_Store diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 38989cfa6df641bd7e5faf47c0c975149e603d6b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKze~eF6nq+ByMqqS;^HE@ z2!c52B>26%tGQ=dM-{mTcVF^;B2sZTqU zP?K7+AH_;Blat~0^RzAc5m{M3le1($74D~iPP){TeFP6##N~f@UmV>NTh~o8-h6Zn z-!xX8fOC&pbRby^uV58%m%CbA8gYNg$vT*nGp3IM_0U6Gc5r&G*R3Nicm29{yjaw~ zMO){tJ0F#Ncep4OX@C{?G4~~B?Mg;n-t*hf`|Z@b+~YGOf1UY%%dls&mC6SF>=p0| zcm+Nc;P*p_!5CRg4eD11CjAKj4B*y=W4(6)Lv{coi>X0)V9JyNO{ub<7|N8xAKJXg zVrtNolhVw%j%8N%3q@&m_(PpeDl+J2uYgxTDo`+|72f~XCx8FTB7f!;@Cy7Z1ym4k z#%oxT-CJ{uKH#l*4nrGaAs+xEi>X0)VE&JQmcdV6fge@i E14TLEX8-^I diff --git a/.gitignore b/.gitignore index 6dd29b7..00b41c6 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -bin/ \ No newline at end of file +bin/ +.DS_Store diff --git a/pkg/.DS_Store b/pkg/.DS_Store deleted file mode 100644 index 0c8885141335ec4fe931b31e71960ae38997f8fd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKK~BR!475vxg1GdEIPC`z^ar5|AD|yVO%O;lY*nczE_nb#{D>P*-~qgZ@!Bd$ zL&XiD%8tCV@!A__62&nQ@pN5Jh{i;eLj^}i7=9307ahpREV9UHk4TS;yUDDI&2r%F zhX2TbJi9}xC{lxC@chEd_IeY`a=wUj*z?opv(Jz0{c`A2|LU{6@4hD&*aCUf$e^MJ zx}{bBJ$gFdUhn2{{np8&vquxZ^VsFn&*N3{aB&8l0cT*}89>bz$&VF%bOxLOXJEsC zd>;Z-urO>D)29PNY5{;5%tp9{eM5m&zu2gV6Pb9tello+>+JS!Ocmn s4bU5?i1@XNn-EM=DTc3<;uB~R*n>=fg<-1*3&ei}5)D2$1AofE7b?U|P5=M^