Feature/target loading architecture

.gitignore

@@ -26,6 +26,7 @@ Dockerfile.cross
 *~
 private/
 lab/dev/clab-*
+lab/dev/netbox/secrets
 design/
 notes/
 docs/public

Makefile

@@ -1,4 +1,5 @@
 include test.mk
+include netbox.mk
 
 # Image URL to use all building/pushing image targets
 IMG ?= controller:latest
@@ -250,10 +251,10 @@ configure-nodes-dev-lab: ## Configure the nodes in the development lab cluster
 	gnmic -a clab-3-nodes-leaf2:57400 -u $(TARGET_USERNAME) -p $(TARGET_PASSWORD) --skip-verify set --request-file lab/dev/configs/leaf2.yaml
 
 .PHONY: apply-resources-dev-lab
-apply-resources-dev-lab: apply-targets-dev-lab apply-subscriptions-dev-lab apply-outputs-dev-lab apply-pipelines-dev-lab apply-clusters-dev-lab ## Apply the resources for the development lab cluster
+apply-resources-dev-lab: apply-targets-dev-lab apply-subscriptions-dev-lab apply-outputs-dev-lab apply-pipelines-dev-lab apply-clusters-dev-lab apply-targetsources-dev-lab ## Apply the resources for the development lab cluster
 
 .PHONY: delete-resources-dev-lab
-delete-resources-dev-lab: delete-clusters-dev-lab delete-targets-dev-lab delete-subscriptions-dev-lab delete-outputs-dev-lab delete-pipelines-dev-lab ## Delete the resources for the development lab cluster
+delete-resources-dev-lab: delete-clusters-dev-lab delete-targets-dev-lab delete-subscriptions-dev-lab delete-outputs-dev-lab delete-pipelines-dev-lab delete-targetsources-dev-lab ## Delete the resources for the development lab cluster
 
 .PHONY: apply-targets-dev-lab
 apply-targets-dev-lab: ## Apply the targets for the development lab cluster
@@ -287,6 +288,7 @@ apply-pipelines-dev-lab: ## Apply the pipelines for the development lab cluster
 .PHONY: delete-pipelines-dev-lab
 delete-pipelines-dev-lab: ## Delete the pipelines for the development lab cluster
 	kubectl delete -f lab/dev/resources/pipelines
+
 .PHONY: apply-clusters-dev-lab
 apply-clusters-dev-lab: ## Apply the clusters for the development lab cluster
 	kubectl apply -f lab/dev/resources/clusters
@@ -295,6 +297,14 @@ apply-clusters-dev-lab: ## Apply the clusters for the development lab cluster
 delete-clusters-dev-lab: ## Delete the clusters for the development lab cluster
 	kubectl delete -f lab/dev/resources/clusters
 
+.PHONY: apply-targetsources-dev-lab
+apply-targetsources-dev-lab: ## Apply the target sources for the development lab cluster
+	kubectl apply -f lab/dev/resources/targetsources
+
+.PHONY: delete-targetsources-dev-lab
+delete-targetsources-dev-lab: ## Delete the target sources for the development lab cluster
+	kubectl delete -f lab/dev/resources/targetsources
+
 ##@ Testing Lab
 
 .PHONY: run-integration-tests

api/v1alpha1/targetsource_types.go

@@ -21,14 +21,21 @@ import (
 )
 
 // TargetSourceSpec defines the desired state of TargetSource
+// +kubebuilder:validation:Required
 type TargetSourceSpec struct {
-	HTTP            *HTTPConfig          `json:"http,omitempty"`
-	Consul          *ConsulConfig        `json:"consul,omitempty"`
-	ConfigMap       string               `json:"configMap,omitempty"`
-	PodSelector     metav1.LabelSelector `json:"podSelector,omitempty"`
-	ServiceSelector metav1.LabelSelector `json:"serviceSelector,omitempty"`
+	Provider *ProviderSpec `json:"provider"`
 	//
+	Type   string            `json:"type,omitempty"`
 	Labels map[string]string `json:"labels,omitempty"`
+
+	// +kubebuilder:validation:MinLength=1
+	Profile string `json:"profile"`
+}
+
+// +kubebuilder:validation:MaxProperties=1
+type ProviderSpec struct {
+	HTTP   *HTTPConfig   `json:"http,omitempty"`
+	Consul *ConsulConfig `json:"consul,omitempty"`
 }
 
 type HTTPConfig struct {

config/crd/bases/operator.gnmic.dev_targetsources.yaml

@@ -39,120 +39,32 @@ spec:
           spec:
             description: TargetSourceSpec defines the desired state of TargetSource
             properties:
-              configMap:
-                type: string
-              consul:
-                properties:
-                  url:
-                    type: string
-                type: object
-              http:
-                properties:
-                  url:
-                    type: string
-                type: object
               labels:
                 additionalProperties:
                   type: string
                 type: object
-              podSelector:
-                description: |-
-                  A label selector is a label query over a set of resources. The result of matchLabels and
-                  matchExpressions are ANDed. An empty label selector matches all objects. A null
-                  label selector matches no objects.
+              profile:
+                minLength: 1
+                type: string
+              provider:
+                maxProperties: 1
                 properties:
-                  matchExpressions:
-                    description: matchExpressions is a list of label selector requirements.
-                      The requirements are ANDed.
-                    items:
-                      description: |-
-                        A label selector requirement is a selector that contains values, a key, and an operator that
-                        relates the key and values.
-                      properties:
-                        key:
-                          description: key is the label key that the selector applies
-                            to.
-                          type: string
-                        operator:
-                          description: |-
-                            operator represents a key's relationship to a set of values.
-                            Valid operators are In, NotIn, Exists and DoesNotExist.
-                          type: string
-                        values:
-                          description: |-
-                            values is an array of string values. If the operator is In or NotIn,
-                            the values array must be non-empty. If the operator is Exists or DoesNotExist,
-                            the values array must be empty. This array is replaced during a strategic
-                            merge patch.
-                          items:
-                            type: string
-                          type: array
-                          x-kubernetes-list-type: atomic
-                      required:
-                      - key
-                      - operator
-                      type: object
-                    type: array
-                    x-kubernetes-list-type: atomic
-                  matchLabels:
-                    additionalProperties:
-                      type: string
-                    description: |-
-                      matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
-                      map is equivalent to an element of matchExpressions, whose key field is "key", the
-                      operator is "In", and the values array contains only "value". The requirements are ANDed.
+                  consul:
+                    properties:
+                      url:
+                        type: string
                     type: object
-                type: object
-                x-kubernetes-map-type: atomic
-              serviceSelector:
-                description: |-
-                  A label selector is a label query over a set of resources. The result of matchLabels and
-                  matchExpressions are ANDed. An empty label selector matches all objects. A null
-                  label selector matches no objects.
-                properties:
-                  matchExpressions:
-                    description: matchExpressions is a list of label selector requirements.
-                      The requirements are ANDed.
-                    items:
-                      description: |-
-                        A label selector requirement is a selector that contains values, a key, and an operator that
-                        relates the key and values.
-                      properties:
-                        key:
-                          description: key is the label key that the selector applies
-                            to.
-                          type: string
-                        operator:
-                          description: |-
-                            operator represents a key's relationship to a set of values.
-                            Valid operators are In, NotIn, Exists and DoesNotExist.
-                          type: string
-                        values:
-                          description: |-
-                            values is an array of string values. If the operator is In or NotIn,
-                            the values array must be non-empty. If the operator is Exists or DoesNotExist,
-                            the values array must be empty. This array is replaced during a strategic
-                            merge patch.
-                          items:
-                            type: string
-                          type: array
-                          x-kubernetes-list-type: atomic
-                      required:
-                      - key
-                      - operator
-                      type: object
-                    type: array
-                    x-kubernetes-list-type: atomic
-                  matchLabels:
-                    additionalProperties:
-                      type: string
-                    description: |-
-                      matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
-                      map is equivalent to an element of matchExpressions, whose key field is "key", the
-                      operator is "In", and the values array contains only "value". The requirements are ANDed.
+                  http:
+                    properties:
+                      url:
+                        type: string
                     type: object
                 type: object
-                x-kubernetes-map-type: atomic
+              type:
+                type: string
+            required:
+            - profile
+            - provider
             type: object
           status:
             description: TargetSourceStatus defines the observed state of TargetSource

internal/controller/discovery/client.go

@@ -0,0 +1,27 @@
+package discovery
+
+// File may become obsolete, depends on how the logic to compare desired vs. existing state will get implemented
+
+import (
+	"context"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
+)
+
+func FetchExistingTargets(ctx context.Context, c client.Client, ts gnmicv1alpha1.TargetSource) ([]gnmicv1alpha1.Target, error) {
+	var targetList gnmicv1alpha1.TargetList
+
+	err := c.List(ctx, &targetList,
+		client.InNamespace(ts.Namespace),
+		client.MatchingLabels{
+			"gnmic.io/source": ts.Name,
+		},
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return targetList.Items, nil
+}

internal/controller/discovery/core/loader_interface.go

@@ -0,0 +1,23 @@
+package core
+
+import (
+	"context"
+
+	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
+)
+
+// Loader defines a pluggable TargetSource loader interface
+// Loaders observe external Sources of Truth and emit target snapshots through a channel
+type Loader interface {
+	// Name returns the unique loader identifier e.g. "http_pull"
+	Name() string
+
+	// Start begins discovery and pushes target snapshots into the out channel
+	// The loader must stop cleanly when ctx is cancelled
+	Start(
+		ctx context.Context,
+		targetsourceName string,
+		spec gnmicv1alpha1.TargetSourceSpec,
+		out chan<- []DiscoveryMessage,
+	) error
+}

internal/controller/discovery/core/types.go

@@ -0,0 +1,22 @@
+package core
+
+// DiscoveredTarget represents a target discovered from an external source
+// before it is materialized as a Kubernetes Target CR
+type DiscoveredTarget struct {
+	Name    string
+	Address string
+	Labels  map[string]string
+}
+
+const (
+	DELETE DiscoveryEvent = 0
+	CREATE DiscoveryEvent = 1
+	UPDATE DiscoveryEvent = 2
+)
+
+type DiscoveryEvent int
+
+type DiscoveryMessage struct {
+	Target DiscoveredTarget
+	Event  DiscoveryEvent
+}

internal/controller/discovery/loader.go

@@ -0,0 +1,24 @@
+package discovery
+
+import (
+	"fmt"
+
+	gnmicv1alpha1 "github.com/gnmic/operator/api/v1alpha1"
+	"github.com/gnmic/operator/internal/controller/discovery/core"
+	"github.com/gnmic/operator/internal/controller/discovery/loaders/http_pull"
+)
+
+// NewLoader creates a loader by name
+func NewLoader(name string, namespace string, spec gnmicv1alpha1.TargetSourceSpec) (core.Loader, error) {
+	loaderName := namespace + "/" + name
+
+	switch {
+	case spec.Provider.HTTP != nil:
+		return http_pull.New(), nil
+	case spec.Provider.Consul != nil:
+		return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", loaderName)
+	default:
+		return nil, fmt.Errorf("unknown targetsource loader, check TargetSource CRD for %s", loaderName)
+	}
+
+}

internal/controller/discovery/loaders/all/all.go

@@ -0,0 +1,6 @@
+package all
+
+import (
+	_ "github.com/gnmic/operator/internal/controller/discovery/loaders/http_pull"
+	// _ "github.com/gnmic/operator/internal/controller/targetsource/loaders/http_push"
+)