From 41206005997a875918b55907f4c3ffef126e8885 Mon Sep 17 00:00:00 2001 From: rustyclock Date: Sun, 8 Nov 2020 11:19:56 +0900 Subject: [PATCH 1/4] Update makefile Signed-off-by: rustyclock --- Makefile.common | 48 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/Makefile.common b/Makefile.common index 07ece87c..976cb8a8 100644 --- a/Makefile.common +++ b/Makefile.common @@ -45,7 +45,7 @@ ifeq (, $(PRE_GO_111)) ifneq (,$(wildcard vendor)) # Always use the local vendor/ directory to satisfy the dependencies. - GOOPTS := $(GOOPTS) #-mod=vendor + GOOPTS := $(GOOPTS) -mod=vendor endif endif else @@ -69,12 +69,21 @@ else GO_BUILD_PLATFORM ?= $(GOHOSTOS)-$(GOHOSTARCH) endif -PROMU_VERSION ?= 0.4.0 +GOTEST := $(GO) test +GOTEST_DIR := +ifneq ($(CIRCLE_JOB),) +ifneq ($(shell which gotestsum),) + GOTEST_DIR := test-results + GOTEST := gotestsum --junitfile $(GOTEST_DIR)/unit-tests.xml -- +endif +endif + +PROMU_VERSION ?= 0.7.0 PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz GOLANGCI_LINT := GOLANGCI_LINT_OPTS ?= -GOLANGCI_LINT_VERSION ?= v1.21.0 +GOLANGCI_LINT_VERSION ?= v1.18.0 # golangci-lint only supports linux, darwin and windows platforms on i386/amd64. # windows isn't included here because of the path separator being different. ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin)) @@ -86,7 +95,8 @@ endif PREFIX ?= $(shell pwd) BIN_DIR ?= $(shell pwd) DOCKER_IMAGE_TAG ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD)) -DOCKERFILE_PATH ?= ./ +DOCKERFILE_PATH ?= ./Dockerfile +DOCKERBUILD_CONTEXT ?= ./ DOCKER_REPO ?= prom DOCKER_ARCHS ?= amd64 @@ -140,15 +150,29 @@ else $(GO) get $(GOOPTS) -t ./... endif +.PHONY: update-go-deps +update-go-deps: + @echo ">> updating Go dependencies" + @for m in $$($(GO) list -mod=readonly -m -f '{{ if and (not .Indirect) (not .Main)}}{{.Path}}{{end}}' all); do \ + $(GO) get $$m; \ + done + GO111MODULE=$(GO111MODULE) $(GO) mod tidy +ifneq (,$(wildcard vendor)) + GO111MODULE=$(GO111MODULE) $(GO) mod vendor +endif + .PHONY: common-test-short -common-test-short: +common-test-short: $(GOTEST_DIR) @echo ">> running short tests" - GO111MODULE=$(GO111MODULE) $(GO) test -short $(GOOPTS) $(pkgs) + GO111MODULE=$(GO111MODULE) $(GOTEST) -short $(GOOPTS) $(pkgs) .PHONY: common-test -common-test: +common-test: $(GOTEST_DIR) @echo ">> running all tests" - GO111MODULE=$(GO111MODULE) $(GO) test $(test-flags) $(GOOPTS) $(pkgs) + GO111MODULE=$(GO111MODULE) $(GOTEST) $(test-flags) $(GOOPTS) $(pkgs) + +$(GOTEST_DIR): + @mkdir -p $@ .PHONY: common-format common-format: @@ -187,7 +211,6 @@ else ifdef GO111MODULE @echo ">> running check for unused/missing packages in go.mod" GO111MODULE=$(GO111MODULE) $(GO) mod tidy - GO111MODULE=$(GO111MODULE) $(GO) mod vendor ifeq (,$(wildcard vendor)) @git diff --exit-code -- go.sum go.mod else @@ -201,7 +224,7 @@ endif .PHONY: common-build common-build: promu @echo ">> building binaries" - GO111MODULE=$(GO111MODULE) $(PROMU) build --prefix $(PREFIX) + GO111MODULE=$(GO111MODULE) $(PROMU) build --prefix $(PREFIX) $(PROMU_BINARIES) .PHONY: common-tarball common-tarball: promu @@ -212,19 +235,22 @@ common-tarball: promu common-docker: $(BUILD_DOCKER_ARCHS) $(BUILD_DOCKER_ARCHS): common-docker-%: docker build -t "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(DOCKER_IMAGE_TAG)" \ + -f $(DOCKERFILE_PATH) \ --build-arg ARCH="$*" \ --build-arg OS="linux" \ - $(DOCKERFILE_PATH) + $(DOCKERBUILD_CONTEXT) .PHONY: common-docker-publish $(PUBLISH_DOCKER_ARCHS) common-docker-publish: $(PUBLISH_DOCKER_ARCHS) $(PUBLISH_DOCKER_ARCHS): common-docker-publish-%: docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(DOCKER_IMAGE_TAG)" +DOCKER_MAJOR_VERSION_TAG = $(firstword $(subst ., ,$(shell cat VERSION))) .PHONY: common-docker-tag-latest $(TAG_DOCKER_ARCHS) common-docker-tag-latest: $(TAG_DOCKER_ARCHS) $(TAG_DOCKER_ARCHS): common-docker-tag-latest-%: docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(DOCKER_IMAGE_TAG)" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:latest" + docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(DOCKER_IMAGE_TAG)" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:v$(DOCKER_MAJOR_VERSION_TAG)" .PHONY: common-docker-manifest common-docker-manifest: From 8b85e168a4c73802d66aabe819169af8499a95fd Mon Sep 17 00:00:00 2001 From: rustyclock Date: Sun, 8 Nov 2020 08:36:20 +0900 Subject: [PATCH 2/4] Add support for multiple clouds Signed-off-by: rustyclock --- README.md | 47 ++++++++++-------- exporters/exporter.go | 8 ++- main.go | 113 +++++++++++++++++++++++++++--------------- 3 files changed, 104 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index 2546127d..b278fde6 100644 --- a/README.md +++ b/README.md @@ -53,20 +53,22 @@ and must by specified with the `--os-client-config` flag. Other options as the binding address/port can by explored with the --help flag. -By default the openstack\_exporter serves on port `0.0.0.0:9180` at the `/metrics` URL. +By default the openstack\_exporter serves on port `0.0.0.0:9180` at the `/probe` URL. You can build it by yourself by cloning this repository and run: ```sh make common-build -./openstack-exporter --os-client-config /etc/openstack/clouds.yaml region.mycludprovider.org +./openstack-exporter --os-client-config /etc/openstack/clouds.yaml +curl "http://localhost:9180/probe?cloud=region.mycludprovider.org" ``` Or alternatively you can use the docker images, as follows (check the openstack configuration section for configuration details): ```sh -docker run -v "$HOME/.config/openstack/clouds.yml":/etc/openstack/clouds.yaml -it quay.io/niedbalski/openstack-exporter-linux-amd64:master my-cloud.org +docker run -v "$HOME/.config/openstack/clouds.yml":/etc/openstack/clouds.yaml -it -p 9180:9180 quay.io/niedbalski/openstack-exporter-linux-amd64:master +curl "http://localhost:9180/probe?cloud=my-cloud.org" ``` ### Command line options @@ -88,27 +90,30 @@ Flags: --endpoint-type="public" openstack endpoint type to use (i.e: public, internal, admin) --collect-metric-time time spent collecting each metric -d, --disable-metric= ... multiple --disable-metric can be specified in the format: service-metric (i.e: cinder-snapshots) - --disable-service.network Disable the network service exporter - --disable-service.compute Disable the compute service exporter - --disable-service.image Disable the image service exporter - --disable-service.volume Disable the volume service exporter - --disable-service.identity - Disable the identity service exporter - --disable-service.object-store - Disable the object-store service exporter - --disable-service.load-balancer - Disable the load-balancer service exporter - --disable-service.container-infra - Disable the container-infra service exporter - --disable-service.dns Disable the dns service exporter - --disable-service.baremetal - Disable the baremetal service exporter - --disable-service.gnocchi Disable the gnocchi service --disable-slow-metrics disable slow metrics for performance reasons --version Show application version. +``` + +### Scrape options + +Which cloud (name or id from the `clouds.yaml` file) or what services from the cloud to scrape, can be specified as the parameters to http scrape request. + +Query Parameter | Description +--- | --- +`cloud` | Name or id of the cloud to gather metrics from (as specified in the `clouds.yaml`) +`include_services` | A comma separated list of services for which metrics will be scraped. Defaults to all services: "network,compute,image,volume,identity,object-store,load-balancer,container-infra,dns,baremetal,gnocchi" +`exclude_services` | A comma separated list of services for which metrics will *not* be scraped. Default is empty: "" + +Examples: +``` +## Scrape all services from `test.cloud` +curl "https://localhost:9180/probe?cloud=test.cloud" + +## Scrape only `network` and `compute` services from `test.cloud` +curl "https://localhost:9180/probe?cloud=test.cloud&include_services=network,compute" -Args: - name or id of the cloud to gather metrics from +## Scrape all services except `load-balancer` and `dns` from `test.cloud` +curl "https://localhost:9180/probe?cloud=test.cloud&exclude_services=load-balancer,dns" ``` ### OpenStack configuration diff --git a/exporters/exporter.go b/exporters/exporter.go index 6e062a64..b66e5ef9 100644 --- a/exporters/exporter.go +++ b/exporters/exporter.go @@ -3,10 +3,11 @@ package exporters import ( "crypto/tls" "fmt" - "github.com/hashicorp/go-uuid" "net/http" "time" + "github.com/hashicorp/go-uuid" + "github.com/gophercloud/gophercloud" "github.com/gophercloud/utils/openstack/clientconfig" "github.com/prometheus/client_golang/prometheus" @@ -44,7 +45,6 @@ func EnableExporter(service, prefix, cloud string, disabledMetrics []string, end if err != nil { return nil, err } - prometheus.MustRegister(exporter) return &exporter, nil } @@ -198,6 +198,10 @@ func NewExporter(name, prefix, cloud string, disabledMetrics []string, endpointT transport = &http.Transport{TLSClientConfig: tlsConfig} } + if transport != nil { + transport.Proxy = http.ProxyFromEnvironment + } + client, err := NewServiceClient(name, &opts, transport, endpointType) if err != nil { return nil, err diff --git a/main.go b/main.go index f1dd0949..dfba7725 100644 --- a/main.go +++ b/main.go @@ -1,11 +1,13 @@ package main import ( - "fmt" + "context" "net/http" "os" + "strings" "github.com/openstack-exporter/openstack-exporter/exporters" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/common/log" "github.com/prometheus/common/version" @@ -16,27 +18,19 @@ var defaultEnabledServices = []string{"network", "compute", "image", "volume", " var DEFAULT_OS_CLIENT_CONFIG = "/etc/openstack/clouds.yaml" +var ( + logLevel = kingpin.Flag("log.level", "Log level: [debug, info, warn, error, fatal]").Default("info").String() + bind = kingpin.Flag("web.listen-address", "address:port to listen on").Default(":9180").String() + metrics = kingpin.Flag("web.telemetry-path", "uri path to expose metrics").Default("/metrics").String() + osClientConfig = kingpin.Flag("os-client-config", "Path to the cloud configuration file").Default(DEFAULT_OS_CLIENT_CONFIG).String() + prefix = kingpin.Flag("prefix", "Prefix for metrics").Default("openstack").String() + endpointType = kingpin.Flag("endpoint-type", "openstack endpoint type to use (i.e: public, internal, admin)").Default("public").String() + collectTime = kingpin.Flag("collect-metric-time", "time spent collecting each metric").Default("false").Bool() + disabledMetrics = kingpin.Flag("disable-metric", "multiple --disable-metric can be specified in the format: service-metric (i.e: cinder-snapshots)").Default("").Short('d').Strings() + disableSlowMetrics = kingpin.Flag("disable-slow-metrics", "disable slow metrics for performance reasons").Default("false").Bool() +) + func main() { - var ( - logLevel = kingpin.Flag("log.level", "Log level: [debug, info, warn, error, fatal]").Default("info").String() - bind = kingpin.Flag("web.listen-address", "address:port to listen on").Default(":9180").String() - metrics = kingpin.Flag("web.telemetry-path", "uri path to expose metrics").Default("/metrics").String() - osClientConfig = kingpin.Flag("os-client-config", "Path to the cloud configuration file").Default(DEFAULT_OS_CLIENT_CONFIG).String() - prefix = kingpin.Flag("prefix", "Prefix for metrics").Default("openstack").String() - endpointType = kingpin.Flag("endpoint-type", "openstack endpoint type to use (i.e: public, internal, admin)").Default("public").String() - collectTime = kingpin.Flag("collect-metric-time", "time spent collecting each metric").Default("false").Bool() - disabledMetrics = kingpin.Flag("disable-metric", "multiple --disable-metric can be specified in the format: service-metric (i.e: cinder-snapshots)").Default("").Short('d').Strings() - disableSlowMetrics = kingpin.Flag("disable-slow-metrics", "disable slow metrics for performance reasons").Default("false").Bool() - cloud = kingpin.Arg("cloud", "name or id of the cloud to gather metrics from").Required().String() - ) - - services := make(map[string]*bool) - - for _, service := range defaultEnabledServices { - flagName := fmt.Sprintf("disable-service.%s", service) - flagHelp := fmt.Sprintf("Disable the %s service exporter", service) - services[service] = kingpin.Flag(flagName, flagHelp).Default().Bool() - } kingpin.Version(version.Print("openstack-exporter")) kingpin.HelpFlag.Short('h') @@ -48,7 +42,6 @@ func main() { os.Exit(-1) } - log.Infof("Starting openstack exporter version %s for cloud: %s", version.Info(), *cloud) log.Infoln("Build context", version.BuildContext()) if *osClientConfig != DEFAULT_OS_CLIENT_CONFIG { @@ -56,25 +49,6 @@ func main() { os.Setenv("OS_CLIENT_CONFIG_FILE", *osClientConfig) } - enabledExporters := 0 - for service, disabled := range services { - if !*disabled { - _, err := exporters.EnableExporter(service, *prefix, *cloud, *disabledMetrics, *endpointType, *collectTime, *disableSlowMetrics, nil) - if err != nil { - // Log error and continue with enabling other exporters - log.Errorf("enabling exporter for service %s failed: %s", service, err) - continue - } - log.Infof("Enabled exporter for service: %s", service) - enabledExporters++ - } - } - - if enabledExporters == 0 { - log.Errorln("No exporter has been enabled, exiting") - os.Exit(-1) - } - http.Handle(*metrics, promhttp.Handler()) http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { _, err := w.Write([]byte(` @@ -88,7 +62,64 @@ func main() { log.Error(err) } }) + http.HandleFunc("/probe", probeHandler) log.Infoln("Starting HTTP server on", *bind) log.Fatal(http.ListenAndServe(*bind, nil)) } + +func probeHandler(w http.ResponseWriter, r *http.Request) { + + ctx, cancel := context.WithCancel(r.Context()) + defer cancel() + r = r.WithContext(ctx) + + registry := prometheus.NewPedanticRegistry() + + cloud := r.URL.Query().Get("cloud") + if cloud == "" { + http.Error(w, "'cloud' parameter is missing", http.StatusBadRequest) + return + } + + services := defaultEnabledServices + includeServices := r.URL.Query().Get("include_services") + if includeServices != "" { + services = strings.Split(includeServices, ",") + } + + excludeServices := strings.Split(r.URL.Query().Get("exclude_services"), ",") + services = removeElements(services, excludeServices) + + log.Infof("Enabled services: %v", services) + + for _, service := range services { + exp, err := exporters.EnableExporter(service, *prefix, cloud, *disabledMetrics, *endpointType, *collectTime, *disableSlowMetrics, nil) + if err != nil { + log.Errorf("enabling exporter for service %s failed: %s", service, err) + continue + } + registry.MustRegister(*exp) + log.Infof("Enabled exporter for service: %s", service) + } + + h := promhttp.HandlerFor(registry, promhttp.HandlerOpts{}) + h.ServeHTTP(w, r) +} + +func removeElements(slice []string, drop []string) []string { + res := []string{} + for _, s := range slice { + keep := true + for _, d := range drop { + if s == d { + keep = false + break + } + } + if keep { + res = append(res, s) + } + } + return res +} From 8c4e8b4623a4e4461d87757ff37c07f95b061f7e Mon Sep 17 00:00:00 2001 From: JeaNoel Vouilloz Date: Thu, 19 Nov 2020 13:13:56 +0100 Subject: [PATCH 3/4] Separate /metrics and /probe endpoint --- main.go | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index dfba7725..d6ced8dc 100644 --- a/main.go +++ b/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "fmt" "net/http" "os" "strings" @@ -28,14 +29,27 @@ var ( collectTime = kingpin.Flag("collect-metric-time", "time spent collecting each metric").Default("false").Bool() disabledMetrics = kingpin.Flag("disable-metric", "multiple --disable-metric can be specified in the format: service-metric (i.e: cinder-snapshots)").Default("").Short('d').Strings() disableSlowMetrics = kingpin.Flag("disable-slow-metrics", "disable slow metrics for performance reasons").Default("false").Bool() + cloud = kingpin.Arg("cloud", "name or id of the cloud to gather metrics from").String() + allClouds = kingpin.Flag("all-clouds", "Toggle the multiple cloud scrapping mode under /probe?cloud=").Default("false").Bool() ) func main() { + services := make(map[string]*bool) + + for _, service := range defaultEnabledServices { + flagName := fmt.Sprintf("disable-service.%s", service) + flagHelp := fmt.Sprintf("Disable the %s service exporter", service) + services[service] = kingpin.Flag(flagName, flagHelp).Default().Bool() + } + kingpin.Version(version.Print("openstack-exporter")) kingpin.HelpFlag.Short('h') kingpin.Parse() + if *cloud == "" && !*allClouds { + log.Fatalln("openstack-exporter: error: required argument 'cloud' or flag --all-clouds not provided, try --help") + } err := log.Base().SetLevel(*logLevel) if err != nil { log.Errorf("Cannot init set logger level: %s", err) @@ -49,7 +63,6 @@ func main() { os.Setenv("OS_CLIENT_CONFIG_FILE", *osClientConfig) } - http.Handle(*metrics, promhttp.Handler()) http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { _, err := w.Write([]byte(` OpenStack Exporter @@ -62,7 +75,11 @@ func main() { log.Error(err) } }) - http.HandleFunc("/probe", probeHandler) + if *allClouds { + http.HandleFunc("/probe", probeHandler) + } else { + http.HandleFunc("/metrics", metricHandler(services)) + } log.Infoln("Starting HTTP server on", *bind) log.Fatal(http.ListenAndServe(*bind, nil)) @@ -107,6 +124,42 @@ func probeHandler(w http.ResponseWriter, r *http.Request) { h.ServeHTTP(w, r) } +func metricHandler(services map[string]*bool) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + log.Infof("Starting openstack exporter version %s for cloud: %s", version.Info(), *cloud) + log.Infoln("Build context", version.BuildContext()) + + if *osClientConfig != DEFAULT_OS_CLIENT_CONFIG { + log.Debugf("Setting Env var OS_CLIENT_CONFIG_FILE = %s", *osClientConfig) + os.Setenv("OS_CLIENT_CONFIG_FILE", *osClientConfig) + } + + registry := prometheus.NewPedanticRegistry() + enabledExporters := 0 + for service, disabled := range services { + if !*disabled { + exp, err := exporters.EnableExporter(service, *prefix, *cloud, *disabledMetrics, *endpointType, *collectTime, *disableSlowMetrics, nil) + if err != nil { + // Log error and continue with enabling other exporters + log.Errorf("enabling exporter for service %s failed: %s", service, err) + continue + } + registry.MustRegister(*exp) + log.Infof("Enabled exporter for service: %s", service) + enabledExporters++ + } + } + + if enabledExporters == 0 { + log.Errorln("No exporter has been enabled, exiting") + os.Exit(-1) + } + + h := promhttp.HandlerFor(registry, promhttp.HandlerOpts{}) + h.ServeHTTP(w, r) + } +} + func removeElements(slice []string, drop []string) []string { res := []string{} for _, s := range slice { From a3669cf6775454655f84bb14792cc8c8e88f4ab8 Mon Sep 17 00:00:00 2001 From: JeaNoel Vouilloz Date: Fri, 20 Nov 2020 09:15:38 +0100 Subject: [PATCH 4/4] remove duplicate uuid import --- exporters/exporter.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/exporters/exporter.go b/exporters/exporter.go index ccae8e52..cb1b8b46 100644 --- a/exporters/exporter.go +++ b/exporters/exporter.go @@ -6,8 +6,6 @@ import ( "net/http" "time" - "github.com/hashicorp/go-uuid" - "github.com/gophercloud/gophercloud" "github.com/gophercloud/utils/openstack/clientconfig" "github.com/hashicorp/go-uuid"