diff --git a/go.mod b/go.mod index 819218ac1871e..9617c67d67018 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/alitto/pond v1.9.2 github.com/alitto/pond/v2 v2.3.4 github.com/aliyun/alibaba-cloud-sdk-go v1.63.107 - github.com/allegro/bigcache v1.2.1 + github.com/allegro/bigcache/v3 v3.1.0 github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9 github.com/antchfx/jsonquery v1.3.6 github.com/antchfx/xmlquery v1.4.4 @@ -76,8 +76,8 @@ require ( github.com/coreos/go-systemd/v22 v22.5.0 github.com/couchbase/go-couchbase v0.1.1 github.com/datadope-io/go-zabbix/v2 v2.0.1 - github.com/devopsext/tools v0.15.3 - github.com/devopsext/utils v0.4.3 + github.com/devopsext/tools v0.18.7 + github.com/devopsext/utils v0.4.8 github.com/digitalocean/go-libvirt v0.0.0-20250417173424-a6a66ef779d6 github.com/dimchansky/utfbom v1.1.1 github.com/djherbis/times v1.6.0 @@ -244,9 +244,9 @@ require ( gopkg.in/olivere/elastic.v5 v5.0.86 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 gopkg.in/yaml.v2 v2.4.0 - k8s.io/api v0.33.1 - k8s.io/apimachinery v0.33.1 - k8s.io/client-go v0.33.1 + k8s.io/api v0.33.3 + k8s.io/apimachinery v0.33.3 + k8s.io/client-go v0.33.3 layeh.com/radius v0.0.0-20221205141417-e7fbddd11d68 modernc.org/sqlite v1.37.0 software.sslmate.com/src/go-pkcs12 v0.5.0 diff --git a/go.sum b/go.sum index 4cc57c0445858..77e080e6ca25a 100644 --- a/go.sum +++ b/go.sum @@ -828,8 +828,8 @@ github.com/alitto/pond/v2 v2.3.4 h1:hR0bqAwJiI2chu3cLN4gVyNC7rc5mj/l5wg0710nxsY= github.com/alitto/pond/v2 v2.3.4/go.mod h1:xkjYEgQ05RSpWdfSd1nM3OVv7TBhLdy7rMp3+2Nq+yE= github.com/aliyun/alibaba-cloud-sdk-go v1.63.107 h1:qagvUyrgOnBIlVRQWOyCZGVKUIYbMBdGdJ104vBpRFU= github.com/aliyun/alibaba-cloud-sdk-go v1.63.107/go.mod h1:SOSDHfe1kX91v3W5QiBsWSLqeLxImobbMX1mxrFHsVQ= -github.com/allegro/bigcache v1.2.1 h1:hg1sY1raCwic3Vnsvje6TT7/pnZba83LeFck5NrFKSc= -github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= +github.com/allegro/bigcache/v3 v3.1.0 h1:H2Vp8VOvxcrB91o86fUSVJFqeuz8kpyyB02eH3bSzwk= +github.com/allegro/bigcache/v3 v3.1.0/go.mod h1:aPyh7jEvrog9zAwx5N7+JUQX5dZTSGpxF1LAR4dr35I= github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9 h1:FXrPTd8Rdlc94dKccl7KPmdmIbVh/OjelJ8/vgMRzcQ= github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9/go.mod h1:eliMa/PW+RDr2QLWRmLH1R1ZA4RInpmvOzDDXtaIZkc= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= @@ -1137,10 +1137,10 @@ github.com/devigned/tab v0.1.1 h1:3mD6Kb1mUOYeLpJvTVSDwSg5ZsfSxfvxGRTxRsJsITA= github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= github.com/devopsext/sdk v0.9.6 h1:UWU4ZeINWMmgLsuCvdC/FPN+HKu1tcW06K65ler4mT4= github.com/devopsext/sdk v0.9.6/go.mod h1:AHHlOEv1+GGQ3ktHMlhuTUwo3zljV3QJbC0+8o2kn+4= -github.com/devopsext/tools v0.15.3 h1:0zTepwKqDCgcCh85ATx6cE2ri98zfOFsyunFVDTavvg= -github.com/devopsext/tools v0.15.3/go.mod h1:FXft1ABAA2XYDnbg6ElDyQHPUXXp/GRPQRFQ1R1Cz8Y= -github.com/devopsext/utils v0.4.3 h1:PbAlmjtWjppKFBG479xKkiD5U8OUbXcDRivRMW7L4Pg= -github.com/devopsext/utils v0.4.3/go.mod h1:Vg3Jxxr4i9JXPmhaZo7TvY+OEr3W3QlZDXmbN8ScU5k= +github.com/devopsext/tools v0.18.7 h1:dCVTFaylYQ4kJZX2O0zdXtnXErf5hnkayAUTwDAoPL0= +github.com/devopsext/tools v0.18.7/go.mod h1:RNiXytM21qgylYzJWya8dhfCTA+xxvI4GGFBjx0ZxAA= +github.com/devopsext/utils v0.4.8 h1:Ql8fwF36lUcHj5nw/IlZrfHEcPNmh1YPJws3MOnuCKA= +github.com/devopsext/utils v0.4.8/go.mod h1:3Apwsy4/k+baHRxsHuK0ipqdgK/YNHif0fZmO7m0W4Q= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= @@ -3532,12 +3532,12 @@ honnef.co/go/tools v0.2.2 h1:MNh1AVMyVX23VUHE2O27jm6lNj3vjO5DexS4A1xvnzk= honnef.co/go/tools v0.2.2/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY= howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= -k8s.io/api v0.33.1 h1:tA6Cf3bHnLIrUK4IqEgb2v++/GYUtqiu9sRVk3iBXyw= -k8s.io/api v0.33.1/go.mod h1:87esjTn9DRSRTD4fWMXamiXxJhpOIREjWOSjsW1kEHw= -k8s.io/apimachinery v0.33.1 h1:mzqXWV8tW9Rw4VeW9rEkqvnxj59k1ezDUl20tFK/oM4= -k8s.io/apimachinery v0.33.1/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= -k8s.io/client-go v0.33.1 h1:ZZV/Ks2g92cyxWkRRnfUDsnhNn28eFpt26aGc8KbXF4= -k8s.io/client-go v0.33.1/go.mod h1:JAsUrl1ArO7uRVFWfcj6kOomSlCv+JpvIsp6usAGefA= +k8s.io/api v0.33.3 h1:SRd5t//hhkI1buzxb288fy2xvjubstenEKL9K51KBI8= +k8s.io/api v0.33.3/go.mod h1:01Y/iLUjNBM3TAvypct7DIj0M0NIZc+PzAHCIo0CYGE= +k8s.io/apimachinery v0.33.3 h1:4ZSrmNa0c/ZpZJhAgRdcsFcZOw1PQU1bALVQ0B3I5LA= +k8s.io/apimachinery v0.33.3/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= +k8s.io/client-go v0.33.3 h1:M5AfDnKfYmVJif92ngN532gFqakcGi6RvaOF16efrpA= +k8s.io/client-go v0.33.3/go.mod h1:luqKBQggEf3shbxHY4uVENAxrDISLOarxpTKMiUuujg= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= diff --git a/plugins/inputs/prometheus_http/prometheus_http.go b/plugins/inputs/prometheus_http/prometheus_http.go index d0a14c7e7080a..ed4c00c14e621 100644 --- a/plugins/inputs/prometheus_http/prometheus_http.go +++ b/plugins/inputs/prometheus_http/prometheus_http.go @@ -18,7 +18,7 @@ import ( "sync" "time" - "github.com/allegro/bigcache" + "github.com/allegro/bigcache/v3" "github.com/araddon/dateparse" "gopkg.in/yaml.v3" @@ -108,6 +108,11 @@ type PrometheusHttp struct { cache *bigcache.BigCache } +// Panic implements [common.Logger]. +func (p *PrometheusHttp) Panic(obj interface{}, args ...interface{}) { + panic("unimplemented") +} + type PrometheusHttpPushFunc = func(when time.Time, tags map[string]string, stamp time.Time, value float64) type PrometheusHttpDatasourceResponse struct { @@ -527,7 +532,6 @@ func (p *PrometheusHttp) setMetrics(w *sync.WaitGroup, pm *PrometheusHttpMetric, } pm.uniques[hash] = true } - v, err := p.getTemplateValue(pm.template, value) if err != nil { p.Log.Error(err) @@ -643,6 +647,7 @@ func (p *PrometheusHttp) gatherMetrics(gid uint64, ds PrometheusHttpDatasource) fields := p.addFields("requests", p.requests.counter.Value()) fields["errors"] = p.errors.counter.Value() + p.addCacheStats(fields) r1 := float64(p.requests.counter.Value()) r2 := float64(p.errors.counter.Value()) @@ -654,6 +659,22 @@ func (p *PrometheusHttp) gatherMetrics(gid uint64, ds PrometheusHttpDatasource) return nil } +func (p *PrometheusHttp) addCacheStats(fields map[string]interface{}) { + + if p.cache == nil { + return + } + + stats := p.cache.Stats() + fields["cache_hits_count"] = stats.Hits + fields["cache_misses_count"] = stats.Misses + fields["cache_delete_hits_count"] = stats.DelHits + fields["cache_delete_misses_count"] = stats.DelMisses + fields["cache_collisions_count"] = stats.Collisions + fields["cache_len"] = p.cache.Len() + fields["cache_capacity_bytes"] = p.cache.Capacity() +} + func (ptt *PrometheusHttpTextTemplate) fCacheRegexMatchFindKey(obj interface{}, field, value string) string { if obj == nil || utils.IsEmpty(field) || utils.IsEmpty(value) { @@ -662,21 +683,33 @@ func (ptt *PrometheusHttpTextTemplate) fCacheRegexMatchFindKey(obj interface{}, if ptt.input.cache == nil { return "" } - key := fmt.Sprintf("%s.%s.%s", ptt.tag, field, value) + key := fmt.Sprintf("%s.%s.%s.%s", ptt.name, ptt.tag, field, value) entry, err := ptt.input.cache.Get(key) if err == nil { v1 := string(entry) - if !utils.IsEmpty(v1) { - return v1 + if v1 == "nil" { + md := ptt.input.cache.KeyMetadata(key) + if md.RequestCount > 1000 { + ptt.input.cache.Delete(key) + } + return "" } + return v1 + } + var v2 any + keys := ptt.template.RegexMatchFindKeys(obj, field, value) + if len(keys) == 0 { + v2 = "" + } else { + v2 = keys[0] } - v2 := ptt.template.RegexMatchFindKey(obj, field, value) v1 := fmt.Sprintf("%v", v2) if !utils.IsEmpty(v1) { ptt.input.cache.Set(key, []byte(v1)) return v1 } + ptt.input.cache.Set(key, []byte("nil")) return "" } @@ -991,6 +1024,7 @@ func (p *PrometheusHttp) Gather(acc telegraf.Accumulator) error { var ds PrometheusHttpDatasource = nil gid := utils.GoRoutineID() p.readFiles(gid, &globalFiles, &globalHashes, p.Metrics, false) + // Gather data err := p.gatherMetrics(gid, ds) return err @@ -1031,53 +1065,54 @@ func (p *PrometheusHttp) Init() error { return err } - p.Log.Debugf("[%d] %s metrics amount: %d", gid, p.Name, lMetrics) + maxltags := 0 for _, m := range p.Metrics { p.setDefaultMetric(gid, m) + ltags := len(m.Tags) + if maxltags < ltags { + maxltags = ltags + } } + p.Log.Debugf("[%d] %s metrics amount: %d, max tags %d", gid, p.Name, lMetrics, maxltags) + //p.files = &sync.Map{} p.requests = NewRateCounter(time.Duration(p.Interval)) p.errors = NewRateCounter(time.Duration(p.Interval)) p.mtx = &sync.Mutex{} - if len(p.Files) > 0 { + if len(p.Files) > 0 && p.cache == nil { - entries, length := p.readFiles(gid, &globalFiles, &globalHashes, p.Metrics, true) + _, length := p.readFiles(gid, &globalFiles, &globalHashes, p.Metrics, true) seconds := time.Duration(p.Timeout).Seconds() if p.CacheDuration <= 0 { - p.CacheDuration = config.Duration(time.Second * time.Duration(seconds)) + p.CacheDuration = config.Duration(time.Second * time.Duration(seconds) * 60) } config := bigcache.DefaultConfig(time.Duration(p.CacheDuration)) - config.Shards = 256 config.CleanWindow = 0 - /*if seconds > 0 { - t := int(math.Round(seconds / 2)) - if t > 1 { - config.CleanWindow = time.Duration(time.Second * time.Duration(t)) - } - }*/ + config.Shards = 32 - config.MaxEntriesInWindow = entries + config.MaxEntriesInWindow = lMetrics * maxltags config.MaxEntrySize = length - maxSizeInMb := 0 - if p.CacheSize > 0 { - maxSizeInMb = int(p.CacheSize) / (1024 * 1024) - } else { - maxSizeInMb = (entries * length * int(seconds)) / (1024 * 1024) - } - if maxSizeInMb == 0 { - maxSizeInMb = 1 - } + maxSizeInMb := 1 + // if p.CacheSize > 0 { + // maxSizeInMb = int(p.CacheSize) / (1024) + // } else { + // maxSizeInMb = (entries * length * int(seconds)) / (1024) + // } + // if maxSizeInMb == 0 { + // maxSizeInMb = 1 + // } config.HardMaxCacheSize = maxSizeInMb config.Logger = p config.Verbose = true + config.StatsEnabled = true cache, err := bigcache.NewBigCache(config) if err != nil {