Skip to content

Prometheus HTTP plugin memory consumption and templating performance #327

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Husteln merged 17 commits into from
Jul 7, 2026
Merged

Prometheus HTTP plugin memory consumption and templating performance #327

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
6a68597
test
Jun 26, 2026
f067979
test
Jun 26, 2026
d5fc5e6
rollback prev changes + update bigcache
Jul 1, 2026
08e567b
fix
Jul 1, 2026
8ad7199
test increased cache dur
Jul 1, 2026
6167c27
test cache clean window
Jul 1, 2026
e41e566
add cache stats
Jul 1, 2026
87a1810
test findkeys changes
Jul 1, 2026
f22e341
test
Jul 2, 2026
f08fc8d
test
Jul 2, 2026
468e554
test
Jul 2, 2026
fbc17e2
test
Jul 2, 2026
70edcd7
test
Jul 2, 2026
7e39eb6
return blocking behavior
Jul 2, 2026
27b62b0
remove debug
Jul 3, 2026
8e82c81
test cacheget
Jul 4, 2026
7ce3123
rollback
Jul 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ require (
github.com/alitto/pond v1.9.2
github.com/alitto/pond/v2 v2.3.4
github.com/aliyun/alibaba-cloud-sdk-go v1.63.107
github.com/allegro/bigcache v1.2.1
github.com/allegro/bigcache/v3 v3.1.0
github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9
github.com/antchfx/jsonquery v1.3.6
github.com/antchfx/xmlquery v1.4.4
Expand Down Expand Up @@ -76,8 +76,8 @@ require (
github.com/coreos/go-systemd/v22 v22.5.0
github.com/couchbase/go-couchbase v0.1.1
github.com/datadope-io/go-zabbix/v2 v2.0.1
github.com/devopsext/tools v0.15.3
github.com/devopsext/utils v0.4.3
github.com/devopsext/tools v0.18.7
github.com/devopsext/utils v0.4.8
github.com/digitalocean/go-libvirt v0.0.0-20250417173424-a6a66ef779d6
github.com/dimchansky/utfbom v1.1.1
github.com/djherbis/times v1.6.0
Expand Down Expand Up @@ -244,9 +244,9 @@ require (
gopkg.in/olivere/elastic.v5 v5.0.86
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7
gopkg.in/yaml.v2 v2.4.0
k8s.io/api v0.33.1
k8s.io/apimachinery v0.33.1
k8s.io/client-go v0.33.1
k8s.io/api v0.33.3
k8s.io/apimachinery v0.33.3
k8s.io/client-go v0.33.3
layeh.com/radius v0.0.0-20221205141417-e7fbddd11d68
modernc.org/sqlite v1.37.0
software.sslmate.com/src/go-pkcs12 v0.5.0
Expand Down
24 changes: 12 additions & 12 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -828,8 +828,8 @@ github.com/alitto/pond/v2 v2.3.4 h1:hR0bqAwJiI2chu3cLN4gVyNC7rc5mj/l5wg0710nxsY=
github.com/alitto/pond/v2 v2.3.4/go.mod h1:xkjYEgQ05RSpWdfSd1nM3OVv7TBhLdy7rMp3+2Nq+yE=
github.com/aliyun/alibaba-cloud-sdk-go v1.63.107 h1:qagvUyrgOnBIlVRQWOyCZGVKUIYbMBdGdJ104vBpRFU=
github.com/aliyun/alibaba-cloud-sdk-go v1.63.107/go.mod h1:SOSDHfe1kX91v3W5QiBsWSLqeLxImobbMX1mxrFHsVQ=
github.com/allegro/bigcache v1.2.1 h1:hg1sY1raCwic3Vnsvje6TT7/pnZba83LeFck5NrFKSc=
github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
github.com/allegro/bigcache/v3 v3.1.0 h1:H2Vp8VOvxcrB91o86fUSVJFqeuz8kpyyB02eH3bSzwk=
github.com/allegro/bigcache/v3 v3.1.0/go.mod h1:aPyh7jEvrog9zAwx5N7+JUQX5dZTSGpxF1LAR4dr35I=
github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9 h1:FXrPTd8Rdlc94dKccl7KPmdmIbVh/OjelJ8/vgMRzcQ=
github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9/go.mod h1:eliMa/PW+RDr2QLWRmLH1R1ZA4RInpmvOzDDXtaIZkc=
github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
Expand Down Expand Up @@ -1137,10 +1137,10 @@ github.com/devigned/tab v0.1.1 h1:3mD6Kb1mUOYeLpJvTVSDwSg5ZsfSxfvxGRTxRsJsITA=
github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY=
github.com/devopsext/sdk v0.9.6 h1:UWU4ZeINWMmgLsuCvdC/FPN+HKu1tcW06K65ler4mT4=
github.com/devopsext/sdk v0.9.6/go.mod h1:AHHlOEv1+GGQ3ktHMlhuTUwo3zljV3QJbC0+8o2kn+4=
github.com/devopsext/tools v0.15.3 h1:0zTepwKqDCgcCh85ATx6cE2ri98zfOFsyunFVDTavvg=
github.com/devopsext/tools v0.15.3/go.mod h1:FXft1ABAA2XYDnbg6ElDyQHPUXXp/GRPQRFQ1R1Cz8Y=
github.com/devopsext/utils v0.4.3 h1:PbAlmjtWjppKFBG479xKkiD5U8OUbXcDRivRMW7L4Pg=
github.com/devopsext/utils v0.4.3/go.mod h1:Vg3Jxxr4i9JXPmhaZo7TvY+OEr3W3QlZDXmbN8ScU5k=
github.com/devopsext/tools v0.18.7 h1:dCVTFaylYQ4kJZX2O0zdXtnXErf5hnkayAUTwDAoPL0=
github.com/devopsext/tools v0.18.7/go.mod h1:RNiXytM21qgylYzJWya8dhfCTA+xxvI4GGFBjx0ZxAA=
github.com/devopsext/utils v0.4.8 h1:Ql8fwF36lUcHj5nw/IlZrfHEcPNmh1YPJws3MOnuCKA=
github.com/devopsext/utils v0.4.8/go.mod h1:3Apwsy4/k+baHRxsHuK0ipqdgK/YNHif0fZmO7m0W4Q=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
Expand Down Expand Up @@ -3532,12 +3532,12 @@ honnef.co/go/tools v0.2.2 h1:MNh1AVMyVX23VUHE2O27jm6lNj3vjO5DexS4A1xvnzk=
honnef.co/go/tools v0.2.2/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M=
howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0=
k8s.io/api v0.33.1 h1:tA6Cf3bHnLIrUK4IqEgb2v++/GYUtqiu9sRVk3iBXyw=
k8s.io/api v0.33.1/go.mod h1:87esjTn9DRSRTD4fWMXamiXxJhpOIREjWOSjsW1kEHw=
k8s.io/apimachinery v0.33.1 h1:mzqXWV8tW9Rw4VeW9rEkqvnxj59k1ezDUl20tFK/oM4=
k8s.io/apimachinery v0.33.1/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM=
k8s.io/client-go v0.33.1 h1:ZZV/Ks2g92cyxWkRRnfUDsnhNn28eFpt26aGc8KbXF4=
k8s.io/client-go v0.33.1/go.mod h1:JAsUrl1ArO7uRVFWfcj6kOomSlCv+JpvIsp6usAGefA=
k8s.io/api v0.33.3 h1:SRd5t//hhkI1buzxb288fy2xvjubstenEKL9K51KBI8=
k8s.io/api v0.33.3/go.mod h1:01Y/iLUjNBM3TAvypct7DIj0M0NIZc+PzAHCIo0CYGE=
k8s.io/apimachinery v0.33.3 h1:4ZSrmNa0c/ZpZJhAgRdcsFcZOw1PQU1bALVQ0B3I5LA=
k8s.io/apimachinery v0.33.3/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM=
k8s.io/client-go v0.33.3 h1:M5AfDnKfYmVJif92ngN532gFqakcGi6RvaOF16efrpA=
k8s.io/client-go v0.33.3/go.mod h1:luqKBQggEf3shbxHY4uVENAxrDISLOarxpTKMiUuujg=
k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4=
Expand Down
89 changes: 62 additions & 27 deletions plugins/inputs/prometheus_http/prometheus_http.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import (
"sync"
"time"

"github.com/allegro/bigcache"
"github.com/allegro/bigcache/v3"
"github.com/araddon/dateparse"
"gopkg.in/yaml.v3"

Expand Down Expand Up @@ -108,6 +108,11 @@ type PrometheusHttp struct {
cache *bigcache.BigCache
}

// Panic implements [common.Logger].
func (p *PrometheusHttp) Panic(obj interface{}, args ...interface{}) {
panic("unimplemented")
}

type PrometheusHttpPushFunc = func(when time.Time, tags map[string]string, stamp time.Time, value float64)

type PrometheusHttpDatasourceResponse struct {
Expand Down Expand Up @@ -527,7 +532,6 @@ func (p *PrometheusHttp) setMetrics(w *sync.WaitGroup, pm *PrometheusHttpMetric,
}
pm.uniques[hash] = true
}

v, err := p.getTemplateValue(pm.template, value)
if err != nil {
p.Log.Error(err)
Expand Down Expand Up @@ -643,6 +647,7 @@ func (p *PrometheusHttp) gatherMetrics(gid uint64, ds PrometheusHttpDatasource)

fields := p.addFields("requests", p.requests.counter.Value())
fields["errors"] = p.errors.counter.Value()
p.addCacheStats(fields)

r1 := float64(p.requests.counter.Value())
r2 := float64(p.errors.counter.Value())
Expand All @@ -654,6 +659,22 @@ func (p *PrometheusHttp) gatherMetrics(gid uint64, ds PrometheusHttpDatasource)
return nil
}

func (p *PrometheusHttp) addCacheStats(fields map[string]interface{}) {

if p.cache == nil {
return
}

stats := p.cache.Stats()
fields["cache_hits_count"] = stats.Hits
fields["cache_misses_count"] = stats.Misses
fields["cache_delete_hits_count"] = stats.DelHits
fields["cache_delete_misses_count"] = stats.DelMisses
fields["cache_collisions_count"] = stats.Collisions
fields["cache_len"] = p.cache.Len()
fields["cache_capacity_bytes"] = p.cache.Capacity()
}

func (ptt *PrometheusHttpTextTemplate) fCacheRegexMatchFindKey(obj interface{}, field, value string) string {

if obj == nil || utils.IsEmpty(field) || utils.IsEmpty(value) {
Expand All @@ -662,21 +683,33 @@ func (ptt *PrometheusHttpTextTemplate) fCacheRegexMatchFindKey(obj interface{},
if ptt.input.cache == nil {
return ""
}
key := fmt.Sprintf("%s.%s.%s", ptt.tag, field, value)
key := fmt.Sprintf("%s.%s.%s.%s", ptt.name, ptt.tag, field, value)

entry, err := ptt.input.cache.Get(key)
if err == nil {
v1 := string(entry)
if !utils.IsEmpty(v1) {
return v1
if v1 == "nil" {
md := ptt.input.cache.KeyMetadata(key)
if md.RequestCount > 1000 {
ptt.input.cache.Delete(key)
}
return ""
}
return v1
}
var v2 any
keys := ptt.template.RegexMatchFindKeys(obj, field, value)
if len(keys) == 0 {
v2 = ""
} else {
v2 = keys[0]
}
v2 := ptt.template.RegexMatchFindKey(obj, field, value)
v1 := fmt.Sprintf("%v", v2)
if !utils.IsEmpty(v1) {
ptt.input.cache.Set(key, []byte(v1))
return v1
}
ptt.input.cache.Set(key, []byte("nil"))
return ""
}

Expand Down Expand Up @@ -991,6 +1024,7 @@ func (p *PrometheusHttp) Gather(acc telegraf.Accumulator) error {
var ds PrometheusHttpDatasource = nil
gid := utils.GoRoutineID()
p.readFiles(gid, &globalFiles, &globalHashes, p.Metrics, false)

// Gather data
err := p.gatherMetrics(gid, ds)
return err
Expand Down Expand Up @@ -1031,53 +1065,54 @@ func (p *PrometheusHttp) Init() error {
return err
}

p.Log.Debugf("[%d] %s metrics amount: %d", gid, p.Name, lMetrics)
maxltags := 0

for _, m := range p.Metrics {
p.setDefaultMetric(gid, m)
ltags := len(m.Tags)
if maxltags < ltags {
maxltags = ltags
}
}

p.Log.Debugf("[%d] %s metrics amount: %d, max tags %d", gid, p.Name, lMetrics, maxltags)

//p.files = &sync.Map{}
p.requests = NewRateCounter(time.Duration(p.Interval))
p.errors = NewRateCounter(time.Duration(p.Interval))
p.mtx = &sync.Mutex{}

if len(p.Files) > 0 {
if len(p.Files) > 0 && p.cache == nil {

entries, length := p.readFiles(gid, &globalFiles, &globalHashes, p.Metrics, true)
_, length := p.readFiles(gid, &globalFiles, &globalHashes, p.Metrics, true)

seconds := time.Duration(p.Timeout).Seconds()

if p.CacheDuration <= 0 {
p.CacheDuration = config.Duration(time.Second * time.Duration(seconds))
p.CacheDuration = config.Duration(time.Second * time.Duration(seconds) * 60)
}

config := bigcache.DefaultConfig(time.Duration(p.CacheDuration))
config.Shards = 256
config.CleanWindow = 0
/*if seconds > 0 {
t := int(math.Round(seconds / 2))
if t > 1 {
config.CleanWindow = time.Duration(time.Second * time.Duration(t))
}
}*/
config.Shards = 32

config.MaxEntriesInWindow = entries
config.MaxEntriesInWindow = lMetrics * maxltags
config.MaxEntrySize = length

maxSizeInMb := 0
if p.CacheSize > 0 {
maxSizeInMb = int(p.CacheSize) / (1024 * 1024)
} else {
maxSizeInMb = (entries * length * int(seconds)) / (1024 * 1024)
}
if maxSizeInMb == 0 {
maxSizeInMb = 1
}
maxSizeInMb := 1
// if p.CacheSize > 0 {
// maxSizeInMb = int(p.CacheSize) / (1024)
// } else {
// maxSizeInMb = (entries * length * int(seconds)) / (1024)
// }
// if maxSizeInMb == 0 {
// maxSizeInMb = 1
// }
config.HardMaxCacheSize = maxSizeInMb

config.Logger = p
config.Verbose = true
config.StatsEnabled = true

cache, err := bigcache.NewBigCache(config)
if err != nil {
Expand Down
Loading