From eb3c6a49b434bd4afe3fbc1938c135f7e2d299ea Mon Sep 17 00:00:00 2001 From: Elia Renzoni Date: Mon, 29 Dec 2025 16:30:36 +0100 Subject: [PATCH 1/2] feat: add support for loading multiple config files and staleness check Signed-off-by: Elia Renzoni --- pkg/config/config.go | 127 ++++++++++++++++++++++++++++++++++---- pkg/config/config_test.go | 123 +++++++++++++++++++++++++++++++++++- pkg/config/loader.go | 3 +- 3 files changed, 237 insertions(+), 16 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 345b1259f..0954a3052 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -22,6 +22,7 @@ package config import ( "fmt" "os" + "io/fs" "sync" "time" @@ -92,7 +93,9 @@ type MainConfig struct { ServerName string `yaml:"server_name,omitempty"` configFilePath string + configFilesPath []string configLastModified time.Time + configFilesLastMod []time.Time configRateLimitTime time.Time stalenessCheckLock sync.Mutex } @@ -115,6 +118,8 @@ func NewConfig() *Config { Logging: lo.New(), Main: &MainConfig{ ServerName: hn, + configFilesPath: make([]string, 0), + configFilesLastMod: make([]time.Time, 0), }, MgmtConfig: mgmt.New(), Metrics: mo.New(), @@ -132,6 +137,18 @@ func NewConfig() *Config { } } +func (c *Config) loadConfigs(flags *Flags) error { + ok, err := c.isDir(flags) + if err != nil { + return err + } + + if !ok { + return c.loadFile(flags) + } + return c.loadAndMergeFiles(flags) +} + // loadFile loads application configuration from a YAML-formatted file. func (c *Config) loadFile(flags *Flags) error { b, err := os.ReadFile(flags.ConfigPath) @@ -143,10 +160,49 @@ func (c *Config) loadFile(flags *Flags) error { return err } c.Main.configFilePath = flags.ConfigPath - c.Main.configLastModified = c.CheckFileLastModified() + c.Main.configLastModified = c.CheckFileLastModified("") return nil } +// loadAndMergeFiles loads application configuration from multiple YAML files +func (c *Config) loadAndMergeFiles(flags *Flags) error { + files, err := fs.Glob(os.DirFS(flags.ConfigPath), "*.yaml") + if err != nil { + return err + } + + for _, file := range files { + path := flags.ConfigPath + "/" + file + data, err := os.ReadFile(path) + if err != nil { + return err + } + + if err = c.loadYAMLConfig(string(data)); err != nil { + return err + } + + c.Main.configFilesPath = append( + c.Main.configFilesPath, + path, + ) + c.Main.configFilesLastMod = append( + c.Main.configFilesLastMod, + c.CheckFileLastModified(path), + ) + } + return nil +} + +func (c *Config) isDir(flags *Flags) (bool, error) { + finfo, err := os.Stat(flags.ConfigPath) + if err != nil { + return false, err + } + + return finfo.IsDir(), nil +} + // loadYAMLConfig loads application configuration from a YAML-formatted byte slice. func (c *Config) loadYAMLConfig(yml string) error { err := yaml.Unmarshal([]byte(yml), &c) @@ -172,15 +228,24 @@ func (c *Config) loadYAMLConfig(yml string) error { } // CheckFileLastModified returns the last modified date of the running config file, if present -func (c *Config) CheckFileLastModified() time.Time { - if c.Main == nil || c.Main.configFilePath == "" { +func (c *Config) CheckFileLastModified(confFile string) time.Time { + if c.Main == nil { return time.Time{} } - file, err := os.Stat(c.Main.configFilePath) + + path := confFile + if path == "" { + path = c.Main.configFilePath + if path == "" { + return time.Time{} + } + } + + fInfo, err := os.Stat(path) if err != nil { return time.Time{} } - return file.ModTime() + return fInfo.ModTime() } // Process converts various raw config options into internal data structures @@ -228,7 +293,9 @@ func (c *Config) Clone() *Config { nc.MgmtConfig = c.MgmtConfig.Clone() nc.Main.configFilePath = c.Main.configFilePath + nc.Main.configFilesPath = c.Main.configFilesPath nc.Main.configLastModified = c.Main.configLastModified + nc.Main.configFilesLastMod = c.Main.configFilesLastMod nc.Main.configRateLimitTime = c.Main.configRateLimitTime nc.Metrics.ListenAddress = c.Metrics.ListenAddress @@ -287,17 +354,32 @@ func (c *Config) IsStale() bool { c.Main.stalenessCheckLock.Lock() defer c.Main.stalenessCheckLock.Unlock() - if c.Main == nil || c.Main.configFilePath == "" || - time.Now().Before(c.Main.configRateLimitTime) { - return false - } + if c.Main == nil || + (len(c.Main.configFilesPath) == 0 && + (c.Main.configFilePath == "" || + time.Now().Before(c.Main.configRateLimitTime))) { + return false + } if c.MgmtConfig == nil { c.MgmtConfig = mgmt.New() } c.Main.configRateLimitTime = time.Now().Add(c.MgmtConfig.ReloadRateLimit) - t := c.CheckFileLastModified() + if len(c.Main.configFilesPath) > 0 { + for index, file := range c.Main.configFilesPath { + t := c.CheckFileLastModified(file) + if t.IsZero() { + continue + } + if !t.Equal(c.Main.configFilesLastMod[index]) { + return true + } + } + return false + } + + t := c.CheckFileLastModified("") if t.IsZero() { return false } @@ -309,7 +391,8 @@ func (c *Config) IsStale() bool { func (c *Config) CheckAndMarkReloadInProgress() bool { c.Main.stalenessCheckLock.Lock() defer c.Main.stalenessCheckLock.Unlock() - if c.Main == nil || c.Main.configFilePath == "" || + if c.Main == nil || + (c.Main.configFilePath == "" && len(c.Main.configFilesPath) == 0) || time.Now().Before(c.Main.configRateLimitTime) { return false } @@ -317,7 +400,22 @@ func (c *Config) CheckAndMarkReloadInProgress() bool { c.MgmtConfig = mgmt.New() } c.Main.configRateLimitTime = time.Now().Add(c.MgmtConfig.ReloadRateLimit) - t := c.CheckFileLastModified() + + if len(c.Main.configFilesPath) > 0 { + for index, file := range c.Main.configFilesPath { + t := c.CheckFileLastModified(file) + if t.IsZero() { + continue + } + if !t.Equal(c.Main.configFilesLastMod[index]) { + c.Main.configFilesLastMod[index] = t + return true + } + } + return false + } + + t := c.CheckFileLastModified("") if t.IsZero() { return false } @@ -353,7 +451,10 @@ func (c *Config) String() string { // ConfigFilePath returns the file path from which this configuration is based func (c *Config) ConfigFilePath() string { if c.Main != nil { - return c.Main.configFilePath + if len(c.Main.configFilesPath) == 0 { + return c.Main.configFilePath + } + return c.Flags.ConfigPath } return "" } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 87867f4ea..eb44ac1d9 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -114,12 +114,12 @@ func TestCloneBackendOptions(t *testing.T) { func TestCheckFileLastModified(t *testing.T) { c := NewConfig() - if !c.CheckFileLastModified().IsZero() { + if !c.CheckFileLastModified("").IsZero() { t.Error("expected zero time") } c.Main.configFilePath = "\t\n" - if !c.CheckFileLastModified().IsZero() { + if !c.CheckFileLastModified("").IsZero() { t.Error("expected zero time") } } @@ -310,6 +310,125 @@ func TestConfig_defaulting(t *testing.T) { } } + +const ( + dataTrk1 = ` +frontend: + listen_port: 8480 + +caches: + default: + provider: memory + +backends: + default: + provider: prometheus + origin_url: http://prometheus:9090 + cache_name: default + is_default: true +` + + dataTrk2 = ` +backends: + prometheus_backend: + provider: prometheus + origin_url: http://prometheus:9090 + cache_name: mycache + is_default: true + +caches: + mycache: + provider: memory +` +) + +func TestLoadAndMergeFiles(t *testing.T) { + dir := t.TempDir() + err := os.WriteFile(dir + "/trickster1.yaml", []byte(dataTrk1), 0666) + if err != nil { + t.Error(err) + } + err = os.WriteFile(dir + "/trickster2.yaml", []byte(dataTrk2), 0666) + if err != nil { + t.Error(err) + } + + c := NewConfig() + + args := []string{"-config", dir} + sargs := make([]string, 0, len(args)) + for _, v := range args { + if !strings.HasPrefix(v, "-test.") { + sargs = append(sargs, v) + } + } + flags, err := parseFlags(sargs) + if err != nil { + t.Error(err) + } + + err = c.loadAndMergeFiles(flags) + if err != nil { + t.Error(err) + } + + opt1, ok1 := c.Backends["default"] + opt2, ok2 := c.Backends["prometheus_backend"] + + switch { + case !ok1, !ok2: + t.Error("expected true result when performing backend lookup") + case opt1.Provider != "prometheus", opt2.Provider != "prometheus": + t.Error("expected prometheus as provider") + case opt1.OriginURL != "http://prometheus:9090", opt2.OriginURL != "http://prometheus:9090": + t.Error("expected http://prometheus:9090 as origin url") + } + + cOpt1, cOk1 := c.Caches["default"] + cOpt2, cOk2 := c.Caches["mycache"] + + switch { + case !cOk1, !cOk2: + t.Error("expected true result when performing cache lookup") + case cOpt1.Provider != "memory", cOpt2.Provider != "memory": + t.Error("expected memory as cache provider") + } +} + +func TestIsStaleWithMultipleFiles(t *testing.T) { + _, tml := emptyTestConfig() + dir := t.TempDir() + err := os.WriteFile(dir + "/trickster1.yaml", []byte(dataTrk1), 0666) + if err != nil { + t.Error(err) + } + + err = os.WriteFile(dir + "/trickster2.yaml", []byte(dataTrk2), 0666) + if err != nil { + t.Error(err) + } + + configs, err := Load([]string{"-config", dir}) + configs.MgmtConfig.ReloadRateLimit = 0 + if err != nil { + t.Error(err) + } + + if configs.IsStale() { + t.Error("expected non-stale configs") + } + + time.Sleep(400 * time.Millisecond) + err = os.WriteFile(dir + "/trickster1.yaml", []byte(tml), 0666) + if err != nil { + t.Error(err) + } + + if !configs.IsStale() { + t.Error("expected stale configs") + } +} + // remove any values that are non-deterministic func clean(c *Config) { c.Main.ServerName = "trickster-test" diff --git a/pkg/config/loader.go b/pkg/config/loader.go index 52f011385..c395810b6 100644 --- a/pkg/config/loader.go +++ b/pkg/config/loader.go @@ -46,7 +46,8 @@ func Load(args []string) (*Config, error) { c.Flags = flags return c, nil } - if err := c.loadFile(flags); err != nil && flags.customPath { + + if err := c.loadConfigs(flags); err != nil && flags.customPath { // a user-provided path couldn't be loaded. return the error for the application to handle return nil, err } From e2c4f381b782d23141bbf369fc79e43ec26f5350 Mon Sep 17 00:00:00 2001 From: Elia Renzoni Date: Mon, 29 Dec 2025 17:21:08 +0100 Subject: [PATCH 2/2] fix: linter issues Signed-off-by: Elia Renzoni --- pkg/config/config.go | 30 +++++++++++++++--------------- pkg/config/loader.go | 2 +- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 0954a3052..ca25340ac 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -21,8 +21,8 @@ package config import ( "fmt" - "os" "io/fs" + "os" "sync" "time" @@ -38,7 +38,7 @@ import ( auth "github.com/trickstercache/trickster/v2/pkg/proxy/authenticator/options" "github.com/trickstercache/trickster/v2/pkg/proxy/request/rewriter" rwopts "github.com/trickstercache/trickster/v2/pkg/proxy/request/rewriter/options" - "gopkg.in/yaml.v2" + "go.yaml.in/yaml/v2" ) const defaultResourceName = "default" @@ -117,8 +117,8 @@ func NewConfig() *Config { }, Logging: lo.New(), Main: &MainConfig{ - ServerName: hn, - configFilesPath: make([]string, 0), + ServerName: hn, + configFilesPath: make([]string, 0), configFilesLastMod: make([]time.Time, 0), }, MgmtConfig: mgmt.New(), @@ -355,11 +355,11 @@ func (c *Config) IsStale() bool { defer c.Main.stalenessCheckLock.Unlock() if c.Main == nil || - (len(c.Main.configFilesPath) == 0 && - (c.Main.configFilePath == "" || - time.Now().Before(c.Main.configRateLimitTime))) { - return false - } + (len(c.Main.configFilesPath) == 0 && + (c.Main.configFilePath == "" || + time.Now().Before(c.Main.configRateLimitTime))) { + return false + } if c.MgmtConfig == nil { c.MgmtConfig = mgmt.New() @@ -374,11 +374,11 @@ func (c *Config) IsStale() bool { } if !t.Equal(c.Main.configFilesLastMod[index]) { return true - } + } } return false } - + t := c.CheckFileLastModified("") if t.IsZero() { return false @@ -391,8 +391,8 @@ func (c *Config) IsStale() bool { func (c *Config) CheckAndMarkReloadInProgress() bool { c.Main.stalenessCheckLock.Lock() defer c.Main.stalenessCheckLock.Unlock() - if c.Main == nil || - (c.Main.configFilePath == "" && len(c.Main.configFilesPath) == 0) || + if c.Main == nil || + (c.Main.configFilePath == "" && len(c.Main.configFilesPath) == 0) || time.Now().Before(c.Main.configRateLimitTime) { return false } @@ -410,7 +410,7 @@ func (c *Config) CheckAndMarkReloadInProgress() bool { if !t.Equal(c.Main.configFilesLastMod[index]) { c.Main.configFilesLastMod[index] = t return true - } + } } return false } @@ -452,7 +452,7 @@ func (c *Config) String() string { func (c *Config) ConfigFilePath() string { if c.Main != nil { if len(c.Main.configFilesPath) == 0 { - return c.Main.configFilePath + return c.Main.configFilePath } return c.Flags.ConfigPath } diff --git a/pkg/config/loader.go b/pkg/config/loader.go index c395810b6..e87ff45ce 100644 --- a/pkg/config/loader.go +++ b/pkg/config/loader.go @@ -46,7 +46,7 @@ func Load(args []string) (*Config, error) { c.Flags = flags return c, nil } - + if err := c.loadConfigs(flags); err != nil && flags.customPath { // a user-provided path couldn't be loaded. return the error for the application to handle return nil, err