gcs: support custom GCS API endpoint via endpoint config option

CHANGELOG.md

@@ -9,6 +9,7 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan
 We use *breaking :warning:* to mark changes that are not backward compatible (relates only to v0.y.z releases.)
 
 ## Unreleased
+- GCS: Add support for setting custom GCS endpoint via `endpoint` config option, for regional GCS endpoints.
 - [#165](https://github.com/thanos-io/objstore/pull/165) GCS: Upgrade cloud.google.com/go/storage version to `v1.50.0`.
 - [#38](https://github.com/thanos-io/objstore/pull/38) GCS: Upgrade cloud.google.com/go/storage version to `v1.43.0`.
 - [#145](https://github.com/thanos-io/objstore/pull/145) Include content length in the response of Get and GetRange.

README.md

@@ -389,6 +389,7 @@ type: GCS
 config:
   bucket: ""
   service_account: ""
+  endpoint: ""
   use_grpc: false
   grpc_conn_pool_size: 0
   http_config:
@@ -413,6 +414,8 @@ config:
 prefix: ""
 ```
 
+The `endpoint` option overrides the default GCS API endpoint. Leave it empty to use the client library's default, or configure a [GCP regional endpoint](https://cloud.google.com/storage/docs/regional-endpoints).
+
 ###### Using GOOGLE_APPLICATION_CREDENTIALS
 
 Application credentials are configured via JSON file and only the bucket needs to be specified, the client looks for:

providers/gcs/gcs.go

@@ -43,7 +43,11 @@ var DefaultConfig = Config{
 type Config struct {
 	Bucket         string `yaml:"bucket"`
 	ServiceAccount string `yaml:"service_account"`
-	UseGRPC        bool   `yaml:"use_grpc"`
+	// Endpoint overrides the default GCS API endpoint. It is useful for GCP regional endpoints.
+	// See https://cloud.google.com/storage/docs/regional-endpoints.
+	// If empty, the client library's default endpoint is used.
+	Endpoint string `yaml:"endpoint"`
+	UseGRPC  bool   `yaml:"use_grpc"`
 	// GRPCConnPoolSize controls the size of the gRPC connection pool and should only be used
 	// when direct path is not enabled.
 	// See https://pkg.go.dev/cloud.google.com/go/storage#hdr-Experimental_gRPC_API for more details
@@ -116,6 +120,9 @@ func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, comp
 	opts = append(opts,
 		option.WithUserAgent(fmt.Sprintf("thanos-%s/%s (%s)", component, version.Version, runtime.Version())),
 	)
+	if gc.Endpoint != "" {
+		opts = append(opts, option.WithEndpoint(gc.Endpoint))
+	}
 
 	if !gc.UseGRPC {
 		var err error

providers/gcs/gcs_test.go

@@ -104,6 +104,13 @@ chunk_size_bytes: 1024`,
 	}
 }
 
+func TestParseConfig_Endpoint(t *testing.T) {
+	cfg, err := parseConfig([]byte(`bucket: abcd
+endpoint: https://storage.example.com`))
+	testutil.Ok(t, err)
+	testutil.Equals(t, "https://storage.example.com", cfg.Endpoint)
+}
+
 func TestParseConfig_HTTPConfig(t *testing.T) {
 	for _, tc := range []struct {
 		name       string