Skip to content

feat: basic egress tracking prometheus metrics #6

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
volmedo merged 9 commits into from
Oct 8, 2025
Merged

feat: basic egress tracking prometheus metrics #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
a06fffd
feat: add basic prometheus metrics
volmedo Oct 3, 2025
b07cc3c
metrics endpoint does bearer token auth
volmedo Oct 6, 2025
c331e15
use OTel SDK's prom exporter
volmedo Oct 6, 2025
7ce4282
ConsolidatedBytesPerNode is an integer
volmedo Oct 7, 2025
d823d74
go mod tidy
volmedo Oct 7, 2025
c4ae63d
remove references to Grafana
volmedo Oct 7, 2025
270a25e
Merge branch 'main' into vic/feat/basic-metrics
volmedo Oct 8, 2025
59993ff
use WithAttributeSet instead of WithAttribute
volmedo Oct 8, 2025
2121baa
metrics auth token is only an env var
volmedo Oct 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/deploy.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ jobs:
# aws-region: ${{ secrets.STAGING_AWS_REGION }}
# region: ${{ secrets.STAGING_AWS_REGION }}
# private-key: ${{ secrets.STAGING_PRIVATE_KEY }}
# metrics-auth-token: ${{ secrets.STAGING_METRICS_AUTH_TOKEN }}
# cloudflare-zone-id: ${{ secrets.STAGING_CLOUDFLARE_ZONE_ID }}
# cloudflare-api-token: ${{ secrets.STAGING_CLOUDFLARE_API_TOKEN }}

Expand All @@ -48,6 +49,7 @@ jobs:
aws-region: ${{ secrets.WARM_STAGING_AWS_REGION }}
region: ${{ secrets.WARM_STAGING_AWS_REGION }}
private-key: ${{ secrets.WARM_STAGING_PRIVATE_KEY }}
metrics-auth-token: ${{ secrets.WARM_STAGING_METRICS_AUTH_TOKEN }}
cloudflare-zone-id: ${{ secrets.WARM_STAGING_CLOUDFLARE_ZONE_ID }}
cloudflare-api-token: ${{ secrets.WARM_STAGING_CLOUDFLARE_API_TOKEN }}

Expand All @@ -64,5 +66,6 @@ jobs:
# aws-region: ${{ secrets.PROD_AWS_REGION }}
# region: ${{ secrets.PROD_AWS_REGION }}
# private-key: ${{ secrets.PROD_PRIVATE_KEY }}
# metrics-auth-token: ${{ secrets.PROD_METRICS_AUTH_TOKEN }}
# cloudflare-zone-id: ${{ secrets.PROD_CLOUDFLARE_ZONE_ID }}
# cloudflare-api-token: ${{ secrets.PROD_CLOUDFLARE_API_TOKEN }}
9 changes: 6 additions & 3 deletions .github/workflows/terraform.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,13 @@ on:
required: true
region:
required: true
metrics-auth-token:
required: true
cloudflare-zone-id:
required: true
cloudflare-api-token:
required: true

concurrency:
group: ${{ github.workflow }}-${{ inputs.workspace }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
Expand All @@ -46,9 +48,10 @@ env:
TF_VAR_private_key: ${{ secrets.private-key }}
TF_VAR_did: ${{ inputs.did }}
TF_VAR_app: etracker
TF_VAR_domain_base:
TF_VAR_domain_base:
TF_VAR_allowed_account_id: ${{ secrets.aws-account-id }}
TF_VAR_region: ${{ secrets.region }}
TF_VAR_metrics_auth_token: ${{ secrets.metrics-auth-token }}
TF_VAR_cloudflare_zone_id: ${{ secrets.cloudflare-zone-id }}
CLOUDFLARE_API_TOKEN: ${{ secrets.cloudflare-api-token }}
DEPLOY_ENV: ci
Expand All @@ -71,7 +74,7 @@ jobs:

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3

- uses: opentofu/setup-opentofu@v1

- name: Tofu Init
Expand Down
7 changes: 6 additions & 1 deletion .storoku.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,12 @@
"topics": null,
"queues": null,
"buckets": null,
"secrets": null,
"secrets": [
{
"name": "ETRACKER_METRICS_AUTH_TOKEN",
"variable": true
}
],
"tables": [
{
"name": "egress-records",
Expand Down
4 changes: 3 additions & 1 deletion cmd/etracker/start.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ func init() {
)
cobra.CheckErr(viper.BindPFlag("did", startCmd.Flags().Lookup("did")))

cobra.CheckErr(viper.BindEnv("metrics_auth_token"))

startCmd.Flags().String(
"egress-table-name",
"",
Expand Down Expand Up @@ -123,7 +125,7 @@ func startService(cmd *cobra.Command, args []string) error {
}

// Create server
server, err := server.New(id, svc)
server, err := server.New(id, svc, server.WithMetricsEndpoint(cfg.MetricsAuthToken))
if err != nil {
return fmt.Errorf("creating server: %w", err)
}
Expand Down
3 changes: 2 additions & 1 deletion deploy/.env.terraform.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,6 @@ TF_VAR_did= # did for your env
TF_VAR_private_key= # private_key or your env -- do not commit to repo!
TF_VAR_allowed_account_id=505595374361
TF_VAR_region=us-west-2
TF_VAR_metrics_auth_token= # enter a value for METRICS_AUTH_TOKEN secret
TF_VAR_cloudflare_zone_id=37783d6f032b78cd97ce37ab6fd42848
CLOUDFLARE_API_TOKEN= # enter a cloudflare api token
CLOUDFLARE_API_TOKEN= # enter a cloudflare api token
1 change: 1 addition & 0 deletions deploy/app/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ module "app" {
# enter secret values your app will use here -- these will be available
# as env vars in the container at runtime
secrets = {
"ETRACKER_METRICS_AUTH_TOKEN" = var.metrics_auth_token
}
# enter any sqs queues you want to create here
queues = []
Expand Down
5 changes: 5 additions & 0 deletions deploy/app/variables.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,3 +50,8 @@ variable "network" {
type = string
default = "hot"
}

variable "metrics_auth_token" {
description = "value for metrics_auth_token secret"
type = string
}
27 changes: 17 additions & 10 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,15 @@ require (
github.com/ipfs/go-cid v0.5.0
github.com/ipfs/go-log/v2 v2.7.0
github.com/ipld/go-ipld-prime v0.21.1-0.20240917223228-6148356a4c2e
github.com/prometheus/client_golang v1.23.2
github.com/spf13/cobra v1.2.1
github.com/spf13/viper v1.8.1
github.com/storacha/go-libstoracha v0.2.9
github.com/storacha/go-ucanto v0.6.5
go.opentelemetry.io/otel v1.38.0
go.opentelemetry.io/otel/exporters/prometheus v0.60.0
go.opentelemetry.io/otel/metric v1.38.0
go.opentelemetry.io/otel/sdk/metric v1.38.0
)

require (
Expand Down Expand Up @@ -46,6 +51,7 @@ require (
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect
github.com/hashicorp/golang-lru v1.0.2 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
Expand All @@ -66,7 +72,6 @@ require (
github.com/ipfs/go-verifcid v0.0.3 // indirect
github.com/ipld/go-car v0.6.2 // indirect
github.com/ipld/go-codec-dagpb v1.6.0 // indirect
github.com/klauspost/compress v1.18.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/magiconair/properties v1.8.10 // indirect
Expand All @@ -81,6 +86,7 @@ require (
github.com/multiformats/go-multicodec v0.9.1 // indirect
github.com/multiformats/go-multihash v0.2.3 // indirect
github.com/multiformats/go-varint v0.0.7 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/onsi/ginkgo/v2 v2.23.4 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pelletier/go-toml v1.9.3 // indirect
Expand All @@ -96,7 +102,8 @@ require (
github.com/pkg/errors v0.9.1 // indirect
github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.65.0 // indirect
github.com/prometheus/common v0.66.1 // indirect
github.com/prometheus/otlptranslator v0.0.2 // indirect
github.com/prometheus/procfs v0.17.0 // indirect
github.com/quic-go/qpack v0.5.1 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
Expand All @@ -108,20 +115,20 @@ require (
github.com/ucan-wg/go-ucan v0.0.0-20240916120445-37f52863156c // indirect
github.com/whyrusleeping/cbor-gen v0.3.1 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/otel v1.37.0 // indirect
go.opentelemetry.io/otel/metric v1.37.0 // indirect
go.opentelemetry.io/otel/trace v1.37.0 // indirect
go.opentelemetry.io/otel/sdk v1.38.0 // indirect
go.opentelemetry.io/otel/trace v1.38.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/mock v0.5.2 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.0 // indirect
golang.org/x/crypto v0.38.0 // indirect
go.yaml.in/yaml/v2 v2.4.2 // indirect
golang.org/x/crypto v0.41.0 // indirect
golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 // indirect
golang.org/x/net v0.40.0 // indirect
golang.org/x/sys v0.34.0 // indirect
golang.org/x/text v0.25.0 // indirect
golang.org/x/net v0.43.0 // indirect
golang.org/x/sys v0.35.0 // indirect
golang.org/x/text v0.28.0 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
google.golang.org/protobuf v1.36.6 // indirect
google.golang.org/protobuf v1.36.8 // indirect
gopkg.in/ini.v1 v1.62.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
lukechampine.com/blake3 v1.4.0 // indirect
Expand Down
Loading
Loading