From 5977ec889be5fbcea23f77af7560adad4a8e734f Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 26 Mar 2026 14:16:47 +0000 Subject: [PATCH 1/2] RM-251: user & guild exports Signed-off-by: Ben --- cmd/main.go | 66 ++++- go.mod | 38 +-- go.sum | 108 +++---- i18n/messages.go | 4 + internal/callback/callback.go | 89 +++++- internal/config/config.go | 24 +- internal/export/zip.go | 57 ++++ internal/gdprrelay/gdprrelay.go | 2 + internal/processor/processor.go | 493 +++++++++++++++++++++++++++++++- internal/utils/utils.go | 4 + 10 files changed, 775 insertions(+), 110 deletions(-) create mode 100644 internal/export/zip.go diff --git a/cmd/main.go b/cmd/main.go index d8a299f..760611e 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "fmt" "os" "os/signal" "syscall" @@ -16,7 +17,11 @@ import ( "github.com/TicketsBot-cloud/gdpr-worker/internal/heartbeat" "github.com/TicketsBot-cloud/gdpr-worker/internal/processor" "github.com/TicketsBot-cloud/gdpr-worker/internal/utils" + "github.com/TicketsBot-cloud/logarchiver/pkg/s3client" "github.com/go-redis/redis/v8" + "github.com/jackc/pgx/v4/pgxpool" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -61,14 +66,57 @@ func main() { return } - logger.Info("Initializing archiver client") + logger.Info("Initialising archiver client") archiver.Initialize( logger.With(), config.Conf.Archiver.Url, config.Conf.Archiver.AesKey, ) - proc := processor.New(logger.With()) + // Initialise S3 client for export operations + var s3Client *s3client.S3Client + if config.Conf.S3.Endpoint != "" { + logger.Info("Initialising S3 client") + minioClient, err := minio.New(config.Conf.S3.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(config.Conf.S3.AccessKey, config.Conf.S3.SecretKey, ""), + Secure: config.Conf.S3.Secure, + }) + if err != nil { + logger.Fatal("Failed to initialise S3 client", zap.Error(err)) + return + } + s3Client = s3client.NewS3Client(minioClient, config.Conf.S3.Bucket) + logger.Info("S3 client initialised") + } else { + logger.Warn("S3 not configured, export features will be unavailable") + } + + // Initialise cache database connection for user export operations + var cachePool *pgxpool.Pool + if config.Conf.CacheDatabase.Host != "" { + logger.Info("Connecting to cache database") + cacheUri := fmt.Sprintf("postgres://%s:%s@%s/%s?pool_max_conns=%d", + config.Conf.CacheDatabase.Username, + config.Conf.CacheDatabase.Password, + config.Conf.CacheDatabase.Host, + config.Conf.CacheDatabase.Database, + config.Conf.CacheDatabase.Threads, + ) + + var err error + cachePool, err = pgxpool.Connect(context.Background(), cacheUri) + if err != nil { + logger.Fatal("Failed to connect to cache database", zap.Error(err)) + return + } + logger.Info("Connected to cache database") + } else { + logger.Warn("Cache database not configured, user export cache data will be unavailable") + } + + aesKey := []byte(config.Conf.Archiver.AesKey) + + proc := processor.New(logger.With(), s3Client, aesKey, cachePool) callbackHandler := callback.New( logger.With(), @@ -142,12 +190,14 @@ func main() { } callbackData := callback.ResultData{ - TranscriptsDeleted: result.TranscriptsDeleted, - MessagesDeleted: result.MessagesDeleted, - Error: result.Error, - RequestType: req.Request.Type, - GuildIds: req.Request.GuildIds, - TicketIds: req.Request.TicketIds, + TranscriptsDeleted: result.TranscriptsDeleted, + MessagesDeleted: result.MessagesDeleted, + Error: result.Error, + RequestType: req.Request.Type, + GuildIds: req.Request.GuildIds, + TicketIds: req.Request.TicketIds, + ExportData: result.ExportData, + ExportFileName: result.ExportFileName, } callbackCtx, callbackCancel := context.WithTimeout(context.Background(), 30*time.Second) diff --git a/go.mod b/go.mod index 6dfd03e..634fd13 100644 --- a/go.mod +++ b/go.mod @@ -1,37 +1,39 @@ module github.com/TicketsBot-cloud/gdpr-worker -go 1.24.0 +go 1.25.0 //replace github.com/TicketsBot-cloud/database => ../database -//replace github.com/TicketsBot-cloud/gdl => ../gdl +replace github.com/TicketsBot-cloud/gdl => ../gdl //replace github.com/TicketsBot-cloud/archiverclient => ./archiverclient -//replace github.com/TicketsBot-cloud/logarchiver => ../logarchiver +replace github.com/TicketsBot-cloud/logarchiver => ../logarchiver require ( github.com/TicketsBot-cloud/archiverclient v0.0.0-20251015181023-f0b66a074704 - github.com/TicketsBot-cloud/database v0.0.0-20251018202538-7f9567e1aeab - github.com/TicketsBot-cloud/gdl v0.0.0-20251007163257-7e59b92d02dd + github.com/TicketsBot-cloud/common v0.0.0-20260210203202-54154661338e + github.com/TicketsBot-cloud/database v0.0.0-20260308193919-30a698fefa8b + github.com/TicketsBot-cloud/gdl v0.0.0-20260306134952-cccb0116fef6 github.com/TicketsBot-cloud/logarchiver v0.0.0-20250809082842-70aa389bcbdf github.com/caarlos0/env/v10 v10.0.0 github.com/go-redis/redis/v8 v8.11.5 github.com/jackc/pgx/v4 v4.18.3 github.com/joho/godotenv v1.5.1 - go.uber.org/zap v1.27.0 + github.com/minio/minio-go/v7 v7.0.99 + go.uber.org/zap v1.27.1 + golang.org/x/sync v0.20.0 ) require ( - github.com/TicketsBot-cloud/common v0.0.0-20250509064208-a2d357175463 // indirect github.com/TicketsBot/common v0.0.0-20241117150316-ff54c97b45c1 // indirect github.com/TicketsBot/ttlcache v1.6.1-0.20200405150101-acc18e37b261 // indirect + github.com/boltdb/bolt v1.3.1 // indirect github.com/caarlos0/env v3.5.0+incompatible // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/go-ini/ini v1.67.0 // indirect - github.com/goccy/go-json v0.10.5 // indirect github.com/google/uuid v1.6.0 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect github.com/jackc/pgconn v1.14.3 // indirect @@ -41,16 +43,16 @@ require ( github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/pgtype v1.14.4 // indirect github.com/jackc/pgx v3.6.2+incompatible // indirect - github.com/jackc/pgx/v5 v5.7.6 // indirect + github.com/jackc/pgx/v5 v5.9.1 // indirect github.com/jackc/puddle v1.3.0 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/juju/ratelimit v1.0.1 // indirect - github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/compress v1.18.5 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect github.com/minio/crc64nvme v1.1.1 // indirect github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/minio-go/v7 v7.0.95 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pasztorpisti/qs v0.0.0-20171216220353-8d6c33ee906c // indirect @@ -59,12 +61,12 @@ require ( github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/rs/xid v1.6.0 // indirect github.com/sirupsen/logrus v1.9.0 // indirect - github.com/tinylib/msgp v1.4.0 // indirect - go.uber.org/multierr v1.10.0 // indirect - golang.org/x/crypto v0.43.0 // indirect + github.com/tinylib/msgp v1.6.3 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/crypto v0.49.0 // indirect golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect - golang.org/x/net v0.46.0 // indirect - golang.org/x/sync v0.17.0 // indirect - golang.org/x/sys v0.37.0 // indirect - golang.org/x/text v0.30.0 // indirect + golang.org/x/net v0.52.0 // indirect + golang.org/x/sys v0.42.0 // indirect + golang.org/x/text v0.35.0 // indirect ) diff --git a/go.sum b/go.sum index f76ed9d..eaacc9f 100644 --- a/go.sum +++ b/go.sum @@ -1,32 +1,18 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/ReneKroon/ttlcache v1.6.0/go.mod h1:DG6nbhXKUQhrExfwwLuZUdH7UnRDDRA1IW+nBuCssvs= -github.com/TicketsBot-cloud/archiverclient v0.0.0-20250514201416-cf23f65eb3fc h1:2EFgb6gQMedsO7YkqUPH1zY8+L6i3sGuV/YAv0gPqhQ= -github.com/TicketsBot-cloud/archiverclient v0.0.0-20250514201416-cf23f65eb3fc/go.mod h1:Mux1bEPpOHwRw1wo6Fa6qJLJH9Erk9qv1yAIfLi1Wmw= github.com/TicketsBot-cloud/archiverclient v0.0.0-20251015181023-f0b66a074704 h1:liLfvCrzoJ89DXFHzsd1iK3cyP8s4i0CnZPRFEj53zg= github.com/TicketsBot-cloud/archiverclient v0.0.0-20251015181023-f0b66a074704/go.mod h1:Mux1bEPpOHwRw1wo6Fa6qJLJH9Erk9qv1yAIfLi1Wmw= -github.com/TicketsBot-cloud/common v0.0.0-20250208132851-d5083bb04d98 h1:HwtXrqSv6y5E8mTnkOtISTaQr6dsK/lfUC8tUA6SzSU= -github.com/TicketsBot-cloud/common v0.0.0-20250208132851-d5083bb04d98/go.mod h1:iiZhl7w5DeTEGzAeq/hzani8+ILQz60g4JFkOy2vkuM= -github.com/TicketsBot-cloud/common v0.0.0-20250509064208-a2d357175463 h1:ZO2kw9lpsy4umoWFmAt9WKVFb3WvU5oSif2Yw4K7Lj4= -github.com/TicketsBot-cloud/common v0.0.0-20250509064208-a2d357175463/go.mod h1:PL5j/omFvU0NeyTKCESmOF+3GscaEuM0aqmI4yFcCFY= -github.com/TicketsBot-cloud/database v0.0.0-20250918212912-4cc263bc1b41 h1:X+V7IZtmDbE9h5cGSKnhCuisBFZ14Suzsdjnk8UUN3A= -github.com/TicketsBot-cloud/database v0.0.0-20250918212912-4cc263bc1b41/go.mod h1:LPDEn9e5wccH7rq/pUlVcL3UhyLnnwdM2dhj0tp/ljo= -github.com/TicketsBot-cloud/database v0.0.0-20251018202538-7f9567e1aeab h1:J8loePxYfe0oSqwaW0Ub51ku2+/fPiekQxHBYwX5S5A= -github.com/TicketsBot-cloud/database v0.0.0-20251018202538-7f9567e1aeab/go.mod h1:LPDEn9e5wccH7rq/pUlVcL3UhyLnnwdM2dhj0tp/ljo= -github.com/TicketsBot-cloud/gdl v0.0.0-20250917180424-569348f7a55b h1:k4kIIhpD3m0tx8Sz8gefZH8DBbuAwbeJJdV0iRr/x5Y= -github.com/TicketsBot-cloud/gdl v0.0.0-20250917180424-569348f7a55b/go.mod h1:CdwBR2egPtxUXjD2CgC9ZwfuB8dz9HPePM8nuG6dt7Y= -github.com/TicketsBot-cloud/gdl v0.0.0-20251007163257-7e59b92d02dd h1:C4GzdEYarK2V81oAarvRW2RA2wzRGRydCAzBKtb/lGU= -github.com/TicketsBot-cloud/gdl v0.0.0-20251007163257-7e59b92d02dd/go.mod h1:CdwBR2egPtxUXjD2CgC9ZwfuB8dz9HPePM8nuG6dt7Y= -github.com/TicketsBot-cloud/logarchiver v0.0.0-20250514201320-d5141071a6eb h1:BapforRlvTfWP8MX8DTsxVM40oDgQorJVo/cnNGTaKU= -github.com/TicketsBot-cloud/logarchiver v0.0.0-20250514201320-d5141071a6eb/go.mod h1:pZqkzPNNTqnwKZvCT8kCaTHxrG7HJbxZV83S0p7mmzM= -github.com/TicketsBot-cloud/logarchiver v0.0.0-20250809082842-70aa389bcbdf h1:InP2ht90XOkLJKLjVrsCYPJunxlLkdsorikgNRk35bQ= -github.com/TicketsBot-cloud/logarchiver v0.0.0-20250809082842-70aa389bcbdf/go.mod h1:pZqkzPNNTqnwKZvCT8kCaTHxrG7HJbxZV83S0p7mmzM= -github.com/TicketsBot/common v0.0.0-20241104184641-e39c64bdcf3e h1:cYfBjPX/FhD/MCViBI2Wz2YlC2esLiTbDE65Qku2WVg= -github.com/TicketsBot/common v0.0.0-20241104184641-e39c64bdcf3e/go.mod h1:N7zwetwx8B3RK/ZajWwMroJSyv2ZJ+bIOZWv/z8DhaM= +github.com/TicketsBot-cloud/common v0.0.0-20260210203202-54154661338e h1:nFKV7yEm8MWbCP7dtsJ88+agcxDUD0YKIotVHMVvytw= +github.com/TicketsBot-cloud/common v0.0.0-20260210203202-54154661338e/go.mod h1:tGrTHFz09OM3eDWF+62hIi9ELpT4igCFi868FKSvKBg= +github.com/TicketsBot-cloud/database v0.0.0-20260308193919-30a698fefa8b h1:bHkfJWo8T/9TiHuYHxaOz8GAILIiKPugC1k3CzdOq/A= +github.com/TicketsBot-cloud/database v0.0.0-20260308193919-30a698fefa8b/go.mod h1:HQXAgmNSm7/FmBYwcsa6qpZqMrDhbLoEl+AyqFQ+RwY= github.com/TicketsBot/common v0.0.0-20241117150316-ff54c97b45c1 h1:FqC1KGOsmB+ikvbmDkyNQU6bGUWyfYq8Ip9r4KxTveY= github.com/TicketsBot/common v0.0.0-20241117150316-ff54c97b45c1/go.mod h1:N7zwetwx8B3RK/ZajWwMroJSyv2ZJ+bIOZWv/z8DhaM= github.com/TicketsBot/ttlcache v1.6.1-0.20200405150101-acc18e37b261 h1:NHD5GB6cjlkpZFjC76Yli2S63/J2nhr8MuE6KlYJpQM= github.com/TicketsBot/ttlcache v1.6.1-0.20200405150101-acc18e37b261/go.mod h1:2zPxDAN2TAPpxUPjxszjs3QFKreKrQh5al/R3cMXmYk= +github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/caarlos0/env v3.5.0+incompatible h1:Yy0UN8o9Wtr/jGHZDpCBLpNrzcFLLM2yixi/rBrKyJs= github.com/caarlos0/env v3.5.0+incompatible/go.mod h1:tdCsowwCzMLdkqRYDlHpZCp2UooDD3MspDBjZ2AD02Y= github.com/caarlos0/env/v10 v10.0.0 h1:yIHUBZGsyqCnpTkbjk8asUlx6RFhhEs+h7TOBdgdzXA= @@ -54,17 +40,12 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= -github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= -github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= @@ -85,7 +66,6 @@ github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5W github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= @@ -96,7 +76,6 @@ github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwX github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= @@ -104,7 +83,6 @@ github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01C github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= -github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw= github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgtype v1.14.4 h1:fKuNiCumbKTAIxQwXfB/nsrnkEI6bPJrrSiMKgbJ2j8= github.com/jackc/pgtype v1.14.4/go.mod h1:aKeozOde08iifGosdJpz9MBZonJOUJxqNpPBcMJTlVA= @@ -117,17 +95,13 @@ github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgS github.com/jackc/pgx/v4 v4.18.2/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= github.com/jackc/pgx/v4 v4.18.3 h1:dE2/TrEsGX3RBprb3qryqSV9Y60iZN1C6i8IrmW9/BA= github.com/jackc/pgx/v4 v4.18.3/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= -github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= -github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= -github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= -github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= +github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.3.0 h1:eHK/5clGOatcjX3oWGBO/MpxpbHzSwud5EWTSCI+MX0= github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= -github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= @@ -137,15 +111,13 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/juju/ratelimit v1.0.1 h1:+7AIFJVQ0EQgq/K9+0Krm7m530Du7tIz0METWzN0RgY= github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= +github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= -github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -170,10 +142,8 @@ github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.73 h1:qr2vi96Qm7kZ4v7LLebjte+MQh621fFWnv93p12htEo= -github.com/minio/minio-go/v7 v7.0.73/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8= -github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU= -github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo= +github.com/minio/minio-go/v7 v7.0.99 h1:2vH/byrwUkIpFQFOilvTfaUpvAX3fEFhEzO+DR3DlCE= +github.com/minio/minio-go/v7 v7.0.99/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -198,8 +168,6 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= -github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= -github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= @@ -225,10 +193,10 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/tinylib/msgp v1.4.0 h1:SYOeDRiydzOw9kSiwdYp9UcBgPFtLU2WDHaJXyHruf8= -github.com/tinylib/msgp v1.4.0/go.mod h1:cvjFkb4RiC8qSBOPMGPSzSAx47nAsfhLVTCZZNuHv5o= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tinylib/msgp v1.6.3 h1:bCSxiTz386UTgyT1i0MSCvdbWjVW+8sG3PjkGsZQt4s= +github.com/tinylib/msgp v1.6.3/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -241,14 +209,16 @@ go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= -go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= -go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= +go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -261,10 +231,8 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ= -golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= -golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= -golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= -golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo= golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -281,17 +249,13 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= -golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= -golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= +golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= +golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= -golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -310,10 +274,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= -golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -329,10 +291,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= -golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= -golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= -golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= diff --git a/i18n/messages.go b/i18n/messages.go index 7a047cc..008ab8e 100644 --- a/i18n/messages.go +++ b/i18n/messages.go @@ -14,4 +14,8 @@ var ( GdprFollowupError MessageId = "gdpr.followup.error" GdprFollowupNoData MessageId = "gdpr.followup.no_data" GdprFollowupSuccess MessageId = "gdpr.followup.success" + GdprCompletedExportGuild MessageId = "gdpr.completed.export_guild" + GdprCompletedExportGuildMulti MessageId = "gdpr.completed.export_guild_multi" + GdprCompletedExportUser MessageId = "gdpr.completed.export_user" + GdprExportDmMessage MessageId = "gdpr.export.dm_message" ) diff --git a/internal/callback/callback.go b/internal/callback/callback.go index 2daab54..be4b84f 100644 --- a/internal/callback/callback.go +++ b/internal/callback/callback.go @@ -1,6 +1,7 @@ package callback import ( + "bytes" "context" "fmt" "strings" @@ -9,10 +10,11 @@ import ( "github.com/TicketsBot-cloud/gdl/objects/interaction/component" "github.com/TicketsBot-cloud/gdl/rest" "github.com/TicketsBot-cloud/gdl/rest/ratelimit" + "github.com/TicketsBot-cloud/gdl/rest/request" + "github.com/TicketsBot-cloud/gdpr-worker/i18n" "github.com/TicketsBot-cloud/gdpr-worker/internal/config" "github.com/TicketsBot-cloud/gdpr-worker/internal/gdprrelay" "github.com/TicketsBot-cloud/gdpr-worker/internal/utils" - "github.com/TicketsBot-cloud/gdpr-worker/i18n" "go.uber.org/zap" ) @@ -24,6 +26,8 @@ type ResultData struct { RequestType gdprrelay.RequestType // Type of GDPR request that was processed GuildIds []uint64 // Guild IDs affected by this request TicketIds []int // Ticket IDs affected by this request + ExportData []byte // ZIP file bytes for export requests + ExportFileName string // Suggested filename for the export archive } type Callback struct { @@ -69,6 +73,19 @@ func (c *Callback) SendCompletion(ctx context.Context, request gdprrelay.GDPRReq return err } + // For export types, send the ZIP file via DM + isExportType := request.Type == gdprrelay.RequestTypeExportGuild || request.Type == gdprrelay.RequestTypeExportUser + if isExportType && result.Error == nil && len(result.ExportData) > 0 { + if err := c.sendExportViaDM(ctx, request, locale, result); err != nil { + c.logger.Error("Failed to send export via DM", + zap.Error(err), + zap.String("scrambled_user_id", scrambledUserId), + ) + return err + } + return nil + } + if err := c.sendEphemeralFollowup(ctx, request, locale, result); err != nil { if c.isTokenExpired(err) { return nil @@ -139,6 +156,21 @@ func (c *Callback) buildResultMessage(locale *i18n.Locale, result ResultData, gu } else { content = i18n.GetMessage(locale, i18n.GdprCompletedSpecificMessages, "Unknown", result.MessagesDeleted) } + + case gdprrelay.RequestTypeExportGuild: + if len(result.GuildIds) == 1 { + guildDisplay := utils.FormatGuildDisplay(result.GuildIds[0], guildNames) + content = i18n.GetMessage(locale, i18n.GdprCompletedExportGuild, guildDisplay) + } else { + guildDisplays := make([]string, len(result.GuildIds)) + for idx, guildId := range result.GuildIds { + guildDisplays[idx] = utils.FormatGuildDisplay(guildId, guildNames) + } + content = i18n.GetMessage(locale, i18n.GdprCompletedExportGuildMulti, strings.Join(guildDisplays, "\n* ")) + } + + case gdprrelay.RequestTypeExportUser: + content = i18n.GetMessage(locale, i18n.GdprCompletedExportUser) } if result.Error != nil { @@ -233,3 +265,58 @@ func (c *Callback) sendCompletionViaDM(ctx context.Context, request gdprrelay.GD return nil } + +// sendExportViaDM sends the data export ZIP file to the user via a direct message. +func (c *Callback) sendExportViaDM(ctx context.Context, req gdprrelay.GDPRRequest, locale *i18n.Locale, result ResultData) error { + scrambledUserId := utils.ScrambleUserId(req.UserId) + + if config.Conf.Discord.Token == "" { + c.logger.Error("Discord token not configured, cannot send export DM", + zap.String("scrambled_user_id", scrambledUserId), + ) + return fmt.Errorf("discord token not configured") + } + + dmChannel, err := rest.CreateDM(ctx, config.Conf.Discord.Token, c.rateLimiter, req.UserId) + if err != nil { + c.logger.Error("Failed to create DM channel for export", + zap.Error(err), + zap.String("scrambled_user_id", scrambledUserId), + ) + return fmt.Errorf("failed to create DM channel: %w", err) + } + + content := i18n.GetMessage(locale, i18n.GdprExportDmMessage) + + data := rest.CreateMessageData{ + Content: content, + Attachments: []request.Attachment{ + { + Id: 0, + FileName: result.ExportFileName, + File: request.File{ + ContentType: "application/zip", + Reader: bytes.NewReader(result.ExportData), + }, + }, + }, + } + + _, err = rest.CreateMessage(ctx, config.Conf.Discord.Token, c.rateLimiter, dmChannel.Id, data) + if err != nil { + c.logger.Error("Failed to send export DM with attachment", + zap.Error(err), + zap.String("scrambled_user_id", scrambledUserId), + zap.Uint64("channel_id", dmChannel.Id), + zap.String("export_file", result.ExportFileName), + ) + return fmt.Errorf("failed to send export DM: %w", err) + } + + c.logger.Info("Export sent via DM", + zap.String("scrambled_user_id", scrambledUserId), + zap.String("export_file", result.ExportFileName), + ) + + return nil +} diff --git a/internal/config/config.go b/internal/config/config.go index 06ff6c1..da1d74f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -6,10 +6,10 @@ import ( ) type Config struct { - JsonLogs bool `env:"JSON_LOGS" envDefault:"false"` - LogLevel zapcore.Level `env:"LOG_LEVEL" envDefault:"info"` - MaxConcurrency int `env:"MAX_CONCURRENCY" envDefault:"1"` - MaxRetries int `env:"MAX_RETRIES" envDefault:"3"` + JsonLogs bool `env:"JSON_LOGS" envDefault:"false"` + LogLevel zapcore.Level `env:"LOG_LEVEL" envDefault:"info"` + MaxConcurrency int `env:"MAX_CONCURRENCY" envDefault:"1"` + MaxRetries int `env:"MAX_RETRIES" envDefault:"3"` Database struct { Host string `env:"HOST"` @@ -34,6 +34,22 @@ type Config struct { ProxyUrl string `env:"PROXY_URL"` Token string `env:"TOKEN"` } `envPrefix:"DISCORD_"` + + S3 struct { + Endpoint string `env:"ENDPOINT"` + AccessKey string `env:"ACCESS_KEY"` + SecretKey string `env:"SECRET_KEY"` + Bucket string `env:"BUCKET"` + Secure bool `env:"SECURE" envDefault:"true"` + } `envPrefix:"S3_"` + + CacheDatabase struct { + Host string `env:"HOST"` + Database string `env:"NAME"` + Username string `env:"USER"` + Password string `env:"PASSWORD"` + Threads int `env:"THREADS"` + } `envPrefix:"CACHE_"` } var Conf Config diff --git a/internal/export/zip.go b/internal/export/zip.go new file mode 100644 index 0000000..10f584b --- /dev/null +++ b/internal/export/zip.go @@ -0,0 +1,57 @@ +package export + +import ( + "archive/zip" + "bytes" + "fmt" +) + +// ZipBuilder provides an in-memory ZIP archive builder for constructing +// data export archives without writing to disk. +type ZipBuilder struct { + buf *bytes.Buffer + writer *zip.Writer + closed bool +} + +// NewZipBuilder creates a new ZipBuilder that writes to an in-memory buffer. +func NewZipBuilder() *ZipBuilder { + buf := new(bytes.Buffer) + return &ZipBuilder{ + buf: buf, + writer: zip.NewWriter(buf), + } +} + +// AddFile adds a file with the given name and data to the ZIP archive. +func (z *ZipBuilder) AddFile(name string, data []byte) error { + if z.closed { + return fmt.Errorf("zip builder is already closed") + } + + f, err := z.writer.Create(name) + if err != nil { + return fmt.Errorf("failed to create zip entry %q: %w", name, err) + } + + if _, err := f.Write(data); err != nil { + return fmt.Errorf("failed to write zip entry %q: %w", name, err) + } + + return nil +} + +// Close finalises the ZIP archive and returns the resulting bytes. +func (z *ZipBuilder) Close() ([]byte, error) { + if z.closed { + return nil, fmt.Errorf("zip builder is already closed") + } + + z.closed = true + + if err := z.writer.Close(); err != nil { + return nil, fmt.Errorf("failed to close zip writer: %w", err) + } + + return z.buf.Bytes(), nil +} diff --git a/internal/gdprrelay/gdprrelay.go b/internal/gdprrelay/gdprrelay.go index a29be0f..89809f8 100644 --- a/internal/gdprrelay/gdprrelay.go +++ b/internal/gdprrelay/gdprrelay.go @@ -20,6 +20,8 @@ const ( RequestTypeSpecificTranscripts // Delete specific transcript archives by ticket IDs RequestTypeAllMessages // Delete all ticket messages for specified guilds RequestTypeSpecificMessages // Delete specific ticket messages by ticket IDs + RequestTypeExportGuild // Export all transcript data for specified guilds + RequestTypeExportUser // Export all personal data for the requesting user ) // GDPRRequest represents a user's request to delete their data under GDPR regulations diff --git a/internal/processor/processor.go b/internal/processor/processor.go index 8b7447e..62798b5 100644 --- a/internal/processor/processor.go +++ b/internal/processor/processor.go @@ -3,40 +3,61 @@ package processor import ( "context" "encoding/json" + "errors" "fmt" + "strconv" "strings" + "sync" "github.com/TicketsBot-cloud/archiverclient" + "github.com/TicketsBot-cloud/common/encryption" + "github.com/TicketsBot-cloud/gdl/cache" + "github.com/TicketsBot-cloud/gdl/objects/channel/message" "github.com/TicketsBot-cloud/gdl/rest" "github.com/TicketsBot-cloud/gdl/rest/ratelimit" "github.com/TicketsBot-cloud/gdpr-worker/internal/archiver" "github.com/TicketsBot-cloud/gdpr-worker/internal/config" "github.com/TicketsBot-cloud/gdpr-worker/internal/database" + "github.com/TicketsBot-cloud/gdpr-worker/internal/export" "github.com/TicketsBot-cloud/gdpr-worker/internal/gdprrelay" "github.com/TicketsBot-cloud/gdpr-worker/internal/utils" + "github.com/TicketsBot-cloud/logarchiver/pkg/model" + v1 "github.com/TicketsBot-cloud/logarchiver/pkg/model/v1" v2 "github.com/TicketsBot-cloud/logarchiver/pkg/model/v2" + "github.com/TicketsBot-cloud/logarchiver/pkg/export/user" + "github.com/TicketsBot-cloud/logarchiver/pkg/s3client" + "github.com/jackc/pgx/v4/pgxpool" "go.uber.org/zap" + "golang.org/x/sync/errgroup" ) -// Processor handles the execution of GDPR data deletion requests +// Processor handles the execution of GDPR data deletion and export requests type Processor struct { logger *zap.Logger rateLimiter *ratelimit.Ratelimiter + s3Client *s3client.S3Client + aesKey []byte + cachePool *pgxpool.Pool } -func New(logger *zap.Logger) *Processor { +func New(logger *zap.Logger, s3Client *s3client.S3Client, aesKey []byte, cachePool *pgxpool.Pool) *Processor { store := ratelimit.NewMemoryStore() return &Processor{ logger: logger, rateLimiter: ratelimit.NewRateLimiter(store, 0), + s3Client: s3Client, + aesKey: aesKey, + cachePool: cachePool, } } // ProcessResult contains the outcome of processing a GDPR request type ProcessResult struct { - TranscriptsDeleted int // Number of transcript archives deleted from archiver - MessagesDeleted int // Number of ticket messages deleted from database - Error error // Error if the processing failed, nil on success + TranscriptsDeleted int // Number of transcript archives deleted from archiver + MessagesDeleted int // Number of ticket messages deleted from database + Error error // Error if the processing failed, nil on success + ExportData []byte // ZIP file bytes for export requests + ExportFileName string // Suggested filename for the export archive } func (p *Processor) Process(ctx context.Context, request gdprrelay.GDPRRequest) ProcessResult { @@ -49,6 +70,10 @@ func (p *Processor) Process(ctx context.Context, request gdprrelay.GDPRRequest) return p.processAllMessages(ctx, request) case gdprrelay.RequestTypeSpecificMessages: return p.processSpecificMessages(ctx, request) + case gdprrelay.RequestTypeExportGuild: + return p.processExportGuild(ctx, request) + case gdprrelay.RequestTypeExportUser: + return p.processExportUser(ctx, request) default: return ProcessResult{Error: fmt.Errorf("unknown GDPR request type: %d", request.Type)} } @@ -559,3 +584,461 @@ func (p *Processor) storeTranscript(ctx context.Context, guildId uint64, ticketI return archiver.Client.ImportTranscript(ctx, guildId, ticketId, data) } + +// processExportGuild exports all transcript data for the specified guilds into a ZIP archive. +func (p *Processor) processExportGuild(ctx context.Context, request gdprrelay.GDPRRequest) ProcessResult { + if len(request.GuildIds) == 0 { + return ProcessResult{Error: fmt.Errorf("no server ID provided")} + } + + if p.s3Client == nil { + return ProcessResult{Error: fmt.Errorf("S3 client not configured, cannot process export")} + } + + scrambledUserId := utils.ScrambleUserId(request.UserId) + + if err := p.verifyAllGuildsOwnership(ctx, request.GuildIds, request.UserId); err != nil { + p.logger.Error("Guild ownership verification failed for export", + zap.String("scrambled_user_id", scrambledUserId), + zap.Error(err), + ) + return ProcessResult{Error: err} + } + + zipBuilder := export.NewZipBuilder() + + for _, guildId := range request.GuildIds { + keys, err := p.s3Client.GetAllKeysForGuild(ctx, guildId) + if err != nil { + p.logger.Error("Failed to list S3 keys for guild export", + zap.Uint64("guild_id", guildId), + zap.String("scrambled_user_id", scrambledUserId), + zap.Error(err), + ) + continue + } + + if len(keys) == 0 { + p.logger.Info("No transcripts found for guild export", + zap.Uint64("guild_id", guildId), + zap.String("scrambled_user_id", scrambledUserId), + ) + continue + } + + // Use errgroup with bounded concurrency for parallel downloads + type transcriptEntry struct { + ticketId int + data []byte + } + + var mu sync.Mutex + var entries []transcriptEntry + + g, gCtx := errgroup.WithContext(ctx) + g.SetLimit(15) + + for _, key := range keys { + key := key + g.Go(func() error { + // Extract ticket ID from key format: {guildId}/{ticketId} + parts := strings.SplitN(key, "/", 2) + if len(parts) != 2 { + p.logger.Warn("Skipping S3 key with unexpected format", + zap.String("key", key), + zap.Uint64("guild_id", guildId), + ) + return nil + } + + ticketId, err := strconv.Atoi(parts[1]) + if err != nil { + p.logger.Warn("Skipping S3 key with non-numeric ticket ID", + zap.String("key", key), + zap.Uint64("guild_id", guildId), + zap.Error(err), + ) + return nil + } + + rawData, err := p.s3Client.GetTicket(gCtx, guildId, ticketId) + if err != nil { + if errors.Is(err, s3client.ErrTicketNotFound) { + p.logger.Debug("Transcript not found in S3, skipping", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + ) + return nil + } + p.logger.Warn("Failed to download transcript from S3", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Error(err), + ) + return nil + } + + decrypted, err := encryption.Decrypt(p.aesKey, rawData) + if err != nil { + p.logger.Warn("Failed to decrypt transcript, skipping", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Error(err), + ) + return nil + } + + var transcript v2.Transcript + version := model.GetVersion(decrypted) + switch version { + case model.V1: + var messages []message.Message + if err := json.Unmarshal(decrypted, &messages); err != nil { + p.logger.Warn("Failed to unmarshal V1 transcript, skipping", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Error(err), + ) + return nil + } + transcript = v1.ConvertToV2(messages) + case model.V2: + if err := json.Unmarshal(decrypted, &transcript); err != nil { + p.logger.Warn("Failed to unmarshal V2 transcript, skipping", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Error(err), + ) + return nil + } + default: + p.logger.Warn("Unknown transcript version, skipping", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Int("version", version.Int()), + ) + return nil + } + + prettyData, err := json.MarshalIndent(transcript, "", " ") + if err != nil { + p.logger.Warn("Failed to marshal transcript for export", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Error(err), + ) + return nil + } + + mu.Lock() + entries = append(entries, transcriptEntry{ticketId: ticketId, data: prettyData}) + mu.Unlock() + + return nil + }) + } + + if err := g.Wait(); err != nil { + p.logger.Error("Error during parallel transcript download", + zap.Uint64("guild_id", guildId), + zap.Error(err), + ) + } + + // Add all collected entries to the ZIP + for _, entry := range entries { + var fileName string + if len(request.GuildIds) > 1 { + fileName = fmt.Sprintf("%d/%d.json", guildId, entry.ticketId) + } else { + fileName = fmt.Sprintf("%d.json", entry.ticketId) + } + + if err := zipBuilder.AddFile(fileName, entry.data); err != nil { + p.logger.Error("Failed to add transcript to ZIP", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", entry.ticketId), + zap.Error(err), + ) + } + } + } + + zipData, err := zipBuilder.Close() + if err != nil { + return ProcessResult{Error: fmt.Errorf("failed to finalise export ZIP: %w", err)} + } + + var exportFileName string + if len(request.GuildIds) == 1 { + exportFileName = fmt.Sprintf("guild_%d.zip", request.GuildIds[0]) + } else { + exportFileName = "guild_export.zip" + } + + p.logger.Info("Guild export completed", + zap.String("scrambled_user_id", scrambledUserId), + zap.String("export_file", exportFileName), + zap.Int("zip_size_bytes", len(zipData)), + ) + + return ProcessResult{ + ExportData: zipData, + ExportFileName: exportFileName, + } +} + +// processExportUser exports all personal data for the requesting user into a ZIP archive. +func (p *Processor) processExportUser(ctx context.Context, request gdprrelay.GDPRRequest) ProcessResult { + scrambledUserId := utils.ScrambleUserId(request.UserId) + + zipBuilder := export.NewZipBuilder() + + // Export database data + userData, err := user.GetUserData(database.Client, request.UserId) + if err != nil { + p.logger.Error("Failed to get user database data for export", + zap.String("scrambled_user_id", scrambledUserId), + zap.Error(err), + ) + return ProcessResult{Error: fmt.Errorf("failed to retrieve database data: %w", err)} + } + + dbJson, err := json.MarshalIndent(userData, "", " ") + if err != nil { + return ProcessResult{Error: fmt.Errorf("failed to serialise database data: %w", err)} + } + + if err := zipBuilder.AddFile("database.json", dbJson); err != nil { + return ProcessResult{Error: fmt.Errorf("failed to add database.json to export: %w", err)} + } + + // Export cache data if cache pool is available + if p.cachePool != nil { + pgCache := cache.NewPgCache(p.cachePool, cache.CacheOptions{ + Users: true, + Members: true, + }) + + cacheData, err := user.GetCacheData(&pgCache, request.UserId) + if err != nil { + p.logger.Warn("Failed to get user cache data for export, continuing without it", + zap.String("scrambled_user_id", scrambledUserId), + zap.Error(err), + ) + } else { + cacheJson, err := json.MarshalIndent(cacheData, "", " ") + if err != nil { + p.logger.Warn("Failed to serialise cache data", + zap.String("scrambled_user_id", scrambledUserId), + zap.Error(err), + ) + } else { + if err := zipBuilder.AddFile("cache.json", cacheJson); err != nil { + p.logger.Warn("Failed to add cache.json to export", + zap.String("scrambled_user_id", scrambledUserId), + zap.Error(err), + ) + } + } + } + } + + // Collect all ticket IDs where the user participated or is the creator + transcriptIds := make(map[uint64][]int) + + // Query tickets where user is a participant + { + query := `SELECT participant.guild_id, participant.ticket_id FROM participant INNER JOIN tickets ON participant.guild_id = tickets.guild_id AND tickets.id = participant.ticket_id WHERE participant.user_id = $1 AND tickets.has_transcript='t' AND tickets.open='f';` + rows, err := database.Client.Participants.Query(ctx, query, request.UserId) + if err != nil { + p.logger.Error("Failed to query participant tickets for user export", + zap.String("scrambled_user_id", scrambledUserId), + zap.Error(err), + ) + return ProcessResult{Error: fmt.Errorf("failed to query participant tickets: %w", err)} + } + + for rows.Next() { + var guildId uint64 + var ticketId int + if err := rows.Scan(&guildId, &ticketId); err != nil { + p.logger.Warn("Failed to scan participant ticket row", + zap.String("scrambled_user_id", scrambledUserId), + zap.Error(err), + ) + continue + } + transcriptIds[guildId] = append(transcriptIds[guildId], ticketId) + } + rows.Close() + } + + // Query tickets where user is the creator + { + query := `SELECT guild_id, id FROM tickets WHERE user_id = $1 AND has_transcript='t' AND open='f';` + rows, err := database.Client.Tickets.Query(ctx, query, request.UserId) + if err != nil { + p.logger.Error("Failed to query user-created tickets for export", + zap.String("scrambled_user_id", scrambledUserId), + zap.Error(err), + ) + return ProcessResult{Error: fmt.Errorf("failed to query user tickets: %w", err)} + } + + for rows.Next() { + var guildId uint64 + var ticketId int + if err := rows.Scan(&guildId, &ticketId); err != nil { + p.logger.Warn("Failed to scan user ticket row", + zap.String("scrambled_user_id", scrambledUserId), + zap.Error(err), + ) + continue + } + transcriptIds[guildId] = append(transcriptIds[guildId], ticketId) + } + rows.Close() + } + + // Deduplicate ticket IDs per guild + for guildId, ticketIds := range transcriptIds { + seen := make(map[int]struct{}) + deduped := make([]int, 0, len(ticketIds)) + for _, id := range ticketIds { + if _, exists := seen[id]; !exists { + seen[id] = struct{}{} + deduped = append(deduped, id) + } + } + transcriptIds[guildId] = deduped + } + + // Download, decrypt, filter and add transcripts to ZIP + if p.s3Client != nil { + for guildId, ticketIds := range transcriptIds { + for _, ticketId := range ticketIds { + rawData, err := p.s3Client.GetTicket(ctx, guildId, ticketId) + if err != nil { + if errors.Is(err, s3client.ErrTicketNotFound) { + p.logger.Debug("Transcript not found for user export, skipping", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + ) + continue + } + p.logger.Warn("Failed to download transcript for user export", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Error(err), + ) + continue + } + + decrypted, err := encryption.Decrypt(p.aesKey, rawData) + if err != nil { + p.logger.Warn("Failed to decrypt transcript for user export, skipping", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Error(err), + ) + continue + } + + var transcript v2.Transcript + version := model.GetVersion(decrypted) + switch version { + case model.V1: + var messages []message.Message + if err := json.Unmarshal(decrypted, &messages); err != nil { + p.logger.Warn("Failed to unmarshal V1 transcript for user export, skipping", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Error(err), + ) + continue + } + transcript = v1.ConvertToV2(messages) + case model.V2: + if err := json.Unmarshal(decrypted, &transcript); err != nil { + p.logger.Warn("Failed to unmarshal V2 transcript for user export, skipping", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Error(err), + ) + continue + } + default: + p.logger.Warn("Unknown transcript version for user export, skipping", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Int("version", version.Int()), + ) + continue + } + + // Filter to only the user's data + transcript.Entities.Channels = nil + transcript.Entities.Roles = nil + + userEntity, ok := transcript.Entities.Users[request.UserId] + if !ok { + transcript.Entities.Users = nil + } else { + transcript.Entities.Users = map[uint64]v2.User{ + userEntity.Id: userEntity, + } + } + + var filteredMessages []v2.Message + for _, msg := range transcript.Messages { + if msg.AuthorId == request.UserId { + filteredMessages = append(filteredMessages, msg) + } + } + transcript.Messages = filteredMessages + + prettyData, err := json.MarshalIndent(transcript, "", " ") + if err != nil { + p.logger.Warn("Failed to marshal filtered transcript for user export", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Error(err), + ) + continue + } + + fileName := fmt.Sprintf("transcripts/%d-%d.json", guildId, ticketId) + if err := zipBuilder.AddFile(fileName, prettyData); err != nil { + p.logger.Error("Failed to add transcript to user export ZIP", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Error(err), + ) + } + } + } + } else { + p.logger.Warn("S3 client not configured, skipping transcript export for user", + zap.String("scrambled_user_id", scrambledUserId), + ) + } + + zipData, err := zipBuilder.Close() + if err != nil { + return ProcessResult{Error: fmt.Errorf("failed to finalise user export ZIP: %w", err)} + } + + exportFileName := fmt.Sprintf("user_%d.zip", request.UserId) + + p.logger.Info("User export completed", + zap.String("scrambled_user_id", scrambledUserId), + zap.String("export_file", exportFileName), + zap.Int("zip_size_bytes", len(zipData)), + ) + + return ProcessResult{ + ExportData: zipData, + ExportFileName: exportFileName, + } +} diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 9b54513..39c90b4 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -73,6 +73,10 @@ func GetRequestTypeName(requestType int) string { return "AllMessages" case 3: return "SpecificMessages" + case 4: + return "ExportGuild" + case 5: + return "ExportUser" default: return fmt.Sprintf("Unknown(%d)", requestType) } From e9aee44a5063922378c671a8036ae85a4aa1b9ea Mon Sep 17 00:00:00 2001 From: biast12 Date: Thu, 26 Mar 2026 22:07:43 +0100 Subject: [PATCH 2/2] Use archiver client for transcripts (remove S3/minio) Replace direct S3/MinIO access and local decryption with the archiver service client. Removed S3 client initialization and MinIO deps from cmd/main.go and dropped s3client, AES key and decryption/model-version handling from the processor. Processor now queries the tickets DB for transcript IDs and calls archiver.Client.Get to fetch transcripts (handling archiverclient.ErrNotFound), and the Processor constructor signature was simplified to only accept cachePool. Imports and error logs were adjusted accordingly. --- cmd/main.go | 25 +--- internal/processor/processor.go | 252 +++++++++----------------------- 2 files changed, 72 insertions(+), 205 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 760611e..2d987f7 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -17,11 +17,8 @@ import ( "github.com/TicketsBot-cloud/gdpr-worker/internal/heartbeat" "github.com/TicketsBot-cloud/gdpr-worker/internal/processor" "github.com/TicketsBot-cloud/gdpr-worker/internal/utils" - "github.com/TicketsBot-cloud/logarchiver/pkg/s3client" "github.com/go-redis/redis/v8" "github.com/jackc/pgx/v4/pgxpool" - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -73,24 +70,6 @@ func main() { config.Conf.Archiver.AesKey, ) - // Initialise S3 client for export operations - var s3Client *s3client.S3Client - if config.Conf.S3.Endpoint != "" { - logger.Info("Initialising S3 client") - minioClient, err := minio.New(config.Conf.S3.Endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(config.Conf.S3.AccessKey, config.Conf.S3.SecretKey, ""), - Secure: config.Conf.S3.Secure, - }) - if err != nil { - logger.Fatal("Failed to initialise S3 client", zap.Error(err)) - return - } - s3Client = s3client.NewS3Client(minioClient, config.Conf.S3.Bucket) - logger.Info("S3 client initialised") - } else { - logger.Warn("S3 not configured, export features will be unavailable") - } - // Initialise cache database connection for user export operations var cachePool *pgxpool.Pool if config.Conf.CacheDatabase.Host != "" { @@ -114,9 +93,7 @@ func main() { logger.Warn("Cache database not configured, user export cache data will be unavailable") } - aesKey := []byte(config.Conf.Archiver.AesKey) - - proc := processor.New(logger.With(), s3Client, aesKey, cachePool) + proc := processor.New(logger.With(), cachePool) callbackHandler := callback.New( logger.With(), diff --git a/internal/processor/processor.go b/internal/processor/processor.go index 62798b5..5e1ac54 100644 --- a/internal/processor/processor.go +++ b/internal/processor/processor.go @@ -3,16 +3,12 @@ package processor import ( "context" "encoding/json" - "errors" "fmt" - "strconv" "strings" "sync" "github.com/TicketsBot-cloud/archiverclient" - "github.com/TicketsBot-cloud/common/encryption" "github.com/TicketsBot-cloud/gdl/cache" - "github.com/TicketsBot-cloud/gdl/objects/channel/message" "github.com/TicketsBot-cloud/gdl/rest" "github.com/TicketsBot-cloud/gdl/rest/ratelimit" "github.com/TicketsBot-cloud/gdpr-worker/internal/archiver" @@ -21,11 +17,8 @@ import ( "github.com/TicketsBot-cloud/gdpr-worker/internal/export" "github.com/TicketsBot-cloud/gdpr-worker/internal/gdprrelay" "github.com/TicketsBot-cloud/gdpr-worker/internal/utils" - "github.com/TicketsBot-cloud/logarchiver/pkg/model" - v1 "github.com/TicketsBot-cloud/logarchiver/pkg/model/v1" v2 "github.com/TicketsBot-cloud/logarchiver/pkg/model/v2" "github.com/TicketsBot-cloud/logarchiver/pkg/export/user" - "github.com/TicketsBot-cloud/logarchiver/pkg/s3client" "github.com/jackc/pgx/v4/pgxpool" "go.uber.org/zap" "golang.org/x/sync/errgroup" @@ -35,18 +28,14 @@ import ( type Processor struct { logger *zap.Logger rateLimiter *ratelimit.Ratelimiter - s3Client *s3client.S3Client - aesKey []byte cachePool *pgxpool.Pool } -func New(logger *zap.Logger, s3Client *s3client.S3Client, aesKey []byte, cachePool *pgxpool.Pool) *Processor { +func New(logger *zap.Logger, cachePool *pgxpool.Pool) *Processor { store := ratelimit.NewMemoryStore() return &Processor{ logger: logger, rateLimiter: ratelimit.NewRateLimiter(store, 0), - s3Client: s3Client, - aesKey: aesKey, cachePool: cachePool, } } @@ -591,10 +580,6 @@ func (p *Processor) processExportGuild(ctx context.Context, request gdprrelay.GD return ProcessResult{Error: fmt.Errorf("no server ID provided")} } - if p.s3Client == nil { - return ProcessResult{Error: fmt.Errorf("S3 client not configured, cannot process export")} - } - scrambledUserId := utils.ScrambleUserId(request.UserId) if err := p.verifyAllGuildsOwnership(ctx, request.GuildIds, request.UserId); err != nil { @@ -608,9 +593,10 @@ func (p *Processor) processExportGuild(ctx context.Context, request gdprrelay.GD zipBuilder := export.NewZipBuilder() for _, guildId := range request.GuildIds { - keys, err := p.s3Client.GetAllKeysForGuild(ctx, guildId) + query := `SELECT id FROM tickets WHERE guild_id = $1 AND has_transcript = 't' AND open = 'f' ORDER BY id;` + rows, err := database.Client.Tickets.Query(ctx, query, guildId) if err != nil { - p.logger.Error("Failed to list S3 keys for guild export", + p.logger.Error("Failed to query tickets for guild export", zap.Uint64("guild_id", guildId), zap.String("scrambled_user_id", scrambledUserId), zap.Error(err), @@ -618,7 +604,21 @@ func (p *Processor) processExportGuild(ctx context.Context, request gdprrelay.GD continue } - if len(keys) == 0 { + var ticketIds []int + for rows.Next() { + var id int + if err := rows.Scan(&id); err != nil { + p.logger.Warn("Failed to scan ticket ID for guild export", + zap.Uint64("guild_id", guildId), + zap.Error(err), + ) + continue + } + ticketIds = append(ticketIds, id) + } + rows.Close() + + if len(ticketIds) == 0 { p.logger.Info("No transcripts found for guild export", zap.Uint64("guild_id", guildId), zap.String("scrambled_user_id", scrambledUserId), @@ -638,49 +638,19 @@ func (p *Processor) processExportGuild(ctx context.Context, request gdprrelay.GD g, gCtx := errgroup.WithContext(ctx) g.SetLimit(15) - for _, key := range keys { - key := key + for _, ticketId := range ticketIds { + ticketId := ticketId g.Go(func() error { - // Extract ticket ID from key format: {guildId}/{ticketId} - parts := strings.SplitN(key, "/", 2) - if len(parts) != 2 { - p.logger.Warn("Skipping S3 key with unexpected format", - zap.String("key", key), - zap.Uint64("guild_id", guildId), - ) - return nil - } - - ticketId, err := strconv.Atoi(parts[1]) - if err != nil { - p.logger.Warn("Skipping S3 key with non-numeric ticket ID", - zap.String("key", key), - zap.Uint64("guild_id", guildId), - zap.Error(err), - ) - return nil - } - - rawData, err := p.s3Client.GetTicket(gCtx, guildId, ticketId) + transcript, err := archiver.Client.Get(gCtx, guildId, ticketId) if err != nil { - if errors.Is(err, s3client.ErrTicketNotFound) { - p.logger.Debug("Transcript not found in S3, skipping", + if err == archiverclient.ErrNotFound { + p.logger.Debug("Transcript not found in archiver, skipping", zap.Uint64("guild_id", guildId), zap.Int("ticket_id", ticketId), ) return nil } - p.logger.Warn("Failed to download transcript from S3", - zap.Uint64("guild_id", guildId), - zap.Int("ticket_id", ticketId), - zap.Error(err), - ) - return nil - } - - decrypted, err := encryption.Decrypt(p.aesKey, rawData) - if err != nil { - p.logger.Warn("Failed to decrypt transcript, skipping", + p.logger.Warn("Failed to download transcript from archiver", zap.Uint64("guild_id", guildId), zap.Int("ticket_id", ticketId), zap.Error(err), @@ -688,38 +658,6 @@ func (p *Processor) processExportGuild(ctx context.Context, request gdprrelay.GD return nil } - var transcript v2.Transcript - version := model.GetVersion(decrypted) - switch version { - case model.V1: - var messages []message.Message - if err := json.Unmarshal(decrypted, &messages); err != nil { - p.logger.Warn("Failed to unmarshal V1 transcript, skipping", - zap.Uint64("guild_id", guildId), - zap.Int("ticket_id", ticketId), - zap.Error(err), - ) - return nil - } - transcript = v1.ConvertToV2(messages) - case model.V2: - if err := json.Unmarshal(decrypted, &transcript); err != nil { - p.logger.Warn("Failed to unmarshal V2 transcript, skipping", - zap.Uint64("guild_id", guildId), - zap.Int("ticket_id", ticketId), - zap.Error(err), - ) - return nil - } - default: - p.logger.Warn("Unknown transcript version, skipping", - zap.Uint64("guild_id", guildId), - zap.Int("ticket_id", ticketId), - zap.Int("version", version.Int()), - ) - return nil - } - prettyData, err := json.MarshalIndent(transcript, "", " ") if err != nil { p.logger.Warn("Failed to marshal transcript for export", @@ -914,114 +852,66 @@ func (p *Processor) processExportUser(ctx context.Context, request gdprrelay.GDP transcriptIds[guildId] = deduped } - // Download, decrypt, filter and add transcripts to ZIP - if p.s3Client != nil { - for guildId, ticketIds := range transcriptIds { - for _, ticketId := range ticketIds { - rawData, err := p.s3Client.GetTicket(ctx, guildId, ticketId) - if err != nil { - if errors.Is(err, s3client.ErrTicketNotFound) { - p.logger.Debug("Transcript not found for user export, skipping", - zap.Uint64("guild_id", guildId), - zap.Int("ticket_id", ticketId), - ) - continue - } - p.logger.Warn("Failed to download transcript for user export", - zap.Uint64("guild_id", guildId), - zap.Int("ticket_id", ticketId), - zap.Error(err), - ) - continue - } - - decrypted, err := encryption.Decrypt(p.aesKey, rawData) - if err != nil { - p.logger.Warn("Failed to decrypt transcript for user export, skipping", - zap.Uint64("guild_id", guildId), - zap.Int("ticket_id", ticketId), - zap.Error(err), - ) - continue - } - - var transcript v2.Transcript - version := model.GetVersion(decrypted) - switch version { - case model.V1: - var messages []message.Message - if err := json.Unmarshal(decrypted, &messages); err != nil { - p.logger.Warn("Failed to unmarshal V1 transcript for user export, skipping", - zap.Uint64("guild_id", guildId), - zap.Int("ticket_id", ticketId), - zap.Error(err), - ) - continue - } - transcript = v1.ConvertToV2(messages) - case model.V2: - if err := json.Unmarshal(decrypted, &transcript); err != nil { - p.logger.Warn("Failed to unmarshal V2 transcript for user export, skipping", - zap.Uint64("guild_id", guildId), - zap.Int("ticket_id", ticketId), - zap.Error(err), - ) - continue - } - default: - p.logger.Warn("Unknown transcript version for user export, skipping", + // Download, filter and add transcripts to ZIP + for guildId, ticketIds := range transcriptIds { + for _, ticketId := range ticketIds { + transcript, err := archiver.Client.Get(ctx, guildId, ticketId) + if err != nil { + if err == archiverclient.ErrNotFound { + p.logger.Debug("Transcript not found for user export, skipping", zap.Uint64("guild_id", guildId), zap.Int("ticket_id", ticketId), - zap.Int("version", version.Int()), ) continue } + p.logger.Warn("Failed to download transcript for user export", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Error(err), + ) + continue + } - // Filter to only the user's data - transcript.Entities.Channels = nil - transcript.Entities.Roles = nil + // Filter to only the user's data + transcript.Entities.Channels = nil + transcript.Entities.Roles = nil - userEntity, ok := transcript.Entities.Users[request.UserId] - if !ok { - transcript.Entities.Users = nil - } else { - transcript.Entities.Users = map[uint64]v2.User{ - userEntity.Id: userEntity, - } + userEntity, ok := transcript.Entities.Users[request.UserId] + if !ok { + transcript.Entities.Users = nil + } else { + transcript.Entities.Users = map[uint64]v2.User{ + userEntity.Id: userEntity, } + } - var filteredMessages []v2.Message - for _, msg := range transcript.Messages { - if msg.AuthorId == request.UserId { - filteredMessages = append(filteredMessages, msg) - } + var filteredMessages []v2.Message + for _, msg := range transcript.Messages { + if msg.AuthorId == request.UserId { + filteredMessages = append(filteredMessages, msg) } - transcript.Messages = filteredMessages + } + transcript.Messages = filteredMessages - prettyData, err := json.MarshalIndent(transcript, "", " ") - if err != nil { - p.logger.Warn("Failed to marshal filtered transcript for user export", - zap.Uint64("guild_id", guildId), - zap.Int("ticket_id", ticketId), - zap.Error(err), - ) - continue - } + prettyData, err := json.MarshalIndent(transcript, "", " ") + if err != nil { + p.logger.Warn("Failed to marshal filtered transcript for user export", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Error(err), + ) + continue + } - fileName := fmt.Sprintf("transcripts/%d-%d.json", guildId, ticketId) - if err := zipBuilder.AddFile(fileName, prettyData); err != nil { - p.logger.Error("Failed to add transcript to user export ZIP", - zap.Uint64("guild_id", guildId), - zap.Int("ticket_id", ticketId), - zap.Error(err), - ) - } + fileName := fmt.Sprintf("transcripts/%d-%d.json", guildId, ticketId) + if err := zipBuilder.AddFile(fileName, prettyData); err != nil { + p.logger.Error("Failed to add transcript to user export ZIP", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Error(err), + ) } } - } else { - p.logger.Warn("S3 client not configured, skipping transcript export for user", - zap.String("scrambled_user_id", scrambledUserId), - ) } zipData, err := zipBuilder.Close()