From 2ba6d65bbafd570a56d1633c1d9fd9ea7a9532e5 Mon Sep 17 00:00:00 2001 From: Stysusss <158248053+stysus@users.noreply.github.com> Date: Tue, 18 Aug 2026 21:04:06 +0700 Subject: [PATCH] feat: automatic database backup with retention (STY-94) Backend (manager): - internal/backup: SQLite VACUUM INTO online snapshot + gzip (BestCompression) + Prune retention (neuralwire-*.db.gz) - GET /api/admin/backup: admin download backup on demand (auth) - Automatic backup at startup then every BACKUP_INTERVAL_HOURS (default 24h), retention BACKUP_RETENTION (default 7) - Config: BACKUP_DIR/BACKUP_RETENTION/BACKUP_INTERVAL_HOURS; Docker/compose point backups at the persistent volume - NewsRepository.DB() exposes *sql.DB for the backup service Verified: backup startup works (733KB -> 179KB gzip), on-demand download 200, gzip valid SQLite, all tests pass. --- Dockerfile | 1 + backend/.env.example | 8 ++ backend/README.md | 4 + backend/cmd/server/main.go | 29 ++++++ backend/internal/api/handlers.go | 29 ++++++ backend/internal/api/server.go | 12 +++ backend/internal/backup/backup.go | 109 +++++++++++++++++++++++ backend/internal/config/config.go | 19 ++++ backend/internal/repository/news_repo.go | 5 ++ docker-compose.yml | 1 + 10 files changed, 217 insertions(+) create mode 100644 backend/internal/backup/backup.go diff --git a/Dockerfile b/Dockerfile index 88223c1..362e5ac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -71,6 +71,7 @@ RUN chmod +x /usr/local/bin/entrypoint.sh ENV STATIC_DIR=/app/static \ DB_PATH=/app/data/neuralwire.db \ UPLOAD_DIR=/app/data/uploads \ + BACKUP_DIR=/app/data/backups \ PORT=8080 EXPOSE 8080 diff --git a/backend/.env.example b/backend/.env.example index 4211ccb..66590db 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -119,3 +119,11 @@ GLOBAL_RATE_LIMIT=120 # Must be writable by the server. In production point it at the persistent # volume (e.g. /app/data/uploads in the Docker image). #UPLOAD_DIR=./data/uploads + +# Directory for SQLite database backups (gzip snapshots). Automatic backup +# runs at startup then every BACKUP_INTERVAL_HOURS; BACKUP_RETENTION keeps +# the newest N backups and prunes older ones. Admin can also download a +# backup on demand via GET /api/admin/backup. +#BACKUP_DIR=./data/backups +#BACKUP_RETENTION=7 +#BACKUP_INTERVAL_HOURS=24 diff --git a/backend/README.md b/backend/README.md index b5e49c2..c7c232a 100644 --- a/backend/README.md +++ b/backend/README.md @@ -72,6 +72,10 @@ English by default). (jpeg/png/webp/gif, ≤ 5 MiB), stores it under `UPLOAD_DIR` with a random name, and serves it at `/uploads/...`. In the Docker image `UPLOAD_DIR` points at the persistent volume so uploaded images survive redeploys +- **Database backups**: automatic gzip snapshot at startup then every + `BACKUP_INTERVAL_HOURS` (default 24h), kept under `BACKUP_DIR` with + `BACKUP_RETENTION` (default 7). Admin can also download a backup on demand + via `GET /api/admin/backup` ## Requirements diff --git a/backend/cmd/server/main.go b/backend/cmd/server/main.go index 6537626..7d01553 100644 --- a/backend/cmd/server/main.go +++ b/backend/cmd/server/main.go @@ -17,6 +17,7 @@ import ( "neuralwire/backend/internal/ai" "neuralwire/backend/internal/api" "neuralwire/backend/internal/auth" + "neuralwire/backend/internal/backup" "neuralwire/backend/internal/config" "neuralwire/backend/internal/database" "neuralwire/backend/internal/fetcher" @@ -192,8 +193,36 @@ func main() { StaticDir: cfg.StaticDir, UploadDir: cfg.UploadDir, Scheduler: sched, + BackupDir: cfg.BackupDir, + BackupRetain: cfg.BackupRetention, }) + // Automatic database backup (STY-94): run on a timer, prune old backups. + if cfg.BackupDir != "" && cfg.BackupIntervalHours > 0 { + runBackup := func() { + path, err := backup.Create(newsRepo.DB(), cfg.BackupDir) + if err != nil { + slogLogger.Error("backup: automatic backup failed", "error", err) + return + } + slogLogger.Info("backup: automatic backup created", "path", path) + if cfg.BackupRetention > 0 { + if err := backup.Prune(cfg.BackupDir, cfg.BackupRetention); err != nil { + slogLogger.Warn("backup: prune failed", "error", err) + } + } + } + runBackup() // backup on startup, then on interval + interval := time.Duration(cfg.BackupIntervalHours) * time.Hour + go func() { + t := time.NewTicker(interval) + defer t.Stop() + for range t.C { + runBackup() + } + }() + } + httpServer := &http.Server{ Addr: ":" + cfg.Port, Handler: srv.Handler(), diff --git a/backend/internal/api/handlers.go b/backend/internal/api/handlers.go index a0c0842..f06f4ef 100644 --- a/backend/internal/api/handlers.go +++ b/backend/internal/api/handlers.go @@ -17,6 +17,7 @@ import ( "strings" "time" + "neuralwire/backend/internal/backup" "neuralwire/backend/internal/models" "neuralwire/backend/internal/repository" ) @@ -153,6 +154,34 @@ var allowedImageTypes = map[string]string{ "image/gif": ".gif", } +// handleBackup creates a gzip-compressed SQLite backup and streams it to the +// client for download, then prunes old backups. Admin-only. +func (s *Server) handleBackup(w http.ResponseWriter, r *http.Request) { + if s.backupDir == "" { + s.writeError(w, http.StatusServiceUnavailable, "backups are not configured") + return + } + + path, err := backup.Create(s.newsRepo.DB(), s.backupDir) + if err != nil { + s.slog.Error("api: backup failed", "error", err) + s.writeError(w, http.StatusInternalServerError, "failed to create backup") + return + } + defer os.Remove(path) + + if s.backupRetain > 0 { + if err := backup.Prune(s.backupDir, s.backupRetain); err != nil { + s.slog.Warn("api: prune backups", "error", err) + } + } + + name := filepath.Base(path) + w.Header().Set("Content-Type", "application/gzip") + w.Header().Set("Content-Disposition", `attachment; filename="`+name+`"`) + http.ServeFile(w, r, path) +} + // handleUploadImage stores an admin-uploaded image and returns its public // URL. Only image MIME types are accepted; the file is saved with a random // name under the configured upload directory (served at /uploads/). diff --git a/backend/internal/api/server.go b/backend/internal/api/server.go index 3546cf9..54fa570 100644 --- a/backend/internal/api/server.go +++ b/backend/internal/api/server.go @@ -73,6 +73,10 @@ type Server struct { // scheduler controls the auto fetch/publish loop (STY-57/61). Nil // disables the start/stop API. scheduler *scheduler.Scheduler + // backupDir is where admin-created database backups are stored. + backupDir string + // backupRetain is how many backups to keep (0 = no pruning). + backupRetain int } // ServerOptions configures the API server. @@ -123,6 +127,11 @@ type ServerOptions struct { // Scheduler is the auto fetch/publish controller. When non-nil, the // /api/admin/autopublish/start and /stop endpoints toggle it. Scheduler *scheduler.Scheduler + // BackupDir is where database backups are stored. Empty disables the + // backup endpoint. + BackupDir string + // BackupRetain keeps the newest N backups and prunes the rest. + BackupRetain int } // NewServer builds a Server. @@ -162,6 +171,8 @@ func NewServer(opts ServerOptions) *Server { staticDir: opts.StaticDir, uploadDir: opts.UploadDir, scheduler: opts.Scheduler, + backupDir: opts.BackupDir, + backupRetain: opts.BackupRetain, } if opts.ViewRateLimit > 0 { srv.viewLimiter = ratelimit.New(opts.ViewRateLimit, opts.ViewRateWindow) @@ -227,6 +238,7 @@ func (s *Server) Handler() http.Handler { mux.Handle("POST /api/admin/autopublish/start", s.requireAuth(s.csrfProtect(http.HandlerFunc(s.handleStartAutoPublish)))) mux.Handle("POST /api/admin/autopublish/stop", s.requireAuth(s.csrfProtect(http.HandlerFunc(s.handleStopAutoPublish)))) mux.Handle("POST /api/admin/upload-image", s.requireAuth(s.csrfProtect(http.HandlerFunc(s.handleUploadImage)))) + mux.Handle("GET /api/admin/backup", s.requireAuth(http.HandlerFunc(s.handleBackup))) mux.Handle("/api/admin/", s.requireAuth(s.csrfProtect(admin))) // Serve admin-uploaded images under /uploads/ when an upload directory diff --git a/backend/internal/backup/backup.go b/backend/internal/backup/backup.go new file mode 100644 index 0000000..3b9c3b0 --- /dev/null +++ b/backend/internal/backup/backup.go @@ -0,0 +1,109 @@ +// Package backup implements SQLite online backups for Neuralwire (STY-94). +// +// It uses SQLite's VACUUM INTO to snapshot the database without locking +// readers/writers, compresses the snapshot with gzip (text-heavy data +// compresses ~90%), and prunes old backups to a retention limit. +package backup + +import ( + "compress/gzip" + "database/sql" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +// Snapshot writes a consistent online backup of db to dstPath using +// VACUUM INTO. The database keeps serving reads/writes while the snapshot +// runs. It returns the number of bytes written. +func Snapshot(db *sql.DB, dstPath string) (int64, error) { + if _, err := db.Exec(`VACUUM INTO ?`, dstPath); err != nil { + return 0, fmt.Errorf("vacuum into: %w", err) + } + if fi, err := os.Stat(dstPath); err == nil { + return fi.Size(), nil + } + return 0, nil +} + +// Create compresses the database into a gzip snapshot file and returns the +// destination path. The snapshot is stored in dir with the name +// neuralwire-YYYYMMDD-HHMMSS.db.gz. +func Create(db *sql.DB, dir string) (string, error) { + if err := os.MkdirAll(dir, 0o755); err != nil { + return "", fmt.Errorf("create backup dir: %w", err) + } + + ts := time.Now().UTC().Format("20060102-150405") + rawPath := filepath.Join(dir, "snapshot-"+ts+".db") + gzPath := filepath.Join(dir, "neuralwire-"+ts+".db.gz") + + if _, err := Snapshot(db, rawPath); err != nil { + return "", err + } + defer os.Remove(rawPath) + + if err := gzipFile(rawPath, gzPath); err != nil { + return "", err + } + return gzPath, nil +} + +// gzipFile compresses src into dst (best compression for text-heavy SQLite). +func gzipFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return fmt.Errorf("open source: %w", err) + } + defer in.Close() + + out, err := os.Create(dst) + if err != nil { + return fmt.Errorf("create gzip: %w", err) + } + defer out.Close() + + gz, err := gzip.NewWriterLevel(out, gzip.BestCompression) + if err != nil { + return fmt.Errorf("gzip writer: %w", err) + } + if _, err := io.Copy(gz, in); err != nil { + gz.Close() + return fmt.Errorf("gzip copy: %w", err) + } + if err := gz.Close(); err != nil { + return fmt.Errorf("gzip close: %w", err) + } + return nil +} + +// Prune removes backups older than the newest retain count. Only files +// matching the neuralwire-*.db.gz pattern are considered. +func Prune(dir string, retain int) error { + if retain <= 0 { + return nil + } + entries, err := os.ReadDir(dir) + if err != nil { + return fmt.Errorf("read backup dir: %w", err) + } + var backups []string + for _, e := range entries { + if !e.IsDir() && strings.HasPrefix(e.Name(), "neuralwire-") && strings.HasSuffix(e.Name(), ".db.gz") { + backups = append(backups, filepath.Join(dir, e.Name())) + } + } + if len(backups) <= retain { + return nil + } + // Newest first (names are timestamp-sorted). + sort.Sort(sort.Reverse(sort.StringSlice(backups))) + for _, b := range backups[retain:] { + _ = os.Remove(b) + } + return nil +} diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go index 67d72e8..68c0d53 100644 --- a/backend/internal/config/config.go +++ b/backend/internal/config/config.go @@ -95,6 +95,12 @@ type Config struct { // must be writable by the server process; in production it should point // at a persistent volume. Files are served under /uploads/. UploadDir string + // BackupDir is where database backups are stored. Empty disables backups. + BackupDir string + // BackupRetention keeps the newest N backups and prunes older ones. + BackupRetention int + // BackupIntervalHours is how often the automatic backup runs (0 disables). + BackupIntervalHours int } // Load builds a Config from the environment, applying defaults. It first @@ -185,6 +191,9 @@ func Load() (Config, error) { LogFormat: strings.ToLower(strings.TrimSpace(getenv("LOG_FORMAT", "text"))), StaticDir: strings.TrimSpace(getenv("STATIC_DIR", "../frontend/build")), UploadDir: strings.TrimSpace(getenv("UPLOAD_DIR", "./data/uploads")), + BackupDir: strings.TrimSpace(getenv("BACKUP_DIR", "./data/backups")), + BackupRetention: getenvIntOr("BACKUP_RETENTION", 7), + BackupIntervalHours: getenvIntOr("BACKUP_INTERVAL_HOURS", 24), }, nil } @@ -260,6 +269,16 @@ func getenvInt(key string, fallback int) (int, error) { return n, nil } +// getenvIntOr parses an integer environment variable, silently falling back +// to the default when unset or invalid (used for optional settings). +func getenvIntOr(key string, fallback int) int { + n, err := getenvInt(key, fallback) + if err != nil { + return fallback + } + return n +} + // getenvList parses a comma-separated environment variable into a slice, // trimming whitespace and dropping empty entries. It returns the fallback // list when the variable is unset or empty. diff --git a/backend/internal/repository/news_repo.go b/backend/internal/repository/news_repo.go index 2261486..5d89f4f 100644 --- a/backend/internal/repository/news_repo.go +++ b/backend/internal/repository/news_repo.go @@ -29,6 +29,11 @@ func (r *NewsRepository) Ping(ctx context.Context) error { return r.db.PingContext(ctx) } +// DB exposes the underlying *sql.DB, used by the backup service. +func (r *NewsRepository) DB() *sql.DB { + return r.db +} + const newsColumns = `id, title, slug, url, source, category, summary, content, image_url, status, published_at, created_at, value_score, value_breakdown, value_confidence, value_recommendation, diff --git a/docker-compose.yml b/docker-compose.yml index a55585c..4138228 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,6 +26,7 @@ services: DB_PATH: /app/data/neuralwire.db STATIC_DIR: /app/static UPLOAD_DIR: /app/data/uploads + BACKUP_DIR: /app/data/backups # Required in production — never use defaults: ADMIN_USERNAME: ${ADMIN_USERNAME:?set ADMIN_USERNAME in .env} ADMIN_PASSWORD: ${ADMIN_PASSWORD:?set ADMIN_PASSWORD in .env}