SSH transfer optimizations: source-side compression and stale staging purge (v3.10.0)

README.md

@@ -283,6 +283,13 @@ Creates a database backup.
   --from-password-file .secrets/prod.pass \
   --from-keep 3  # Keep last 3 dumps
 
+# Slow uplink? Compress on the source host, then transfer the small file
+./t-pgsql dump \
+  --from "ssh://user@server/postgres@localhost/prod" \
+  --from-password-file .secrets/prod.pass \
+  --compress zstd \
+  --compress-where source  # 20 GB raw dump can shrink to ~1 GB before the copy
+
 # Custom dump name
 ./t-pgsql dump \
   --from "postgres@localhost/mydb" \
@@ -469,7 +476,7 @@ operation:
   exit_code: 0
 
 environment:
-  script_version: "3.9.0"
+  script_version: "3.10.0"
   executed_by: dbadmin
   executed_on: macbookair
   working_dir: /opt/t-pgsql/t-pgsql
@@ -1026,6 +1033,7 @@ Supported keys mirror the flags: `from`, `to`, `password`/`from_password`/`to_pa
 | `--compress <type>` | Compression algorithm | `gzip` | No | `zstd`, `xz`, `bzip2`, `none` |
 | `--compress-level <1-9>` | Compression level | `6` | No | `9` |
 | `--pg-compress-level <0-9>` | pg_dump internal compression | `6` | No | `0` (no compression) |
+| `--compress-where <where>` | For SSH dumps: run zstd/xz/bzip2 on the `source` host before the copy, or on the `target` after it. `source` moves a much smaller file over slow uplinks; falls back to `target` if the tool is missing on the source | `target` | No | `source` |
 
 ### Storage Parameters
 
@@ -1034,6 +1042,7 @@ Supported keys mirror the flags: `from`, `to`, `password`/`from_password`/`to_pa
 | `--output <dir>` | Output directory for dumps | `<script dir>/../data/dumps` | No | `/backups/daily` |
 | `--keep <N>` | Number of local dumps to keep | `-1` (all) | No | `7`, `0` (delete), `-1` (all) |
 | `--from-keep <N>` | Number of dumps to keep on source | `1` | No | `3`, `0` (delete), `-1` (all) |
+| `--from-stale <time>` | With `--from-keep 0`: purge this job's leftover dumps older than `<time>` from the source staging dir before dumping (failed runs never reach the normal cleanup) | `72h` | No | `48h`, `2d`, `0` (off) |
 | `--dump-name <name>` | Custom dump filename (without timestamp) | Database name | No | `myapp-backup` |
 | `--skip-if-recent <time>` | Skip if dump exists within timeframe | - | No | `24h`, `12h`, `1d`, `today` |
 | `--file <path>` | Specific dump file for restore | - | No | `./dumps/backup.tar.gz` |

README_DE.md

@@ -1026,6 +1026,7 @@ Die unterstützten Schlüssel spiegeln die Flags wider: `from`, `to`, `password`
 | `--compress <type>` | Komprimierungsalgorithmus | `gzip` | Nein | `zstd`, `xz`, `bzip2`, `none` |
 | `--compress-level <1-9>` | Komprimierungsstufe | `6` | Nein | `9` |
 | `--pg-compress-level <0-9>` | Interne Komprimierung von pg_dump | `6` | Nein | `0` (keine Komprimierung) |
+| `--compress-where <where>` | Bei SSH-Dumps: zstd/xz/bzip2 auf dem `source`-Host vor dem Kopieren ausführen oder auf dem `target` danach. `source` überträgt eine viel kleinere Datei über langsame Uplinks; fällt auf `target` zurück, wenn das Tool auf der Quelle fehlt | `target` | Nein | `source` |
 
 ### Speicherparameter
 
@@ -1034,6 +1035,7 @@ Die unterstützten Schlüssel spiegeln die Flags wider: `from`, `to`, `password`
 | `--output <dir>` | Ausgabeverzeichnis für Dumps | `<script dir>/../data/dumps` | Nein | `/backups/daily` |
 | `--keep <N>` | Anzahl der lokal zu behaltenden Dumps | `-1` (alle) | Nein | `7`, `0` (löschen), `-1` (alle) |
 | `--from-keep <N>` | Anzahl der auf der Quelle zu behaltenden Dumps | `1` | Nein | `3`, `0` (löschen), `-1` (alle) |
+| `--from-stale <time>` | Mit `--from-keep 0`: vor dem Dump liegengebliebene Dumps dieses Jobs, die älter als `<time>` sind, aus dem Staging-Verzeichnis der Quelle löschen (fehlgeschlagene Läufe erreichen die normale Bereinigung nie) | `72h` | Nein | `48h`, `2d`, `0` (aus) |
 | `--dump-name <name>` | Benutzerdefinierter Dump-Dateiname (ohne Zeitstempel) | Datenbankname | Nein | `myapp-backup` |
 | `--skip-if-recent <time>` | Überspringen, wenn innerhalb des Zeitrahmens ein Dump existiert | - | Nein | `24h`, `12h`, `1d`, `today` |
 | `--file <path>` | Bestimmte Dump-Datei für die Wiederherstellung | - | Nein | `./dumps/backup.tar.gz` |

README_ES.md

@@ -1026,6 +1026,7 @@ Las claves admitidas reflejan las opciones: `from`, `to`, `password`/`from_passw
 | `--compress <type>` | Algoritmo de compresión | `gzip` | No | `zstd`, `xz`, `bzip2`, `none` |
 | `--compress-level <1-9>` | Nivel de compresión | `6` | No | `9` |
 | `--pg-compress-level <0-9>` | Compresión interna de pg_dump | `6` | No | `0` (sin compresión) |
+| `--compress-where <where>` | Para dumps SSH: ejecutar zstd/xz/bzip2 en el host `source` antes de la copia, o en el `target` después. `source` transfiere un archivo mucho más pequeño por enlaces lentos; recurre a `target` si la herramienta falta en el origen | `target` | No | `source` |
 
 ### Parámetros de almacenamiento
 
@@ -1034,6 +1035,7 @@ Las claves admitidas reflejan las opciones: `from`, `to`, `password`/`from_passw
 | `--output <dir>` | Directorio de salida para los dumps | `<script dir>/../data/dumps` | No | `/backups/daily` |
 | `--keep <N>` | Número de dumps locales a conservar | `-1` (todos) | No | `7`, `0` (eliminar), `-1` (todos) |
 | `--from-keep <N>` | Número de dumps a conservar en el origen | `1` | No | `3`, `0` (eliminar), `-1` (todos) |
+| `--from-stale <time>` | Con `--from-keep 0`: antes del dump, purgar del directorio de staging del origen los dumps de este job más antiguos que `<time>` (las ejecuciones fallidas nunca llegan a la limpieza normal) | `72h` | No | `48h`, `2d`, `0` (desactivado) |
 | `--dump-name <name>` | Nombre de archivo de dump personalizado (sin marca de tiempo) | Nombre de la base de datos | No | `myapp-backup` |
 | `--skip-if-recent <time>` | Omite si existe un dump dentro del periodo | - | No | `24h`, `12h`, `1d`, `today` |
 | `--file <path>` | Archivo de dump específico para restaurar | - | No | `./dumps/backup.tar.gz` |

README_RU.md

@@ -1026,6 +1026,7 @@ verbose: true
 | `--compress <type>` | Алгоритм сжатия | `gzip` | Нет | `zstd`, `xz`, `bzip2`, `none` |
 | `--compress-level <1-9>` | Уровень сжатия | `6` | Нет | `9` |
 | `--pg-compress-level <0-9>` | Внутреннее сжатие pg_dump | `6` | Нет | `0` (без сжатия) |
+| `--compress-where <where>` | Для SSH-дампов: запускать zstd/xz/bzip2 на хосте `source` до копирования или на `target` после. `source` передаёт значительно меньший файл по медленным каналам; при отсутствии инструмента на источнике откат к `target` | `target` | Нет | `source` |
 
 ### Параметры хранилища
 
@@ -1034,6 +1035,7 @@ verbose: true
 | `--output <dir>` | Каталог вывода для дампов | `<script dir>/../data/dumps` | Нет | `/backups/daily` |
 | `--keep <N>` | Количество локальных дампов для хранения | `-1` (все) | Нет | `7`, `0` (удалить), `-1` (все) |
 | `--from-keep <N>` | Количество дампов для хранения на источнике | `1` | Нет | `3`, `0` (удалить), `-1` (все) |
+| `--from-stale <time>` | С `--from-keep 0`: перед дампом удалять из staging-каталога источника оставшиеся дампы этого задания старше `<time>` (неудачные запуски никогда не доходят до обычной очистки) | `72h` | Нет | `48h`, `2d`, `0` (выкл) |
 | `--dump-name <name>` | Пользовательское имя файла дампа (без временной метки) | Имя базы данных | Нет | `myapp-backup` |
 | `--skip-if-recent <time>` | Пропустить, если дамп существует в пределах временного окна | - | Нет | `24h`, `12h`, `1d`, `today` |
 | `--file <path>` | Конкретный файл дампа для восстановления | - | Нет | `./dumps/backup.tar.gz` |

debian/changelog

@@ -1,3 +1,14 @@
+t-pgsql (3.10.0-1) unstable; urgency=medium
+
+  * New features for SSH dumps:
+    - --compress-where source|target: compress on the source host before
+      the transfer (falls back to target-side if the tool is missing)
+    - --from-stale <time>: purge leftover staging dumps from failed runs
+      on the source before dumping (default 72h, with --from-keep 0)
+    - Widened retention/fetch globs to match compressed staging artifacts
+
+ -- Asim Atasert <asimatasert@outlook.com>  Wed, 16 Jul 2026 15:00:00 +0300
+
 t-pgsql (3.9.0-1) unstable; urgency=medium
 
   * Packaging and audit remediation:

src/00-header.sh

@@ -21,7 +21,7 @@
 # ==============================================================================
 # VERSION & PATHS
 # ==============================================================================
-VERSION="3.9.0"
+VERSION="3.10.0"
 SCRIPT_NAME="t-pgsql"
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 

src/01-globals.sh

@@ -81,6 +81,20 @@ BWLIMIT_KBIT=""               # derived: scp -l wants Kbit/s
 PV_RATE=""                    # derived: pv -L rate for streaming clone
 RETRIES=0                     # extra retries for scp transfers on failure
 
+# Where external compression (zstd/xz/bzip2) runs for SSH dumps:
+#   target (default) = copy the raw dump, compress locally (old behavior)
+#   source           = compress on the source host first, then copy the small
+#                      file — a 20x+ transfer saving on slow uplinks
+COMPRESS_WHERE="target"
+COMPRESS_WHERE_SET=false      # true if --compress-where passed on CLI
+
+# Before an SSH dump with --from-keep 0, purge leftover dumps of the same job
+# older than this from the source temp dir. Failed or interrupted runs never
+# reach the post-transfer cleanup, so their dumps would otherwise pile up in
+# /tmp forever. 0 disables.
+FROM_STALE="72h"
+FROM_STALE_SET=false          # true if --from-stale passed on CLI
+
 # Migration / upgrade
 GLOBALS=false                 # also migrate cluster globals (roles, tablespaces)
 PG_BINDIR=""                  # dir prepended to PATH so a specific PostgreSQL

src/30-help.sh

@@ -72,6 +72,12 @@ COMPRESSION:
                                   compression is disabled so data is compressed once.
     --pg-compress-level <0-9>     Advanced: override pg_dump's built-in -Z level
                                   (only affects the gzip type; default: 6)
+    --compress-where <where>      For SSH dumps with zstd/xz/bzip2 (default: target):
+                                  target = copy the raw dump, compress locally
+                                  source = compress on the source host first, then
+                                  copy the much smaller file (best on slow uplinks).
+                                  Falls back to target if the tool is missing there.
+                                  gzip is unaffected (pg_dump -Z compresses in-place).
 
 STORAGE:
     --output <dir>                Output directory (default: env T_PGSQL_OUTPUT_DIR,
@@ -80,6 +86,11 @@ STORAGE:
     --keep <N>                    Keep last N local dumps (-1=all, 0=none)
     --from-keep <N>               For SSH sources: how many dumps to keep in the remote
                                   /tmp staging dir (default: 1, -1=all, 0=delete)
+    --from-stale <time>           With --from-keep 0: before dumping, purge this job's
+                                  leftover dumps older than <time> from the remote
+                                  staging dir — failed runs never reach the normal
+                                  cleanup and would pile up there (default: 72h,
+                                  0 disables). Examples: 48h, 2d
     --skip-if-recent <time>       Skip if dump exists within timeframe
                                   Examples: 24h, 12h, 1d, today
     --from-file [pattern]         Fetch existing dump (for fetch command)

src/50-config.sh

@@ -49,7 +49,9 @@ load_config() {
             output) [ "$OUTPUT_DIR_SET" != true ] && OUTPUT_DIR="$value" ;;
             keep) [ "$KEEP_SET" != true ] && KEEP="$value" ;;
             from_keep|from-keep) [ "$FROM_KEEP_SET" != true ] && FROM_KEEP="$value" ;;
+            from_stale|from-stale) [ "$FROM_STALE_SET" != true ] && FROM_STALE="$value" ;;
             compress) [ "$COMPRESS_SET" != true ] && COMPRESS="$value" ;;
+            compress_where|compress-where) [ "$COMPRESS_WHERE_SET" != true ] && COMPRESS_WHERE="$value" ;;
             exclude_table|exclude-table) [ -z "$EXCLUDE_TABLES" ] && EXCLUDE_TABLES="$value" ;;
             exclude_schema|exclude-schema) [ -z "$EXCLUDE_SCHEMAS" ] && EXCLUDE_SCHEMAS="$value" ;;
             exclude_data|exclude-data) [ -z "$EXCLUDE_DATA" ] && EXCLUDE_DATA="$value" ;;

src/55-dump.sh

@@ -127,6 +127,7 @@ cmd_dump() {
     esac
 
     local result=0
+    local remote_compressed=false comp_ext=""
 
     if [ "$FROM_TYPE" = "ssh" ]; then
         log_info "Dumping via SSH..."
@@ -135,6 +136,20 @@ cmd_dump() {
         local dump_base_name="${DUMP_NAME:-$FROM_DATABASE}"
         local remote_file="${remote_dump_dir}/${dump_base_name}_${ts}.dump"
 
+        # Purge leftovers from failed/interrupted runs (--from-stale). Post-transfer
+        # cleanup never runs when a run dies mid-way, so its dump would sit in the
+        # source temp dir forever. Only with --from-keep 0: a positive keep policy
+        # manages its own retention, and -1 means keep everything on the source.
+        local stale_min=$(( $(parse_time_to_seconds "$FROM_STALE") / 60 ))
+        if [ "$FROM_KEEP" -eq 0 ] && [ "$stale_min" -gt 0 ]; then
+            local stale_out
+            stale_out=$(ssh -p "$FROM_SSH_PORT" "${SSH_OPTS[@]}" "${FROM_SSH_USER}@${FROM_SSH_HOST}" \
+                "find $(pq "$remote_dump_dir") -maxdepth 1 -name $(pq "${dump_base_name}_*.dump*") -mmin +${stale_min} -print -delete 2>/dev/null")
+            if [ -n "$stale_out" ]; then
+                log_info "Purged $(echo "$stale_out" | wc -l | tr -d ' ') stale source dump(s) older than ${FROM_STALE}"
+            fi
+        fi
+
         local cmd=""
         if [ "$SUDO" = true ]; then
             # Use sudo -u postgres (peer auth)
@@ -153,13 +168,64 @@ cmd_dump() {
         ssh_exec "$FROM_SSH_PORT" "${FROM_SSH_USER}@${FROM_SSH_HOST}" "$cmd"
         result=$?
 
+        # Source-side compression (--compress-where source): shrink the dump on
+        # the source host BEFORE the copy, so a slow uplink moves the compressed
+        # artifact instead of the raw dump. Falls back to the old transfer-then-
+        # compress path if the tool is missing or compression fails — never
+        # blocks the backup. gzip needs nothing here: pg_dump -Z already
+        # compressed it in-format on the source.
+        if [ $result -eq 0 ] && [ "$COMPRESS_WHERE" = "source" ]; then
+            case "$COMPRESS" in
+                zstd|xz|bzip2)
+                    local clvl="$COMPRESS_LEVEL" ccmd=""
+                    [ "$clvl" -lt 1 ] && clvl=1
+                    case "$COMPRESS" in
+                        zstd)  [ "$clvl" -gt 19 ] && clvl=19; comp_ext="zst"; ccmd="zstd -q -${clvl} --rm -f $(pq "$remote_file")" ;;
+                        xz)    [ "$clvl" -gt 9 ]  && clvl=9;  comp_ext="xz";  ccmd="xz -${clvl} -f $(pq "$remote_file")" ;;
+                        bzip2) [ "$clvl" -gt 9 ]  && clvl=9;  comp_ext="bz2"; ccmd="bzip2 -${clvl} -f $(pq "$remote_file")" ;;
+                    esac
+                    if ssh -p "$FROM_SSH_PORT" "${SSH_OPTS[@]}" "${FROM_SSH_USER}@${FROM_SSH_HOST}" "command -v $(pq "$COMPRESS") >/dev/null 2>&1"; then
+                        log_info "Compressing on source ($COMPRESS)..."
+                        if ssh -p "$FROM_SSH_PORT" "${SSH_OPTS[@]}" "${FROM_SSH_USER}@${FROM_SSH_HOST}" "$ccmd"; then
+                            remote_file="${remote_file}.${comp_ext}"
+                            remote_compressed=true
+                        else
+                            log_warn "Source-side compression failed; transferring raw and compressing on target"
+                        fi
+                    else
+                        log_warn "'$COMPRESS' not found on source host; transferring raw and compressing on target"
+                    fi
+                    ;;
+            esac
+        fi
+
         if [ $result -eq 0 ]; then
             log_info "Transferring..."
             if ! scp_transfer "$FROM_SSH_PORT" "${FROM_SSH_USER}@${FROM_SSH_HOST}:${remote_file}" "$dump_file"; then
                 log_error "Transfer failed; leaving the source dump in place"
                 result=1
             fi
 
+            # The local name was reserved as a plain .dump path; when the bytes
+            # arrived already compressed, rename to the real extension so
+            # metadata, retention and restore see what the file actually is.
+            if [ $result -eq 0 ] && [ "$remote_compressed" = true ]; then
+                local final_file="${dump_file}.${comp_ext}" _m=0
+                while [ -e "$final_file" ]; do
+                    _m=$((_m + 1))
+                    [ "$_m" -gt 10000 ] && { log_error "Too many colliding dumps for ${dump_file}.${comp_ext}"; result=1; break; }
+                    final_file="${dump_file}.${_m}.${comp_ext}"
+                done
+                if [ $result -eq 0 ]; then
+                    if mv "$dump_file" "$final_file"; then
+                        dump_file="$final_file"
+                    else
+                        log_error "Could not rename transferred dump to $final_file"
+                        result=1
+                    fi
+                fi
+            fi
+
             # Cleanup based on FROM_KEEP — ONLY after a successful transfer, so a
             # failed copy never deletes the only remaining copy on the source.
             if [ $result -eq 0 ] && [ "$FROM_KEEP" -eq 0 ]; then
@@ -168,7 +234,7 @@ cmd_dump() {
             elif [ $result -eq 0 ] && [ "$FROM_KEEP" -gt 0 ]; then
                 log_info "Keeping last $FROM_KEEP dump(s) on source..."
                 local dump_base_name="${DUMP_NAME:-$FROM_DATABASE}"
-                local cleanup_cmd="cd $(pq "$remote_dump_dir") && ls -t $(pq "$dump_base_name")_*.dump 2>/dev/null | tail -n +$((FROM_KEEP + 1)) | xargs -r rm -f"
+                local cleanup_cmd="cd $(pq "$remote_dump_dir") && ls -t $(pq "$dump_base_name")_*.dump* 2>/dev/null | tail -n +$((FROM_KEEP + 1)) | xargs -r rm -f"
                 ssh -p "$FROM_SSH_PORT" "${SSH_OPTS[@]}" "${FROM_SSH_USER}@${FROM_SSH_HOST}" "$cleanup_cmd"
             elif [ $result -eq 0 ]; then
                 log_info "Keeping source dump (--from-keep -1)"
@@ -193,7 +259,8 @@ cmd_dump() {
         # External compression must happen BEFORE metadata is written, and
         # dump_file must be advanced to the compressed artifact so meta_write
         # sees an existing file (and DUMP_FILE reflects what really exists).
-        if [ "$COMPRESS" != "gzip" ] && [ "$COMPRESS" != "none" ]; then
+        # Skipped when the file already arrived compressed from the source.
+        if [ "$remote_compressed" != true ] && [ "$COMPRESS" != "gzip" ] && [ "$COMPRESS" != "none" ]; then
             if compress_file "$dump_file"; then
                 case "$COMPRESS" in
                     zstd)  dump_file="${dump_file}.zst" ;;

src/62-fetch.sh

@@ -41,7 +41,7 @@ cmd_fetch() {
         # Auto-find latest dump for database (--from-file or --from-file latest)
         log_info "Finding latest dump for: $FROM_DATABASE"
         remote_file=$(ssh -p "$FROM_SSH_PORT" "${SSH_OPTS[@]}" "${FROM_SSH_USER}@${FROM_SSH_HOST}" \
-            "ls -t /tmp/t-pgsql/$(pq "$FROM_DATABASE")_*.dump 2>/dev/null | head -1")
+            "ls -t /tmp/t-pgsql/$(pq "$FROM_DATABASE")_*.dump* 2>/dev/null | head -1")
     fi
 
     if [ -z "$remote_file" ]; then