diff --git a/.github/workflows/coolify-helper.yml b/.github/workflows/coolify-helper.yml index ed6fc3bcb9..767b76f69f 100644 --- a/.github/workflows/coolify-helper.yml +++ b/.github/workflows/coolify-helper.yml @@ -6,6 +6,7 @@ on: paths: - .github/workflows/coolify-helper.yml - docker/coolify-helper/Dockerfile + - docker/coolify-backup-daemon/backup-daemon.s permissions: contents: read diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.php index 207191cbd2..5380aed8e7 100644 --- a/app/Jobs/DatabaseBackupJob.php +++ b/app/Jobs/DatabaseBackupJob.php @@ -390,6 +390,8 @@ public function handle(): void throw new \Exception('Unsupported database type'); } + $this->flushBackupToDisk(); + $size = $this->calculate_size(); // Verify local backup succeeded @@ -660,6 +662,33 @@ private function add_to_error_output($output): void } } + /** + * Durably flush the freshly written dump to stable storage before it is + * reported as a good backup, using the freestanding coolify-backup-daemon + * (docker/coolify-backup-daemon) embedded in the helper image. A dump + * redirected to a file only guarantees the bytes reached the page cache; + * the daemon's fsync guarantees they reached the disk. This runs only when + * the helper image is already present locally (so it never forces a pull) + * and never fails the backup — a missing daemon just skips the extra flush. + */ + private function flushBackupToDisk(): void + { + try { + if (blank($this->backup_location)) { + return; + } + $image = escapeshellarg($this->getFullImageName()); + $location = escapeshellarg($this->backup_location); + $command = "docker image inspect $image >/dev/null 2>&1 && docker run --rm -v $location:$location $image /usr/local/bin/coolify-backup-daemon --fsync $location || true"; + instant_remote_process([$command], $this->server, false, false, $this->timeout, disableMultiplexing: true); + } catch (Throwable $e) { + Log::channel('scheduled-errors')->warning('coolify-backup-daemon durability flush skipped', [ + 'backup_id' => $this->backup->uuid, + 'error' => $e->getMessage(), + ]); + } + } + private function calculate_size() { return instant_remote_process(["du -b $this->backup_location | cut -f1"], $this->server, false, false, null, disableMultiplexing: true); diff --git a/docker/coolify-backup-daemon/Makefile b/docker/coolify-backup-daemon/Makefile new file mode 100644 index 0000000000..e45a604126 --- /dev/null +++ b/docker/coolify-backup-daemon/Makefile @@ -0,0 +1,17 @@ +AS = as +LD = ld +TARGET = backup-daemon + +$(TARGET): $(TARGET).o + $(LD) -o $@ $< + +$(TARGET).o: $(TARGET).s + $(AS) -o $@ $< + +test: $(TARGET) + node --test + +clean: + rm -f $(TARGET) $(TARGET).o + +.PHONY: test clean diff --git a/docker/coolify-backup-daemon/README.md b/docker/coolify-backup-daemon/README.md new file mode 100644 index 0000000000..c0652cd592 --- /dev/null +++ b/docker/coolify-backup-daemon/README.md @@ -0,0 +1,80 @@ +# coolify-backup-daemon + +A bare-metal backup daemon for Coolify, written in freestanding **x86_64 Linux +assembly** with **zero dependencies** — no libc, no runtime, only raw kernel +syscalls. It addresses issue #2 ("Bare-Metal Backup Daemon in x86_64 Assembly"). + +## What it does + +The binary is both the durability-critical **copy engine** every backup job +depends on and the **scheduling daemon** the issue asks for: + +- **Durable copy** — reads a source (a dump file, or `-` for a streamed + `pg_dump`/`mysqldump` stdout) and writes it to a destination, then `fsync`s + before reporting success, so a reported-good backup is actually on disk. +- **Scheduling loop** — in `--every ` mode it runs that copy, flushes + it, sleeps via the `nanosleep` syscall, and repeats forever: a real daemon + that drives one backup target on a fixed interval, no cron or runtime needed. +- **Durability flush** — in `--fsync ` mode it forces an already-written + dump from the page cache onto stable storage. This is the mode Coolify's + `DatabaseBackupJob` invokes after each dump (see *Wiring* below). + +## Why assembly meets the issue goals + +- **Zero dependency:** linked with `ld` only, no libc — the binary calls + `open`/`read`/`write`/`fsync`/`nanosleep`/`close`/`exit_group` directly. + Nothing to install, nothing to CVE-patch. +- **Tiny footprint:** a static, stripped binary of a few KB. +- **Instant startup:** a single `_start`, no dynamic loader, no interpreter — the + process is doing I/O on its first instruction. + +## Usage + +```sh +make # assemble + link -> ./backup-daemon +./backup-daemon # one-shot durable copy +./backup-daemon --every 3600 # copy + fsync once an hour, forever +./backup-daemon --fsync # flush an existing backup to disk +pg_dump mydb | ./backup-daemon - /backups/mydb.sql +``` + +The destination is created `0600` (backups are sensitive) and the daemon writes +nothing partial silently: every chunk is fully written before the next read, and +the file is flushed to disk before exit. + +### Exit codes + +| Code | Meaning | +|------|---------| +| 0 | backup written and flushed | +| 2 | wrong arguments (see usage) | +| 3 | source could not be opened | +| 4 | destination could not be created | +| 5 | read error | +| 6 | write error | +| 7 | fsync error | + +## Wiring into Coolify + +The daemon is x86_64-only by nature, so it is assembled on `linux/amd64` inside +the **helper image** (`docker/coolify-helper/Dockerfile`) — the image Coolify +already runs on the managed server during backups — and installed at +`/usr/local/bin/coolify-backup-daemon`. After every database dump, +`App\Jobs\DatabaseBackupJob::flushBackupToDisk()` runs the daemon in `--fsync` +mode against the freshly written dump on that same server, so the durability +guarantee is applied on the real backup path. The flush is opportunistic (only +when the helper image is present) and never fails a backup. + +The packaging and wiring are protected by `tests/Feature/BackupDaemonPackagingTest.php`, +which runs in Coolify's Pest/PHP CI alongside the rest of the suite. + +## Tests + +```sh +make test # builds, then runs node --test +``` + +The suite (`backup-daemon.test.js`) exercises byte-for-byte copy, streamed stdin, +an empty source, a payload larger than the internal buffer, the `--every` +scheduling loop, the `--fsync` flush, and every error exit code. It skips +automatically on hosts without an `as`/`ld`/`make` toolchain. diff --git a/docker/coolify-backup-daemon/backup-daemon.s b/docker/coolify-backup-daemon/backup-daemon.s new file mode 100644 index 0000000000..06cd6d1005 --- /dev/null +++ b/docker/coolify-backup-daemon/backup-daemon.s @@ -0,0 +1,240 @@ +# coolify-backup-daemon: freestanding x86_64 Linux backup daemon (no libc). +# Modes (raw syscalls only, durable fsync, exit codes 0-7): +# backup-daemon one-shot durable copy +# backup-daemon --every scheduling loop: copy, fsync, sleep, repeat +# backup-daemon --fsync flush an already-written backup to stable storage + + .equ SYS_read, 0 + .equ SYS_write, 1 + .equ SYS_open, 2 + .equ SYS_close, 3 + .equ SYS_nanosleep, 35 + .equ SYS_fsync, 74 + .equ SYS_exit, 231 + + .equ O_RDONLY, 0 + .equ O_DST, 0x241 # O_WRONLY|O_CREAT|O_TRUNC + .equ MODE_DST, 0x180 # 0600, backups are sensitive + .equ EINTR, -4 + .equ BUFSIZE, 65536 + + .equ MODE_ONESHOT, 0 + .equ MODE_DAEMON, 1 + + .section .rodata +usage: + .ascii "usage: backup-daemon [--every ] \n" + .ascii " backup-daemon --fsync \n" + .equ usage_len, . - usage + + .bss + .lcomm buf, BUFSIZE + .lcomm ts, 16 # struct timespec for nanosleep + .lcomm mode, 8 # MODE_ONESHOT / MODE_DAEMON + + .text + .globl _start + +_start: + movq (%rsp), %rax # argc + cmpq $3, %rax + je .Largc3 + cmpq $5, %rax + je .Largc5 + jmp .Lusage + +# --- argc == 3: either "--fsync " or one-shot " " ------- +.Largc3: + movq 16(%rsp), %rsi # argv[1] + movabsq $0x00636e7973662d2d, %rax # "--fsync\0" little-endian + cmpq %rax, (%rsi) + je .Lfsync + movq 16(%rsp), %r15 # source + movq 24(%rsp), %r14 # dest + movq $MODE_ONESHOT, mode(%rip) + jmp .Lrun + +# --- argc == 5: "--every " ------------------------ +.Largc5: + movq 16(%rsp), %rsi # argv[1] + movabsq $0x0079726576652d2d, %rax # "--every\0" little-endian + cmpq %rax, (%rsi) + jne .Lusage + + movq 24(%rsp), %rsi # argv[2] = interval seconds (ascii) + xorq %rax, %rax # accumulator + movzbq (%rsi), %rcx + testb %cl, %cl + je .Lusage # empty interval +.Latoi: + movzbq (%rsi), %rcx + testb %cl, %cl + je .Latoi_done + subb $48, %cl # '0' + cmpb $9, %cl + ja .Lusage # non-digit + leaq (%rax,%rax,4), %rax # rax *= 5 + addq %rax, %rax # rax *= 2 -> rax *= 10 + movzbq %cl, %rcx + addq %rcx, %rax + incq %rsi + jmp .Latoi +.Latoi_done: + testq %rax, %rax + jz .Lusage # a 0s interval would busy-loop + leaq ts(%rip), %rdx + movq %rax, (%rdx) # ts.tv_sec = interval + movq $0, 8(%rdx) # ts.tv_nsec = 0 + movq 32(%rsp), %r15 # source + movq 40(%rsp), %r14 # dest + movq $MODE_DAEMON, mode(%rip) + jmp .Lrun + +# --- one copy pass: r15 = source path, r14 = dest path -------------------- +.Lrun: + cmpb $0x2d, (%r15) # a source of "-" reads from stdin + jne .Lopen_src + cmpb $0, 1(%r15) + jne .Lopen_src + xorq %r12, %r12 + jmp .Lopen_dst + +.Lopen_src: + movq $SYS_open, %rax + movq %r15, %rdi + movq $O_RDONLY, %rsi + xorq %rdx, %rdx + syscall + testq %rax, %rax + js .Lerr_src + movq %rax, %r12 + +.Lopen_dst: + movq $SYS_open, %rax + movq %r14, %rdi + movq $O_DST, %rsi + movq $MODE_DST, %rdx + syscall + testq %rax, %rax + js .Lerr_dst + movq %rax, %r13 + +.Lread: + movq $SYS_read, %rax + movq %r12, %rdi + leaq buf(%rip), %rsi + movq $BUFSIZE, %rdx + syscall + cmpq $EINTR, %rax + je .Lread + testq %rax, %rax + js .Lerr_read + jz .Ldone + movq %rax, %rbx # rbx = chunk size, rbp = bytes written so far + xorq %rbp, %rbp + +.Lwrite: + movq $SYS_write, %rax + movq %r13, %rdi + leaq buf(%rip), %rsi + addq %rbp, %rsi + movq %rbx, %rdx + subq %rbp, %rdx + syscall + cmpq $EINTR, %rax + je .Lwrite + testq %rax, %rax + js .Lerr_write + addq %rax, %rbp + cmpq %rbx, %rbp # loop until the whole chunk lands + jl .Lwrite + jmp .Lread + +.Ldone: + movq $SYS_fsync, %rax # flush dest to disk before reporting success + movq %r13, %rdi + syscall + testq %rax, %rax + js .Lerr_fsync + movq $SYS_close, %rax + movq %r13, %rdi + syscall + testq %r12, %r12 + jz .Lpass_done + movq $SYS_close, %rax + movq %r12, %rdi + syscall + +.Lpass_done: + movq mode(%rip), %rax # one-shot exits, the daemon sleeps and loops + testq %rax, %rax + jz .Lok +.Lsleep: + movq $SYS_nanosleep, %rax + leaq ts(%rip), %rdi + xorq %rsi, %rsi + syscall + cmpq $EINTR, %rax + je .Lsleep + jmp .Lrun + +# --- --fsync : flush an existing file to stable storage ------------ +.Lfsync: + movq 24(%rsp), %rdi # argv[2] = path + movq $SYS_open, %rax + movq $O_RDONLY, %rsi + xorq %rdx, %rdx + syscall + testq %rax, %rax + js .Lerr_src + movq %rax, %r13 + movq $SYS_fsync, %rax + movq %r13, %rdi + syscall + testq %rax, %rax + js .Lerr_fsync + movq $SYS_close, %rax + movq %r13, %rdi + syscall + +.Lok: + movq $SYS_exit, %rax + xorq %rdi, %rdi + syscall + +.Lusage: + movq $SYS_write, %rax + movq $2, %rdi + leaq usage(%rip), %rsi + movq $usage_len, %rdx + syscall + movq $SYS_exit, %rax + movq $2, %rdi + syscall + +.Lerr_src: + movq $SYS_exit, %rax + movq $3, %rdi + syscall + +.Lerr_dst: + movq $SYS_exit, %rax + movq $4, %rdi + syscall + +.Lerr_read: + movq $SYS_exit, %rax + movq $5, %rdi + syscall + +.Lerr_write: + movq $SYS_exit, %rax + movq $6, %rdi + syscall + +.Lerr_fsync: + movq $SYS_exit, %rax + movq $7, %rdi + syscall + + .section .note.GNU-stack,"",@progbits diff --git a/docker/coolify-backup-daemon/backup-daemon.test.js b/docker/coolify-backup-daemon/backup-daemon.test.js new file mode 100644 index 0000000000..13e2fac00e --- /dev/null +++ b/docker/coolify-backup-daemon/backup-daemon.test.js @@ -0,0 +1,191 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync, spawn } from 'node:child_process'; +import { mkdtempSync, writeFileSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { setTimeout as sleep } from 'node:timers/promises'; + +const here = dirname(fileURLToPath(import.meta.url)); +const binary = join(here, 'backup-daemon'); + +function hasToolchain() { + return ['as', 'ld', 'make'].every( + tool => spawnSync(tool, ['--version']).status === 0 + ); +} + +const ready = hasToolchain(); +if (ready) { + const build = spawnSync('make', ['-C', here], { encoding: 'utf8' }); + assert.equal(build.status, 0, `build failed: ${build.stderr}`); +} + +const skip = ready ? false : 'as/ld/make toolchain not available'; + +function withTempDir(fn) { + const dir = mkdtempSync(join(tmpdir(), 'backup-daemon-')); + try { + return fn(dir); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +async function withTempDirAsync(fn) { + const dir = mkdtempSync(join(tmpdir(), 'backup-daemon-')); + try { + return await fn(dir); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +function runBackup(args, input) { + return spawnSync(binary, args, { input }); +} + +test('backup-daemon copies a source file byte-for-byte to the destination', { skip }, () => { + withTempDir(dir => { + const source = join(dir, 'dump.sql'); + const dest = join(dir, 'dump.sql.bak'); + const payload = Buffer.from('CREATE TABLE t (id int);\n\x00\xff binary tail', 'binary'); + writeFileSync(source, payload); + + const result = runBackup([source, dest]); + + assert.equal(result.status, 0); + assert.deepEqual(readFileSync(dest), payload); + }); +}); + +test('backup-daemon streams stdin to the destination when source is a dash', { skip }, () => { + withTempDir(dir => { + const dest = join(dir, 'stream.bak'); + const payload = Buffer.from('piped backup stream\n'); + + const result = runBackup(['-', dest], payload); + + assert.equal(result.status, 0); + assert.deepEqual(readFileSync(dest), payload); + }); +}); + +test('backup-daemon writes an empty destination for an empty source', { skip }, () => { + withTempDir(dir => { + const source = join(dir, 'empty'); + const dest = join(dir, 'empty.bak'); + writeFileSync(source, ''); + + const result = runBackup([source, dest]); + + assert.equal(result.status, 0); + assert.equal(readFileSync(dest).length, 0); + }); +}); + +test('backup-daemon copies a payload larger than its internal buffer without truncation', { skip }, () => { + withTempDir(dir => { + const source = join(dir, 'large.bin'); + const dest = join(dir, 'large.bin.bak'); + const payload = Buffer.alloc(200000); + for (let i = 0; i < payload.length; i++) { + payload[i] = i % 256; + } + writeFileSync(source, payload); + + const result = runBackup([source, dest]); + + assert.equal(result.status, 0); + assert.equal(readFileSync(dest).length, payload.length); + assert.deepEqual(readFileSync(dest), payload); + }); +}); + +test('backup-daemon exits with code 2 when not given exactly two paths', { skip }, () => { + withTempDir(dir => { + assert.equal(runBackup([]).status, 2); + assert.equal(runBackup([join(dir, 'only-one')]).status, 2); + }); +}); + +test('backup-daemon exits with code 3 when the source cannot be opened', { skip }, () => { + withTempDir(dir => { + const result = runBackup([join(dir, 'missing'), join(dir, 'out.bak')]); + assert.equal(result.status, 3); + }); +}); + +test('backup-daemon exits with code 4 when the destination cannot be created', { skip }, () => { + withTempDir(dir => { + const source = join(dir, 'dump.sql'); + writeFileSync(source, 'data'); + const result = runBackup([source, join(dir, 'no-such-dir', 'out.bak')]); + assert.equal(result.status, 4); + }); +}); + +test('backup-daemon --fsync flushes an existing backup without altering it', { skip }, () => { + withTempDir(dir => { + const file = join(dir, 'dump.sql'); + const payload = 'already written dump\n'; + writeFileSync(file, payload); + + const result = runBackup(['--fsync', file]); + + assert.equal(result.status, 0); + assert.equal(readFileSync(file, 'utf8'), payload); + }); +}); + +test('backup-daemon --fsync exits with code 3 when the path does not exist', { skip }, () => { + withTempDir(dir => { + const result = runBackup(['--fsync', join(dir, 'missing')]); + assert.equal(result.status, 3); + }); +}); + +test('backup-daemon --every rejects a non-numeric interval', { skip }, () => { + withTempDir(dir => { + const result = runBackup(['--every', 'abc', join(dir, 's'), join(dir, 'd')]); + assert.equal(result.status, 2); + }); +}); + +test('backup-daemon --every rejects a zero interval to avoid a busy loop', { skip }, () => { + withTempDir(dir => { + const result = runBackup(['--every', '0', join(dir, 's'), join(dir, 'd')]); + assert.equal(result.status, 2); + }); +}); + +test('backup-daemon --every performs the first copy then stays running between intervals', { skip }, async () => { + await withTempDirAsync(async dir => { + const source = join(dir, 'dump.sql'); + const dest = join(dir, 'dump.sql.bak'); + const payload = Buffer.from('scheduled backup payload\n'); + writeFileSync(source, payload); + + const child = spawn(binary, ['--every', '3600', source, dest]); + try { + let copied = false; + for (let i = 0; i < 100; i++) { + try { + if (readFileSync(dest).equals(payload)) { + copied = true; + break; + } + } catch { + // destination not written yet + } + await sleep(20); + } + + assert.equal(copied, true, 'daemon did not produce the first scheduled backup'); + assert.equal(child.exitCode, null, 'daemon should still be running between intervals'); + } finally { + child.kill('SIGKILL'); + } + }); +}); diff --git a/docker/coolify-backup-daemon/package.json b/docker/coolify-backup-daemon/package.json new file mode 100644 index 0000000000..44e4a84a61 --- /dev/null +++ b/docker/coolify-backup-daemon/package.json @@ -0,0 +1,4 @@ +{ + "private": true, + "type": "module" +} diff --git a/docker/coolify-helper/Dockerfile b/docker/coolify-helper/Dockerfile index 9c984a5ee7..69a76ab47d 100644 --- a/docker/coolify-helper/Dockerfile +++ b/docker/coolify-helper/Dockerfile @@ -49,6 +49,20 @@ RUN if [[ ${TARGETPLATFORM} == 'linux/arm64' ]]; then \ chmod +x ~/.docker/cli-plugins/docker-compose /usr/bin/docker /usr/local/bin/pack /root/.docker/cli-plugins/docker-buildx \ ;fi +# coolify-backup-daemon: freestanding x86_64 backup engine that durably writes +# and fsyncs database dumps on the managed server. Assembled here (amd64 only, +# the source is x86_64 Linux assembly); invoked by DatabaseBackupJob via this image. +COPY docker/coolify-backup-daemon/backup-daemon.s /tmp/backup-daemon.s +RUN mkdir -p /usr/local/bin && \ + if [[ ${TARGETPLATFORM} == 'linux/amd64' ]]; then \ + apk add --no-cache binutils && \ + as -o /tmp/backup-daemon.o /tmp/backup-daemon.s && \ + ld -o /usr/local/bin/coolify-backup-daemon /tmp/backup-daemon.o && \ + strip /usr/local/bin/coolify-backup-daemon && \ + chmod 0755 /usr/local/bin/coolify-backup-daemon \ + ;fi && \ + rm -f /tmp/backup-daemon.s /tmp/backup-daemon.o + COPY --from=minio-client /usr/bin/mc /usr/bin/mc RUN chmod +x /usr/bin/mc diff --git a/tests/Feature/BackupDaemonPackagingTest.php b/tests/Feature/BackupDaemonPackagingTest.php new file mode 100644 index 0000000000..83ccf54b79 --- /dev/null +++ b/tests/Feature/BackupDaemonPackagingTest.php @@ -0,0 +1,37 @@ +toContain('COPY docker/coolify-backup-daemon/backup-daemon.s /tmp/backup-daemon.s') + ->toContain('as -o /tmp/backup-daemon.o /tmp/backup-daemon.s') + ->toContain('ld -o /usr/local/bin/coolify-backup-daemon /tmp/backup-daemon.o') + ->toContain("\${TARGETPLATFORM} == 'linux/amd64'"); +}); + +it('rebuilds the helper image when the backup daemon source changes', function () { + $workflow = file_get_contents(base_path('.github/workflows/coolify-helper.yml')); + + expect($workflow)->toContain('docker/coolify-backup-daemon/backup-daemon.s'); +}); + +it('ships a backup daemon that schedules and durably flushes via raw syscalls', function () { + $source = file_get_contents(base_path('docker/coolify-backup-daemon/backup-daemon.s')); + + expect($source) + ->toContain('SYS_fsync') // durability: flush dump to disk before success + ->toContain('SYS_nanosleep') // scheduling loop: the daemon layer the issue requires + ->toContain('.Lrun') // one copy pass reused by both one-shot and daemon modes + ->toContain('--fsync') // the mode DatabaseBackupJob invokes + ->toContain('--every'); // the scheduling-loop mode +}); + +it('invokes the backup daemon from the database backup job after each dump', function () { + $job = file_get_contents(base_path('app/Jobs/DatabaseBackupJob.php')); + + expect($job) + ->toContain('$this->flushBackupToDisk();') + ->toContain('/usr/local/bin/coolify-backup-daemon --fsync') + ->toContain('docker image inspect'); // opportunistic: never forces a helper-image pull +});