Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 14 additions & 1 deletion .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -24,5 +24,18 @@ SRTL_COOKIE_SECURE=false
SRTL_ALLOWED_ORIGINS=
SRTL_TRUST_PROXY=false

# Reserved for future multi-worker support. The current release enforces one worker.
# Parallel job slots hosted by the single worker service. One preserves the
# existing serial behavior. The total limit defaults to SRTL_WORKER_COUNT and
# per-type limits default to that total. Copy file concurrency defaults to one;
# the independent process-wide active-file limit defaults to the worker count.
SRTL_WORKER_COUNT=1
# SRTL_MAX_RUNNING_JOBS=1
# SRTL_MAX_RUNNING_SCANS=1
# SRTL_MAX_RUNNING_AUDITS=1
# SRTL_MAX_RUNNING_COPIES=1
# SRTL_COPY_FILE_CONCURRENCY=1
# SRTL_MAX_ACTIVE_COPY_FILES=1

# Terminal job, event, audit, and scan history retention. Zero preserves all
# history. Jobs with unresolved copy recovery state are never removed.
SRTL_JOB_HISTORY_RETENTION_DAYS=90
74 changes: 72 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,12 @@ on:
push:
branches: [beta, main]
workflow_dispatch:
inputs:
publish_beta:
description: Publish the current beta prerelease container
required: false
type: boolean
default: false

permissions:
contents: read
Expand Down Expand Up @@ -35,7 +41,7 @@ jobs:
timeout-minutes: 30
services:
postgres:
image: postgres:17-alpine
image: postgres:17-alpine@sha256:742f40ea20b9ff2ff31db5458d127452988a2164df9e17441e191f3b72252193
env:
POSTGRES_DB: postgres
POSTGRES_USER: srtl_test
Expand Down Expand Up @@ -75,7 +81,7 @@ jobs:
npm run start:api > /tmp/srtl-api.log 2>&1 &
echo $! > /tmp/srtl-api.pid
for _ in {1..30}; do
curl -fsS http://127.0.0.1:3010/api/health && exit 0
curl -fsS http://127.0.0.1:3010/api/health/live && exit 0
sleep 1
done
cat /tmp/srtl-api.log
Expand Down Expand Up @@ -142,3 +148,67 @@ jobs:
run: |
test ! -f /tmp/srtl-api.log || tail -200 /tmp/srtl-api.log
test ! -f /tmp/srtl-worker.log || tail -200 /tmp/srtl-worker.log

publish-beta:
if: github.event_name == 'workflow_dispatch' && inputs.publish_beta == true
needs: verify
runs-on: ubuntu-latest
timeout-minutes: 45
permissions:
contents: read
packages: write
id-token: write
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- id: version
name: Validate beta publication
run: |
if [[ "$GITHUB_REF_NAME" != "beta" ]]; then
echo "Beta images may only be published from the beta branch."
exit 1
fi
version="$(node -p 'require("./package.json").version')"
if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+-beta\.[1-9][0-9]*$ ]]; then
echo "Beta versions must use the form 0.1.2-beta.1."
exit 1
fi
echo "value=$version" >> "$GITHUB_OUTPUT"
- name: Validate container definition
run: docker build --check .
- uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4
- uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4
- uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- id: metadata
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6
with:
images: ghcr.io/${{ github.repository_owner }}/srtl-manager
tags: |
type=raw,value=${{ steps.version.outputs.value }}
- uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
with:
context: .
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ steps.metadata.outputs.tags }}
labels: ${{ steps.metadata.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
provenance: mode=max
sbom: true
- name: Scan the published beta image
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
with:
image-ref: ghcr.io/${{ github.repository_owner }}/srtl-manager:${{ steps.version.outputs.value }}
format: table
exit-code: "1"
ignore-unfixed: true
severity: CRITICAL,HIGH
- name: Promote the verified image to beta
env:
IMAGE_REF: ghcr.io/${{ github.repository_owner }}/srtl-manager
VERSION: ${{ steps.version.outputs.value }}
run: docker buildx imagetools create --tag "$IMAGE_REF:beta" "$IMAGE_REF:$VERSION"
2 changes: 1 addition & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ jobs:
run: docker build --check .
- uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4
- uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4
- uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4
- uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
with:
registry: ghcr.io
username: ${{ github.actor }}
Expand Down
50 changes: 50 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,56 @@ All notable changes are documented here. The project follows Semantic Versioning

## [Unreleased]

## [0.1.2] - 2026-08-04

### Added

- Added immutable database-backed job selections so large jobs retain exact, bounded title details without embedding unbounded link arrays in progress payloads.
- Added operator-assisted copy-reconciliation status and safe automatic recovery for journal entries whose current filesystem identity proves their final state.
- Added separate liveness and readiness health checks, periodic configurable terminal-history and expired-session retention, and indexes for worker, audit, and cleanup workloads.

### Changed

- Paginated audit findings, pushed library filters and inventory counts into Postgres, and removed per-result scanner lookups that made large libraries progressively slower.
- Frozen copy behavior when jobs are admitted, including explicit per-job overrides for source-title mismatches, so later settings changes cannot alter queued work.
- Hardened development synchronization with destination ownership checks, argv-safe transfer options, and a non-mutating dry-run mode.
- Decoupled per-file copy concurrency from job-slot count while retaining an explicit process-wide active-file ceiling, validated legacy integrity constraints, and pinned the PostgreSQL runtime image.
- Added configurable in-process worker slots and independent global, per-job-type, and copy-transfer concurrency limits without an arbitrary worker-count ceiling.
- Kept example deployments at one worker slot by default while honoring any positive `SRTL_WORKER_COUNT` value from `.env`.
- Allowed non-overlapping copy, audit, and targeted title-rescan work to run concurrently while broad scans and path migrations remain exclusive.
- Made targeted title rescans validate readable symlink targets, reconcile their exact storage files, and report persistent read failures.
- Retried transient source and transfer I/O failures before failing a copy.
- Matched administrator usernames case-insensitively for login and account conflicts while preserving display capitalization.
- Replaced layout-shifting dashboard action messages with responsive overlay notifications.
- Displayed a copy job's title directly when all selected links belong to one title, while retaining the title list for multi-title jobs.

### Fixed

- Isolated concurrent copy progress so one file transfer cannot leak totals or current-file details into another job or transfer.
- Prevented replaced destination files from being automatically reconciled unless their recorded identity still matches the durable copy journal.
- Closed legacy destination-only reconciliation journals when the original symlink is intact and no temporary or displaced artifacts remain, preserving the unlinked destination for normal conflict handling instead of indefinitely blocking retries.
- Persisted automatic copy-reconciliation resolutions so later service restarts and path checks cannot reactivate already-settled legacy journals.
- Cleared failed copy-submission state whenever the copy dialog selection changes so an earlier title's admission error cannot appear on a later title.
- Allowed superseded recovery records to age out with their terminal jobs while continuing to preserve genuinely unresolved copy state.
- Prevented queued jobs from being reinterpreted after section settings change, rejected malformed password hashes safely, expired stale sessions promptly, and surfaced corrupt stored settings instead of silently substituting defaults.
- Accepted selections beyond the former 1,000-link request limit and loaded large audit result sets incrementally in the interface.
- Prevented superseded legacy copy-reconciliation records from blocking newly scanned media while retaining exact media and managed-path safeguards for genuinely unresolved filesystem state.
- Limited newly queued scoped copy jobs to their actionable media so already satisfied title links no longer inflate job totals or selected-title details.
- Batched large selected-link title lookups and restored title tooltips for multi-link jobs without exceeding the API request limit.
- Derived storage-file assignment exclusively from current linked symlinks so unlinked files cannot remain assigned to a storage location.
- Reconciled legacy storage-file policies during migration and after scans or policy updates.
- Made queue admission, worker claims, stale-job recovery, and job updates lease-aware so overlapping or superseded workers cannot mutate the same job.
- Preserved immutable job resource scopes so later inventory changes cannot remove an active job's overlap protection.
- Scoped legacy failed-copy reconciliation locks to their exact media records and managed paths so newly scanned items from the same title can still be queued.
- Allowed filesystem-read-only scans and audits to run while terminal legacy copy records await reconciliation; path migration and exact conflicting mutations remain fenced.
- Loaded every page of dashboard work lists and made show and season copy actions server-scoped so large sections are never truncated to the first 250 links.
- Accepted routine FUSE and NFS remounts without a false path migration when the canonical path and stable mount signature are unchanged, while retaining exact identity checks during active mutations.

### Security

- Patched current high- and moderate-severity transitive dependency advisories in `fast-uri`, `brace-expansion`, `undici`, and `postcss`.
- Updated the pinned Docker registry login action to its hardened 4.6.0 release.

## [0.1.1] - 2026-07-25

### Added
Expand Down
25 changes: 20 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,13 @@ SRTL Manager is a local-first web app for inventorying and maintaining a symlink
## Highlights

- Guided first-run account, path, section, policy, and initial-scan setup.
- Symlink, local-root, remote-root, and orphan inventory with targeted title rescans.
- Symlink, local-root, remote-root, and orphan inventory with targeted title rescans and exact target checks.
- Per-location storage policies using editable friendly names, plus an Unassigned queue for newly discovered titles.
- Fast and deep audits across local and remote targets.
- Bidirectional copies with live progress, conflict handling, configurable verification, and source/title risk checks.
- Bidirectional copies with live progress, transient transfer retry, conflict handling, configurable verification, and source/title risk checks.
- Safe job termination, complete event timelines, restart recovery, and durable per-file copy journals.
- Required path-change review before changed mounts can affect managed links.
- Configurable parallel job and copy-file execution with database-backed overlap protection.
- Required path-change review before different configured roots or storage mount sources can affect managed links.
- Editable storage-location names while deployment paths remain environment-managed.
- Postgres-backed API and worker services in a hardened container stack.
- Dark, light, and system themes with responsive administration views.
Expand Down Expand Up @@ -73,9 +74,21 @@ The default configuration follows the current stable `latest` image. Pin `SRTL_I

The API receives read-only root mounts. The worker alone receives writable roots for copy and path-migration jobs. Postgres is reachable only inside the Compose network.

### Worker concurrency

The single Compose worker service can host any positive number of independent job slots. Set `SRTL_WORKER_COUNT` to the desired slot count; there is no fixed two-worker ceiling. Jobs still pass database-backed admission, resource-claim, and lease checks, so adding slots does not allow two jobs to mutate the same managed resources.

The optional `SRTL_MAX_RUNNING_JOBS` setting limits total simultaneous jobs and must not exceed `SRTL_WORKER_COUNT`. `SRTL_MAX_RUNNING_SCANS`, `SRTL_MAX_RUNNING_AUDITS`, and `SRTL_MAX_RUNNING_COPIES` apply per-type limits, may be zero to pause that job type, and must not exceed the total-job limit. When omitted, the total limit follows the configured worker count and the per-type limits follow that total limit.

`SRTL_COPY_FILE_CONCURRENCY` controls how many files one copy job may transfer at once. `SRTL_MAX_ACTIVE_COPY_FILES` is the independent worker process-wide copy-file ceiling and must be at least the per-job value. Their defaults keep one file active per copy job while allowing separate copy jobs to use separate slots. Setting both values above the worker count is supported, including parallel file transfers from one copy job in a one-slot worker; start conservatively and raise copy limits only when the storage endpoints and network can sustain the additional I/O.

Scale with `SRTL_WORKER_COUNT`; do not simultaneously run `docker compose up --scale worker=...`. The supported deployment model keeps job slots and the active-copy-file safeguard inside one worker process. Compose gives that process two minutes to stop active jobs and perform safe rollback during shutdown.

`SRTL_JOB_HISTORY_RETENTION_DAYS` controls automatic cleanup of terminal job, event, audit, and scan history. The default is 90 days; set it to `0` to preserve all history. Jobs with unresolved copy-recovery state are always retained.

The API checks the public GitHub Releases endpoint for stable and beta version information at startup and when version status is refreshed. This request does not include credentials, paths, or inventory data.

When a configured root changes, restart the stack. The UI enters maintenance mode until it validates and applies a path migration or the prior value is restored. This rebases managed paths; it does not move stored content.
When a configured root changes, restart the stack. The UI enters maintenance mode until it validates and applies a path migration or the prior value is restored. This rebases managed paths; it does not move stored content. Routine Linux remounts are accepted automatically when the canonical path, mount point, filesystem type, and mount source are unchanged; exact device and inode checks still fence active filesystem mutations.

## Backup And Restore

Expand Down Expand Up @@ -121,6 +134,8 @@ npm run test:e2e

Stop the disposable database with `docker stop srtl-manager-dev-postgres`. To use an existing Postgres instance instead, provide the `SRTL_POSTGRES_*` settings or `SRTL_DATABASE_URL` in a local `.env`.

The optional `npm run sync:server` helper synchronizes source to an existing development server selected by `SRTL_REMOTE` (or `SRTL_REMOTE_HOST`). It preserves remote `.env`, data, dependencies, and logs, refuses an unrelated non-empty destination, and supports a non-mutating preview with `SRTL_SYNC_DRY_RUN=1`. A normal first sync initializes the destination marker that later syncs require.

## Releases And Contributions

- Pull requests target `beta`.
Expand All @@ -142,4 +157,4 @@ SRTL Manager is available under the [MIT License](LICENSE). Contributions are ac
- Optional remote source-availability preflight checks where providers expose reliable metadata.
- Event-driven targeted refresh hooks.
- Additional numbered storage locations and per-location assignment policies.
- Controlled multi-worker execution after single-worker recovery semantics are fully proven.
- Per-location throughput telemetry and concurrency recommendations.
7 changes: 5 additions & 2 deletions docker-compose.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
services:
postgres:
image: postgres:17-alpine
image: postgres:17-alpine@sha256:742f40ea20b9ff2ff31db5458d127452988a2164df9e17441e191f3b72252193
restart: unless-stopped
environment:
POSTGRES_DB: ${SRTL_POSTGRES_DB:?Set SRTL_POSTGRES_DB in .env}
Expand Down Expand Up @@ -65,7 +65,7 @@ services:
tmpfs:
- /tmp:size=256m,mode=1777
healthcheck:
test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:3010/api/health').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"]
test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:3010/api/health/live').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"]
interval: 10s
timeout: 5s
retries: 12
Expand All @@ -84,6 +84,9 @@ services:
image: ${SRTL_IMAGE:?Set SRTL_IMAGE in .env}
command: ["node", "dist/server/worker.js"]
restart: unless-stopped
# SRTL_WORKER_COUNT scales safe job slots inside this service. Do not also
# use `docker compose --scale worker=...` for the same deployment.
stop_grace_period: 2m
user: "${SRTL_UID:?Set SRTL_UID in .env}:${SRTL_GID:?Set SRTL_GID in .env}"
read_only: true
env_file:
Expand Down
Loading