From 0c84d9f07506f5f34fed5a8f10063865e97e15b5 Mon Sep 17 00:00:00 2001 From: Aleksey Dmitriev Date: Sat, 25 Jul 2026 19:09:25 -0400 Subject: [PATCH 01/11] Start 0.1.2 beta 1 --- package-lock.json | 4 ++-- package.json | 2 +- tests/app.test.ts | 16 ++++++++-------- tests/e2e/app-smoke.spec.ts | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/package-lock.json b/package-lock.json index df0f5ac..db0d4b8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "srtl-manager", - "version": "0.1.1", + "version": "0.1.2-beta.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "srtl-manager", - "version": "0.1.1", + "version": "0.1.2-beta.1", "license": "MIT", "dependencies": { "@fastify/compress": "^9.1.0", diff --git a/package.json b/package.json index a98664f..ac94c35 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "srtl-manager", - "version": "0.1.1", + "version": "0.1.2-beta.1", "private": true, "license": "MIT", "homepage": "https://github.com/ramphex/SRTL-Manager#readme", diff --git a/tests/app.test.ts b/tests/app.test.ts index 3b8f01b..c8fa223 100644 --- a/tests/app.test.ts +++ b/tests/app.test.ts @@ -791,9 +791,9 @@ describe("api app", () => { expect(version.statusCode).toBe(200); expect(version.json()).toMatchObject({ - currentVersion: "0.1.1", - currentChannel: "stable", - currentChannelLabel: "Stable", + currentVersion: "0.1.2-beta.1", + currentChannel: "beta", + currentChannelLabel: "Beta", latestVersion: null, updateAvailable: false, status: "unavailable", @@ -858,13 +858,13 @@ describe("api app", () => { expect(version.statusCode).toBe(200); expect(version.json()).toMatchObject({ - currentVersion: "0.1.1", - currentChannel: "stable", - currentChannelLabel: "Stable", - latestVersion: "0.1.1", + currentVersion: "0.1.2-beta.1", + currentChannel: "beta", + currentChannelLabel: "Beta", + latestVersion: "0.2.0-beta.1", updateAvailable: true, status: "update_available", - releaseUrl: "https://github.com/ramphex/srtl-manager/releases/tag/v0.1.1", + releaseUrl: "https://github.com/ramphex/srtl-manager/releases/tag/v0.2.0-beta.1", message: "Beta v0.2.0-beta.1 available", checkedAt: expect.any(String), stable: { diff --git a/tests/e2e/app-smoke.spec.ts b/tests/e2e/app-smoke.spec.ts index 01cc4a0..751d713 100644 --- a/tests/e2e/app-smoke.spec.ts +++ b/tests/e2e/app-smoke.spec.ts @@ -131,7 +131,7 @@ test("refreshes an open work list when an inventory job finishes", async ({ page else if (url.pathname === "/api/settings/storage-locations") body = { locations: [{ key: "location_1", rootType: "local", displayName: "Local", path: "/mnt/local" }, { key: "location_2", rootType: "remote", displayName: "Remote", path: "/mnt/remote" }] }; else if (url.pathname === "/api/system/version") { const unavailableRelease = { latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, releaseNotes: null, message: "Unavailable" }; - body = { currentVersion: "0.1.1", currentChannel: "stable", currentChannelLabel: "Stable", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; + body = { currentVersion: "0.1.2-beta.1", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; } else if (url.pathname === "/api/settings/sections") body = { sections: ["shows"], sectionTitles: { shows: "Shows" }, sectionTypes: { shows: "shows" } }; else if (url.pathname === "/api/settings/paths") body = { symlinkDir: "/mnt/links", localDir: "/mnt/local", remoteDir: "/mnt/remote" }; @@ -1003,7 +1003,7 @@ test("copy progress opens a persistent, scrollable completed item summary", asyn else if (url.pathname === "/api/settings/storage-locations") body = { locations: [{ key: "location_1", rootType: "local", displayName: "Local", path: "/mnt/local" }, { key: "location_2", rootType: "remote", displayName: "Remote", path: "/mnt/remote" }] }; else if (url.pathname === "/api/system/version") { const unavailableRelease = { latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, releaseNotes: null, message: "Unavailable" }; - body = { currentVersion: "0.1.1", currentChannel: "stable", currentChannelLabel: "Stable", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; + body = { currentVersion: "0.1.2-beta.1", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; } else if (url.pathname === `/api/jobs/${jobId}/events/page`) body = { events, total: events.length, hasOlder: false }; else if (url.pathname === `/api/jobs/${jobId}`) body = job; else if (url.pathname === "/api/jobs") body = [job]; From 6cb67578daea4cb0192f34bfa725d4fc19e43049 Mon Sep 17 00:00:00 2001 From: Aleksey Dmitriev Date: Sun, 26 Jul 2026 21:40:16 -0400 Subject: [PATCH 02/11] Replace dashboard notices with overlay toasts --- src/client/libraryRoutes.tsx | 35 +++++++++++-- src/client/styles.css | 97 ++++++++++++++++++++++++++++++++++++ tests/e2e/app-smoke.spec.ts | 89 +++++++++++++++++++++++++++++++++ 3 files changed, 218 insertions(+), 3 deletions(-) diff --git a/src/client/libraryRoutes.tsx b/src/client/libraryRoutes.tsx index 1864145..e685685 100644 --- a/src/client/libraryRoutes.tsx +++ b/src/client/libraryRoutes.tsx @@ -1,7 +1,7 @@ import { useEffect, useId, useMemo, useRef, useState, type ReactNode } from "react"; import { createPortal } from "react-dom"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import { Activity, ArrowLeft, ChevronRight, Copy, Database, File, FileText, Folder, HardDrive, HardDriveDownload, Library, Link2, ListChecks, RefreshCw, Search, Settings, Shield, Trash2, TriangleAlert, Unlink, X } from "lucide-react"; +import { Activity, ArrowLeft, CheckCircle2, ChevronRight, Copy, Database, File, FileText, Folder, HardDrive, HardDriveDownload, Library, Link2, ListChecks, RefreshCw, Search, Settings, Shield, Trash2, TriangleAlert, Unlink, X } from "lucide-react"; import { api } from "./api"; import { evaluateSourceTitleRisk, type SourceTitleRiskResult } from "../shared/sourceTitleRisk"; import { activeJobForLink, activeJobNotice, activeJobsForLinks, activeJobsForStoragePolicyTitle, isActiveQueueJob, normalizeAuditTargets } from "./jobScopeLocks"; @@ -34,6 +34,35 @@ function titleScopeIsPending(scopes: ScanTitleScope[] | undefined, section: stri return Boolean(scopes?.some((scope) => scope.section === section && scope.itemName === itemName)); } +function ActionToast({ message, tone }: { message: string | null; tone: "success" | "error" }) { + const [visibleMessage, setVisibleMessage] = useState(null); + + useEffect(() => { + setVisibleMessage(message); + if (!message) return; + + const timeout = window.setTimeout(() => setVisibleMessage(null), tone === "error" ? 10_000 : 6_000); + return () => window.clearTimeout(timeout); + }, [message, tone]); + + if (!visibleMessage || typeof document === "undefined") return null; + const Icon = tone === "error" ? TriangleAlert : CheckCircle2; + + return createPortal( +
+ +
+ {tone === "error" ? "Action failed" : "Task queued"} + {visibleMessage} +
+ +
, + document.body + ); +} + export function DashboardPage() { const queryClient = useQueryClient(); const { autoOpenTaskStatus, recentJobsCompletedWindowMinutes } = useUserPreferences(); @@ -197,6 +226,7 @@ export function DashboardPage() { : startScan.data ? `Inventory scan job #${startScan.data.jobId} queued.` : null; + const actionToastMessage = actionError?.message ?? actionMessage; useEffect(() => { if (scanSettings.data) setScanOptions(stripLegacyScanSections(scanSettings.data)); @@ -419,8 +449,7 @@ export function DashboardPage() { - {actionError ?

{actionError.message}

: null} - {!actionError && actionMessage ?

{actionMessage}

: null} +
diff --git a/src/client/styles.css b/src/client/styles.css index ae9ea08..8cd9a35 100644 --- a/src/client/styles.css +++ b/src/client/styles.css @@ -1361,6 +1361,91 @@ tbody tr:has(.job-link-title-tooltip:focus-within) > td:has(.job-link-title-tool color: var(--bad); } +.action-toast { + align-items: start; + animation: action-toast-enter 160ms ease-out; + background: color-mix(in srgb, var(--surface) 94%, var(--bg)); + border: 1px solid var(--border); + border-left-width: 3px; + border-radius: 8px; + bottom: max(20px, env(safe-area-inset-bottom)); + box-shadow: 0 18px 48px rgb(0 0 0 / 0.28); + color: var(--text); + display: grid; + gap: 11px; + grid-template-columns: auto minmax(0, 1fr) auto; + max-width: calc(100vw - 32px); + padding: 13px 12px 13px 14px; + position: fixed; + right: max(20px, env(safe-area-inset-right)); + width: min(380px, calc(100vw - 32px)); + z-index: 90; +} + +.action-toast-success { + border-left-color: var(--good); +} + +.action-toast-error { + border-left-color: var(--bad); +} + +.action-toast-icon { + color: var(--good); + margin-top: 1px; +} + +.action-toast-error .action-toast-icon { + color: var(--bad); +} + +.action-toast-content { + display: grid; + gap: 3px; + min-width: 0; +} + +.action-toast-content strong { + color: var(--text-strong); + font-size: 13px; + line-height: 1.25; +} + +.action-toast-content span { + color: var(--muted-strong); + font-size: 13px; + line-height: 1.4; + overflow-wrap: anywhere; +} + +.action-toast-dismiss { + background: transparent; + border-color: transparent; + color: var(--muted); + min-height: 28px; + padding: 0; + width: 28px; +} + +.action-toast-dismiss:hover, +.action-toast-dismiss:focus-visible { + background: var(--secondary-hover); + border-color: var(--border); + color: var(--text-strong); +} + +@keyframes action-toast-enter { + from { + opacity: 0; + transform: translateY(8px); + } + + to { + opacity: 1; + transform: translateY(0); + } +} + .dashboard-summary { display: grid; gap: 12px; @@ -4198,8 +4283,12 @@ button.danger-button:hover:not(:disabled) { } @media (prefers-reduced-motion: reduce) { + .action-toast, .scan-progress-track.is-live span { animation: none; + } + + .scan-progress-track.is-live span { transform: translateX(130%); } } @@ -5493,6 +5582,14 @@ button.danger-button:hover:not(:disabled) { } @media (max-width: 820px) { + .action-toast { + bottom: max(12px, env(safe-area-inset-bottom)); + left: max(12px, env(safe-area-inset-left)); + max-width: none; + right: max(12px, env(safe-area-inset-right)); + width: auto; + } + .responsive-table { display: block; max-width: 100%; diff --git a/tests/e2e/app-smoke.spec.ts b/tests/e2e/app-smoke.spec.ts index 751d713..65eaddc 100644 --- a/tests/e2e/app-smoke.spec.ts +++ b/tests/e2e/app-smoke.spec.ts @@ -49,6 +49,95 @@ test("loads the authenticated dashboard when a development session is provided", await expect(page.getByText("Library Summary", { exact: true })).toBeVisible(); }); +test("dashboard task notifications overlay without shifting content", async ({ page }) => { + const timestamp = "2026-07-26T12:00:00.000Z"; + const inventory = { + totalLinks: 1, + remoteLinks: 1, + localLinks: 0, + brokenLinks: 0, + otherLinks: 0, + nonMediaLinks: 0, + actionableRemoteLinks: 1, + actionableLocalLinks: 0, + assignedRemoteLinks: 0, + unassignedRemoteLinks: 0, + unassignedLocalLinks: 0, + localFiles: 0, + remoteFiles: 0, + actionableRemoteFiles: 0, + actionableLocalFiles: 0, + assignedRemoteFiles: 0, + unassignedRemoteFiles: 0, + unassignedLocalFiles: 0, + localOrphanFiles: 0, + remoteOrphanFiles: 0, + missingLinks: 0, + missingLocalFiles: 0, + missingRemoteFiles: 0 + }; + + await page.setViewportSize({ width: 1280, height: 800 }); + await page.route("**/api/**", async (route) => { + const url = new URL(route.request().url()); + let body: unknown; + + if (url.pathname === "/api/auth/me") body = { setupRequired: false, authenticated: true, user: { id: 1, username: "admin" } }; + else if (url.pathname === "/api/system/path-migration") body = { status: "ready", blocking: false, activePaths: {}, detectedPaths: {}, environmentErrors: [], changes: [], migration: null }; + else if (url.pathname === "/api/onboarding") body = { required: false, phase: "completed" }; + else if (url.pathname === "/api/settings/user-preferences") body = { timeFormat: "12h", autoOpenTaskStatus: false, recentJobsCompletedWindowMinutes: 1440 }; + else if (url.pathname === "/api/settings/storage-locations") body = { locations: [{ key: "location_1", rootType: "local", displayName: "Local", path: "/mnt/local" }, { key: "location_2", rootType: "remote", displayName: "Remote", path: "/mnt/remote" }] }; + else if (url.pathname === "/api/system/version") { + const unavailableRelease = { latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, releaseNotes: null, message: "Unavailable" }; + body = { currentVersion: "0.1.2-beta.1", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; + } else if (url.pathname === "/api/settings/sections") body = { sections: ["shows"], sectionTitles: { shows: "Shows" }, sectionTypes: { shows: "shows" } }; + else if (url.pathname === "/api/settings/paths") body = { symlinkDir: "/mnt/links", localDir: "/mnt/local", remoteDir: "/mnt/remote" }; + else if (url.pathname === "/api/settings/scan") body = { scanSymlinks: true, scanLocal: false, scanRemote: false, symlinkSections: ["shows"], localSections: [] }; + else if (url.pathname === "/api/settings/audit") body = { sections: ["shows"], targets: ["local", "remote"] }; + else if (url.pathname === "/api/sections") body = [{ section: "shows", title: "Shows", type: "shows", totalLinks: 1, itemCount: 1, seasonCount: 1, episodeCount: 1, remoteLinks: 1, localLinks: 0, brokenLinks: 0, otherLinks: 0, nonMediaLinks: 0, actionableRemoteLinks: 1, actionableLocalLinks: 0, assignedRemoteLinks: 0, unassignedRemoteLinks: 0, unassignedLocalLinks: 0 }]; + else if (url.pathname === "/api/inventory/summary") body = inventory; + else if (url.pathname === "/api/inventory/scan-timestamps") body = { symlinkSections: { shows: timestamp }, localSections: { shows: null }, remoteRoot: null }; + else if (url.pathname === "/api/scans" && route.request().method() === "POST") body = { jobId: 321 }; + else if (url.pathname === "/api/jobs") body = []; + else body = {}; + + await route.fulfill({ status: 200, contentType: "application/json", body: JSON.stringify(body) }); + }); + + await page.goto(baseUrl!); + const summary = page.locator(".dashboard-summary"); + const runScan = page.getByRole("button", { name: "Run Inventory Scan", exact: true }); + await expect(summary).toBeVisible(); + await expect(runScan).toBeEnabled(); + const before = await summary.boundingBox(); + + await runScan.click(); + const toast = page.locator(".action-toast"); + await expect(toast).toContainText("Inventory scan job #321 queued."); + await expect(toast).toHaveCSS("position", "fixed"); + const after = await summary.boundingBox(); + const toastBox = await toast.boundingBox(); + + expect(before).not.toBeNull(); + expect(after).not.toBeNull(); + expect(toastBox).not.toBeNull(); + expect(after!.y).toBe(before!.y); + expect(toastBox!.x + toastBox!.width).toBeGreaterThan(1200); + expect(toastBox!.y + toastBox!.height).toBeGreaterThan(700); + + await page.getByRole("button", { name: "Dismiss notification" }).click(); + await expect(toast).not.toBeVisible(); + + await page.setViewportSize({ width: 390, height: 844 }); + await runScan.click(); + await expect(toast).toBeVisible(); + const mobileToastBox = await toast.boundingBox(); + expect(mobileToastBox).not.toBeNull(); + expect(mobileToastBox!.x).toBeGreaterThanOrEqual(11); + expect(mobileToastBox!.x + mobileToastBox!.width).toBeLessThanOrEqual(379); + expect(await page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth)).toBe(true); +}); + test("refreshes an open work list when an inventory job finishes", async ({ page }) => { test.setTimeout(15_000); let inventoryJobPolls = 0; From 549db4ae16a71c93f234f494848b8cd973b0447d Mon Sep 17 00:00:00 2001 From: Aleksey Dmitriev Date: Mon, 27 Jul 2026 05:03:45 -0400 Subject: [PATCH 03/11] Show single copy job titles in recent jobs --- src/client/jobPresentation.tsx | 4 +- src/client/jobPresentationUtils.ts | 16 ++++++ tests/e2e/app-smoke.spec.ts | 92 ++++++++++++++++++++++++++++++ tests/jobPresentationUtils.test.ts | 52 ++++++++++++++++- 4 files changed, 161 insertions(+), 3 deletions(-) diff --git a/src/client/jobPresentation.tsx b/src/client/jobPresentation.tsx index a8d6e4a..8676cc0 100644 --- a/src/client/jobPresentation.tsx +++ b/src/client/jobPresentation.tsx @@ -11,7 +11,7 @@ import { normalizeRecentJobsCompletedWindowMinutes, recentJobsCompletedWindowOpt import { type AuditMode, type AuditResultRecord, type AuditRunRecord, type CopyConflictPreview, type JobEventRecord, type JobRecord, type CopyLocalConflictStrategy, type MediaLinkRow, type TimeFormatPreference } from "../shared/types"; import { JobStatusTerminateAction, LogChipList, Panel, ScanProgressPanel, StatusPill, TerminateJobDialog } from "./App"; import { AuditPrompt, AuditStatusPrompt, canTerminateJob, copyElapsedLabel, CopyPrompt, finiteNumberFromUnknown, formatBytes, formatDate, formatNumber, formatTime, invalidateCopyJobData, recordFromUnknown, scanAgeLabel, ScanStatusPrompt, sectionDisplayTitle, storageLocationName, useJobEventTimeline, useStartCopyJob, useStorageLocations, useTerminateJobMutation, useUserPreferences } from "./appShared"; -import { auditProgressFromJob, auditProgressPercent, auditStageLabel, auditStatusDetail, basenameFromPath, copyCompletedCount, copyCompletedItemSummaries, copyCurrentItem, copyEventChips, copyFailedItemSummaries, copyOverallProgressPercent, copyProgressFromJob, copyRemainingLabel, copyStageLabel, copyStagePercent, copySymlinkedCount, copyThroughputLabel, copyTransferSpeedLabel, copyTransferSpeedSecondaryLabel, formatAuditScope, formatCopyScope, formatScopedFolderParts, formatTitleScanJobDetail, jobDurationLabel, scanFolderScopeParts, scanScopeLabels, selectedLinkIdsFromJobs, selectedLinkTitleSummaries } from "./jobPresentationUtils"; +import { auditProgressFromJob, auditProgressPercent, auditStageLabel, auditStatusDetail, basenameFromPath, copyCompletedCount, copyCompletedItemSummaries, copyCurrentItem, copyEventChips, copyFailedItemSummaries, copyOverallProgressPercent, copyProgressFromJob, copyRemainingLabel, copyStageLabel, copyStagePercent, copySymlinkedCount, copyThroughputLabel, copyTransferSpeedLabel, copyTransferSpeedSecondaryLabel, formatAuditScope, formatCopyScope, formatScopedFolderParts, formatTitleScanJobDetail, jobDurationLabel, scanFolderScopeParts, scanScopeLabels, selectedLinkIdsFromJobs, selectedLinkTitleSummaries, singleSelectedLinkTitle } from "./jobPresentationUtils"; function JobEventsHeader({ label, jobId, @@ -1072,6 +1072,8 @@ function JobScopeDetail({ linkRowsError?: string | null; }) { if (selectedLinkIds.length === 0) return {text}; + const singleTitle = !linkRowsLoading && !linkRowsError ? singleSelectedLinkTitle(selectedLinkIds, linkRowsById) : null; + if (singleTitle) return {singleTitle}; const canShowTitleLookup = Boolean(linkRowsById || linkRowsLoading || linkRowsError); return ( diff --git a/src/client/jobPresentationUtils.ts b/src/client/jobPresentationUtils.ts index 64b09b9..d02ecbd 100644 --- a/src/client/jobPresentationUtils.ts +++ b/src/client/jobPresentationUtils.ts @@ -520,6 +520,22 @@ export function selectedLinkTitleSummaries(linkIds: number[], linkRowsById: Map< return summaries; } +export function singleSelectedLinkTitle(linkIds: number[], linkRowsById: Map | undefined): string | null { + if (!linkRowsById || linkIds.length === 0) return null; + let selectedTitle: { key: string; label: string } | null = null; + + for (const id of linkIds) { + const link = linkRowsById.get(id); + const label = link?.itemName.trim(); + if (!link || !label) return null; + const key = `${link.section}\0${label}`; + if (selectedTitle && selectedTitle.key !== key) return null; + selectedTitle = { key, label }; + } + + return selectedTitle?.label ?? null; +} + export function copyPromptFromJob( job: JobRecord, availableSections: Array<{ section: string; title?: string | null }>, diff --git a/tests/e2e/app-smoke.spec.ts b/tests/e2e/app-smoke.spec.ts index 65eaddc..6568b22 100644 --- a/tests/e2e/app-smoke.spec.ts +++ b/tests/e2e/app-smoke.spec.ts @@ -805,6 +805,98 @@ test("recent jobs identifies a targeted scan by title instead of only its parent await expect(row.locator(".job-scope-cell > small")).toContainText("Movies 4K"); }); +test("recent copy jobs show a single title directly and retain the title list for multi-title jobs", async ({ page }) => { + test.skip(!sessionToken, "Set SRTL_E2E_SESSION_TOKEN to exercise authenticated pages."); + const timestamp = new Date().toISOString(); + const singleMovieTitle = "Single Copy Title (2026)"; + const singleSeriesTitle = "Single Series Title (2026)"; + const jobs = [ + { + id: 999991, + type: "copy", + status: "completed", + createdAt: timestamp, + startedAt: timestamp, + finishedAt: timestamp, + progress: { options: { direction: "to_local", linkIds: [9101] }, stage: "completed", total: 1, current: 1, copied: 1 } + }, + { + id: 999992, + type: "copy", + status: "completed", + createdAt: timestamp, + startedAt: timestamp, + finishedAt: timestamp, + progress: { options: { direction: "to_local", linkIds: [9201, 9202] }, stage: "completed", total: 2, current: 2, copied: 2 } + }, + { + id: 999993, + type: "copy", + status: "completed", + createdAt: timestamp, + startedAt: timestamp, + finishedAt: timestamp, + progress: { options: { direction: "to_local", linkIds: [9301, 9302] }, stage: "completed", total: 2, current: 2, copied: 2 } + } + ]; + const link = (id: number, section: string, itemName: string) => ({ + id, + section, + itemName, + relativePath: `${itemName}/item-${id}.mkv`, + linkPath: `/links/${itemName}/item-${id}.mkv`, + targetPath: `/remote/${itemName}/item-${id}.mkv`, + kind: "remote", + targetExists: true, + isMedia: true, + storagePolicy: "location_1", + resolvedStorageFileId: null, + sizeBytes: 1, + firstSeenAt: timestamp, + lastSeenAt: timestamp, + lastChangedAt: timestamp, + missingSince: null, + updatedAt: timestamp + }); + const links = [ + link(9101, "movies", singleMovieTitle), + link(9201, "shows", singleSeriesTitle), + link(9202, "shows", singleSeriesTitle), + link(9301, "movies", "Alpha Multi Title (2026)"), + link(9302, "movies", "Zulu Multi Title (2026)") + ]; + + await page.route("**/api/jobs?*", async (route) => { + await route.fulfill({ status: 200, contentType: "application/json", body: JSON.stringify(jobs) }); + }); + await page.route("**/api/media-links/by-ids", async (route) => { + await route.fulfill({ status: 200, contentType: "application/json", body: JSON.stringify(links) }); + }); + await page.route("**/api/settings/storage-locations", async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ locations: [{ key: "location_1", rootType: "local", displayName: "Local", path: "/local" }, { key: "location_2", rootType: "remote", displayName: "Remote", path: "/remote" }] }) + }); + }); + + await page.goto(baseUrl!); + const singleMovieRow = page.locator("tbody tr").filter({ hasText: "#999991" }); + const singleSeriesRow = page.locator("tbody tr").filter({ hasText: "#999992" }); + const multiTitleRow = page.locator("tbody tr").filter({ hasText: "#999993" }); + + await expect(singleMovieRow.locator(".job-scope-cell > small")).toHaveText(singleMovieTitle); + await expect(singleMovieRow.getByLabel("View selected titles")).toHaveCount(0); + await expect(singleSeriesRow.locator(".job-scope-cell > small")).toHaveText(singleSeriesTitle); + await expect(singleSeriesRow.getByLabel("View selected titles")).toHaveCount(0); + + await expect(multiTitleRow.locator(".job-scope-detail-line > span:first-child")).toHaveText("2 selected links"); + const multiTitleTrigger = multiTitleRow.getByLabel("View selected titles"); + await expect(multiTitleTrigger).toHaveCount(1); + await multiTitleTrigger.hover(); + await expect(multiTitleTrigger.locator("li")).toHaveText(["Alpha Multi Title (2026)", "Zulu Multi Title (2026)"]); +}); + test("job progress shows the complete event timeline and opens the selected full log", async ({ page }) => { test.skip(!sessionToken, "Set SRTL_E2E_SESSION_TOKEN to exercise authenticated pages."); const jobId = 999998; diff --git a/tests/jobPresentationUtils.test.ts b/tests/jobPresentationUtils.test.ts index 18cf848..1d85eb5 100644 --- a/tests/jobPresentationUtils.test.ts +++ b/tests/jobPresentationUtils.test.ts @@ -1,11 +1,59 @@ import { describe, expect, it } from "vitest"; -import { copyCompletedItemSummaries, copyFailedItemSummaries } from "../src/client/jobPresentationUtils"; -import type { JobEventRecord } from "../src/shared/types"; +import { copyCompletedItemSummaries, copyFailedItemSummaries, singleSelectedLinkTitle } from "../src/client/jobPresentationUtils"; +import type { JobEventRecord, MediaLinkRow } from "../src/shared/types"; function event(id: number, message: string, data: unknown, level: JobEventRecord["level"] = "error"): JobEventRecord { return { id, jobId: 42, timestamp: "2026-07-17T12:00:00.000Z", level, message, data }; } +function mediaLink(id: number, section: string, itemName: string): MediaLinkRow { + const timestamp = "2026-07-17T12:00:00.000Z"; + return { + id, + section, + itemName, + relativePath: `${itemName}/item-${id}.mkv`, + linkPath: `/links/${itemName}/item-${id}.mkv`, + targetPath: `/remote/${itemName}/item-${id}.mkv`, + kind: "remote", + targetExists: true, + isMedia: true, + storagePolicy: "location_1", + resolvedStorageFileId: null, + sizeBytes: null, + firstSeenAt: timestamp, + lastSeenAt: timestamp, + lastChangedAt: timestamp, + missingSince: null, + updatedAt: timestamp + }; +} + +describe("selected link title display", () => { + it("returns a title when every selected link resolves to the same section and title", () => { + const rows = new Map([ + [1, mediaLink(1, "shows", "Single Series (2026)")], + [2, mediaLink(2, "shows", "Single Series (2026)")] + ]); + + expect(singleSelectedLinkTitle([1], rows)).toBe("Single Series (2026)"); + expect(singleSelectedLinkTitle([1, 2], rows)).toBe("Single Series (2026)"); + }); + + it("falls back when titles differ, sections differ, or inventory rows are incomplete", () => { + const rows = new Map([ + [1, mediaLink(1, "movies", "Shared Title (2026)")], + [2, mediaLink(2, "movies", "Another Title (2026)")], + [3, mediaLink(3, "movies4k", "Shared Title (2026)")] + ]); + + expect(singleSelectedLinkTitle([1, 2], rows)).toBeNull(); + expect(singleSelectedLinkTitle([1, 3], rows)).toBeNull(); + expect(singleSelectedLinkTitle([1, 4], rows)).toBeNull(); + expect(singleSelectedLinkTitle([], rows)).toBeNull(); + }); +}); + describe("copy failure summaries", () => { it("collects identifiable item failures, deduplicates retries, and sorts titles", () => { const summaries = copyFailedItemSummaries([ From d177576ec1bfa6042d891024109a7fec9c4b6e3c Mon Sep 17 00:00:00 2001 From: Aleksey Dmitriev Date: Mon, 27 Jul 2026 17:15:25 -0400 Subject: [PATCH 04/11] Add controlled beta image publication --- .github/workflows/ci.yml | 70 ++++++++++++++++++++++++++++++++++++++++ CHANGELOG.md | 7 ++++ 2 files changed, 77 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a64dea..251322e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,12 @@ on: push: branches: [beta, main] workflow_dispatch: + inputs: + publish_beta: + description: Publish the current beta prerelease container + required: false + type: boolean + default: false permissions: contents: read @@ -142,3 +148,67 @@ jobs: run: | test ! -f /tmp/srtl-api.log || tail -200 /tmp/srtl-api.log test ! -f /tmp/srtl-worker.log || tail -200 /tmp/srtl-worker.log + + publish-beta: + if: github.event_name == 'workflow_dispatch' && inputs.publish_beta == true + needs: verify + runs-on: ubuntu-latest + timeout-minutes: 45 + permissions: + contents: read + packages: write + id-token: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - id: version + name: Validate beta publication + run: | + if [[ "$GITHUB_REF_NAME" != "beta" ]]; then + echo "Beta images may only be published from the beta branch." + exit 1 + fi + version="$(node -p 'require("./package.json").version')" + if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+-beta\.[1-9][0-9]*$ ]]; then + echo "Beta versions must use the form 0.1.2-beta.1." + exit 1 + fi + echo "value=$version" >> "$GITHUB_OUTPUT" + - name: Validate container definition + run: docker build --check . + - uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4 + - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4 + - uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - id: metadata + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6 + with: + images: ghcr.io/${{ github.repository_owner }}/srtl-manager + tags: | + type=raw,value=${{ steps.version.outputs.value }} + - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.metadata.outputs.tags }} + labels: ${{ steps.metadata.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + provenance: mode=max + sbom: true + - name: Scan the published beta image + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + with: + image-ref: ghcr.io/${{ github.repository_owner }}/srtl-manager:${{ steps.version.outputs.value }} + format: table + exit-code: "1" + ignore-unfixed: true + severity: CRITICAL,HIGH + - name: Promote the verified image to beta + env: + IMAGE_REF: ghcr.io/${{ github.repository_owner }}/srtl-manager + VERSION: ${{ steps.version.outputs.value }} + run: docker buildx imagetools create --tag "$IMAGE_REF:beta" "$IMAGE_REF:$VERSION" diff --git a/CHANGELOG.md b/CHANGELOG.md index aca9a32..14cb8e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes are documented here. The project follows Semantic Versioning ## [Unreleased] +## [0.1.2-beta.1] - 2026-07-27 + +### Changed + +- Replaced layout-shifting dashboard action messages with responsive overlay notifications. +- Displayed a copy job's title directly when all selected links belong to one title, while retaining the title list for multi-title jobs. + ## [0.1.1] - 2026-07-25 ### Added From 7e33f99df2e7f1142ea1effd80f6c8fe6ce7a8bf Mon Sep 17 00:00:00 2001 From: Aleksey Dmitriev Date: Tue, 28 Jul 2026 22:41:19 -0400 Subject: [PATCH 05/11] Harden title rescans and username matching --- README.md | 4 +- src/client/App.tsx | 4 +- src/server/auth.ts | 12 +++- src/server/jobs/jobRunner.ts | 12 +++- src/server/lib/copier.ts | 103 ++++++++++++++++++++++++----- src/server/lib/filesystemSafety.ts | 73 ++++++++++++++++++++ src/server/lib/scanner.ts | 88 ++++++++++++++++++++---- src/server/routes/authRoutes.ts | 4 +- tests/app.test.ts | 26 ++++++++ tests/copier.test.ts | 64 ++++++++++++++++++ tests/scanner.test.ts | 53 +++++++++++++++ 11 files changed, 405 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 2073aec..5dd0230 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,10 @@ SRTL Manager is a local-first web app for inventorying and maintaining a symlink ## Highlights - Guided first-run account, path, section, policy, and initial-scan setup. -- Symlink, local-root, remote-root, and orphan inventory with targeted title rescans. +- Symlink, local-root, remote-root, and orphan inventory with targeted title rescans and exact target checks. - Per-location storage policies using editable friendly names, plus an Unassigned queue for newly discovered titles. - Fast and deep audits across local and remote targets. -- Bidirectional copies with live progress, conflict handling, configurable verification, and source/title risk checks. +- Bidirectional copies with live progress, transient transfer retry, conflict handling, configurable verification, and source/title risk checks. - Safe job termination, complete event timelines, restart recovery, and durable per-file copy journals. - Required path-change review before changed mounts can affect managed links. - Editable storage-location names while deployment paths remain environment-managed. diff --git a/src/client/App.tsx b/src/client/App.tsx index adaf9bb..b8fe08f 100644 --- a/src/client/App.tsx +++ b/src/client/App.tsx @@ -5,7 +5,7 @@ import { Activity, ArrowDown, ArrowLeft, ArrowRight, ArrowUp, Blocks, CheckCircl import { api } from "./api"; import { formatJobType, jobProgressChips } from "./logDisplay"; import { formatCurrentVersionDisplay } from "./versionDisplay"; -import { inventoryJobRefreshKey } from "./recentJobs"; +import { inventoryJobRefreshKey, isActiveDashboardJob } from "./recentJobs"; import { inferSectionContentType } from "../shared/sections"; import { type AppReleaseInfo, type AppVersionInfo, type JobRecord, type OnboardingPolicyMode, type OnboardingState, type PathConfigurationState, type SectionContentType, type StorageLocationKey } from "../shared/types"; import { SectionDraft, SidebarGroup, StorageLocationsContext, UserPreferencesContext, canTerminateJob, copyElapsedLabel, createEmptySectionDraft, defaultStorageLocations, defaultUserPreferences, formatNumber, historySections, onboardingScanVisibleStats, parseLogsRouteSearch, pathMigrationProgress, scanOptionsFromProgress, scanProgressFromJob, scanStageLabel, scanStagePercent, scanStatusDetail, scanVisibleIndexedItemCount, scanVisibleStats, sectionDraftsToSettings, sectionSettingsToDrafts, sectionTypeOptions, settingsSections, storageLocationName, themeOptions, useLiveTimestamp, useStorageLocations, useTerminateJobMutation, useThemePreference, versionChannelLabel, versionCheckIntervalMs, visibleVersionReleases } from "./appShared"; @@ -588,7 +588,7 @@ function InventoryJobDataRefresher() { const jobs = useQuery({ queryKey: ["jobs", "inventory-refresh"], queryFn: () => api.jobs({ completedWithinMinutes: 15 }), - refetchInterval: 3000 + refetchInterval: (query) => (query.state.data?.some((job) => (job.type === "scan" || job.type === "copy") && isActiveDashboardJob(job)) ? 500 : 3000) }); const refreshKey = inventoryJobRefreshKey(jobs.data ?? []); diff --git a/src/server/auth.ts b/src/server/auth.ts index f6f21fc..279bc53 100644 --- a/src/server/auth.ts +++ b/src/server/auth.ts @@ -36,8 +36,18 @@ export async function createAdmin(db: Db, username: string, password: string): P }); } +export async function findAdminByUsername(db: Db, username: string): Promise { + return first( + db + .select() + .from(schema.adminUsers) + .where(sql`lower(${schema.adminUsers.username}) = lower(${username.trim()})`) + .limit(1) + ); +} + export async function login(db: Db, username: string, password: string): Promise { - const user = await first(db.select().from(schema.adminUsers).where(eq(schema.adminUsers.username, username)).limit(1)); + const user = await findAdminByUsername(db, username); if (!user) return null; if (!(await verifyPassword(password, user.passwordHash))) return null; const token = crypto.randomBytes(32).toString("base64url"); diff --git a/src/server/jobs/jobRunner.ts b/src/server/jobs/jobRunner.ts index 1f37770..83a3489 100644 --- a/src/server/jobs/jobRunner.ts +++ b/src/server/jobs/jobRunner.ts @@ -1898,6 +1898,16 @@ export class JobWorker { for (const issue of result.storageScanIssues) { await ctx.event("warn", "Remote storage directory remained unreadable after retry", issue); } + for (const link of result.links) { + if (!link.targetReadError) continue; + await ctx.event("warn", "Symlink target remained unreadable after retry", { + section: link.section, + itemName: link.itemName, + linkPath: link.linkPath, + targetPath: link.targetPath, + message: link.targetReadError + }); + } if (await ctx.isCancelled()) { await this.db.update(schema.scanRuns).set({ status: "cancelled", finishedAt: nowIso(), errorMessage: "Job cancelled" }).where(eq(schema.scanRuns.id, scanRun.id)); await ctx.setProgress(scanProgressPayload(normalizedOptions, "cancelled", "Scan cancelled before inventory results were written", result.inventory)); @@ -2220,7 +2230,7 @@ export class JobWorker { async (update) => { activeUpdate = update; await setCopyProgress(update.stage, update.message, link, activeUpdate); - if (update.stage === "copying" || update.stage === "preparing") return; + if (update.stage === "copying" || (update.stage === "preparing" && !/retry/i.test(update.message))) return; const progressEventKey = `${update.stage}:${update.message}`; if (progressEventKey === lastProgressEventKey) return; lastProgressEventKey = progressEventKey; diff --git a/src/server/lib/copier.ts b/src/server/lib/copier.ts index e5dc6ff..3662e46 100644 --- a/src/server/lib/copier.ts +++ b/src/server/lib/copier.ts @@ -7,7 +7,7 @@ import { pipeline } from "node:stream/promises"; import { defaultCopyJobBehaviorSettings } from "../../shared/advancedSettings"; import type { AuditMode, CopyDirection, CopyJobBehaviorSettings, CopyLocalConflictStrategy, MediaLinkRow, PathsSettings, StorageRootType } from "../../shared/types"; import { isMediaFile, isPathInside } from "./media"; -import { assertDestinationPathInside, assertExistingPathInside, assertPathParentInside } from "./filesystemSafety"; +import { assertDestinationPathInside, assertExistingPathInside, assertPathParentInside, assertReadableRegularFile } from "./filesystemSafety"; import { appendBoundedOutput, commandTimeoutMs, terminateChildProcess } from "./processSafety"; const copyDirectoryMode = 0o755; @@ -471,6 +471,54 @@ function isMissingPathError(error: unknown): boolean { return Boolean(error && typeof error === "object" && "code" in error && error.code === "ENOENT"); } +const retryableTransferErrorCodes = new Set(["EAGAIN", "EBUSY", "EIO", "ENETDOWN", "ENETRESET", "ENETUNREACH", "ENOTCONN", "EREMOTEIO", "ESTALE", "ETIMEDOUT"]); + +function transferErrorCode(error: unknown): string | null { + if (!error || typeof error !== "object") return null; + if ("code" in error && typeof error.code === "string") return error.code; + if ("cause" in error) return transferErrorCode(error.cause); + return null; +} + +function isRetryableTransferError(error: unknown): boolean { + const code = transferErrorCode(error); + if (code && retryableTransferErrorCodes.has(code)) return true; + return error instanceof Error && /timed out|temporarily unavailable/i.test(error.message); +} + +async function waitForSourceRetry(signal?: AbortSignal): Promise { + throwIfAborted(signal); + await new Promise((resolve, reject) => { + const finish = () => { + signal?.removeEventListener("abort", abort); + resolve(); + }; + const timeout = setTimeout(finish, 500); + const abort = () => { + clearTimeout(timeout); + signal?.removeEventListener("abort", abort); + reject(abortError()); + }; + signal?.addEventListener("abort", abort, { once: true }); + }); +} + +async function assertExistingSourcePathInside(root: string, sourcePath: string, signal?: AbortSignal): Promise { + let lastError: unknown; + for (let attempt = 1; attempt <= 3; attempt += 1) { + throwIfAborted(signal); + try { + await assertExistingPathInside(root, sourcePath, "Source path"); + return; + } catch (error) { + lastError = error; + if (attempt === 3 || !isRetryableTransferError(error)) throw error; + await waitForSourceRetry(signal); + } + } + throw lastError; +} + async function createDestinationParent(destinationRoot: string, destinationPath: string): Promise { const destinationDirectory = path.dirname(destinationPath); const missingDirectories: string[] = []; @@ -703,12 +751,17 @@ export async function copyMediaLink( const sourcePath = path.resolve(link.targetPath); await reportCopyProgress(reportProgress, { stage: "preparing", message: "Checking source, destination, and symlink state", sourcePath, linkPath: link.linkPath }); assertInside(sourceRoot, sourcePath, "Source path"); - await assertExistingPathInside(sourceRoot, sourcePath, "Source path"); + await assertExistingSourcePathInside(sourceRoot, sourcePath, signal); await assertPathParentInside(paths.symlinkDir, link.linkPath, "Symlink path"); ensureMediaCandidate(sourcePath, link.relativePath, link.linkPath); await validateLinkStillPointsTo(link, sourcePath); - const sourceStatBefore = await statRegularFile(sourcePath, "Source file"); + const sourceStatBefore = await assertReadableRegularFile(sourcePath, "Source file", { + attempts: 3, + retryDelayMs: 500, + signal, + onRetry: () => reportCopyProgress(reportProgress, { stage: "preparing", message: "Source is temporarily unreadable; retrying preflight", sourcePath, linkPath: link.linkPath }) + }); const destinationPath = copyDestinationPath(link, paths, destinationRootType); assertInside(destinationRoot, destinationPath, "Destination path"); await assertDestinationPathInside(destinationRoot, destinationPath, "Destination path"); @@ -783,21 +836,39 @@ export async function copyMediaLink( bytesPerSecond: 0, remainingSeconds: null }); - await runner.copyFile( - sourcePath, - tempPath, - (progress) => - reportCopyProgress(reportProgress, { - stage: "copying", - message: transferMessage, + for (let transferAttempt = 1; transferAttempt <= 2; transferAttempt += 1) { + try { + await runner.copyFile( + sourcePath, + tempPath, + (progress) => + reportCopyProgress(reportProgress, { + stage: "copying", + message: transferMessage, + sourcePath, + destinationPath, + linkPath: link.linkPath, + sizeBytes: sourceStatBefore.size, + ...progress + }), + signal + ); + break; + } catch (error) { + if (transferAttempt === 2 || !isRetryableTransferError(error) || signal?.aborted) throw error; + await fs.rm(tempPath, { force: true }).catch(() => undefined); + await reportCopyProgress(reportProgress, { + stage: "preparing", + message: "Transfer hit a temporary I/O error; retrying once", sourcePath, destinationPath, linkPath: link.linkPath, - sizeBytes: sourceStatBefore.size, - ...progress - }), - signal - ); + sizeBytes: sourceStatBefore.size + }); + await waitForSourceRetry(signal); + await assertReadableRegularFile(sourcePath, "Source file", { attempts: 3, retryDelayMs: 500, signal }); + } + } throwIfAborted(signal); await fs.chmod(tempPath, copyFileMode); const tempStat = await statRegularFile(tempPath, "Temporary copy"); @@ -807,7 +878,7 @@ export async function copyMediaLink( await verifyCopiedFile(runner, sourcePath, tempPath, reportProgress, baseProgress, behavior, signal); await reportOperation?.({ stage: "verified", tempPath, sizeBytes: tempStat.size }); throwIfAborted(signal); - const sourceStatAfter = await statRegularFile(sourcePath, "Source file"); + const sourceStatAfter = await assertReadableRegularFile(sourcePath, "Source file", { attempts: 3, retryDelayMs: 500, signal }); if (sourceStatAfter.size !== sourceStatBefore.size || sourceStatAfter.mtimeMs !== sourceStatBefore.mtimeMs) { throw new Error("Source file changed during copy; destination was not promoted"); } diff --git a/src/server/lib/filesystemSafety.ts b/src/server/lib/filesystemSafety.ts index f94d69b..3fb0ca9 100644 --- a/src/server/lib/filesystemSafety.ts +++ b/src/server/lib/filesystemSafety.ts @@ -4,6 +4,19 @@ import { isPathInside } from "./media"; const defaultFilesystemTimeoutMs = 15_000; +export interface ReadableFileSnapshot { + size: number; + mtimeMs: number; +} + +export interface ReadableFileOptions { + attempts?: number; + retryDelayMs?: number; + signal?: AbortSignal; + timeoutMs?: number; + onRetry?: (attempt: number, error: unknown) => Promise | void; +} + function configuredTimeoutMs(): number { const parsed = Number(process.env.SRTL_FILESYSTEM_TIMEOUT_MS); if (!Number.isFinite(parsed) || parsed < 1_000) return defaultFilesystemTimeoutMs; @@ -22,6 +35,66 @@ export async function withFilesystemTimeout(operation: Promise, descriptio } } +function abortError(): Error { + return new Error("Job terminated"); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +async function waitForRetry(delayMs: number, signal?: AbortSignal): Promise { + if (signal?.aborted) throw abortError(); + if (delayMs <= 0) return; + await new Promise((resolve, reject) => { + const finish = () => { + signal?.removeEventListener("abort", abort); + resolve(); + }; + const timeout = setTimeout(finish, delayMs); + const abort = () => { + clearTimeout(timeout); + signal?.removeEventListener("abort", abort); + reject(abortError()); + }; + signal?.addEventListener("abort", abort, { once: true }); + }); +} + +export async function assertReadableRegularFile(filePath: string, label: string, options: ReadableFileOptions = {}): Promise { + const attempts = Math.max(1, Math.min(Math.trunc(options.attempts ?? 1), 5)); + const retryDelayMs = Math.max(0, Math.min(Math.trunc(options.retryDelayMs ?? 250), 5_000)); + let lastError: unknown = new Error(`${label} could not be read`); + + for (let attempt = 1; attempt <= attempts; attempt += 1) { + if (options.signal?.aborted) throw abortError(); + let fileHandle: Awaited> | null = null; + try { + const stat = await withFilesystemTimeout(fs.stat(filePath), `${label} metadata check`, options.timeoutMs); + if (!stat.isFile()) throw new Error(`${label} is not a regular file`); + fileHandle = await withFilesystemTimeout(fs.open(filePath, "r"), `${label} read check`, options.timeoutMs); + if (stat.size > 0) { + const buffer = Buffer.allocUnsafe(1); + const read = await withFilesystemTimeout(fileHandle.read(buffer, 0, 1, 0), `${label} first-byte read`, options.timeoutMs); + if (read.bytesRead !== 1) throw new Error(`${label} returned no data during its read check`); + } + return { size: stat.size, mtimeMs: stat.mtimeMs }; + } catch (error) { + lastError = error; + } finally { + if (fileHandle) await fileHandle.close().catch(() => undefined); + } + + if (attempt < attempts) { + await options.onRetry?.(attempt, lastError); + await waitForRetry(retryDelayMs, options.signal); + } + } + + const attemptsLabel = attempts === 1 ? "read check" : `${attempts} read attempts`; + throw new Error(`${label} is missing or unreadable after ${attemptsLabel}: ${errorMessage(lastError)}`, { cause: lastError }); +} + async function realPath(filePath: string, label: string): Promise { return withFilesystemTimeout(fs.realpath(filePath), `${label} realpath`); } diff --git a/src/server/lib/scanner.ts b/src/server/lib/scanner.ts index 8050574..6862a82 100644 --- a/src/server/lib/scanner.ts +++ b/src/server/lib/scanner.ts @@ -9,7 +9,7 @@ import { inferSectionContentType } from "../../shared/sections"; import { canonicalTitleKey } from "./storagePolicies"; import { applyPendingOnboardingPolicy } from "./onboarding"; import { isMediaFile, isPathInside, safeRelativePath } from "./media"; -import { withFilesystemTimeout } from "./filesystemSafety"; +import { assertReadableRegularFile, withFilesystemTimeout } from "./filesystemSafety"; import type { InventorySummary, InventoryScanTimestamps, @@ -46,6 +46,8 @@ export interface ClassifiedLink { storagePolicy: StoragePolicyKind; resolvedStorageFileId?: number | null; sizeBytes: number | null; + targetMtimeMs: number | null; + targetReadError: string | null; } export interface ClassifiedStorageFile { @@ -73,6 +75,7 @@ export interface ScanResult { options: ScanOptions; links: ClassifiedLink[]; storageFiles: ClassifiedStorageFile[]; + reconciledStorageFiles: ClassifiedStorageFile[]; storageScanIssues: StorageScanIssue[]; summaries: SectionSummary[]; inventory: InventorySummary; @@ -106,7 +109,8 @@ export async function classifySymlink( sectionRoot: string, paths: PathsSettings, section: string, - storagePolicies: StoragePolicyLookup + storagePolicies: StoragePolicyLookup, + verifyTargetReadability = false ): Promise { const rawTargetPath = await withFilesystemTimeout(fs.readlink(linkPath), `Symlink target read for ${linkPath}`); const targetPath = path.isAbsolute(rawTargetPath) ? rawTargetPath : path.resolve(path.dirname(linkPath), rawTargetPath); @@ -116,17 +120,28 @@ export async function classifySymlink( const targetRootType: StorageRootType | "other" = isPathInside(paths.remoteDir, targetPath) ? "remote" : isPathInside(paths.localDir, targetPath) ? "local" : "other"; let targetExists = false; let sizeBytes: number | null = null; + let targetMtimeMs: number | null = null; + let targetReadError: string | null = null; - const attempts = targetRootType === "remote" ? 2 : 1; + const attempts = targetRootType === "remote" ? (verifyTargetReadability ? 3 : 2) : 1; for (let attempt = 1; attempt <= attempts; attempt += 1) { try { - const stat = await withFilesystemTimeout(fs.stat(targetPath), `Target check for ${targetPath}`); + if (verifyTargetReadability && media && targetRootType !== "other") { + const snapshot = await assertReadableRegularFile(targetPath, `Symlink target ${targetPath}`, { attempts: 1, timeoutMs: 5_000 }); + sizeBytes = snapshot.size; + targetMtimeMs = Math.trunc(snapshot.mtimeMs); + } else { + const stat = await withFilesystemTimeout(fs.stat(targetPath), `Target check for ${targetPath}`); + sizeBytes = stat.isFile() ? stat.size : null; + targetMtimeMs = Math.trunc(stat.mtimeMs); + } targetExists = true; - sizeBytes = stat.isFile() ? stat.size : null; + targetReadError = null; break; - } catch { + } catch (error) { targetExists = false; - if (attempt < attempts) await new Promise((resolve) => setTimeout(resolve, 100)); + targetReadError = verifyTargetReadability ? describeError(error) : null; + if (attempt < attempts) await new Promise((resolve) => setTimeout(resolve, verifyTargetReadability ? 350 : 100)); } } @@ -154,7 +169,9 @@ export async function classifySymlink( targetExists, isMedia: media, storagePolicy, - sizeBytes + sizeBytes, + targetMtimeMs, + targetReadError }; } @@ -458,7 +475,7 @@ export async function scanLibrary( if (checkingUpdate) await checkingUpdate; for (const linkPath of symlinks) { await throwIfScanCancelled(isCancelled); - links.push(await classifySymlink(linkPath, sectionRoot, paths, section, storagePolicies)); + links.push(await classifySymlink(linkPath, sectionRoot, paths, section, storagePolicies, Boolean(titleScopesBySection))); checkedLinks += 1; if (!shouldReportSymlinkActivity()) continue; const progressUpdate = reportSymlinkActivity( @@ -488,8 +505,11 @@ export async function scanLibrary( } const classifiedStorageFiles = applyStorageFilePolicies(storageFiles, settings, storagePolicies); + const reconciledStorageFiles = titleScopesBySection + ? uniqueStorageFiles(links.map((link) => storageFileFromTargetedLink(link, paths)).filter((file): file is ClassifiedStorageFile => file !== null)) + : []; const summaries = summarizeLinks(links, scopedSettings.sections, scopedSettings.sectionTitles, scopedSettings.sectionTypes); - return { options, links, storageFiles: classifiedStorageFiles, storageScanIssues, summaries, inventory: summarizeInventory(links, classifiedStorageFiles) }; + return { options, links, storageFiles: classifiedStorageFiles, reconciledStorageFiles, storageScanIssues, summaries, inventory: summarizeInventory(links, classifiedStorageFiles) }; } export function summarizeLinks( @@ -587,6 +607,23 @@ function applyStorageFilePolicies(files: ClassifiedStorageFile[], settings: Sect }); } +function storageFileFromTargetedLink(link: ClassifiedLink, paths: PathsSettings): ClassifiedStorageFile | null { + if (!link.targetExists || !link.isMedia || link.sizeBytes === null || link.targetMtimeMs === null) return null; + if (link.kind !== "local" && link.kind !== "remote") return null; + const rootPath = link.kind === "local" ? paths.localDir : paths.remoteDir; + return { + rootType: link.kind, + rootPath, + section: link.section, + itemName: link.itemName, + relativePath: safeRelativePath(rootPath, link.targetPath), + filePath: link.targetPath, + storagePolicy: link.storagePolicy, + sizeBytes: link.sizeBytes, + mtimeMs: link.targetMtimeMs + }; +} + export function summarizeInventory(links: ClassifiedLink[], storageFiles: ClassifiedStorageFile[]): InventorySummary { const linkedTargets = new Set(links.filter((link) => link.targetExists && link.isMedia).map((link) => link.targetPath)); const localFiles = storageFiles.filter((file) => file.rootType === "local"); @@ -670,6 +707,7 @@ async function throwIfPersistenceCancelled(isCancelled?: ScanCancellationCheck): export async function persistScanResult(db: Db, result: ScanResult, jobId: number, isCancelled?: ScanCancellationCheck): Promise { await throwIfPersistenceCancelled(isCancelled); const timestamp = nowIso(); + const filesToReconcile = uniqueStorageFiles([...result.storageFiles, ...result.reconciledStorageFiles]); const seenStorageFilePaths = new Set(result.storageFiles.map((file) => file.filePath)); const seenLinkPaths = new Set(result.links.map((link) => link.linkPath)); const scannedStorageRootTypes = new Set(); @@ -684,7 +722,7 @@ export async function persistScanResult(db: Db, result: ScanResult, jobId: numbe if (result.options.scanLocal) scannedStorageRootTypes.add("local"); if (result.options.scanRemote) scannedStorageRootTypes.add("remote"); - for (const file of result.storageFiles) { + for (const file of filesToReconcile) { await throwIfPersistenceCancelled(isCancelled); const existing = await first(db.select().from(schema.storageFiles).where(eq(schema.storageFiles.filePath, file.filePath)).limit(1)); const firstSeenAt = existing?.firstSeenAt ?? timestamp; @@ -725,7 +763,25 @@ export async function persistScanResult(db: Db, result: ScanResult, jobId: numbe const existing = await first(db.select().from(schema.mediaLinks).where(eq(schema.mediaLinks.linkPath, link.linkPath)).limit(1)); const firstSeenAt = existing?.firstSeenAt ?? existing?.updatedAt ?? timestamp; const lastChangedAt = linkChanged(existing, link, resolvedStorageFileId) ? timestamp : existing?.lastChangedAt ?? existing?.updatedAt ?? timestamp; - const values = { ...link, resolvedStorageFileId, firstSeenAt, lastSeenAt: timestamp, lastChangedAt, missingSince: null, lastSeenJobId: jobId, updatedAt: timestamp }; + const values = { + section: link.section, + itemName: link.itemName, + relativePath: link.relativePath, + linkPath: link.linkPath, + targetPath: link.targetPath, + kind: link.kind, + targetExists: link.targetExists, + isMedia: link.isMedia, + storagePolicy: link.storagePolicy, + sizeBytes: link.sizeBytes, + resolvedStorageFileId, + firstSeenAt, + lastSeenAt: timestamp, + lastChangedAt, + missingSince: null, + lastSeenJobId: jobId, + updatedAt: timestamp + }; await db .insert(schema.mediaLinks) .values(values) @@ -776,7 +832,9 @@ export async function persistScanResult(db: Db, result: ScanResult, jobId: numbe isMedia: link.isMedia, storagePolicy: normalizeStoragePolicy(link.storagePolicy), resolvedStorageFileId: link.resolvedStorageFileId, - sizeBytes: link.sizeBytes + sizeBytes: link.sizeBytes, + targetMtimeMs: null, + targetReadError: null })), [] ); @@ -1652,7 +1710,9 @@ export async function listSectionSummaries(db: Db): Promise { targetExists: row.targetExists, isMedia: row.isMedia, storagePolicy: row.storagePolicy, - sizeBytes: row.sizeBytes + sizeBytes: row.sizeBytes, + targetMtimeMs: null, + targetReadError: null })); return summarizeLinks(links, sections, sectionTitles, sectionTypes); } diff --git a/src/server/routes/authRoutes.ts b/src/server/routes/authRoutes.ts index ceb6496..7864f71 100644 --- a/src/server/routes/authRoutes.ts +++ b/src/server/routes/authRoutes.ts @@ -1,7 +1,7 @@ import { z } from "zod"; import type { FastifyInstance, FastifyReply } from "fastify"; import { eq } from "drizzle-orm"; -import { createAdmin, getSessionUser, hashPassword, hasAdmin, login, logout, verifyPassword } from "../auth"; +import { createAdmin, findAdminByUsername, getSessionUser, hashPassword, hasAdmin, login, logout, verifyPassword } from "../auth"; import { first, type Db } from "../db/database"; import * as schema from "../db/schema"; import { markOnboardingAccountCreated } from "../lib/onboarding"; @@ -98,7 +98,7 @@ export function registerAuthRoutes(app: FastifyInstance, db: Db, options: AuthRo } const username = body.username.trim(); - const conflictingUser = await first(db.select({ id: schema.adminUsers.id }).from(schema.adminUsers).where(eq(schema.adminUsers.username, username)).limit(1)); + const conflictingUser = await findAdminByUsername(db, username); if (conflictingUser && conflictingUser.id !== user.id) { return reply.code(409).send({ error: "Username is already in use" }); } diff --git a/tests/app.test.ts b/tests/app.test.ts index c8fa223..de8ac22 100644 --- a/tests/app.test.ts +++ b/tests/app.test.ts @@ -568,6 +568,32 @@ describe("api app", () => { expect(sections.json()).toEqual({ sections: ["movies", "shows"], sectionTitles: {}, sectionTypes: { movies: "movies", shows: "shows" } }); }); + it("matches usernames case-insensitively while preserving their display capitalization", async () => { + const setup = await ctx.app.inject({ + method: "POST", + url: "/api/auth/setup", + payload: { username: "MixedCaseAdmin", password: "password123", confirmPassword: "password123" } + }); + expect(setup.statusCode).toBe(200); + + const login = await ctx.app.inject({ + method: "POST", + url: "/api/auth/login", + payload: { username: "mixedcaseadmin", password: "password123" } + }); + expect(login.statusCode).toBe(200); + + const me = await ctx.app.inject({ + method: "GET", + url: "/api/auth/me", + headers: { cookie: String(login.headers["set-cookie"]) } + }); + expect(me.json()).toMatchObject({ + authenticated: true, + user: { username: "MixedCaseAdmin" } + }); + }); + it("saves friendly storage location names without exposing path mutation", async () => { const denied = await ctx.app.inject({ method: "GET", url: "/api/settings/storage-locations" }); expect(denied.statusCode).toBe(401); diff --git a/tests/copier.test.ts b/tests/copier.test.ts index 33f467b..3271ed8 100644 --- a/tests/copier.test.ts +++ b/tests/copier.test.ts @@ -88,6 +88,70 @@ describe("copy runner", () => { } }); + it("retries one transient source read failure before installing the copy", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "srtl-copier-retry-")); + try { + const symlinkDir = path.join(directory, "symlinks"); + const localDir = path.join(directory, "local"); + const remoteDir = path.join(directory, "remote"); + const relativePath = path.join("Retry Title", "retry.mkv"); + const sourcePath = path.join(remoteDir, "items", relativePath); + const linkPath = path.join(symlinkDir, "items", relativePath); + const destinationPath = path.join(localDir, "items", relativePath); + await Promise.all([ + fs.mkdir(path.dirname(sourcePath), { recursive: true }), + fs.mkdir(path.dirname(linkPath), { recursive: true }), + fs.mkdir(path.join(localDir, "items"), { recursive: true }) + ]); + await fs.writeFile(sourcePath, "retry source"); + await fs.symlink(sourcePath, linkPath); + const timestamp = new Date().toISOString(); + let copyAttempts = 0; + const runner = { + ...defaultCopyRunner, + async copyFile(source: string, destination: string, reportProgress: Parameters[2], signal: AbortSignal | undefined) { + copyAttempts += 1; + if (copyAttempts === 1) throw Object.assign(new Error("temporary remote read failure"), { code: "EIO" }); + return defaultCopyRunner.copyFile(source, destination, reportProgress, signal); + } + }; + + const result = await copyMediaLink( + { + id: 1, + section: "items", + itemName: "Retry Title", + relativePath, + linkPath, + targetPath: sourcePath, + kind: "remote", + targetExists: true, + isMedia: true, + storagePolicy: "location_1", + resolvedStorageFileId: null, + sizeBytes: 12, + firstSeenAt: timestamp, + lastSeenAt: timestamp, + lastChangedAt: timestamp, + missingSince: null, + updatedAt: timestamp + }, + { symlinkDir, localDir, remoteDir }, + "to_local", + runner, + undefined, + { profile: "off", byteCompare: false, mediaValidation: "off" } + ); + + expect(copyAttempts).toBe(2); + expect(result).toMatchObject({ status: "copied", destinationPath }); + await expect(fs.readFile(destinationPath, "utf8")).resolves.toBe("retry source"); + await expect(fs.readlink(linkPath)).resolves.toBe(destinationPath); + } finally { + await fs.rm(directory, { recursive: true, force: true }); + } + }); + it("creates readable destination directories and files under a restrictive process umask", async () => { const directory = await fs.mkdtemp(path.join(os.tmpdir(), "srtl-copier-modes-")); const previousUmask = process.umask(); diff --git a/tests/scanner.test.ts b/tests/scanner.test.ts index 02d7b31..295875b 100644 --- a/tests/scanner.test.ts +++ b/tests/scanner.test.ts @@ -1,6 +1,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; +import { eq } from "drizzle-orm"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { openTestDatabase } from "./testDb"; import * as schema from "../src/server/db/schema"; @@ -447,6 +448,15 @@ describe("scanner", () => { const summary = await persistScanResult(database.db, titleRescan, 2); expect(titleRescan.links.map((link) => link.linkPath)).toEqual([newFirstLink]); + expect(titleRescan.reconciledStorageFiles).toEqual([ + expect.objectContaining({ + rootType: "remote", + section: "shows", + itemName: "First Title", + filePath: newFirstTarget, + sizeBytes: 3 + }) + ]); expect(summary).toMatchObject({ totalLinks: 1, remoteLinks: 1, missingLinks: 1 }); const persistedLinks = (await database.db.select().from(schema.mediaLinks)).filter((link) => !link.missingSince); expect(persistedLinks.map((link) => [link.linkPath, link.lastSeenJobId]).sort()).toEqual( @@ -455,12 +465,55 @@ describe("scanner", () => { [secondLink, 1] ].sort() ); + const rescannedLink = persistedLinks.find((link) => link.linkPath === newFirstLink); + expect(rescannedLink?.resolvedStorageFileId).toEqual(expect.any(Number)); + expect(await listStorageFiles(database.db, "remote")).toEqual([ + expect.objectContaining({ filePath: newFirstTarget, section: "shows", itemName: "First Title" }) + ]); + await expect(database.db.select().from(schema.storageFiles).where(eq(schema.storageFiles.filePath, newFirstTarget))).resolves.toEqual([ + expect.objectContaining({ filePath: newFirstTarget, lastSeenJobId: 2 }) + ]); expect(await listMediaLinks(database.db, undefined, "missing")).toMatchObject([{ linkPath: oldFirstLink, missingSince: expect.any(String) }]); } finally { await database.close(); } }); + it("marks a targeted symlink broken when its target cannot pass the bounded read preflight", async () => { + const symlinkDir = path.join(tmpDir, "links"); + const localDir = path.join(tmpDir, "local"); + const remoteDir = path.join(tmpDir, "remote"); + const titleRoot = path.join(symlinkDir, "movies", "Unreadable Title"); + const invalidTarget = path.join(remoteDir, "invalid-target.mkv"); + await fs.mkdir(titleRoot, { recursive: true }); + await fs.mkdir(invalidTarget, { recursive: true }); + await fs.mkdir(localDir, { recursive: true }); + await fs.symlink(invalidTarget, path.join(titleRoot, "Unreadable Title.mkv")); + + const result = await scanLibrary( + { symlinkDir, localDir, remoteDir }, + { sections: ["movies"], sectionTypes: { movies: "movies" } }, + new Map(), + { + scanSymlinks: true, + scanLocal: false, + scanRemote: false, + symlinkSections: ["movies"], + titleScopes: [{ section: "movies", itemName: "Unreadable Title" }] + } + ); + + expect(result.links).toEqual([ + expect.objectContaining({ + itemName: "Unreadable Title", + kind: "broken", + targetExists: false, + targetReadError: expect.stringContaining("not a regular file") + }) + ]); + expect(result.reconciledStorageFiles).toEqual([]); + }); + it("uses separate section scopes for symlink and local file scans", async () => { const symlinkDir = path.join(tmpDir, "plex"); const localDir = path.join(tmpDir, "local"); From 8c04d0eebe1cd83af54c229d91d64c458a5489dd Mon Sep 17 00:00:00 2001 From: Aleksey Dmitriev Date: Tue, 28 Jul 2026 22:46:21 -0400 Subject: [PATCH 06/11] Stabilize inventory refresh smoke test --- tests/e2e/app-smoke.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/app-smoke.spec.ts b/tests/e2e/app-smoke.spec.ts index 6568b22..6b3bb08 100644 --- a/tests/e2e/app-smoke.spec.ts +++ b/tests/e2e/app-smoke.spec.ts @@ -235,7 +235,7 @@ test("refreshes an open work list when an inventory job finishes", async ({ page body = { rows, total: 2, limit: 250, offset: 0, hasMore: rows.length < 2 }; } else if (url.pathname === "/api/jobs" && url.searchParams.get("completedWithinMinutes") === "15") { inventoryJobPolls += 1; - const completed = inventoryJobPolls > 1; + const completed = workListRequests > 0 && inventoryJobPolls > 1; body = [{ id: 900, type: "scan", status: completed ? "completed" : "running", createdAt: timestamp, startedAt: timestamp, finishedAt: completed ? timestamp : null, progress: {} }]; } else if (url.pathname === "/api/jobs") body = []; else body = {}; From 30590834e0fb7c08eebeed64248131db0b2c9039 Mon Sep 17 00:00:00 2001 From: Aleksey Dmitriev Date: Wed, 29 Jul 2026 13:35:05 -0400 Subject: [PATCH 07/11] Release 0.1.2 beta 2 --- CHANGELOG.md | 8 ++++++++ package-lock.json | 4 ++-- package.json | 2 +- tests/app.test.ts | 4 ++-- tests/e2e/app-smoke.spec.ts | 6 +++--- 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14cb8e3..5ba7b34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ All notable changes are documented here. The project follows Semantic Versioning ## [Unreleased] +## [0.1.2-beta.2] - 2026-07-29 + +### Changed + +- Made targeted title rescans validate readable symlink targets, reconcile their exact storage files, and report persistent read failures. +- Retried transient source and transfer I/O failures before failing a copy. +- Matched administrator usernames case-insensitively for login and account conflicts while preserving display capitalization. + ## [0.1.2-beta.1] - 2026-07-27 ### Changed diff --git a/package-lock.json b/package-lock.json index db0d4b8..2e58d25 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "srtl-manager", - "version": "0.1.2-beta.1", + "version": "0.1.2-beta.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "srtl-manager", - "version": "0.1.2-beta.1", + "version": "0.1.2-beta.2", "license": "MIT", "dependencies": { "@fastify/compress": "^9.1.0", diff --git a/package.json b/package.json index ac94c35..d171eb9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "srtl-manager", - "version": "0.1.2-beta.1", + "version": "0.1.2-beta.2", "private": true, "license": "MIT", "homepage": "https://github.com/ramphex/SRTL-Manager#readme", diff --git a/tests/app.test.ts b/tests/app.test.ts index de8ac22..caddb2c 100644 --- a/tests/app.test.ts +++ b/tests/app.test.ts @@ -817,7 +817,7 @@ describe("api app", () => { expect(version.statusCode).toBe(200); expect(version.json()).toMatchObject({ - currentVersion: "0.1.2-beta.1", + currentVersion: "0.1.2-beta.2", currentChannel: "beta", currentChannelLabel: "Beta", latestVersion: null, @@ -884,7 +884,7 @@ describe("api app", () => { expect(version.statusCode).toBe(200); expect(version.json()).toMatchObject({ - currentVersion: "0.1.2-beta.1", + currentVersion: "0.1.2-beta.2", currentChannel: "beta", currentChannelLabel: "Beta", latestVersion: "0.2.0-beta.1", diff --git a/tests/e2e/app-smoke.spec.ts b/tests/e2e/app-smoke.spec.ts index 6b3bb08..e0b3f75 100644 --- a/tests/e2e/app-smoke.spec.ts +++ b/tests/e2e/app-smoke.spec.ts @@ -89,7 +89,7 @@ test("dashboard task notifications overlay without shifting content", async ({ p else if (url.pathname === "/api/settings/storage-locations") body = { locations: [{ key: "location_1", rootType: "local", displayName: "Local", path: "/mnt/local" }, { key: "location_2", rootType: "remote", displayName: "Remote", path: "/mnt/remote" }] }; else if (url.pathname === "/api/system/version") { const unavailableRelease = { latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, releaseNotes: null, message: "Unavailable" }; - body = { currentVersion: "0.1.2-beta.1", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; + body = { currentVersion: "0.1.2-beta.2", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; } else if (url.pathname === "/api/settings/sections") body = { sections: ["shows"], sectionTitles: { shows: "Shows" }, sectionTypes: { shows: "shows" } }; else if (url.pathname === "/api/settings/paths") body = { symlinkDir: "/mnt/links", localDir: "/mnt/local", remoteDir: "/mnt/remote" }; else if (url.pathname === "/api/settings/scan") body = { scanSymlinks: true, scanLocal: false, scanRemote: false, symlinkSections: ["shows"], localSections: [] }; @@ -220,7 +220,7 @@ test("refreshes an open work list when an inventory job finishes", async ({ page else if (url.pathname === "/api/settings/storage-locations") body = { locations: [{ key: "location_1", rootType: "local", displayName: "Local", path: "/mnt/local" }, { key: "location_2", rootType: "remote", displayName: "Remote", path: "/mnt/remote" }] }; else if (url.pathname === "/api/system/version") { const unavailableRelease = { latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, releaseNotes: null, message: "Unavailable" }; - body = { currentVersion: "0.1.2-beta.1", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; + body = { currentVersion: "0.1.2-beta.2", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; } else if (url.pathname === "/api/settings/sections") body = { sections: ["shows"], sectionTitles: { shows: "Shows" }, sectionTypes: { shows: "shows" } }; else if (url.pathname === "/api/settings/paths") body = { symlinkDir: "/mnt/links", localDir: "/mnt/local", remoteDir: "/mnt/remote" }; @@ -1184,7 +1184,7 @@ test("copy progress opens a persistent, scrollable completed item summary", asyn else if (url.pathname === "/api/settings/storage-locations") body = { locations: [{ key: "location_1", rootType: "local", displayName: "Local", path: "/mnt/local" }, { key: "location_2", rootType: "remote", displayName: "Remote", path: "/mnt/remote" }] }; else if (url.pathname === "/api/system/version") { const unavailableRelease = { latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, releaseNotes: null, message: "Unavailable" }; - body = { currentVersion: "0.1.2-beta.1", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; + body = { currentVersion: "0.1.2-beta.2", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; } else if (url.pathname === `/api/jobs/${jobId}/events/page`) body = { events, total: events.length, hasOlder: false }; else if (url.pathname === `/api/jobs/${jobId}`) body = job; else if (url.pathname === "/api/jobs") body = [job]; From 16b03ceeff9799f0dddf6a14f5fefb43d400ebb9 Mon Sep 17 00:00:00 2001 From: Aleksey Dmitriev Date: Fri, 31 Jul 2026 23:03:55 -0400 Subject: [PATCH 08/11] Release 0.1.2 beta 3 --- .env.example | 11 +- CHANGELOG.md | 17 + README.md | 17 +- docker-compose.yml | 3 + package-lock.json | 4 +- package.json | 2 +- src/client/App.tsx | 17 +- src/client/appShared.ts | 18 +- src/client/jobPresentation.tsx | 8 +- src/client/libraryRoutes.tsx | 72 +- src/server/app.ts | 27 +- src/server/config.ts | 80 +- src/server/db/database.ts | 65 +- src/server/db/schema.ts | 45 +- src/server/jobs/copyLimiter.ts | 75 + src/server/jobs/copyPool.ts | 43 + src/server/jobs/jobRunner.ts | 2765 +++++++++++++++++----- src/server/jobs/resourceMutationGuard.ts | 123 + src/server/jobs/scheduling.ts | 3 + src/server/lib/copier.ts | 343 ++- src/server/lib/env.ts | 12 + src/server/lib/filesystemSafety.ts | 16 + src/server/lib/mountIdentity.ts | 62 + src/server/lib/pathConfiguration.ts | 1105 +++++++-- src/server/lib/scanner.ts | 6 +- src/server/lib/storagePolicies.ts | 60 +- src/server/lib/workerHeartbeats.ts | 44 + src/server/routes/libraryRoutes.ts | 77 +- src/server/worker.ts | 75 +- src/shared/types.ts | 10 +- tests/app.test.ts | 1190 +++++++++- tests/config.test.ts | 72 +- tests/copier.test.ts | 591 ++++- tests/copyLimiter.test.ts | 48 + tests/copyPool.test.ts | 69 + tests/database.test.ts | 193 +- tests/e2e/app-smoke.spec.ts | 181 +- tests/env.test.ts | 24 +- tests/jobScheduler.test.ts | 943 ++++++++ tests/mountIdentity.test.ts | 60 + tests/pathConfiguration.test.ts | 1293 +++++++++- tests/pathMigrationDisplay.test.ts | 18 + tests/workerHeartbeats.test.ts | 222 ++ 43 files changed, 9056 insertions(+), 1053 deletions(-) create mode 100644 src/server/jobs/copyLimiter.ts create mode 100644 src/server/jobs/copyPool.ts create mode 100644 src/server/jobs/resourceMutationGuard.ts create mode 100644 src/server/jobs/scheduling.ts create mode 100644 src/server/lib/mountIdentity.ts create mode 100644 src/server/lib/workerHeartbeats.ts create mode 100644 tests/copyLimiter.test.ts create mode 100644 tests/copyPool.test.ts create mode 100644 tests/jobScheduler.test.ts create mode 100644 tests/mountIdentity.test.ts create mode 100644 tests/pathMigrationDisplay.test.ts create mode 100644 tests/workerHeartbeats.test.ts diff --git a/.env.example b/.env.example index 5a76dc0..26d2096 100644 --- a/.env.example +++ b/.env.example @@ -24,5 +24,14 @@ SRTL_COOKIE_SECURE=false SRTL_ALLOWED_ORIGINS= SRTL_TRUST_PROXY=false -# Reserved for future multi-worker support. The current release enforces one worker. +# Parallel job slots hosted by the single worker service. One preserves the +# existing serial behavior. The total limit defaults to SRTL_WORKER_COUNT and +# per-type limits default to that total. Copy file concurrency defaults to one; +# the active-file limit defaults to the worker count. SRTL_WORKER_COUNT=1 +# SRTL_MAX_RUNNING_JOBS=1 +# SRTL_MAX_RUNNING_SCANS=1 +# SRTL_MAX_RUNNING_AUDITS=1 +# SRTL_MAX_RUNNING_COPIES=1 +# SRTL_COPY_FILE_CONCURRENCY=1 +# SRTL_MAX_ACTIVE_COPY_FILES=1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ba7b34..8d249de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,23 @@ All notable changes are documented here. The project follows Semantic Versioning ## [Unreleased] +## [0.1.2-beta.3] - 2026-07-31 + +### Changed + +- Added configurable in-process worker slots and independent global, per-job-type, and copy-transfer concurrency limits without an arbitrary worker-count ceiling. +- Kept example deployments at one worker slot by default while honoring any positive `SRTL_WORKER_COUNT` value from `.env`. +- Allowed non-overlapping copy, audit, and targeted title-rescan work to run concurrently while broad scans and path migrations remain exclusive. + +### Fixed + +- Made queue admission, worker claims, stale-job recovery, and job updates lease-aware so overlapping or superseded workers cannot mutate the same job. +- Preserved immutable job resource scopes so later inventory changes cannot remove an active job's overlap protection. +- Scoped legacy failed-copy reconciliation locks to their exact media records and managed paths so newly scanned items from the same title can still be queued. +- Allowed filesystem-read-only scans and audits to run while terminal legacy copy records await reconciliation; path migration and exact conflicting mutations remain fenced. +- Loaded every page of dashboard work lists and made show and season copy actions server-scoped so large sections are never truncated to the first 250 links. +- Accepted routine FUSE and NFS remounts without a false path migration when the canonical path and stable mount signature are unchanged, while retaining exact identity checks during active mutations. + ## [0.1.2-beta.2] - 2026-07-29 ### Changed diff --git a/README.md b/README.md index 5dd0230..ed8a6cd 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,8 @@ SRTL Manager is a local-first web app for inventorying and maintaining a symlink - Fast and deep audits across local and remote targets. - Bidirectional copies with live progress, transient transfer retry, conflict handling, configurable verification, and source/title risk checks. - Safe job termination, complete event timelines, restart recovery, and durable per-file copy journals. -- Required path-change review before changed mounts can affect managed links. +- Configurable parallel job and copy-file execution with database-backed overlap protection. +- Required path-change review before different configured roots or storage mount sources can affect managed links. - Editable storage-location names while deployment paths remain environment-managed. - Postgres-backed API and worker services in a hardened container stack. - Dark, light, and system themes with responsive administration views. @@ -73,9 +74,19 @@ The default configuration follows the current stable `latest` image. Pin `SRTL_I The API receives read-only root mounts. The worker alone receives writable roots for copy and path-migration jobs. Postgres is reachable only inside the Compose network. +### Worker concurrency + +The single Compose worker service can host any positive number of independent job slots. Set `SRTL_WORKER_COUNT` to the desired slot count; there is no fixed two-worker ceiling. Jobs still pass database-backed admission, resource-claim, and lease checks, so adding slots does not allow two jobs to mutate the same managed resources. + +The optional `SRTL_MAX_RUNNING_JOBS` setting limits total simultaneous jobs and must not exceed `SRTL_WORKER_COUNT`. `SRTL_MAX_RUNNING_SCANS`, `SRTL_MAX_RUNNING_AUDITS`, and `SRTL_MAX_RUNNING_COPIES` apply per-type limits, may be zero to pause that job type, and must not exceed the total-job limit. When omitted, the total limit follows the configured worker count and the per-type limits follow that total limit. + +`SRTL_COPY_FILE_CONCURRENCY` controls how many files one copy job may transfer at once. `SRTL_MAX_ACTIVE_COPY_FILES` is the worker process-wide copy-file ceiling and must be at least the per-job value. Their defaults keep one file active per copy job while allowing separate copy jobs to use separate slots. Start conservatively and raise copy limits only when the storage endpoints and network can sustain the additional I/O. + +Scale with `SRTL_WORKER_COUNT`; do not simultaneously run `docker compose up --scale worker=...`. The supported deployment model keeps job slots and the active-copy-file safeguard inside one worker process. Compose gives that process two minutes to stop active jobs and perform safe rollback during shutdown. + The API checks the public GitHub Releases endpoint for stable and beta version information at startup and when version status is refreshed. This request does not include credentials, paths, or inventory data. -When a configured root changes, restart the stack. The UI enters maintenance mode until it validates and applies a path migration or the prior value is restored. This rebases managed paths; it does not move stored content. +When a configured root changes, restart the stack. The UI enters maintenance mode until it validates and applies a path migration or the prior value is restored. This rebases managed paths; it does not move stored content. Routine Linux remounts are accepted automatically when the canonical path, mount point, filesystem type, and mount source are unchanged; exact device and inode checks still fence active filesystem mutations. ## Backup And Restore @@ -142,4 +153,4 @@ SRTL Manager is available under the [MIT License](LICENSE). Contributions are ac - Optional remote source-availability preflight checks where providers expose reliable metadata. - Event-driven targeted refresh hooks. - Additional numbered storage locations and per-location assignment policies. -- Controlled multi-worker execution after single-worker recovery semantics are fully proven. +- Per-location throughput telemetry and concurrency recommendations. diff --git a/docker-compose.yml b/docker-compose.yml index 9384866..51706a6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -84,6 +84,9 @@ services: image: ${SRTL_IMAGE:?Set SRTL_IMAGE in .env} command: ["node", "dist/server/worker.js"] restart: unless-stopped + # SRTL_WORKER_COUNT scales safe job slots inside this service. Do not also + # use `docker compose --scale worker=...` for the same deployment. + stop_grace_period: 2m user: "${SRTL_UID:?Set SRTL_UID in .env}:${SRTL_GID:?Set SRTL_GID in .env}" read_only: true env_file: diff --git a/package-lock.json b/package-lock.json index 2e58d25..65b165f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "srtl-manager", - "version": "0.1.2-beta.2", + "version": "0.1.2-beta.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "srtl-manager", - "version": "0.1.2-beta.2", + "version": "0.1.2-beta.3", "license": "MIT", "dependencies": { "@fastify/compress": "^9.1.0", diff --git a/package.json b/package.json index d171eb9..2573586 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "srtl-manager", - "version": "0.1.2-beta.2", + "version": "0.1.2-beta.3", "private": true, "license": "MIT", "homepage": "https://github.com/ramphex/SRTL-Manager#readme", diff --git a/src/client/App.tsx b/src/client/App.tsx index b8fe08f..1511ef7 100644 --- a/src/client/App.tsx +++ b/src/client/App.tsx @@ -8,7 +8,7 @@ import { formatCurrentVersionDisplay } from "./versionDisplay"; import { inventoryJobRefreshKey, isActiveDashboardJob } from "./recentJobs"; import { inferSectionContentType } from "../shared/sections"; import { type AppReleaseInfo, type AppVersionInfo, type JobRecord, type OnboardingPolicyMode, type OnboardingState, type PathConfigurationState, type SectionContentType, type StorageLocationKey } from "../shared/types"; -import { SectionDraft, SidebarGroup, StorageLocationsContext, UserPreferencesContext, canTerminateJob, copyElapsedLabel, createEmptySectionDraft, defaultStorageLocations, defaultUserPreferences, formatNumber, historySections, onboardingScanVisibleStats, parseLogsRouteSearch, pathMigrationProgress, scanOptionsFromProgress, scanProgressFromJob, scanStageLabel, scanStagePercent, scanStatusDetail, scanVisibleIndexedItemCount, scanVisibleStats, sectionDraftsToSettings, sectionSettingsToDrafts, sectionTypeOptions, settingsSections, storageLocationName, themeOptions, useLiveTimestamp, useStorageLocations, useTerminateJobMutation, useThemePreference, versionChannelLabel, versionCheckIntervalMs, visibleVersionReleases } from "./appShared"; +import { SectionDraft, SidebarGroup, StorageLocationsContext, UserPreferencesContext, canTerminateJob, copyElapsedLabel, createEmptySectionDraft, defaultStorageLocations, defaultUserPreferences, formatNumber, historySections, isActivePathMigrationStatus, onboardingScanVisibleStats, parseLogsRouteSearch, pathMigrationProgress, pathMigrationProgressTitle, pathMigrationStatusLabel, scanOptionsFromProgress, scanProgressFromJob, scanStageLabel, scanStagePercent, scanStatusDetail, scanVisibleIndexedItemCount, scanVisibleStats, sectionDraftsToSettings, sectionSettingsToDrafts, sectionTypeOptions, settingsSections, storageLocationName, themeOptions, useLiveTimestamp, useStorageLocations, useTerminateJobMutation, useThemePreference, versionChannelLabel, versionCheckIntervalMs, visibleVersionReleases } from "./appShared"; export function SectionDraftList({ drafts, onChange, disabled = false }: { drafts: SectionDraft[]; onChange: (drafts: SectionDraft[]) => void; disabled?: boolean }) { const updateDraft = (id: string, patch: Partial) => { onChange(drafts.map((draft) => (draft.id === id ? { ...draft, ...patch } : draft))); @@ -898,7 +898,7 @@ function PathMigrationGate({ state }: { state: PathConfigurationState }) { queryKey: ["job", migration?.jobId], queryFn: () => api.job(migration?.jobId ?? 0), enabled: Boolean(migration?.jobId), - refetchInterval: migration?.status === "queued" || migration?.status === "running" ? 1000 : false + refetchInterval: isActivePathMigrationStatus(migration?.status) ? 1000 : false }); const progress = pathMigrationProgress(job.data ?? null); const progressPercent = progress.total > 0 ? Math.min(100, Math.max(0, (progress.current / progress.total) * 100)) : 0; @@ -949,7 +949,7 @@ function PathMigrationGate({ state }: { state: PathConfigurationState }) {
{change.changed ? ( - {change.identityMatch === "same" ? "Same root detected" : change.identityMatch === "different" ? "Different root identity" : "Root identity unavailable"} + {change.identityMatch === "same" ? "Same storage mount" : change.identityMatch === "different" ? "Different storage mount" : "Storage mount unavailable"} ) : null}
@@ -961,15 +961,15 @@ function PathMigrationGate({ state }: { state: PathConfigurationState }) {
Validating every affected symlink and mapped target...
) : null} - {migration && ["planned", "queued", "running", "failed"].includes(migration.status) ? ( + {migration && ["planned", "queued", "running", "rollback_pending", "failed"].includes(migration.status) ? (

Migration analysis

Only path references are migrated. Storage content is never moved by this workflow.

- 0 || migration.status === "failed" ? "failed" : migration.status === "running" || migration.status === "queued" ? "running" : "completed"}`}> - {migration.status === "planned" ? "Ready" : migration.status === "queued" ? "Queued" : migration.status === "running" ? "Running" : "Needs attention"} + 0 || migration.status === "failed" ? "failed" : isActivePathMigrationStatus(migration.status) ? "running" : "completed"}`}> + {pathMigrationStatusLabel(migration.status)}
@@ -997,10 +997,11 @@ function PathMigrationGate({ state }: { state: PathConfigurationState }) {
) : null} - {migration && (migration.status === "queued" || migration.status === "running") ? ( + {migration && isActivePathMigrationStatus(migration.status) ? (
-
{progress.message}{formatNumber(progress.current)} / {formatNumber(progress.total)}
+
{pathMigrationProgressTitle(migration.status, progress.message)}{formatNumber(progress.current)} / {formatNumber(progress.total)}
+ {migration.status === "rollback_pending" && progress.message !== "Rolling back paths" ?

{progress.message}

: null}

Do not change mounts or edit symlinks while this migration is running. This page will unlock automatically when reconciliation completes.

diff --git a/src/client/appShared.ts b/src/client/appShared.ts index 033e3d2..4a084c6 100644 --- a/src/client/appShared.ts +++ b/src/client/appShared.ts @@ -6,7 +6,7 @@ import { scanOptionsFromJob } from "./jobScopeLocks"; import { mergeJobEventPages } from "./jobEvents"; import { defaultRecentJobsCompletedWindowMinutes } from "./recentJobs"; import { inferSectionContentType } from "../shared/sections"; -import { type AuditMode, type AuditOptions, type AppReleaseInfo, type AppVersionInfo, type CopyConflictPreview, type InventorySummary, type JobRecord, type JobStatus, type CopyMediaValidationMode, type CopyOptions, type CopyVerificationProfile, type MediaLinkTreeKindFilter, type ScanOptions, type SectionContentType, type SectionSettings, type SectionSummary, type StoragePolicyCategory, type StoragePolicyKind, type StorageLocationsSettings, type StorageRootType, type TimeFormatPreference, type UserPreferences } from "../shared/types"; +import { type AuditMode, type AuditOptions, type AppReleaseInfo, type AppVersionInfo, type CopyConflictPreview, type InventorySummary, type JobRecord, type JobStatus, type CopyMediaValidationMode, type CopyOptions, type CopyVerificationProfile, type MediaLinkTreeKindFilter, type PathMigrationStatus, type ScanOptions, type SectionContentType, type SectionSettings, type SectionSummary, type StoragePolicyCategory, type StoragePolicyKind, type StorageLocationsSettings, type StorageRootType, type TimeFormatPreference, type UserPreferences } from "../shared/types"; export type ThemePreference = "light" | "dark" | "system"; @@ -687,6 +687,22 @@ export function pathMigrationProgress(job: JobRecord | null): { current: number; }; } +export function isActivePathMigrationStatus(status: PathMigrationStatus | null | undefined): boolean { + return status === "queued" || status === "running" || status === "rollback_pending"; +} + +export function pathMigrationStatusLabel(status: PathMigrationStatus): string { + if (status === "planned") return "Ready"; + if (status === "queued") return "Queued"; + if (status === "running") return "Running"; + if (status === "rollback_pending") return "Rolling back"; + return "Needs attention"; +} + +export function pathMigrationProgressTitle(status: PathMigrationStatus, message: string): string { + return status === "rollback_pending" ? "Rolling back paths" : message; +} + export function dateTimeFormatOptions(timeFormat: TimeFormatPreference): Intl.DateTimeFormatOptions { return { year: "numeric", month: "numeric", day: "numeric", hour: "numeric", minute: "2-digit", hour12: timeFormat === "12h" }; } diff --git a/src/client/jobPresentation.tsx b/src/client/jobPresentation.tsx index 8676cc0..7e920a2 100644 --- a/src/client/jobPresentation.tsx +++ b/src/client/jobPresentation.tsx @@ -241,7 +241,9 @@ export function CopyDialog({ {startCopy.error ?

{startCopy.error.message}

: null} - {needsLocalConflictResolution && prompt.conflicts ? ( + {startCopy.error && !jobId ? ( +

The copy job was not queued. No files were changed.

+ ) : needsLocalConflictResolution && prompt.conflicts ? ( setLocalConflictStrategy("keep_both")} onReplace={() => setLocalConflictStrategy("replace")} /> ) : ( )} -
+ {startCopy.error && !jobId ? null :
{jobId && events.isLoading ?

Loading copy events...

: null} {events.error ?

{events.error.message}

: null} @@ -268,7 +270,7 @@ export function CopyDialog({ ))}
) : null} -
+ } [0], "limit" | "offset"> +): Promise>> { + const rows: MediaLinkRow[] = []; + const seenIds = new Set(); + let offset = 0; + let total: number; + + while (true) { + const page = await api.mediaLinksPage({ ...params, limit: dashboardRemoteWorkLinkLimit, offset }); + total = page.total; + for (const row of page.rows) { + if (seenIds.has(row.id)) continue; + seenIds.add(row.id); + rows.push(row); + } + if (!page.hasMore) break; + const nextOffset = page.offset + page.rows.length; + if (nextOffset <= offset) throw new Error("The work list did not advance while loading additional results"); + offset = nextOffset; + } + + return { rows, total, limit: dashboardRemoteWorkLinkLimit, offset: 0, hasMore: false }; +} + type ActionableEpisode = { episodeName: string; link: MediaLinkRow; @@ -802,15 +827,13 @@ function RemoteWorkLinksTable({ const remoteWorkLinks = useQuery({ queryKey: ["dashboard-remote-work-links", selection?.kind, selectedSection, selectedPrefix, trimmedSearch], queryFn: () => - api.mediaLinksPage({ + loadAllRemoteWorkLinks({ kind: detail?.kind, section: selectedSection ?? "", storagePolicy: detail?.storagePolicy ?? "unassigned", relativePathPrefix: selectedPrefix, - search: trimmedSearch, - limit: dashboardRemoteWorkLinkLimit, - offset: 0 - }), + search: trimmedSearch + }), enabled: Boolean(selectedSection && detail) }); const jobs = useQuery({ queryKey: ["jobs", "active"], queryFn: () => api.jobs({ activeOnly: true }), refetchInterval: 3000 }); @@ -939,17 +962,24 @@ function RemoteWorkLinksTable({ } function queueShowCopy(show: ActionableShowGroup) { - if (!copyDirection) return; - const links = showLinks(show); - if (links.length === 0) return; - queueCopy(`Copy ${show.showName} to ${copyDestinationLabel}`, `${title} / ${show.showName}`, { direction: copyDirection, linkIds: links.map((link) => link.id) }); + if (!copyDirection || !selectedSection) return; + queueCopy(`Copy ${show.showName} to ${copyDestinationLabel}`, `${title} / ${show.showName}`, { + direction: copyDirection, + section: selectedSection, + itemName: show.showName, + ...(selectedPrefix ? { relativePathPrefix: selectedPrefix } : {}) + }); } function queueSeasonCopy(showName: string, season: ActionableSeasonGroup) { - if (!copyDirection) return; - const links = seasonLinks(season); - if (links.length === 0) return; - queueCopy(`Copy ${showName} / ${season.seasonName} to ${copyDestinationLabel}`, `${title} / ${showName} / ${season.seasonName}`, { direction: copyDirection, linkIds: links.map((link) => link.id) }); + if (!copyDirection || !selectedSection) return; + const prefix = scopedRelativePrefix(selectedPrefix, seasonRelativePrefix(season)); + queueCopy(`Copy ${showName} / ${season.seasonName} to ${copyDestinationLabel}`, `${title} / ${showName} / ${season.seasonName}`, { + direction: copyDirection, + section: selectedSection, + itemName: showName, + ...(prefix ? { relativePathPrefix: prefix } : {}) + }); } function queueLinkCopy(link: MediaLinkRow) { @@ -964,7 +994,7 @@ function RemoteWorkLinksTable({ function queueSeasonAudit(showName: string, season: ActionableSeasonGroup) { if (!selectedSection) return; - const prefix = seasonRelativePrefix(season); + const prefix = scopedRelativePrefix(selectedPrefix, seasonRelativePrefix(season)); queueAudit(`Audit ${showName} / ${season.seasonName}`, `${title} / ${showName} / ${season.seasonName}`, { section: selectedSection, itemName: showName, @@ -999,8 +1029,8 @@ function RemoteWorkLinksTable({ {remoteWorkLinks.data ? ( {isShowSection - ? `Showing ${formatNumber(rows.length)} links across ${formatNumber(showGroups.length)} shows of ${formatNumber(shownTotal)}` - : `Showing ${formatNumber(rows.length)} of ${formatNumber(shownTotal)}`} + ? `Showing all ${formatNumber(rows.length)} links across ${formatNumber(showGroups.length)} shows` + : `Showing all ${formatNumber(rows.length)}`} ) : null} {canCopy ? ( @@ -1278,6 +1308,16 @@ function seasonRelativePrefix(season: ActionableSeasonGroup): string | null { return parts.slice(0, 2).join("/"); } +function scopedRelativePrefix(selectedPrefix: string, actionPrefix: string | null): string | undefined { + const selected = splitMediaRelativePath(selectedPrefix).join("/"); + const action = actionPrefix ? splitMediaRelativePath(actionPrefix).join("/") : ""; + if (!selected) return action || undefined; + if (!action) return selected; + if (selected === action || selected.startsWith(`${action}/`)) return selected; + if (action.startsWith(`${selected}/`)) return action; + return action; +} + function remoteWorkDetail(kind: SectionWorkKind, storageLocations: StorageLocationsSettings): { title: string; emptyName: string; diff --git a/src/server/app.ts b/src/server/app.ts index 58c4ba4..bef5fe9 100644 --- a/src/server/app.ts +++ b/src/server/app.ts @@ -16,7 +16,7 @@ import { loadConfig } from "./config"; import { hasAdmin, requireAuth } from "./auth"; import { openDatabase, type DatabaseContext } from "./db/database"; import * as schema from "./db/schema"; -import { desc } from "drizzle-orm"; +import { desc, eq } from "drizzle-orm"; import { isPathConfigurationBlocked, reconcileEnvironmentPaths } from "./lib/pathConfiguration"; import { canAdoptEnvironmentPathsBeforeInitialScan, isOnboardingComplete, reconcileOnboardingState } from "./lib/onboarding"; import { JobRunner } from "./jobs/jobRunner"; @@ -121,13 +121,30 @@ export async function createApp(overrides: Partial = {}): Promise { await database.pool.query("select 1"); - const worker = (await database.db.select().from(schema.workerHeartbeats).orderBy(desc(schema.workerHeartbeats.heartbeatAt)).limit(1))[0] ?? null; - const workerAgeMs = worker ? Math.max(0, Date.now() - Date.parse(worker.heartbeatAt)) : null; + const workers = await database.db + .select() + .from(schema.workerHeartbeats) + .where(eq(schema.workerHeartbeats.status, "running")) + .orderBy(desc(schema.workerHeartbeats.heartbeatAt)); + const latestWorker = workers[0] ?? null; + const now = Date.now(); + const readyWorkerCount = workers.reduce((capacity, worker) => { + const heartbeatAt = Date.parse(worker.heartbeatAt); + return Number.isFinite(heartbeatAt) && Math.max(0, now - heartbeatAt) <= 30_000 ? capacity + worker.capacity : capacity; + }, 0); + const staleWorkerCount = workers.reduce((capacity, worker) => { + const heartbeatAt = Date.parse(worker.heartbeatAt); + return !Number.isFinite(heartbeatAt) || Math.max(0, now - heartbeatAt) > 30_000 ? capacity + worker.capacity : capacity; + }, 0); + const expectedWorkerCount = config.jobConcurrency.maxRunningJobs; return { ok: true, database: "ready", - worker: worker && worker.status === "running" && workerAgeMs != null && workerAgeMs <= 30_000 ? "ready" : worker ? "stale" : "not_started", - workerHeartbeatAt: worker?.heartbeatAt ?? null + worker: readyWorkerCount >= expectedWorkerCount ? "ready" : workers.length > 0 ? "stale" : "not_started", + workerHeartbeatAt: latestWorker?.heartbeatAt ?? null, + expectedWorkerCount, + readyWorkerCount, + staleWorkerCount }; }); registerAuthRoutes(app, database.db, { diff --git a/src/server/config.ts b/src/server/config.ts index ff84a69..d004814 100644 --- a/src/server/config.ts +++ b/src/server/config.ts @@ -26,12 +26,10 @@ export interface JobConcurrencySettings { maxRunningScans: number; maxRunningAudits: number; maxRunningCopies: number; + copyFileConcurrency: number; + maxActiveCopyFiles: number; } -// Placeholder until multi-worker setup ships in future updates. Keep the -// effective worker count hard-capped at 1 while scan/copy/audit behavior is -// hardened. -const currentWorkerCountHardLimit = 1; const exampleDatabasePassword = "replace-with-your-password"; function booleanSetting(value: string | undefined, fallback: boolean): boolean { @@ -103,23 +101,73 @@ export function resolveDatabaseUrl(envFile: ReturnType = {}) return databaseUrlSetting(url.toString()); } -function normalizeWorkerCount(value: number | string | undefined): number { - const parsed = typeof value === "number" ? value : Number(value); - if (!Number.isInteger(parsed) || parsed < 1) return 1; - return Math.min(parsed, currentWorkerCountHardLimit); +function integerSetting(value: number | string | undefined, fallback: number, name: string, minimum: 0 | 1): number { + if (value == null) return fallback; + const normalized = typeof value === "string" ? value.trim() : value; + if (typeof normalized === "string" && !/^(0|[1-9]\d*)$/.test(normalized)) { + throw new Error(`${name} must be a ${minimum === 0 ? "non-negative" : "positive"} safe integer`); + } + const parsed = typeof normalized === "number" ? normalized : Number(normalized); + if (!Number.isSafeInteger(parsed) || parsed < minimum) { + throw new Error(`${name} must be a ${minimum === 0 ? "non-negative" : "positive"} safe integer`); + } + return parsed; } function loadJobConcurrency(envFile: ReturnType, overrides: Partial | undefined = {}): JobConcurrencySettings { - // Every running-job limit derives from the effective worker count, which is - // temporarily capped above. - const workerCount = normalizeWorkerCount(overrides.workerCount ?? process.env.SRTL_WORKER_COUNT ?? envFile.SRTL_WORKER_COUNT); - return { + const workerCount = integerSetting(overrides.workerCount ?? process.env.SRTL_WORKER_COUNT ?? envFile.SRTL_WORKER_COUNT, 1, "SRTL_WORKER_COUNT", 1); + const maxRunningJobs = integerSetting( + overrides.maxRunningJobs ?? process.env.SRTL_MAX_RUNNING_JOBS ?? envFile.SRTL_MAX_RUNNING_JOBS, workerCount, - maxRunningJobs: workerCount, - maxRunningScans: workerCount, - maxRunningAudits: workerCount, - maxRunningCopies: workerCount + "SRTL_MAX_RUNNING_JOBS", + 1 + ); + const settings: JobConcurrencySettings = { + workerCount, + maxRunningJobs, + maxRunningScans: integerSetting( + overrides.maxRunningScans ?? process.env.SRTL_MAX_RUNNING_SCANS ?? envFile.SRTL_MAX_RUNNING_SCANS, + maxRunningJobs, + "SRTL_MAX_RUNNING_SCANS", + 0 + ), + maxRunningAudits: integerSetting( + overrides.maxRunningAudits ?? process.env.SRTL_MAX_RUNNING_AUDITS ?? envFile.SRTL_MAX_RUNNING_AUDITS, + maxRunningJobs, + "SRTL_MAX_RUNNING_AUDITS", + 0 + ), + maxRunningCopies: integerSetting( + overrides.maxRunningCopies ?? process.env.SRTL_MAX_RUNNING_COPIES ?? envFile.SRTL_MAX_RUNNING_COPIES, + maxRunningJobs, + "SRTL_MAX_RUNNING_COPIES", + 0 + ), + copyFileConcurrency: integerSetting( + overrides.copyFileConcurrency ?? process.env.SRTL_COPY_FILE_CONCURRENCY ?? envFile.SRTL_COPY_FILE_CONCURRENCY, + 1, + "SRTL_COPY_FILE_CONCURRENCY", + 1 + ), + maxActiveCopyFiles: integerSetting( + overrides.maxActiveCopyFiles ?? process.env.SRTL_MAX_ACTIVE_COPY_FILES ?? envFile.SRTL_MAX_ACTIVE_COPY_FILES, + workerCount, + "SRTL_MAX_ACTIVE_COPY_FILES", + 1 + ) }; + if (settings.maxRunningJobs > settings.workerCount) throw new Error("SRTL_MAX_RUNNING_JOBS must not exceed SRTL_WORKER_COUNT"); + for (const [name, value] of [ + ["SRTL_MAX_RUNNING_SCANS", settings.maxRunningScans], + ["SRTL_MAX_RUNNING_AUDITS", settings.maxRunningAudits], + ["SRTL_MAX_RUNNING_COPIES", settings.maxRunningCopies] + ] as const) { + if (value > settings.maxRunningJobs) throw new Error(`${name} must not exceed SRTL_MAX_RUNNING_JOBS`); + } + if (settings.copyFileConcurrency > settings.maxActiveCopyFiles) { + throw new Error("SRTL_COPY_FILE_CONCURRENCY must not exceed SRTL_MAX_ACTIVE_COPY_FILES"); + } + return settings; } export function loadConfig(overrides: Partial = {}): AppConfig { diff --git a/src/server/db/database.ts b/src/server/db/database.ts index 9a6a05d..5ea09e0 100644 --- a/src/server/db/database.ts +++ b/src/server/db/database.ts @@ -6,6 +6,7 @@ import type { SectionContentType, SectionSettings } from "../../shared/types"; import { inferSectionContentType, normalizeSectionContentType } from "../../shared/sections"; export type Db = NodePgDatabase; +export type DbExecutor = Db; export { inferSectionContentType } from "../../shared/sections"; export interface DatabaseContext { @@ -20,13 +21,13 @@ export interface DatabaseOpenOptions { pool?: Pool; } -export const currentSchemaVersion = 4; +export const currentSchemaVersion = 7; const ddl = [ `CREATE TABLE IF NOT EXISTS app_settings (key TEXT PRIMARY KEY, value TEXT NOT NULL, updated_at TEXT NOT NULL)`, `CREATE TABLE IF NOT EXISTS path_configurations (id SERIAL PRIMARY KEY, status TEXT NOT NULL, symlink_dir TEXT NOT NULL, local_dir TEXT NOT NULL, remote_dir TEXT NOT NULL, symlink_identity TEXT NOT NULL, local_identity TEXT NOT NULL, remote_identity TEXT NOT NULL, created_at TEXT NOT NULL, applied_at TEXT)`, `CREATE TABLE IF NOT EXISTS path_migrations (id SERIAL PRIMARY KEY, source_config_id INTEGER, target_config_id INTEGER NOT NULL, status TEXT NOT NULL, job_id INTEGER, error_message TEXT, created_at TEXT NOT NULL, planned_at TEXT, started_at TEXT, finished_at TEXT)`, - `CREATE TABLE IF NOT EXISTS path_migration_items (id SERIAL PRIMARY KEY, migration_id INTEGER NOT NULL, media_link_id INTEGER NOT NULL, item_name TEXT NOT NULL, current_link_path TEXT NOT NULL, link_path_before TEXT NOT NULL, link_path_after TEXT NOT NULL, target_path_before TEXT NOT NULL, target_path_after TEXT NOT NULL, target_changed BOOLEAN NOT NULL, expected_size_bytes BIGINT, validation_status TEXT NOT NULL, message TEXT NOT NULL, applied_at TEXT, rolled_back_at TEXT)`, + `CREATE TABLE IF NOT EXISTS path_migration_items (id SERIAL PRIMARY KEY, migration_id INTEGER NOT NULL, media_link_id INTEGER NOT NULL, item_name TEXT NOT NULL, current_link_path TEXT NOT NULL, link_path_before TEXT NOT NULL, link_path_after TEXT NOT NULL, target_path_before TEXT NOT NULL, target_path_after TEXT NOT NULL, target_changed BOOLEAN NOT NULL, expected_size_bytes BIGINT, target_identity TEXT, validation_status TEXT NOT NULL, message TEXT NOT NULL, applied_at TEXT, rolled_back_at TEXT)`, `CREATE TABLE IF NOT EXISTS admin_users (id SERIAL PRIMARY KEY, username TEXT NOT NULL UNIQUE, password_hash TEXT NOT NULL, created_at TEXT NOT NULL)`, `CREATE TABLE IF NOT EXISTS sessions (token_hash TEXT PRIMARY KEY, user_id INTEGER NOT NULL, expires_at TEXT NOT NULL, created_at TEXT NOT NULL)`, `CREATE TABLE IF NOT EXISTS sections (id SERIAL PRIMARY KEY, name TEXT NOT NULL UNIQUE, display_name TEXT, content_type TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL)`, @@ -61,7 +62,7 @@ const ddl = [ ]; const hardeningDdl = [ - `CREATE TABLE IF NOT EXISTS copy_operations (id SERIAL PRIMARY KEY, job_id INTEGER NOT NULL, media_link_id INTEGER NOT NULL, link_path TEXT NOT NULL, source_path TEXT NOT NULL, destination_path TEXT NOT NULL, original_target_path TEXT NOT NULL, original_link_state TEXT NOT NULL, previous_copy_source TEXT, temp_path TEXT, displaced_path TEXT, stage TEXT NOT NULL, result_status TEXT, local_conflict_strategy TEXT, size_bytes BIGINT, error_message TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, completed_at TEXT, UNIQUE(job_id, media_link_id))`, + `CREATE TABLE IF NOT EXISTS copy_operations (id SERIAL PRIMARY KEY, job_id INTEGER NOT NULL, media_link_id INTEGER NOT NULL, link_path TEXT NOT NULL, source_path TEXT NOT NULL, destination_path TEXT NOT NULL, original_target_path TEXT NOT NULL, original_link_state TEXT NOT NULL, previous_copy_source TEXT, temp_path TEXT, displaced_path TEXT, temp_identity TEXT, destination_identity TEXT, displaced_identity TEXT, stage TEXT NOT NULL, result_status TEXT, local_conflict_strategy TEXT, size_bytes BIGINT, error_message TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, completed_at TEXT, UNIQUE(job_id, media_link_id))`, `CREATE INDEX IF NOT EXISTS copy_operations_job_stage_idx ON copy_operations(job_id, stage, id)`, `CREATE UNIQUE INDEX IF NOT EXISTS admin_users_singleton_idx ON admin_users ((true))`, `DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'sessions_user_fk') THEN ALTER TABLE sessions ADD CONSTRAINT sessions_user_fk FOREIGN KEY (user_id) REFERENCES admin_users(id) ON DELETE CASCADE NOT VALID; END IF; END $$`, @@ -123,7 +124,9 @@ async function initializeDatabase(pool: Pool): Promise { if (!applied.has(3)) { await client.query("BEGIN"); try { - await client.query(`CREATE TABLE IF NOT EXISTS worker_heartbeats (worker_id TEXT PRIMARY KEY, started_at TEXT NOT NULL, heartbeat_at TEXT NOT NULL, status TEXT NOT NULL)`); + await client.query( + `CREATE TABLE IF NOT EXISTS worker_heartbeats (worker_id TEXT PRIMARY KEY, started_at TEXT NOT NULL, heartbeat_at TEXT NOT NULL, status TEXT NOT NULL, capacity BIGINT NOT NULL DEFAULT 1)` + ); await client.query(`DROP TABLE IF EXISTS legacy_policy_import_tombstones`); await client.query(`DROP TABLE IF EXISTS integration_sync_runs`); await client.query(`DROP TABLE IF EXISTS integration_configs`); @@ -171,6 +174,60 @@ async function initializeDatabase(pool: Pool): Promise { throw error; } } + + if (!applied.has(5)) { + await client.query("BEGIN"); + try { + await client.query(`ALTER TABLE jobs ADD COLUMN IF NOT EXISTS lease_version INTEGER NOT NULL DEFAULT 0`); + await client.query(`ALTER TABLE jobs ADD COLUMN IF NOT EXISTS exclusive BOOLEAN NOT NULL DEFAULT TRUE`); + await client.query(` + CREATE TABLE IF NOT EXISTS job_resource_claims ( + id SERIAL PRIMARY KEY, + job_id INTEGER NOT NULL REFERENCES jobs(id) ON DELETE CASCADE, + resource_type TEXT NOT NULL, + resource_key TEXT NOT NULL, + access TEXT NOT NULL DEFAULT 'exclusive', + created_at TEXT NOT NULL, + CONSTRAINT job_resource_claims_job_resource_idx UNIQUE (job_id, resource_type, resource_key), + CONSTRAINT job_resource_claims_access_check CHECK (access IN ('shared', 'exclusive')) + ) + `); + await client.query(`CREATE INDEX IF NOT EXISTS job_resource_claims_lookup_idx ON job_resource_claims(resource_type, resource_key, access)`); + await client.query(`ALTER TABLE worker_heartbeats ADD COLUMN IF NOT EXISTS capacity BIGINT NOT NULL DEFAULT 1`); + await client.query(`CREATE INDEX IF NOT EXISTS worker_heartbeats_status_heartbeat_idx ON worker_heartbeats(status, heartbeat_at)`); + await client.query(`INSERT INTO schema_migrations (version, name, applied_at) VALUES (5, 'multi_worker_job_claims', $1)`, [nowIso()]); + await client.query("COMMIT"); + } catch (error) { + await client.query("ROLLBACK"); + throw error; + } + } + + if (!applied.has(6)) { + await client.query("BEGIN"); + try { + await client.query(`ALTER TABLE copy_operations ADD COLUMN IF NOT EXISTS temp_identity TEXT`); + await client.query(`ALTER TABLE copy_operations ADD COLUMN IF NOT EXISTS destination_identity TEXT`); + await client.query(`ALTER TABLE copy_operations ADD COLUMN IF NOT EXISTS displaced_identity TEXT`); + await client.query(`INSERT INTO schema_migrations (version, name, applied_at) VALUES (6, 'copy_operation_file_identities', $1)`, [nowIso()]); + await client.query("COMMIT"); + } catch (error) { + await client.query("ROLLBACK"); + throw error; + } + } + + if (!applied.has(7)) { + await client.query("BEGIN"); + try { + await client.query(`ALTER TABLE path_migration_items ADD COLUMN IF NOT EXISTS target_identity TEXT`); + await client.query(`INSERT INTO schema_migrations (version, name, applied_at) VALUES (7, 'path_migration_target_identities', $1)`, [nowIso()]); + await client.query("COMMIT"); + } catch (error) { + await client.query("ROLLBACK"); + throw error; + } + } } finally { await client.query("select pg_advisory_unlock($1)", [bootstrapLockKey]).catch(() => undefined); client.release(); diff --git a/src/server/db/schema.ts b/src/server/db/schema.ts index 317de6a..beb2980 100644 --- a/src/server/db/schema.ts +++ b/src/server/db/schema.ts @@ -1,4 +1,5 @@ -import { bigint, boolean, integer, pgTable, serial, text, uniqueIndex } from "drizzle-orm/pg-core"; +import { sql } from "drizzle-orm"; +import { bigint, boolean, check, index, integer, pgTable, serial, text, uniqueIndex } from "drizzle-orm/pg-core"; export const appSettings = pgTable("app_settings", { key: text("key").primaryKey(), @@ -44,6 +45,7 @@ export const pathMigrationItems = pgTable("path_migration_items", { targetPathAfter: text("target_path_after").notNull(), targetChanged: boolean("target_changed").notNull(), expectedSizeBytes: bigint("expected_size_bytes", { mode: "number" }), + targetIdentity: text("target_identity"), validationStatus: text("validation_status").notNull(), message: text("message").notNull(), appliedAt: text("applied_at"), @@ -156,16 +158,42 @@ export const jobs = pgTable("jobs", { lockedBy: text("locked_by"), lockedAt: text("locked_at"), heartbeatAt: text("heartbeat_at"), + leaseVersion: integer("lease_version").notNull().default(0), + exclusive: boolean("exclusive").notNull().default(true), cancelRequestedAt: text("cancel_requested_at"), progress: text("progress").notNull() }); -export const workerHeartbeats = pgTable("worker_heartbeats", { - workerId: text("worker_id").primaryKey(), - startedAt: text("started_at").notNull(), - heartbeatAt: text("heartbeat_at").notNull(), - status: text("status").notNull() -}); +export const jobResourceClaims = pgTable( + "job_resource_claims", + { + id: serial("id").primaryKey(), + jobId: integer("job_id") + .notNull() + .references(() => jobs.id, { onDelete: "cascade" }), + resourceType: text("resource_type").notNull(), + resourceKey: text("resource_key").notNull(), + access: text("access").notNull().default("exclusive"), + createdAt: text("created_at").notNull() + }, + (table) => [ + uniqueIndex("job_resource_claims_job_resource_idx").on(table.jobId, table.resourceType, table.resourceKey), + index("job_resource_claims_lookup_idx").on(table.resourceType, table.resourceKey, table.access), + check("job_resource_claims_access_check", sql`${table.access} IN ('shared', 'exclusive')`) + ] +); + +export const workerHeartbeats = pgTable( + "worker_heartbeats", + { + workerId: text("worker_id").primaryKey(), + startedAt: text("started_at").notNull(), + heartbeatAt: text("heartbeat_at").notNull(), + status: text("status").notNull(), + capacity: bigint("capacity", { mode: "number" }).notNull().default(1) + }, + (table) => [index("worker_heartbeats_status_heartbeat_idx").on(table.status, table.heartbeatAt)] +); export const copyOperations = pgTable( "copy_operations", @@ -181,6 +209,9 @@ export const copyOperations = pgTable( previousCopySource: text("previous_copy_source"), tempPath: text("temp_path"), displacedPath: text("displaced_path"), + tempIdentity: text("temp_identity"), + destinationIdentity: text("destination_identity"), + displacedIdentity: text("displaced_identity"), stage: text("stage").notNull(), resultStatus: text("result_status"), localConflictStrategy: text("local_conflict_strategy"), diff --git a/src/server/jobs/copyLimiter.ts b/src/server/jobs/copyLimiter.ts new file mode 100644 index 0000000..6b21a95 --- /dev/null +++ b/src/server/jobs/copyLimiter.ts @@ -0,0 +1,75 @@ +function abortReason(signal: AbortSignal): Error { + return signal.reason instanceof Error ? signal.reason : new Error("Copy transfer was cancelled"); +} + +interface WaitingTransfer { + signal: AbortSignal; + resolve(release: () => void): void; + reject(error: Error): void; + abort(): void; +} + +export class CopyTransferLimiter { + private active = 0; + private readonly waiting: WaitingTransfer[] = []; + + constructor(readonly maximum: number) { + if (!Number.isSafeInteger(maximum) || maximum < 1) throw new Error("Copy transfer limit must be a positive safe integer"); + } + + get activeCount(): number { + return this.active; + } + + get waitingCount(): number { + return this.waiting.length; + } + + acquire(signal: AbortSignal): Promise<() => void> { + if (signal.aborted) return Promise.reject(abortReason(signal)); + if (this.active < this.maximum) { + this.active += 1; + return Promise.resolve(this.releaseOnce()); + } + + return new Promise<() => void>((resolve, reject) => { + const waiting: WaitingTransfer = { + signal, + resolve, + reject, + abort: () => { + const index = this.waiting.indexOf(waiting); + if (index >= 0) this.waiting.splice(index, 1); + signal.removeEventListener("abort", waiting.abort); + reject(abortReason(signal)); + } + }; + this.waiting.push(waiting); + signal.addEventListener("abort", waiting.abort, { once: true }); + }); + } + + private releaseOnce(): () => void { + let released = false; + return () => { + if (released) return; + released = true; + this.active -= 1; + this.startNext(); + }; + } + + private startNext(): void { + while (this.active < this.maximum) { + const next = this.waiting.shift(); + if (!next) return; + next.signal.removeEventListener("abort", next.abort); + if (next.signal.aborted) { + next.reject(abortReason(next.signal)); + continue; + } + this.active += 1; + next.resolve(this.releaseOnce()); + } + } +} diff --git a/src/server/jobs/copyPool.ts b/src/server/jobs/copyPool.ts new file mode 100644 index 0000000..83a65be --- /dev/null +++ b/src/server/jobs/copyPool.ts @@ -0,0 +1,43 @@ +export async function runKeyedPool( + items: readonly T[], + maximum: number, + keyFor: (item: T) => string, + run: (item: T) => Promise, + shouldContinue: () => boolean = () => true +): Promise { + if (!Number.isSafeInteger(maximum) || maximum < 1) throw new Error("Copy file concurrency must be a positive safe integer"); + + const lanes = new Map(); + for (const item of items) { + const key = keyFor(item); + const lane = lanes.get(key); + if (lane) lane.push(item); + else lanes.set(key, [item]); + } + + const pendingLanes = [...lanes.values()]; + let nextLane = 0; + let stopStarting = false; + const worker = async () => { + while (!stopStarting && shouldContinue()) { + const laneIndex = nextLane; + nextLane += 1; + const lane = pendingLanes[laneIndex]; + if (!lane) return; + for (const item of lane) { + if (stopStarting || !shouldContinue()) return; + try { + await run(item); + } catch (error) { + stopStarting = true; + throw error; + } + } + } + }; + + const workers = Array.from({ length: Math.min(maximum, pendingLanes.length) }, () => worker()); + const results = await Promise.allSettled(workers); + const failure = results.find((result): result is PromiseRejectedResult => result.status === "rejected"); + if (failure) throw failure.reason; +} diff --git a/src/server/jobs/jobRunner.ts b/src/server/jobs/jobRunner.ts index 83a3489..a3a55ff 100644 --- a/src/server/jobs/jobRunner.ts +++ b/src/server/jobs/jobRunner.ts @@ -1,19 +1,34 @@ -import { and, asc, count, desc, eq, gt, gte, inArray, lt, ne } from "drizzle-orm"; +import { and, asc, count, desc, eq, gt, gte, inArray, isNull, lt, ne, sql } from "drizzle-orm"; import type { Dirent } from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; import type { JobConcurrencySettings } from "../config"; -import { first, getJsonSetting, getSectionSettings, nowIso, type Db } from "../db/database"; +import { dbGet, first, getJsonSetting, getSectionSettings, nowIso, type Db, type DbExecutor } from "../db/database"; import * as schema from "../db/schema"; import { auditMediaLink, defaultAuditRunner, type AuditCommandRunner } from "../lib/auditor"; -import { copyMediaLink, defaultCopyRunner, type CopyCommandRunner, type CopyMediaResult, type CopyOperationUpdate, type CopyProgressUpdate } from "../lib/copier"; -import { assertDestinationPathInside, assertExistingPathInside, assertPathParentInside, withFilesystemTimeout } from "../lib/filesystemSafety"; +import { + copyFileIdentitiesMatch, + copyMediaLink, + CopyReconciliationRequiredError, + defaultCopyRunner, + parseCopyFileIdentity, + readCopyFileIdentity, + type CopyFileIdentity, + type CopyCommandRunner, + type CopyMediaResult, + type CopyOperationUpdate, + type CopyProgressUpdate +} from "../lib/copier"; +import { assertDestinationPathInside, assertExistingPathInside, assertPathParentInside, canonicalPathForClaim, withFilesystemTimeout } from "../lib/filesystemSafety"; import { isMediaFile, isPathInside } from "../lib/media"; -import { assertPathMigrationReady, isPathConfigurationBlocked, runPathMigration } from "../lib/pathConfiguration"; +import { isPathConfigurationBlocked, runPathMigration } from "../lib/pathConfiguration"; import { completeOnboardingScan } from "../lib/onboarding"; import { defaultScanOptions, getStoragePolicyMap, listMediaLinks, persistScanResult, scanLibrary, type ScanActivity } from "../lib/scanner"; import { normalizeAdvancedSettings } from "../../shared/advancedSettings"; import { evaluateSourceTitleRisk } from "../../shared/sourceTitleRisk"; +import { CopyTransferLimiter } from "./copyLimiter"; +import { runKeyedPool } from "./copyPool"; +import { schedulerLockKey } from "./scheduling"; import type { AuditMode, AuditOptions, @@ -46,6 +61,13 @@ class WorkerShutdownError extends Error { } } +export class LeaseLostError extends Error { + constructor(jobId: number, options?: ErrorOptions) { + super(`Worker lease lost for job #${jobId}`, options); + this.name = "LeaseLostError"; + } +} + class PartialJobFailureError extends Error { constructor(message: string) { super(message); @@ -58,15 +80,43 @@ const defaultJobConcurrency: JobConcurrencySettings = { maxRunningJobs: 1, maxRunningScans: 1, maxRunningAudits: 1, - maxRunningCopies: 1 + maxRunningCopies: 1, + copyFileConcurrency: 1, + maxActiveCopyFiles: 1 }; +type ResourceClaimAccess = "shared" | "exclusive"; + +interface ResourceClaim { + resourceType: string; + resourceKey: string; + access: ResourceClaimAccess; +} + +interface PreparedJob { + progress: unknown; + exclusive: boolean; + claims: ResourceClaim[]; +} + +type LeasedJob = JobRecord & { leaseVersion: number; exclusive: boolean }; + +interface ClaimedJob { + job: LeasedJob; + reclaimed: boolean; +} + export interface JobContext { jobId: number; signal: AbortSignal; event(level: JobEventRecord["level"], message: string, data?: unknown): Promise; setProgress(progress: unknown): Promise; isCancelled(): Promise; + assertLease(): Promise; + withLease(action: () => Promise): Promise; + withLeaseDb(action: (db: DbExecutor) => Promise): Promise; + finishCompleted(action: (db: DbExecutor) => Promise): Promise; + finishCompletedIsolated(action: (db: DbExecutor) => Promise): Promise; } export interface JobListOptions { @@ -85,6 +135,8 @@ export interface JobWorkerOptions { copyRunner?: CopyCommandRunner; auditRunner?: AuditCommandRunner; concurrency?: JobConcurrencySettings; + copyTransferLimiter?: CopyTransferLimiter; + dispatchConcurrency?: number; } function normalizeSelectedSections(selectedSections: string[] | undefined, configuredSections: string[], label: string): string[] { @@ -230,7 +282,7 @@ async function normalizeAuditOptions(db: Db, input: AuditMode | AuditOptions): P if (typeof input !== "string" && input.linkIds && linkIds?.length !== input.linkIds.length) throw new Error("Audit link IDs must be positive integers"); const itemName = typeof input === "string" ? undefined : input.itemName?.trim(); const relativePathPrefix = typeof input === "string" ? undefined : normalizeRelativePrefix(input.relativePathPrefix); - const hasScopedAudit = Boolean(linkIds?.length || section || itemName || relativePathPrefix); + const hasScopedAudit = Boolean(linkIds !== undefined || section || itemName || relativePathPrefix); const hasRequestedTargets = Array.isArray(requestedOptions.targets); const targets = normalizeAuditTargets(requestedOptions.targets); const selectedSections = requestedOptions.sections @@ -245,7 +297,7 @@ async function normalizeAuditOptions(db: Db, input: AuditMode | AuditOptions): P ...(selectedSections ? { sections: selectedSections } : {}), ...(!hasScopedAudit ? { targets } : hasRequestedTargets ? { targets } : {}), ...(section ? { section } : {}), - ...(linkIds && linkIds.length > 0 ? { linkIds } : {}), + ...(linkIds !== undefined ? { linkIds } : {}), ...(itemName ? { itemName } : {}), ...(relativePathPrefix ? { relativePathPrefix } : {}), ...(requestedOptions.byteCompare === false ? { byteCompare: false } : {}) @@ -331,7 +383,7 @@ function relativePathMatchesPrefix(relativePath: string, prefix: string): boolea } function hasScopedAuditOptions(options: AuditOptions): boolean { - return Boolean(options.linkIds?.length || options.section || options.itemName || options.relativePathPrefix); + return Boolean(options.linkIds !== undefined || options.section || options.itemName || options.relativePathPrefix); } function filterScanLinks(links: MediaLinkRow[], options: ScanOptions): MediaLinkRow[] { @@ -348,7 +400,7 @@ function filterScanLinks(links: MediaLinkRow[], options: ScanOptions): MediaLink } function filterAuditLinks(links: MediaLinkRow[], options: AuditOptions): MediaLinkRow[] { - const requestedIds = options.linkIds?.length ? new Set(options.linkIds) : null; + const requestedIds = options.linkIds === undefined ? null : new Set(options.linkIds); const requestedTargetSet = options.targets ? new Set(normalizeAuditTargets(options.targets)) : null; if (hasScopedAuditOptions(options)) { return links.filter((link) => { @@ -374,7 +426,7 @@ function filterAuditLinks(links: MediaLinkRow[], options: AuditOptions): MediaLi } function filterCopyLinks(links: MediaLinkRow[], options: CopyOptions): MediaLinkRow[] { - const requestedIds = options.linkIds?.length ? new Set(options.linkIds) : null; + const requestedIds = options.linkIds === undefined ? null : new Set(options.linkIds); const sourceKind = options.direction === "to_local" ? "remote" : "local"; const storagePolicy = options.direction === "to_local" ? "location_1" : "location_2"; return links.filter((link) => { @@ -396,7 +448,7 @@ function copyStoragePolicy(direction: CopyOptions["direction"]): StoragePolicyKi } function filterCopySelectedLinks(links: MediaLinkRow[], options: CopyOptions): MediaLinkRow[] { - const requestedIds = options.linkIds?.length ? new Set(options.linkIds) : null; + const requestedIds = options.linkIds === undefined ? null : new Set(options.linkIds); const sourceKind = options.direction === "to_local" ? "remote" : "local"; const destinationKind = copyDestinationKind(options.direction); const storagePolicy = copyStoragePolicy(options.direction); @@ -410,32 +462,365 @@ function filterCopySelectedLinks(links: MediaLinkRow[], options: CopyOptions): M }); } -function activeJobLinks(job: JobRecord, links: MediaLinkRow[]): MediaLinkRow[] { - if (job.type === "scan") return filterScanLinks(links, jobProgressOptions(job) ?? defaultScanOptions); - if (job.type === "copy") return filterCopyLinks(links, readCopyOptions(job)); - if (job.type === "audit") return filterAuditLinks(links, readAuditOptions(job)); - return []; +function orderedCopySelection(links: MediaLinkRow[], options: CopyOptions): MediaLinkRow[] { + const selected = filterCopySelectedLinks(links, options); + const requestedOrder = options.linkIds?.length ? new Map(options.linkIds.map((id, index) => [id, index])) : null; + return requestedOrder + ? [...selected].sort((firstLink, secondLink) => (requestedOrder.get(firstLink.id) ?? 0) - (requestedOrder.get(secondLink.id) ?? 0)) + : selected; +} + +function copyAdmissionFingerprint(link: MediaLinkRow): string { + return JSON.stringify([ + link.id, + link.section, + link.itemName, + link.relativePath, + path.resolve(link.linkPath), + path.resolve(link.targetPath), + link.kind, + link.targetExists, + link.isMedia, + link.storagePolicy, + link.resolvedStorageFileId, + link.sizeBytes, + link.missingSince + ]); +} + +function resourceClaimKey(claim: Pick): string { + return `${claim.resourceType}\0${claim.resourceKey}`; +} + +function normalizeResourceClaims(claims: ResourceClaim[]): ResourceClaim[] { + const normalized = new Map(); + for (const claim of claims) { + const key = resourceClaimKey(claim); + const current = normalized.get(key); + if (!current || claim.access === "exclusive") normalized.set(key, claim); + } + return [...normalized.values()]; +} + +async function managedPathResourceClaims( + root: string | null, + candidate: string, + label: string, + access: ResourceClaimAccess, + preserveLeaf = false +): Promise { + const lexicalPath = path.resolve(candidate); + if (!root) return [{ resourceType: "path", resourceKey: lexicalPath, access }]; + const canonicalPath = await canonicalPathForClaim(root, lexicalPath, label, preserveLeaf); + return [...new Set([lexicalPath, canonicalPath])].map((resourceKey) => ({ resourceType: "path", resourceKey, access })); +} + +function managedRootForTarget(paths: PathsSettings, targetPath: string): string | null { + if (isPathInside(paths.localDir, targetPath)) return paths.localDir; + if (isPathInside(paths.remoteDir, targetPath)) return paths.remoteDir; + return null; +} + +async function mediaLinkResourceClaims(link: MediaLinkRow, paths: PathsSettings, access: ResourceClaimAccess): Promise { + const [linkPathClaims, targetPathClaims] = await Promise.all([ + managedPathResourceClaims(paths.symlinkDir, link.linkPath, "Library symlink claim", access, true), + managedPathResourceClaims(managedRootForTarget(paths, link.targetPath), link.targetPath, "Media target claim", access) + ]); + return [ + { resourceType: "media", resourceKey: String(link.id), access }, + ...linkPathClaims, + ...targetPathClaims, + { resourceType: "title", resourceKey: JSON.stringify([link.section, link.itemName]), access } + ]; +} + +async function batchedMediaLinkResourceClaims( + links: MediaLinkRow[], + paths: PathsSettings, + access: ResourceClaimAccess +): Promise { + const claims: ResourceClaim[] = []; + for (let offset = 0; offset < links.length; offset += 16) { + const batch = await Promise.all(links.slice(offset, offset + 16).map((link) => mediaLinkResourceClaims(link, paths, access))); + claims.push(...batch.flat()); + } + return claims; +} + +async function titleScanResourceClaims(options: ScanOptions, links: MediaLinkRow[], paths: PathsSettings): Promise { + const claims: ResourceClaim[] = []; + for (const scope of options.titleScopes ?? []) { + claims.push({ resourceType: "title", resourceKey: JSON.stringify([scope.section, scope.itemName]), access: "exclusive" }); + } + claims.push(...(await batchedMediaLinkResourceClaims(links, paths, "exclusive"))); + return normalizeResourceClaims(claims); +} + +async function auditResourceClaims(links: MediaLinkRow[], paths: PathsSettings): Promise { + return normalizeResourceClaims(await batchedMediaLinkResourceClaims(links, paths, "shared")); +} + +type CopyPathBindingRole = "link" | "source" | "destination"; + +interface CopyPathBinding { + linkId: number; + role: CopyPathBindingRole; + lexicalPath: string; + canonicalPath: string; +} + +interface CopySelectedDestination { + linkId: number; + lexicalPath: string; + canonicalPath: string; +} + +interface CopySelectedDestinationIndex { + entries: CopySelectedDestination[]; + lexicalOwners: Map>; + canonicalOwners: Map>; +} + +function addCopyDestinationOwner(owners: Map>, filePath: string, linkId: number): void { + const existing = owners.get(filePath); + if (existing) existing.add(linkId); + else owners.set(filePath, new Set([linkId])); +} + +function indexCopySelectedDestinations(entries: CopySelectedDestination[]): CopySelectedDestinationIndex { + const lexicalOwners = new Map>(); + const canonicalOwners = new Map>(); + for (const entry of entries) { + addCopyDestinationOwner(lexicalOwners, entry.lexicalPath, entry.linkId); + addCopyDestinationOwner(canonicalOwners, entry.canonicalPath, entry.linkId); + } + return { entries, lexicalOwners, canonicalOwners }; +} + +async function copySelectedDestinationForLink( + link: MediaLinkRow, + paths: PathsSettings, + direction: CopyOptions["direction"] +): Promise { + const destinationRoot = storageRootForDirection(paths, direction); + const lexicalPath = path.resolve(copyDestinationPathForLink(link, paths, direction)); + const canonicalPath = await canonicalPathForClaim(destinationRoot, lexicalPath, "Copy destination claim"); + return { linkId: link.id, lexicalPath, canonicalPath }; +} + +async function copySelectedDestinationsForLinks( + links: MediaLinkRow[], + paths: PathsSettings, + direction: CopyOptions["direction"] +): Promise { + const entries: CopySelectedDestination[] = []; + for (let offset = 0; offset < links.length; offset += 16) { + entries.push(...(await Promise.all(links.slice(offset, offset + 16).map((link) => copySelectedDestinationForLink(link, paths, direction))))); + } + return indexCopySelectedDestinations(entries); +} + +async function isOtherSelectedCopyDestination( + paths: PathsSettings, + filePath: string, + currentLinkId: number, + selectedDestinations: CopySelectedDestinationIndex +): Promise { + if (selectedDestinations.entries.length === 0) return false; + const lexicalPath = path.resolve(filePath); + const canonicalPath = await canonicalPathForClaim(paths.localDir, lexicalPath, "Local replacement candidate"); + const owners = new Set([ + ...(selectedDestinations.lexicalOwners.get(lexicalPath) ?? []), + ...(selectedDestinations.canonicalOwners.get(canonicalPath) ?? []) + ]); + return owners.size > 0 && !owners.has(currentLinkId); +} + +function copyPathBindingMapKey(linkId: number, role: CopyPathBindingRole): string { + return `${linkId}\0${role}`; +} + +function copyPathBindingResourceKey(binding: CopyPathBinding): string { + return JSON.stringify([binding.linkId, binding.role, binding.lexicalPath, binding.canonicalPath]); +} + +function parseCopyPathBindingResourceKey(value: string): CopyPathBinding | null { + const parsed = parseJson(value); + if ( + !Array.isArray(parsed) || + parsed.length !== 4 || + !Number.isSafeInteger(parsed[0]) || + Number(parsed[0]) < 1 || + !["link", "source", "destination"].includes(String(parsed[1])) || + typeof parsed[2] !== "string" || + typeof parsed[3] !== "string" + ) { + return null; + } + return { + linkId: Number(parsed[0]), + role: parsed[1] as CopyPathBindingRole, + lexicalPath: path.resolve(parsed[2]), + canonicalPath: path.resolve(parsed[3]) + }; +} + +async function copyPathBindingsForLink( + link: MediaLinkRow, + paths: PathsSettings, + direction: CopyOptions["direction"] +): Promise { + const sourceRoot = direction === "to_local" ? paths.remoteDir : paths.localDir; + const linkPath = path.resolve(link.linkPath); + const sourcePath = path.resolve(link.targetPath); + const [canonicalLinkPath, canonicalSourcePath, destination] = await Promise.all([ + canonicalPathForClaim(paths.symlinkDir, linkPath, "Library symlink claim", true), + canonicalPathForClaim(sourceRoot, sourcePath, "Media target claim"), + copySelectedDestinationForLink(link, paths, direction) + ]); + return [ + { linkId: link.id, role: "link", lexicalPath: linkPath, canonicalPath: canonicalLinkPath }, + { linkId: link.id, role: "source", lexicalPath: sourcePath, canonicalPath: canonicalSourcePath }, + { linkId: link.id, role: "destination", lexicalPath: destination.lexicalPath, canonicalPath: destination.canonicalPath } + ]; +} + +async function copyResourceClaims(links: MediaLinkRow[], paths: PathsSettings, direction: CopyOptions["direction"]): Promise { + const claims = await batchedMediaLinkResourceClaims(links, paths, "exclusive"); + const eligibleLinkIds = new Set( + filterCopyLinks(links, { direction, linkIds: links.map((link) => link.id) }).map((link) => link.id) + ); + for (let offset = 0; offset < links.length; offset += 16) { + const batch = links.slice(offset, offset + 16); + const [destinationClaims, pathBindings] = await Promise.all([ + Promise.all( + batch.map((link) => + managedPathResourceClaims( + storageRootForDirection(paths, direction), + copyDestinationPathForLink(link, paths, direction), + "Copy destination claim", + "exclusive" + ) + ) + ), + Promise.all(batch.filter((link) => eligibleLinkIds.has(link.id)).map((link) => copyPathBindingsForLink(link, paths, direction))) + ]); + claims.push(...destinationClaims.flat()); + claims.push( + ...pathBindings.flat().map((binding) => ({ + resourceType: "copy_path_binding", + resourceKey: copyPathBindingResourceKey(binding), + access: "exclusive" as const + })) + ); + } + return normalizeResourceClaims(claims); +} + +interface CopyCleanupIdentity { + device: string; + inode: string; + size: string; + modifiedNs: string; + changedNs: string; +} + +async function copyCleanupIdentity(filePath: string): Promise { + const stat = await fs.stat(filePath, { bigint: true }).catch(() => null); + if (!stat?.isFile()) return null; + return { + device: stat.dev.toString(), + inode: stat.ino.toString(), + size: stat.size.toString(), + modifiedNs: stat.mtimeNs.toString(), + changedNs: stat.ctimeNs.toString() + }; +} + +function sameCopyCleanupIdentity(first: CopyCleanupIdentity | null | undefined, second: CopyCleanupIdentity | null | undefined): boolean { + return Boolean( + first && + second && + first.device === second.device && + first.inode === second.inode && + first.size === second.size && + first.modifiedNs === second.modifiedNs && + first.changedNs === second.changedNs + ); +} + +function copyCleanupMarkerKey(linkId: number, filePath: string, identity: CopyCleanupIdentity): string { + return JSON.stringify([linkId, path.resolve(filePath), identity]); } -function overlappingLinkCount(first: MediaLinkRow[], second: MediaLinkRow[]): number { - const secondIds = new Set(second.map((link) => link.id)); - return first.filter((link) => secondIds.has(link.id)).length; +function isCopyCleanupIdentity(value: unknown): value is CopyCleanupIdentity { + if (!isRecord(value)) return false; + return ["device", "inode", "size", "modifiedNs", "changedNs"].every( + (key) => typeof value[key] === "string" && /^\d+$/.test(value[key]) + ); } -async function assertNoActiveJobOverlap(db: Db, links: MediaLinkRow[], requestedLinks: MediaLinkRow[]): Promise { - if (requestedLinks.length === 0) return; - const activeJobs = (await db.select().from(schema.jobs)) - .map(toJobRecord) - .filter((job) => (job.type === "scan" || job.type === "copy" || job.type === "audit") && (job.status === "queued" || job.status === "running")); +function parseCopyCleanupMarker(value: string): { linkId: number; filePath: string; identity: CopyCleanupIdentity | null } | null { + const parsed = parseJson(value); + if (!Array.isArray(parsed) || (parsed.length !== 2 && parsed.length !== 3) || !Number.isSafeInteger(parsed[0]) || Number(parsed[0]) < 1 || typeof parsed[1] !== "string") { + return null; + } + const identity = isCopyCleanupIdentity(parsed[2]) ? parsed[2] : null; + return { linkId: Number(parsed[0]), filePath: path.resolve(parsed[1]), identity }; +} - for (const job of activeJobs) { - const overlapCount = overlappingLinkCount(requestedLinks, activeJobLinks(job, links)); - if (overlapCount > 0) { - throw new Error( - `Job #${job.id} is already ${job.status} for ${overlapCount} matching media item${overlapCount === 1 ? "" : "s"}. Wait for it to finish or terminate it before queuing another action.` +async function copyReplacementResourceClaims( + db: Db, + links: MediaLinkRow[], + paths: PathsSettings, + options: CopyOptions +): Promise { + const selectedDestinations = await copySelectedDestinationsForLinks(links, paths, options.direction); + const destinations = new Map(); + const libraryLinks = new Map(); + for (const destination of selectedDestinations.entries) { + const existingLinkId = destinations.get(destination.canonicalPath); + if (existingLinkId != null && existingLinkId !== destination.linkId) { + throw new Error(`Selected media #${existingLinkId} and #${destination.linkId} resolve to the same copy destination: ${destination.lexicalPath}`); + } + destinations.set(destination.canonicalPath, destination.linkId); + } + for (const link of links) { + const canonicalLinkPath = await canonicalPathForClaim(paths.symlinkDir, link.linkPath, "Library symlink claim", true); + const existingLibraryLinkId = libraryLinks.get(canonicalLinkPath); + if (existingLibraryLinkId != null && existingLibraryLinkId !== link.id) { + throw new Error(`Selected media #${existingLibraryLinkId} and #${link.id} resolve to the same library symlink: ${link.linkPath}`); + } + libraryLinks.set(canonicalLinkPath, link.id); + } + if (options.direction !== "to_local" || options.localConflictStrategy !== "replace") return []; + + const claims: ResourceClaim[] = []; + const cleanupOwners = new Map(); + const eligibleLinks = filterCopyLinks(links, { ...options, linkIds: links.map((link) => link.id) }); + for (const link of eligibleLinks) { + const conflict = await copyLocalConflictForLink(db, link, paths, selectedDestinations); + for (const candidate of conflict?.candidates ?? []) { + const candidatePath = path.resolve(candidate.filePath); + const canonicalCandidatePath = await canonicalPathForClaim(paths.localDir, candidatePath, "Local replacement claim"); + const destinationOwner = destinations.get(canonicalCandidatePath); + if (destinationOwner != null && destinationOwner !== link.id) { + throw new Error(`Replacement cleanup for media #${link.id} overlaps selected destination for media #${destinationOwner}: ${candidatePath}`); + } + const cleanupOwner = cleanupOwners.get(canonicalCandidatePath); + if (cleanupOwner != null && cleanupOwner !== link.id) { + throw new Error(`Replacement cleanup for media #${link.id} overlaps cleanup for media #${cleanupOwner}: ${candidatePath}`); + } + cleanupOwners.set(canonicalCandidatePath, link.id); + const identity = await copyCleanupIdentity(candidatePath); + if (!identity) continue; + claims.push( + ...(await managedPathResourceClaims(paths.localDir, candidatePath, "Local replacement claim", "exclusive")), + { resourceType: "copy_cleanup", resourceKey: copyCleanupMarkerKey(link.id, candidatePath, identity), access: "exclusive" } ); } } + return normalizeResourceClaims(claims); } function storageRootForDirection(paths: PathsSettings, direction: CopyOptions["direction"]): string { @@ -556,7 +941,12 @@ function addUniqueLocalCandidate(candidates: Map { +async function localConflictCandidatesForLink( + db: Db, + link: MediaLinkRow, + paths: PathsSettings, + selectedDestinations: CopySelectedDestinationIndex +): Promise { const destinationPath = copyDestinationPathForLink(link, paths, "to_local"); const candidates = new Map(); const scope = localConflictSearchScope(link); @@ -588,11 +978,22 @@ async function localConflictCandidatesForLink(db: Db, link: MediaLinkRow, paths: } } - return [...candidates.values()].sort((first, second) => first.relativePath.localeCompare(second.relativePath, undefined, { numeric: true, sensitivity: "base" })); + const filteredCandidates: CopyLocalConflictCandidate[] = []; + for (const candidate of candidates.values()) { + if (!(await isOtherSelectedCopyDestination(paths, candidate.filePath, link.id, selectedDestinations))) { + filteredCandidates.push(candidate); + } + } + return filteredCandidates.sort((first, second) => first.relativePath.localeCompare(second.relativePath, undefined, { numeric: true, sensitivity: "base" })); } -async function copyLocalConflictForLink(db: Db, link: MediaLinkRow, paths: PathsSettings): Promise { - const candidates = await localConflictCandidatesForLink(db, link, paths); +async function copyLocalConflictForLink( + db: Db, + link: MediaLinkRow, + paths: PathsSettings, + selectedDestinations: CopySelectedDestinationIndex +): Promise { + const candidates = await localConflictCandidatesForLink(db, link, paths, selectedDestinations); if (candidates.length === 0) return null; return { linkId: link.id, @@ -607,10 +1008,12 @@ async function copyLocalConflictForLink(db: Db, link: MediaLinkRow, paths: Paths async function previewCopyConflicts(db: Db, paths: PathsSettings, options: CopyOptions): Promise { if (options.direction !== "to_local") return { conflicts: [], totalConflicts: 0, totalCandidates: 0 }; - const links = filterCopyLinks(await listMediaLinks(db), options); + const selectedLinks = orderedCopySelection(await listMediaLinks(db), options); + const links = filterCopyLinks(selectedLinks, { ...options, linkIds: selectedLinks.map((link) => link.id) }); + const selectedDestinations = await copySelectedDestinationsForLinks(selectedLinks, paths, options.direction); const conflicts: CopyLocalConflict[] = []; for (const link of links) { - const conflict = await copyLocalConflictForLink(db, link, paths); + const conflict = await copyLocalConflictForLink(db, link, paths, selectedDestinations); if (conflict) conflicts.push(conflict); } return { @@ -620,16 +1023,30 @@ async function previewCopyConflicts(db: Db, paths: PathsSettings, options: CopyO }; } -async function removeLocalConflictCandidates(db: Db, paths: PathsSettings, candidates: CopyLocalConflictCandidate[], preservedPath: string): Promise { +async function removeLocalConflictCandidates( + db: DbExecutor, + paths: PathsSettings, + candidates: CopyLocalConflictCandidate[], + preservedPath: string, + expectedIdentities: Map, + currentLinkId: number, + selectedDestinations: CopySelectedDestinationIndex +): Promise { const timestamp = nowIso(); const removed: string[] = []; const preserved = path.resolve(preservedPath); for (const candidate of candidates) { const candidatePath = path.resolve(candidate.filePath); if (candidatePath === preserved) continue; + if (await isOtherSelectedCopyDestination(paths, candidatePath, currentLinkId, selectedDestinations)) continue; await assertExistingPathInside(paths.localDir, candidatePath, "Local replacement candidate"); const stat = await fs.stat(candidatePath).catch(() => null); if (!stat?.isFile()) continue; + const expectedIdentity = expectedIdentities.get(candidatePath); + const actualIdentity = await copyCleanupIdentity(candidatePath); + if (!sameCopyCleanupIdentity(expectedIdentity, actualIdentity) || stat.size !== candidate.sizeBytes) { + throw new Error(`Local replacement candidate changed after copy admission: ${candidatePath}`); + } await fs.rm(candidatePath, { force: true }); await db.update(schema.storageFiles).set({ missingSince: timestamp, updatedAt: timestamp }).where(eq(schema.storageFiles.filePath, candidatePath)); removed.push(candidatePath); @@ -641,13 +1058,23 @@ type CopySourceRow = typeof schema.copySources.$inferSelect; type CopyOperationRow = typeof schema.copyOperations.$inferSelect; async function prepareCopyOperation( - db: Db, + db: DbExecutor, jobId: number, link: MediaLinkRow, destinationPath: string, previousCopySource: CopySourceRow | null, localConflictStrategy: CopyLocalConflictStrategy | undefined ): Promise { + const existing = await first( + db + .select({ id: schema.copyOperations.id, stage: schema.copyOperations.stage, errorMessage: schema.copyOperations.errorMessage }) + .from(schema.copyOperations) + .where(and(eq(schema.copyOperations.jobId, jobId), eq(schema.copyOperations.mediaLinkId, link.id))) + .limit(1) + ); + if (existing?.stage === "reconciliation_required") { + throw new Error(`Copy operation #${existing.id} requires manual reconciliation: ${existing.errorMessage ?? "filesystem state is uncertain"}`); + } const timestamp = nowIso(); const row = await first( db @@ -663,6 +1090,9 @@ async function prepareCopyOperation( previousCopySource: previousCopySource ? JSON.stringify(previousCopySource) : null, tempPath: null, displacedPath: null, + tempIdentity: null, + destinationIdentity: null, + displacedIdentity: null, stage: "planned", resultStatus: null, localConflictStrategy: localConflictStrategy ?? null, @@ -683,6 +1113,9 @@ async function prepareCopyOperation( previousCopySource: previousCopySource ? JSON.stringify(previousCopySource) : null, tempPath: null, displacedPath: null, + tempIdentity: null, + destinationIdentity: null, + displacedIdentity: null, stage: "planned", resultStatus: null, localConflictStrategy: localConflictStrategy ?? null, @@ -698,65 +1131,83 @@ async function prepareCopyOperation( return row; } -async function updateCopyOperation(db: Db, operationId: number, update: CopyOperationUpdate): Promise { +async function updateCopyOperation(db: DbExecutor, operationId: number, update: CopyOperationUpdate): Promise { + const row = await first( + db + .update(schema.copyOperations) + .set({ + stage: update.stage, + ...(update.tempPath !== undefined ? { tempPath: update.tempPath } : {}), + ...(update.displacedPath !== undefined ? { displacedPath: update.displacedPath } : {}), + ...(update.tempIdentity !== undefined ? { tempIdentity: update.tempIdentity } : {}), + ...(update.destinationIdentity !== undefined ? { destinationIdentity: update.destinationIdentity } : {}), + ...(update.displacedIdentity !== undefined ? { displacedIdentity: update.displacedIdentity } : {}), + ...(update.sizeBytes !== undefined ? { sizeBytes: update.sizeBytes } : {}), + ...(update.resultStatus !== undefined ? { resultStatus: update.resultStatus } : {}), + updatedAt: nowIso() + }) + .where(eq(schema.copyOperations.id, operationId)) + .returning({ id: schema.copyOperations.id }) + ); + if (!row) throw new Error(`Copy operation #${operationId} disappeared before its journal could be updated`); +} + +async function commitCopyOperation(db: DbExecutor, operationId: number, link: MediaLinkRow, result: CopyMediaResult): Promise { + const timestamp = nowIso(); + await db + .insert(schema.copySources) + .values({ destinationPath: result.destinationPath, sourcePath: result.sourcePath, linkPath: result.linkPath, recordedAt: timestamp }) + .onConflictDoUpdate({ + target: schema.copySources.destinationPath, + set: { sourcePath: result.sourcePath, linkPath: result.linkPath, recordedAt: timestamp } + }); await db + .update(schema.mediaLinks) + .set({ + targetPath: result.destinationPath, + kind: result.destinationRootType, + targetExists: true, + sizeBytes: result.sizeBytes, + updatedAt: timestamp + }) + .where(eq(schema.mediaLinks.id, link.id)); + const operation = await first(db .update(schema.copyOperations) .set({ - stage: update.stage, - ...(update.tempPath !== undefined ? { tempPath: update.tempPath } : {}), - ...(update.displacedPath !== undefined ? { displacedPath: update.displacedPath } : {}), - ...(update.sizeBytes !== undefined ? { sizeBytes: update.sizeBytes } : {}), - ...(update.resultStatus !== undefined ? { resultStatus: update.resultStatus } : {}), - updatedAt: nowIso() + stage: "committed", + resultStatus: result.status, + sizeBytes: result.sizeBytes, + tempPath: null, + errorMessage: null, + updatedAt: timestamp, + completedAt: timestamp }) - .where(eq(schema.copyOperations.id, operationId)); + .where(eq(schema.copyOperations.id, operationId)) + .returning({ id: schema.copyOperations.id })); + if (!operation) throw new Error(`Copy operation #${operationId} disappeared before commit`); } -async function commitCopyOperation(db: Db, operationId: number, link: MediaLinkRow, result: CopyMediaResult): Promise { - const timestamp = nowIso(); - await db.transaction(async (transaction) => { - await transaction - .insert(schema.copySources) - .values({ destinationPath: result.destinationPath, sourcePath: result.sourcePath, linkPath: result.linkPath, recordedAt: timestamp }) - .onConflictDoUpdate({ - target: schema.copySources.destinationPath, - set: { sourcePath: result.sourcePath, linkPath: result.linkPath, recordedAt: timestamp } - }); - await transaction - .update(schema.mediaLinks) - .set({ - targetPath: result.destinationPath, - kind: result.destinationRootType, - targetExists: true, - sizeBytes: result.sizeBytes, - updatedAt: timestamp - }) - .where(eq(schema.mediaLinks.id, link.id)); - await transaction - .update(schema.copyOperations) - .set({ - stage: "committed", - resultStatus: result.status, - sizeBytes: result.sizeBytes, - tempPath: null, - errorMessage: null, - updatedAt: timestamp, - completedAt: timestamp - }) - .where(eq(schema.copyOperations.id, operationId)); - }); +async function failCopyOperation(db: DbExecutor, operationId: number, message: string): Promise { + const row = await first(db + .update(schema.copyOperations) + .set({ stage: "failed", errorMessage: message, updatedAt: nowIso(), completedAt: nowIso() }) + .where(eq(schema.copyOperations.id, operationId)) + .returning({ id: schema.copyOperations.id })); + if (!row) throw new Error(`Copy operation #${operationId} disappeared before failure could be recorded`); } -async function failCopyOperation(db: Db, operationId: number, message: string): Promise { - await db +async function requireCopyOperationReconciliation(db: DbExecutor, operationId: number, message: string): Promise { + const row = await first(db .update(schema.copyOperations) - .set({ stage: "failed", errorMessage: message, updatedAt: nowIso(), completedAt: nowIso() }) - .where(eq(schema.copyOperations.id, operationId)); + .set({ stage: "reconciliation_required", errorMessage: message, updatedAt: nowIso(), completedAt: null }) + .where(eq(schema.copyOperations.id, operationId)) + .returning({ id: schema.copyOperations.id })); + if (!row) throw new Error(`Copy operation #${operationId} disappeared before reconciliation could be recorded`); } -async function completeCopyOperationWithoutMutation(db: Db, operationId: number, result: CopyMediaResult): Promise { +async function completeCopyOperationWithoutMutation(db: DbExecutor, operationId: number, result: CopyMediaResult): Promise { const timestamp = nowIso(); - await db + const row = await first(db .update(schema.copyOperations) .set({ stage: "committed", @@ -767,13 +1218,18 @@ async function completeCopyOperationWithoutMutation(db: Db, operationId: number, updatedAt: timestamp, completedAt: timestamp }) - .where(eq(schema.copyOperations.id, operationId)); + .where(eq(schema.copyOperations.id, operationId)) + .returning({ id: schema.copyOperations.id })); + if (!row) throw new Error(`Copy operation #${operationId} disappeared before completion could be recorded`); } interface CopyRollbackEntry { link: MediaLinkRow; result: CopyMediaResult; previousCopySource: CopySourceRow | null; + displacedPath: string | null; + destinationIdentity: string | null; + displacedIdentity: string | null; } async function currentSymlinkTarget(linkPath: string): Promise { @@ -893,7 +1349,7 @@ async function replaceSymlinkTarget(linkRoot: string, linkPath: string, targetPa } } -async function restoreCopySource(db: Db, entry: CopyRollbackEntry): Promise { +async function restoreCopySource(db: DbExecutor, entry: CopyRollbackEntry): Promise { if (entry.previousCopySource) { await db .insert(schema.copySources) @@ -916,12 +1372,45 @@ async function restoreCopySource(db: Db, entry: CopyRollbackEntry): Promise { +function requiredJournalIdentity(rawIdentity: string | null, label: string): CopyFileIdentity { + if (!rawIdentity) throw new Error(`${label} has no durable file identity; refusing automatic filesystem recovery`); + try { + return parseCopyFileIdentity(rawIdentity); + } catch (error) { + throw new Error(`${label} has an invalid durable file identity`, { cause: error }); + } +} + +async function currentJournalFileIdentity(filePath: string, label: string): Promise { + const stat = await fs.lstat(filePath).catch((error: unknown) => { + if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") return null; + throw error; + }); + if (!stat) return null; + if (!stat.isFile()) throw new Error(`${label} is not a regular file`); + const identity = await readCopyFileIdentity(filePath); + if (!identity) throw new Error(`${label} disappeared while its identity was being checked`); + return identity; +} + +function assertJournalIdentity(actual: CopyFileIdentity, rawExpected: string | null, label: string): void { + const expected = requiredJournalIdentity(rawExpected, label); + if (!copyFileIdentitiesMatch(actual, expected)) throw new Error(`${label} changed after its file identity was journaled`); +} + +async function rollbackCopiedMediaLink( + db: Db, + entry: CopyRollbackEntry, + paths: PathsSettings, + ctx: Pick +): Promise<{ rolledBack: boolean; warning?: string }> { const currentTarget = await currentSymlinkTarget(entry.link.linkPath); - if (currentTarget !== path.resolve(entry.result.destinationPath)) { + const resolvedDestination = path.resolve(entry.result.destinationPath); + const resolvedOriginal = path.resolve(entry.link.targetPath); + if (currentTarget !== resolvedDestination && currentTarget !== resolvedOriginal) { return { rolledBack: false, - warning: `Skipped rollback for ${entry.link.linkPath}; symlink no longer points to the job destination` + warning: `Skipped rollback for ${entry.link.linkPath}; symlink no longer points to the job destination or original source` }; } @@ -929,27 +1418,64 @@ async function rollbackCopiedMediaLink(db: Db, entry: CopyRollbackEntry, paths: if (!originalRoot) return { rolledBack: false, warning: `Skipped rollback for ${entry.link.linkPath}; original target root is unknown` }; await assertExistingPathInside(originalRoot, entry.link.targetPath, "Original copy source"); await assertDestinationPathInside(entry.result.destinationRootType === "local" ? paths.localDir : paths.remoteDir, entry.result.destinationPath, "Copy destination"); - await replaceSymlinkTarget(paths.symlinkDir, entry.link.linkPath, entry.link.targetPath); - await db - .update(schema.mediaLinks) - .set({ - targetPath: entry.link.targetPath, - kind: entry.link.kind, - targetExists: entry.link.targetExists, - resolvedStorageFileId: entry.link.resolvedStorageFileId, - sizeBytes: entry.link.sizeBytes, - updatedAt: nowIso() - }) - .where(eq(schema.mediaLinks.id, entry.link.id)); - await restoreCopySource(db, entry); + if (currentTarget === resolvedDestination) { + await ctx.withLease(async () => { + const lockedTarget = await currentSymlinkTarget(entry.link.linkPath); + if (lockedTarget !== resolvedDestination) throw new Error("Symlink changed while copy rollback was waiting for its lease"); + await replaceSymlinkTarget(paths.symlinkDir, entry.link.linkPath, entry.link.targetPath); + }); + } + await ctx.withLeaseDb(async (leaseDb) => { + await leaseDb + .update(schema.mediaLinks) + .set({ + targetPath: entry.link.targetPath, + kind: entry.link.kind, + targetExists: entry.link.targetExists, + resolvedStorageFileId: entry.link.resolvedStorageFileId, + sizeBytes: entry.link.sizeBytes, + updatedAt: nowIso() + }) + .where(eq(schema.mediaLinks.id, entry.link.id)); + await restoreCopySource(leaseDb, entry); + }); if (entry.result.status === "copied") { try { - const stat = await fs.stat(entry.result.destinationPath); - if (stat.isFile() && stat.size === entry.result.sizeBytes) { - await assertExistingPathInside(entry.result.destinationRootType === "local" ? paths.localDir : paths.remoteDir, entry.result.destinationPath, "Copy destination"); - await fs.rm(entry.result.destinationPath, { force: true }); - } else { + const restored = await ctx.withLease(async () => { + const destinationIdentity = await currentJournalFileIdentity(entry.result.destinationPath, "Copy destination"); + const displacedIdentity = entry.displacedPath + ? await currentJournalFileIdentity(entry.displacedPath, "Displaced copy destination") + : null; + + if ( + currentTarget === resolvedOriginal && + entry.displacedPath && + destinationIdentity && + !displacedIdentity && + copyFileIdentitiesMatch(destinationIdentity, requiredJournalIdentity(entry.displacedIdentity, "Displaced copy destination")) + ) { + return true; + } + if (currentTarget === resolvedOriginal && !entry.displacedPath && !destinationIdentity) return true; + + if (destinationIdentity) { + assertJournalIdentity(destinationIdentity, entry.destinationIdentity, "Copy destination"); + await assertExistingPathInside(entry.result.destinationRootType === "local" ? paths.localDir : paths.remoteDir, entry.result.destinationPath, "Copy destination"); + await fs.rm(entry.result.destinationPath, { force: true }); + } + if (entry.displacedPath) { + if (!displacedIdentity) throw new Error("Journaled displaced destination is missing"); + assertJournalIdentity(displacedIdentity, entry.displacedIdentity, "Displaced copy destination"); + const occupiedDestination = await currentJournalFileIdentity(entry.result.destinationPath, "Copy destination restore path"); + if (occupiedDestination) throw new Error("Copy destination became occupied before displaced-file restoration"); + await assertExistingPathInside(entry.result.destinationRootType === "local" ? paths.localDir : paths.remoteDir, entry.displacedPath, "Displaced copy destination"); + await assertDestinationPathInside(entry.result.destinationRootType === "local" ? paths.localDir : paths.remoteDir, entry.result.destinationPath, "Copy destination restore path"); + await fs.rename(entry.displacedPath, entry.result.destinationPath); + } + return true; + }); + if (!restored) { return { rolledBack: true, warning: `Restored symlink for ${entry.link.linkPath}, but left ${entry.result.destinationPath} because it changed after copy` @@ -963,28 +1489,31 @@ async function rollbackCopiedMediaLink(db: Db, entry: CopyRollbackEntry, paths: return { rolledBack: true }; } -async function removeJournalFile(root: string, filePath: string | null, expectedSize: number | null, label: string): Promise { +async function removeJournalFile(root: string, filePath: string | null, expectedIdentity: string | null, label: string): Promise { if (!filePath) return; - try { - await assertExistingPathInside(root, filePath, label); - const stat = await fs.stat(filePath); - if (!stat.isFile()) throw new Error(`${label} is not a regular file`); - if (expectedSize != null && stat.size !== expectedSize) throw new Error(`${label} changed size after it was journaled`); - await fs.rm(filePath, { force: true }); - } catch (error) { - if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") return; - throw error; - } + await assertDestinationPathInside(root, filePath, label); + const actualIdentity = await currentJournalFileIdentity(filePath, label); + if (!actualIdentity) return; + assertJournalIdentity(actualIdentity, expectedIdentity, label); + await assertExistingPathInside(root, filePath, label); + await fs.rm(filePath, { force: true }); } async function reconcileCopyOperationsForJob( db: Db, jobId: number, paths: PathsSettings, - event: JobContext["event"] + ctx: JobContext ): Promise { - const terminalStages = new Set(["committed", "rolled_back", "failed", "reconciliation_required"]); - const operations = (await db.select().from(schema.copyOperations).where(eq(schema.copyOperations.jobId, jobId))).filter( + const allOperations = await db.select().from(schema.copyOperations).where(eq(schema.copyOperations.jobId, jobId)); + const blockedOperation = allOperations.find((operation) => operation.stage === "reconciliation_required"); + if (blockedOperation) { + throw new Error( + `Copy operation #${blockedOperation.id} requires manual reconciliation: ${blockedOperation.errorMessage ?? "filesystem state is uncertain"}` + ); + } + const terminalStages = new Set(["committed", "rolled_back", "failed"]); + const operations = allOperations.filter( (operation) => !terminalStages.has(operation.stage) ); @@ -997,18 +1526,21 @@ async function reconcileCopyOperationsForJob( const resolvedOriginal = path.resolve(operation.originalTargetPath); if (currentTarget === resolvedDestination) { - await assertExistingPathInside(destinationRoot, operation.destinationPath, "Recovered copy destination"); - const stat = await fs.stat(operation.destinationPath); - if (!stat.isFile()) throw new Error("Recovered copy destination is not a regular file"); - if (operation.sizeBytes != null && stat.size !== operation.sizeBytes) throw new Error("Recovered copy destination changed size"); const link = copyOperationLink(operation); - const result = copyOperationResult(operation, paths, stat.size); - await commitCopyOperation(db, operation.id, link, result); - await removeJournalFile(destinationRoot, operation.tempPath, operation.sizeBytes, "Temporary copy"); - if (operation.displacedPath && operation.localConflictStrategy === "replace") { - await removeJournalFile(destinationRoot, operation.displacedPath, null, "Displaced destination"); - } - await event("warn", "Recovered copy operation after worker interruption", { + await ctx.withLeaseDb(async (leaseDb) => { + if ((await currentSymlinkTarget(operation.linkPath)) !== resolvedDestination) { + throw new Error("Recovered symlink changed while copy recovery was waiting for its lease"); + } + await assertExistingPathInside(destinationRoot, operation.destinationPath, "Recovered copy destination"); + const destinationIdentity = await currentJournalFileIdentity(operation.destinationPath, "Recovered copy destination"); + if (!destinationIdentity) throw new Error("Recovered copy destination is missing"); + assertJournalIdentity(destinationIdentity, operation.destinationIdentity, "Recovered copy destination"); + const stat = await fs.stat(operation.destinationPath); + const result = copyOperationResult(operation, paths, stat.size); + await removeJournalFile(destinationRoot, operation.tempPath, operation.tempIdentity, "Temporary copy"); + await commitCopyOperation(leaseDb, operation.id, link, result); + }); + await ctx.event("warn", "Recovered copy operation after worker interruption", { operationId: operation.id, linkPath: operation.linkPath, destinationPath: operation.destinationPath, @@ -1018,26 +1550,53 @@ async function reconcileCopyOperationsForJob( } if (currentTarget === resolvedOriginal) { - await removeJournalFile(destinationRoot, operation.tempPath, operation.sizeBytes, "Temporary copy"); - if (operation.stage === "promoted" || operation.stage === "repointed") { - await removeJournalFile(destinationRoot, operation.destinationPath, operation.sizeBytes, "Uncommitted promoted copy"); - } - if (operation.displacedPath) { - const destinationExistsNow = await fs.stat(operation.destinationPath).then(() => true).catch((error: unknown) => { - if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") return false; - throw error; - }); - if (destinationExistsNow) throw new Error("Cannot restore displaced destination because its original path is occupied"); - await assertExistingPathInside(destinationRoot, operation.displacedPath, "Displaced destination"); - await assertDestinationPathInside(destinationRoot, operation.destinationPath, "Destination restore path"); - await fs.rename(operation.displacedPath, operation.destinationPath); - } + await ctx.withLease(async () => { + if ((await currentSymlinkTarget(operation.linkPath)) !== resolvedOriginal) { + throw new Error("Recovered symlink changed while copy rollback was waiting for its lease"); + } + const tempIdentity = operation.tempPath + ? await currentJournalFileIdentity(operation.tempPath, "Temporary copy") + : null; + if ((operation.stage === "promoted" || operation.stage === "repointed") && operation.resultStatus !== "repointed" && !tempIdentity) { + await removeJournalFile(destinationRoot, operation.destinationPath, operation.destinationIdentity, "Uncommitted promoted copy"); + } + await removeJournalFile(destinationRoot, operation.tempPath, operation.tempIdentity, "Temporary copy"); + if (operation.displacedPath) { + const destinationIdentity = await currentJournalFileIdentity(operation.destinationPath, "Destination restore path"); + const displacedIdentity = await currentJournalFileIdentity(operation.displacedPath, "Displaced destination"); + if (destinationIdentity && displacedIdentity) { + throw new Error("Cannot restore displaced destination because its original path is occupied"); + } + if (!destinationIdentity && displacedIdentity) { + assertJournalIdentity(displacedIdentity, operation.displacedIdentity, "Displaced destination"); + await assertExistingPathInside(destinationRoot, operation.displacedPath, "Displaced destination"); + await assertDestinationPathInside(destinationRoot, operation.destinationPath, "Destination restore path"); + await fs.rename(operation.displacedPath, operation.destinationPath); + } else if (destinationIdentity && operation.displacedIdentity) { + assertJournalIdentity(destinationIdentity, operation.displacedIdentity, "Restored displaced destination"); + } else if (!destinationIdentity) { + throw new Error("Journaled displaced destination is missing from both paths"); + } + } + }); const timestamp = nowIso(); - await db - .update(schema.copyOperations) - .set({ stage: "rolled_back", tempPath: null, displacedPath: null, errorMessage: null, updatedAt: timestamp, completedAt: timestamp }) - .where(eq(schema.copyOperations.id, operation.id)); - await event("warn", "Rolled back incomplete copy operation after worker interruption", { + await ctx.withLeaseDb(async (leaseDb) => { + await leaseDb + .update(schema.copyOperations) + .set({ + stage: "rolled_back", + tempPath: null, + displacedPath: null, + tempIdentity: null, + destinationIdentity: null, + displacedIdentity: null, + errorMessage: null, + updatedAt: timestamp, + completedAt: timestamp + }) + .where(eq(schema.copyOperations.id, operation.id)); + }); + await ctx.event("warn", "Rolled back incomplete copy operation after worker interruption", { operationId: operation.id, linkPath: operation.linkPath, resolution: "rolled_back" @@ -1047,12 +1606,15 @@ async function reconcileCopyOperationsForJob( throw new Error("Symlink no longer points to either the original target or the journaled destination"); } catch (error: unknown) { + if (error instanceof LeaseLostError || (error instanceof Error && error.name === "LeaseLostError")) throw error; const message = errorMessage(error); - await db - .update(schema.copyOperations) - .set({ stage: "reconciliation_required", errorMessage: message, updatedAt: nowIso() }) - .where(eq(schema.copyOperations.id, operation.id)); - await event("error", "Copy operation requires manual reconciliation", { + await ctx.withLeaseDb(async (leaseDb) => { + await leaseDb + .update(schema.copyOperations) + .set({ stage: "reconciliation_required", errorMessage: message, updatedAt: nowIso() }) + .where(eq(schema.copyOperations.id, operation.id)); + }); + await ctx.event("error", "Copy operation requires manual reconciliation", { operationId: operation.id, linkPath: operation.linkPath, destinationPath: operation.destinationPath, @@ -1067,19 +1629,21 @@ async function rollbackDurableCopyOperations( db: Db, jobId: number, paths: PathsSettings, - event: JobContext["event"] + ctx: JobContext ): Promise<{ rolledBack: number; warnings: string[] }> { - await reconcileCopyOperationsForJob(db, jobId, paths, event); + await reconcileCopyOperationsForJob(db, jobId, paths, ctx); const operations = (await db.select().from(schema.copyOperations).where(and(eq(schema.copyOperations.jobId, jobId), eq(schema.copyOperations.stage, "committed")))).reverse(); let rolledBack = 0; const warnings: string[] = []; for (const operation of operations) { if (operation.resultStatus !== "copied" && operation.resultStatus !== "repointed") { - await db - .update(schema.copyOperations) - .set({ stage: "rolled_back", updatedAt: nowIso(), completedAt: nowIso() }) - .where(eq(schema.copyOperations.id, operation.id)); + await ctx.withLeaseDb(async (leaseDb) => { + await leaseDb + .update(schema.copyOperations) + .set({ stage: "rolled_back", updatedAt: nowIso(), completedAt: nowIso() }) + .where(eq(schema.copyOperations.id, operation.id)); + }); continue; } try { @@ -1087,24 +1651,37 @@ async function rollbackDurableCopyOperations( const result = copyOperationResult(operation, paths, operation.sizeBytes ?? link.sizeBytes ?? 0); const rollback = await rollbackCopiedMediaLink( db, - { link, result, previousCopySource: copyOperationPreviousSource(operation) }, - paths + { + link, + result, + previousCopySource: copyOperationPreviousSource(operation), + displacedPath: operation.displacedPath, + destinationIdentity: operation.destinationIdentity, + displacedIdentity: operation.displacedIdentity + }, + paths, + ctx ); if (rollback.rolledBack) { rolledBack += 1; - await db - .update(schema.copyOperations) - .set({ stage: "rolled_back", errorMessage: null, updatedAt: nowIso(), completedAt: nowIso() }) - .where(eq(schema.copyOperations.id, operation.id)); + await ctx.withLeaseDb(async (leaseDb) => { + await leaseDb + .update(schema.copyOperations) + .set({ stage: "rolled_back", errorMessage: null, updatedAt: nowIso(), completedAt: nowIso() }) + .where(eq(schema.copyOperations.id, operation.id)); + }); } if (rollback.warning) warnings.push(rollback.warning); } catch (error: unknown) { + if (error instanceof LeaseLostError || (error instanceof Error && error.name === "LeaseLostError")) throw error; const warning = `Rollback failed for ${operation.linkPath}: ${errorMessage(error)}`; warnings.push(warning); - await db - .update(schema.copyOperations) - .set({ stage: "reconciliation_required", errorMessage: warning, updatedAt: nowIso() }) - .where(eq(schema.copyOperations.id, operation.id)); + await ctx.withLeaseDb(async (leaseDb) => { + await leaseDb + .update(schema.copyOperations) + .set({ stage: "reconciliation_required", errorMessage: warning, updatedAt: nowIso() }) + .where(eq(schema.copyOperations.id, operation.id)); + }); } } return { rolledBack, warnings }; @@ -1245,7 +1822,7 @@ async function readCopyResumeState(db: Db, jobId: number, selectedLinks: MediaLi return { copied, repointed, skipped, alreadyCompleted, startedTotal }; } -function toJobRecord(row: JobRow): JobRecord { +function toJobRecord(row: JobRow): LeasedJob { const progress = parseJson(row.progress); const normalizedProgress = normalizeJobProgress(row.type, row.status, progress); const status = normalizeJobStatus(row.type, row.status, progress); @@ -1253,7 +1830,9 @@ function toJobRecord(row: JobRow): JobRecord { ...row, type: row.type as JobRecord["type"], status, - progress: normalizedProgress + progress: normalizedProgress, + leaseVersion: row.leaseVersion, + exclusive: row.exclusive }; } @@ -1289,67 +1868,150 @@ function isStaleRunningJob(job: Pick= staleAfterMs; } -async function addEvent(db: Db, jobId: number, level: JobEventRecord["level"], message: string, data: unknown = {}): Promise { - await db.insert(schema.jobEvents).values({ jobId, timestamp: nowIso(), level, message, data: JSON.stringify(data) }); -} - -async function setProgress(db: Db, jobId: number, progress: unknown): Promise { - await db.update(schema.jobs).set({ progress: JSON.stringify(progress) }).where(eq(schema.jobs.id, jobId)); -} - export class JobRunner { constructor(private readonly db: Db) {} async createJob(type: JobRecord["type"], progress: unknown = {}): Promise { - if (type !== "path_migration" && (await isPathConfigurationBlocked(this.db))) { - throw new Error("Managed storage paths changed. Resolve the required path migration before starting another job."); - } - const row = await first(this.db - .insert(schema.jobs) - .values({ - type, - status: "queued", - createdAt: nowIso(), - startedAt: null, - finishedAt: null, - lockedBy: null, - lockedAt: null, - heartbeatAt: null, - cancelRequestedAt: null, - progress: JSON.stringify(progress) - }) - .returning({ id: schema.jobs.id })); - if (!row) throw new Error("Job was not queued"); - await addEvent(this.db, row.id, "info", "Job queued", { type }); - return row.id; + return this.enqueueJob(type, progress, true, []); } - async listJobs(options: JobListOptions = {}): Promise { - const limit = Math.min(Math.max(options.limit ?? 500, 1), 1000); - const activeStatuses: JobStatus[] = ["queued", "running"]; - const terminalStatuses: JobStatus[] = ["completed", "partially_failed", "failed", "cancelled"]; - const activeRows = await this.db.select().from(schema.jobs).where(inArray(schema.jobs.status, activeStatuses)).orderBy(desc(schema.jobs.id)); - if (options.activeOnly) return activeRows.map(toJobRecord); - - const terminalRows = options.completedSince - ? await this.db - .select() - .from(schema.jobs) - .where(and(inArray(schema.jobs.status, terminalStatuses), gte(schema.jobs.finishedAt, options.completedSince))) - .orderBy(desc(schema.jobs.id)) - .limit(limit) - : await this.db - .select() - .from(schema.jobs) - .where(inArray(schema.jobs.status, terminalStatuses)) - .orderBy(desc(schema.jobs.id)) - .limit(limit); - return [...activeRows, ...terminalRows].sort((a, b) => b.id - a.id).map(toJobRecord); + private async enqueueJob(type: JobRecord["type"], progress: unknown, exclusive: boolean, requestedClaims: ResourceClaim[]): Promise { + return this.enqueuePreparedJob(type, async () => ({ progress, exclusive, claims: requestedClaims })); } - async getJob(jobId: number): Promise { - const row = await first(this.db.select().from(schema.jobs).where(eq(schema.jobs.id, jobId)).limit(1)); - return row ? toJobRecord(row) : null; + private async enqueuePreparedJob(type: JobRecord["type"], prepare: (db: DbExecutor) => Promise): Promise { + if (type !== "path_migration" && (await isPathConfigurationBlocked(this.db))) { + throw new Error("Managed storage paths changed. Resolve the required path migration before starting another job."); + } + return this.db.transaction(async (transaction) => { + await transaction.execute(sql`select pg_advisory_xact_lock(${schedulerLockKey})`); + if (type !== "path_migration" && (await isPathConfigurationBlocked(transaction))) { + throw new Error("Managed storage paths changed. Resolve the required path migration before starting another job."); + } + const prepared = await prepare(transaction); + const claims = normalizeResourceClaims(prepared.claims); + + if (!prepared.exclusive && claims.length > 0) { + const conflict = await dbGet<{ jobId: number; status: string; overlapCount: number }>(transaction, sql` + WITH requested_claims AS ( + SELECT "resourceType" AS resource_type, "resourceKey" AS resource_key, access + FROM jsonb_to_recordset(${JSON.stringify(claims)}::jsonb) + AS requested("resourceType" text, "resourceKey" text, access text) + ), blocking_claims AS ( + SELECT active.job_id, active.resource_type, active.resource_key, active.access, jobs.status + FROM job_resource_claims AS active + JOIN jobs ON jobs.id = active.job_id + WHERE jobs.status IN ('queued', 'running') + UNION + SELECT active.job_id, active.resource_type, active.resource_key, active.access, 'reconciliation_required'::text AS status + FROM job_resource_claims AS active + JOIN copy_operations AS operation ON operation.job_id = active.job_id + WHERE operation.stage = 'reconciliation_required' + AND active.resource_type <> 'title' + UNION + SELECT operation.job_id, 'media'::text, operation.media_link_id::text, 'exclusive'::text, 'reconciliation_required'::text AS status + FROM copy_operations AS operation + WHERE operation.stage = 'reconciliation_required' + UNION + SELECT operation.job_id, 'path'::text, paths.resource_key, 'exclusive'::text, 'reconciliation_required'::text AS status + FROM copy_operations AS operation + CROSS JOIN LATERAL unnest(ARRAY[ + operation.link_path, + operation.source_path, + operation.destination_path, + operation.temp_path, + operation.displaced_path + ]) AS paths(resource_key) + WHERE operation.stage = 'reconciliation_required' + AND paths.resource_key IS NOT NULL + ) + SELECT active.job_id AS "jobId", + active.status, + greatest(1, count(*) FILTER (WHERE requested.resource_type = 'media'))::integer AS "overlapCount" + FROM requested_claims AS requested + JOIN blocking_claims AS active + ON active.resource_type = requested.resource_type + AND active.resource_key = requested.resource_key + AND (active.access = 'exclusive' OR requested.access = 'exclusive') + GROUP BY active.job_id, active.status + ORDER BY active.job_id + LIMIT 1 + `); + if (conflict) { + if (conflict.status === "reconciliation_required") { + throw new Error( + `Copy data from job #${conflict.jobId} requires manual reconciliation before another action can touch the same media item or managed path.` + ); + } + throw new Error( + `Job #${conflict.jobId} is already ${conflict.status} for ${conflict.overlapCount} matching media item${conflict.overlapCount === 1 ? "" : "s"}. Wait for it to finish or terminate it before queuing another action.` + ); + } + } + + const timestamp = nowIso(); + const row = await first( + transaction + .insert(schema.jobs) + .values({ + type, + status: "queued", + createdAt: timestamp, + startedAt: null, + finishedAt: null, + lockedBy: null, + lockedAt: null, + heartbeatAt: null, + leaseVersion: 0, + exclusive: prepared.exclusive, + cancelRequestedAt: null, + progress: JSON.stringify(prepared.progress) + }) + .returning({ id: schema.jobs.id }) + ); + if (!row) throw new Error("Job was not queued"); + for (let offset = 0; offset < claims.length; offset += 500) { + await transaction.insert(schema.jobResourceClaims).values( + claims.slice(offset, offset + 500).map((claim) => ({ + jobId: row.id, + resourceType: claim.resourceType, + resourceKey: claim.resourceKey, + access: claim.access, + createdAt: timestamp + })) + ); + } + await transaction.insert(schema.jobEvents).values({ jobId: row.id, timestamp, level: "info", message: "Job queued", data: JSON.stringify({ type }) }); + return row.id; + }); + } + + async listJobs(options: JobListOptions = {}): Promise { + const limit = Math.min(Math.max(options.limit ?? 500, 1), 1000); + const activeStatuses: JobStatus[] = ["queued", "running"]; + const terminalStatuses: JobStatus[] = ["completed", "partially_failed", "failed", "cancelled"]; + const activeRows = await this.db.select().from(schema.jobs).where(inArray(schema.jobs.status, activeStatuses)).orderBy(desc(schema.jobs.id)); + if (options.activeOnly) return activeRows.map(toJobRecord); + + const terminalRows = options.completedSince + ? await this.db + .select() + .from(schema.jobs) + .where(and(inArray(schema.jobs.status, terminalStatuses), gte(schema.jobs.finishedAt, options.completedSince))) + .orderBy(desc(schema.jobs.id)) + .limit(limit) + : await this.db + .select() + .from(schema.jobs) + .where(inArray(schema.jobs.status, terminalStatuses)) + .orderBy(desc(schema.jobs.id)) + .limit(limit); + return [...activeRows, ...terminalRows].sort((a, b) => b.id - a.id).map(toJobRecord); + } + + async getJob(jobId: number): Promise { + const row = await first(this.db.select().from(schema.jobs).where(eq(schema.jobs.id, jobId)).limit(1)); + return row ? toJobRecord(row) : null; } async listEvents(jobId: number, afterId = 0, limit = 100): Promise { @@ -1393,29 +2055,43 @@ export class JobRunner { } async terminate(jobId: number): Promise { - const job = await this.getJob(jobId); - if (!job) return false; - const timestamp = nowIso(); - if (job.status === "queued") { - await this.db - .update(schema.jobs) - .set({ status: "cancelled", finishedAt: timestamp, cancelRequestedAt: timestamp }) - .where(eq(schema.jobs.id, jobId)); - if (job.type === "path_migration" && isRecord(job.progress) && Number.isInteger(job.progress.migrationId)) { - await this.db - .update(schema.pathMigrations) - .set({ status: "failed", finishedAt: timestamp, errorMessage: "Path migration was terminated before it started. Analyze the path change again or restore the previous environment paths." }) - .where(and(eq(schema.pathMigrations.id, Number(job.progress.migrationId)), eq(schema.pathMigrations.status, "queued"))); + return this.db.transaction(async (transaction) => { + const row = await first(transaction.select().from(schema.jobs).where(eq(schema.jobs.id, jobId)).for("update").limit(1)); + if (!row) return false; + const job = toJobRecord(row); + const timestamp = nowIso(); + if (job.status === "queued") { + const terminated = await first( + transaction + .update(schema.jobs) + .set({ status: "cancelled", finishedAt: timestamp, cancelRequestedAt: timestamp }) + .where(and(eq(schema.jobs.id, jobId), eq(schema.jobs.status, "queued"))) + .returning({ id: schema.jobs.id }) + ); + if (!terminated) return false; + if (job.type === "path_migration" && isRecord(job.progress) && Number.isInteger(job.progress.migrationId)) { + await transaction + .update(schema.pathMigrations) + .set({ status: "failed", finishedAt: timestamp, errorMessage: "Path migration was terminated before it started. Analyze the path change again or restore the previous environment paths." }) + .where(and(eq(schema.pathMigrations.id, Number(job.progress.migrationId)), eq(schema.pathMigrations.status, "queued"))); + } + await transaction.insert(schema.jobEvents).values({ jobId, timestamp, level: "warn", message: "Queued job terminated", data: "{}" }); + return true; } - await addEvent(this.db, jobId, "warn", "Queued job terminated"); - return true; - } - if (job.status === "running") { - await this.db.update(schema.jobs).set({ cancelRequestedAt: timestamp }).where(eq(schema.jobs.id, jobId)); - await addEvent(this.db, jobId, "warn", "Termination requested"); - return true; - } - return false; + if (job.status === "running") { + const requested = await first( + transaction + .update(schema.jobs) + .set({ cancelRequestedAt: timestamp }) + .where(and(eq(schema.jobs.id, jobId), eq(schema.jobs.status, "running"))) + .returning({ id: schema.jobs.id }) + ); + if (!requested) return false; + await transaction.insert(schema.jobEvents).values({ jobId, timestamp, level: "warn", message: "Termination requested", data: "{}" }); + return true; + } + return false; + }); } async cancel(jobId: number): Promise { @@ -1424,17 +2100,25 @@ export class JobRunner { async startScan(options: ScanOptions = defaultScanOptions): Promise { const normalizedOptions = await normalizeScanOptions(this.db, options); - const links = await listMediaLinks(this.db, undefined, "current"); - const scanLinks = filterScanLinks(links, normalizedOptions); - if (normalizedOptions.titleScopes?.length) { + const targeted = Boolean(normalizedOptions.titleScopes?.length); + if (!targeted) return this.enqueueJob("scan", { options: normalizedOptions }, true, []); + return this.enqueuePreparedJob("scan", async (transaction) => { + const scanLinks = filterScanLinks(await listMediaLinks(transaction, undefined, "current"), normalizedOptions); + const paths = await getJsonSetting(transaction, "paths", { symlinkDir: "", localDir: "", remoteDir: "" }); + if (!paths.symlinkDir || !paths.localDir || !paths.remoteDir) throw new Error("Path settings are incomplete"); const availableScopes = new Set(scanLinks.map((link) => `${link.section}\0${link.itemName}`)); - const unavailableScopes = normalizedOptions.titleScopes.filter((scope) => !availableScopes.has(`${scope.section}\0${scope.itemName}`)); + const unavailableScopes = (normalizedOptions.titleScopes ?? []).filter( + (scope) => !availableScopes.has(`${scope.section}\0${scope.itemName}`) + ); if (unavailableScopes.length > 0) { throw new Error(`Title is not available in the current symlink inventory: ${unavailableScopes.map((scope) => scope.itemName).join(", ")}`); } - } - await assertNoActiveJobOverlap(this.db, links, scanLinks); - return this.createJob("scan", { options: normalizedOptions }); + return { + progress: { options: normalizedOptions }, + exclusive: false, + claims: await titleScanResourceClaims(normalizedOptions, scanLinks, paths) + }; + }); } async startAudit(input: AuditMode | AuditOptions): Promise { @@ -1445,20 +2129,50 @@ export class JobRunner { ...normalizedOptions, ...(requestedOptions.byteCompare === undefined && !advancedSettings.audit.byteCompareWhenSourceKnown ? { byteCompare: false } : {}) }; - const links = await listMediaLinks(this.db, undefined, "current"); - await assertNoActiveJobOverlap(this.db, links, filterAuditLinks(links, optionsWithDefaults)); - return this.createJob("audit", { options: optionsWithDefaults }); + const scoped = hasScopedAuditOptions(optionsWithDefaults); + if (!scoped) return this.enqueueJob("audit", { options: optionsWithDefaults }, true, []); + return this.enqueuePreparedJob("audit", async (transaction) => { + const auditLinks = filterAuditLinks(await listMediaLinks(transaction, undefined, "current"), optionsWithDefaults); + const paths = await getJsonSetting(transaction, "paths", { symlinkDir: "", localDir: "", remoteDir: "" }); + if (!paths.symlinkDir || !paths.localDir || !paths.remoteDir) throw new Error("Path settings are incomplete"); + const frozenOptions = { ...optionsWithDefaults, linkIds: auditLinks.map((link) => link.id) }; + return { + progress: { options: frozenOptions }, + exclusive: false, + claims: await auditResourceClaims(auditLinks, paths) + }; + }); } async startCopy(input: CopyOptions): Promise { const normalizedOptions = await normalizeCopyOptions(this.db, input); const links = await listMediaLinks(this.db, undefined, "current"); - const copyLinks = filterCopyLinks(links, normalizedOptions); - await assertNoActiveJobOverlap(this.db, links, copyLinks); - const requestedOrder = normalizedOptions.linkIds?.length ? new Map(normalizedOptions.linkIds.map((id, index) => [id, index])) : null; - const orderedCopyLinks = requestedOrder ? [...copyLinks].sort((firstLink, secondLink) => (requestedOrder.get(firstLink.id) ?? 0) - (requestedOrder.get(secondLink.id) ?? 0)) : copyLinks; - const optionsWithResolvedLinks = orderedCopyLinks.length > 0 ? { ...normalizedOptions, linkIds: orderedCopyLinks.map((link) => link.id) } : normalizedOptions; - return this.createJob("copy", { options: optionsWithResolvedLinks }); + const orderedSelectedLinks = orderedCopySelection(links, normalizedOptions); + const optionsWithResolvedLinks = { ...normalizedOptions, linkIds: orderedSelectedLinks.map((link) => link.id) }; + const paths = await getJsonSetting(this.db, "paths", { symlinkDir: "", localDir: "", remoteDir: "" }); + if (!paths.localDir || !paths.remoteDir) throw new Error("Path settings are incomplete"); + const replacementClaims = await copyReplacementResourceClaims(this.db, orderedSelectedLinks, paths, optionsWithResolvedLinks); + const expectedSelection = orderedSelectedLinks.map(copyAdmissionFingerprint); + return this.enqueuePreparedJob("copy", async (transaction) => { + const currentSelection = orderedCopySelection( + await listMediaLinks(transaction, undefined, "current"), + normalizedOptions + ); + const currentPaths = await getJsonSetting(transaction, "paths", { symlinkDir: "", localDir: "", remoteDir: "" }); + if ( + JSON.stringify(currentSelection.map(copyAdmissionFingerprint)) !== JSON.stringify(expectedSelection) || + currentPaths.symlinkDir !== paths.symlinkDir || + currentPaths.localDir !== paths.localDir || + currentPaths.remoteDir !== paths.remoteDir + ) { + throw new Error("Copy selection changed while the job was being prepared. Review the current inventory and queue it again."); + } + return { + progress: { options: optionsWithResolvedLinks }, + exclusive: false, + claims: [...(await copyResourceClaims(orderedSelectedLinks, paths, normalizedOptions.direction)), ...replacementClaims] + }; + }); } async previewCopyConflicts(input: CopyOptions): Promise { @@ -1472,20 +2186,32 @@ export class JobRunner { } async startPathMigration(migrationId: number): Promise { - await assertPathMigrationReady(this.db, migrationId); const jobId = await this.db.transaction(async (transaction) => { + await transaction.execute(sql`select pg_advisory_xact_lock(${schedulerLockKey})`); + const readyMigration = await first(transaction.select().from(schema.pathMigrations).where(eq(schema.pathMigrations.id, migrationId)).for("update").limit(1)); + if (!readyMigration || readyMigration.status !== "planned") throw new Error("Analyze the path change before starting migration"); + const blocked = await first( + transaction + .select({ value: count() }) + .from(schema.pathMigrationItems) + .where(and(eq(schema.pathMigrationItems.migrationId, migrationId), eq(schema.pathMigrationItems.validationStatus, "blocked"))) + ); + if (Number(blocked?.value ?? 0) > 0) throw new Error("Resolve every blocked symlink before starting migration"); + const timestamp = nowIso(); const row = await first( transaction .insert(schema.jobs) .values({ type: "path_migration", status: "queued", - createdAt: nowIso(), + createdAt: timestamp, startedAt: null, finishedAt: null, lockedBy: null, lockedAt: null, heartbeatAt: null, + leaseVersion: 0, + exclusive: true, cancelRequestedAt: null, progress: JSON.stringify({ migrationId, stage: "queued", current: 0, total: 0, message: "Path migration queued" }) }) @@ -1500,7 +2226,7 @@ export class JobRunner { .returning({ id: schema.pathMigrations.id }) ); if (!migration) throw new Error("Path migration is no longer ready to start"); - await transaction.insert(schema.jobEvents).values({ jobId: row.id, timestamp: nowIso(), level: "info", message: "Job queued", data: JSON.stringify({ type: "path_migration", migrationId }) }); + await transaction.insert(schema.jobEvents).values({ jobId: row.id, timestamp, level: "info", message: "Job queued", data: JSON.stringify({ type: "path_migration", migrationId }) }); return row.id; }); return jobId; @@ -1508,66 +2234,112 @@ export class JobRunner { } export class JobWorker { - private readonly queue: JobRunner; private readonly workerId: string; private readonly pollIntervalMs: number; private readonly heartbeatIntervalMs: number; private readonly reclaimStaleAfterMs: number; private readonly reclaimOwnInterruptedAfterMs: number; + private readonly dispatchConcurrency: number; private readonly logger: Pick; private readonly copyRunner: CopyCommandRunner; private readonly auditRunner: AuditCommandRunner; private readonly concurrency: JobConcurrencySettings; + private readonly copyTransferLimiter: CopyTransferLimiter; private readonly activeAbortControllers = new Map(); - private stopped = true; + private readonly activeRuns = new Set>(); + private stopRequested = false; + private loopRunning = false; private sleepTimer: NodeJS.Timeout | null = null; private resolveSleep: (() => void) | null = null; constructor(private readonly db: Db, options: JobWorkerOptions = {}) { - this.queue = new JobRunner(db); this.workerId = options.workerId ?? `worker-${process.pid}`; this.pollIntervalMs = options.pollIntervalMs ?? 2000; this.heartbeatIntervalMs = options.heartbeatIntervalMs ?? 10_000; this.reclaimStaleAfterMs = options.reclaimStaleAfterMs ?? 15 * 60_000; this.reclaimOwnInterruptedAfterMs = options.reclaimOwnInterruptedAfterMs ?? Math.max(30_000, this.heartbeatIntervalMs * 2); + this.dispatchConcurrency = options.dispatchConcurrency ?? 1; + if (!Number.isSafeInteger(this.dispatchConcurrency) || this.dispatchConcurrency < 1) throw new Error("Worker dispatch concurrency must be a positive safe integer"); this.logger = options.logger ?? console; this.copyRunner = options.copyRunner ?? defaultCopyRunner; this.auditRunner = options.auditRunner ?? defaultAuditRunner; this.concurrency = options.concurrency ?? defaultJobConcurrency; + this.copyTransferLimiter = options.copyTransferLimiter ?? new CopyTransferLimiter(this.concurrency.maxActiveCopyFiles); } async start(): Promise { - if (!this.stopped) return; - this.stopped = false; + if (this.loopRunning) return; + this.loopRunning = true; + this.stopRequested = false; this.logger.info( `SRTL worker ${this.workerId} started with limits: jobs=${this.concurrency.maxRunningJobs}, scans=${this.concurrency.maxRunningScans}, audits=${this.concurrency.maxRunningAudits}, copies=${this.concurrency.maxRunningCopies}` ); - while (!this.stopped) { - const ranJob = await this.runOnce(); - if (!ranJob) await this.sleep(this.pollIntervalMs); + try { + while (!this.stopRequested) { + let claimedAny = false; + while (!this.stopRequested && this.activeRuns.size < this.dispatchConcurrency) { + if (await isPathConfigurationBlocked(this.db)) await this.requeueInterruptedJobsForPathMigration(); + const claimed = await this.claimNextJob(); + if (this.stopRequested) { + if (claimed) await this.requeueInterruptedJob(claimed.job); + break; + } + if (!claimed) break; + claimedAny = true; + const run = this.runClaimedJob(claimed.job) + .catch((error: unknown) => { + this.logger.error(`SRTL worker ${this.workerId} dispatcher failed job #${claimed.job.id}: ${errorMessage(error)}`); + }) + .finally(() => { + this.activeRuns.delete(run); + this.wake(); + }); + this.activeRuns.add(run); + } + if (!this.stopRequested && (!claimedAny || this.activeRuns.size >= this.dispatchConcurrency)) { + await this.sleep(this.pollIntervalMs); + } + } + } finally { + this.stopRequested = true; + for (const abortController of this.activeAbortControllers.values()) { + if (!abortController.signal.aborted) abortController.abort(new WorkerShutdownError()); + } + await Promise.allSettled([...this.activeRuns]); + this.loopRunning = false; + this.logger.info(`SRTL worker ${this.workerId} stopped`); } - this.logger.info(`SRTL worker ${this.workerId} stopped`); } stop(): void { - this.stopped = true; + this.stopRequested = true; for (const abortController of this.activeAbortControllers.values()) { if (!abortController.signal.aborted) abortController.abort(new WorkerShutdownError()); } - if (this.sleepTimer) clearTimeout(this.sleepTimer); - this.sleepTimer = null; - this.resolveSleep?.(); - this.resolveSleep = null; + this.wake(); } async runOnce(): Promise { + if (this.stopRequested) return false; if (await isPathConfigurationBlocked(this.db)) await this.requeueInterruptedJobsForPathMigration(); - const job = await this.claimNextJob(); - if (!job) return false; - await this.runClaimedJob(job); + if (this.stopRequested) return false; + const claimed = await this.claimNextJob(); + if (!claimed) return false; + if (this.stopRequested) { + await this.requeueInterruptedJob(claimed.job); + return false; + } + await this.runClaimedJob(claimed.job); return true; } + private wake(): void { + if (this.sleepTimer) clearTimeout(this.sleepTimer); + this.sleepTimer = null; + this.resolveSleep?.(); + this.resolveSleep = null; + } + private sleep(ms: number): Promise { return new Promise((resolve) => { this.resolveSleep = resolve; @@ -1579,110 +2351,187 @@ export class JobWorker { }); } - private async claimNextJob(): Promise { - const pathConfigurationBlocked = await isPathConfigurationBlocked(this.db); - const allowedType = pathConfigurationBlocked ? "path_migration" : undefined; - return (await this.claimInterruptedOwnJob(allowedType)) ?? (await this.claimStaleRunningJob(allowedType)) ?? (await this.claimQueuedJob(allowedType)); - } - - private async claimQueuedJob(allowedType?: JobRecord["type"]): Promise { - const queuedRows = await this.db - .select() - .from(schema.jobs) - .where(eq(schema.jobs.status, "queued")) - .orderBy(asc(schema.jobs.id)); - let row: JobRow | undefined; - for (const queuedJob of queuedRows) { - if (allowedType && queuedJob.type !== allowedType) continue; - if (await this.canStartJobType(queuedJob.type as JobRecord["type"])) { - row = queuedJob; - break; - } + private isReclaimableWithHeartbeat( + job: LeasedJob, + ownerHeartbeat: typeof schema.workerHeartbeats.$inferSelect | undefined + ): boolean { + if (job.lockedBy === this.workerId) { + if (this.activeAbortControllers.has(job.id)) return false; + return isStaleRunningJob(job, this.reclaimOwnInterruptedAfterMs); } - if (!row) return null; - const timestamp = nowIso(); - const claimedRow = await first(this.db - .update(schema.jobs) - .set({ status: "running", startedAt: row.startedAt ?? timestamp, lockedBy: this.workerId, lockedAt: timestamp, heartbeatAt: timestamp }) - .where(and(eq(schema.jobs.id, row.id), eq(schema.jobs.status, "queued"))) - .returning()); - if (!claimedRow) return null; - const claimed = await this.queue.getJob(row.id); - return claimed?.status === "running" && claimed.lockedBy === this.workerId ? claimed : null; - } - - private async claimStaleRunningJob(allowedType?: JobRecord["type"]): Promise { - const runningRows = await this.db - .select() - .from(schema.jobs) - .where(eq(schema.jobs.status, "running")) - .orderBy(asc(schema.jobs.id)); - let staleJob: JobRecord | null = null; - for (const job of runningRows.map(toJobRecord)) { - if (allowedType && job.type !== allowedType) continue; - if (isStaleRunningJob(job, this.reclaimStaleAfterMs) && (await this.canStartJobType(job.type, job.id))) { - staleJob = job; - break; + if (!ownerHeartbeat) return isStaleRunningJob(job, this.reclaimStaleAfterMs); + if (ownerHeartbeat.status !== "running") return true; + const ownerHeartbeatAt = Date.parse(ownerHeartbeat.heartbeatAt); + const processHeartbeatStale = + !Number.isFinite(ownerHeartbeatAt) || Date.now() - ownerHeartbeatAt >= this.reclaimOwnInterruptedAfterMs; + return processHeartbeatStale && isStaleRunningJob(job, this.reclaimOwnInterruptedAfterMs); + } + + private async claimNextJob(): Promise { + return this.db.transaction(async (transaction) => { + await transaction.execute(sql`select pg_advisory_xact_lock(${schedulerLockKey})`); + const pathConfigurationBlocked = await isPathConfigurationBlocked(transaction); + const recoveryCopyJobIds = pathConfigurationBlocked + ? new Set( + ( + await transaction.execute<{ jobId: number }>(sql` + select distinct jobs.id as "jobId" + from jobs + join copy_operations on copy_operations.job_id = jobs.id + where jobs.type = 'copy' + and jobs.status in ('queued', 'running') + and copy_operations.stage not in ('rolled_back', 'failed') + `) + ).rows.map((row) => row.jobId) + ) + : new Set(); + const allowedWhilePathsBlocked = (job: Pick): boolean => + job.type === "path_migration" || (job.type === "copy" && recoveryCopyJobIds.has(job.id)); + const initialRunningRows = await transaction.select().from(schema.jobs).where(eq(schema.jobs.status, "running")).orderBy(asc(schema.jobs.id)); + const initialRunningJobs = initialRunningRows.map(toJobRecord); + const ownerIds = [...new Set(initialRunningJobs.map((job) => job.lockedBy).filter((ownerId): ownerId is string => Boolean(ownerId)))]; + const ownerHeartbeats = ownerIds.length > 0 + ? await transaction.select().from(schema.workerHeartbeats).where(inArray(schema.workerHeartbeats.workerId, ownerIds)) + : []; + const ownerHeartbeatById = new Map(ownerHeartbeats.map((heartbeat) => [heartbeat.workerId, heartbeat])); + const isReclaimable = (job: LeasedJob): boolean => + this.isReclaimableWithHeartbeat(job, job.lockedBy ? ownerHeartbeatById.get(job.lockedBy) : undefined); + const eligibleRunning = pathConfigurationBlocked ? initialRunningJobs.filter(allowedWhilePathsBlocked) : initialRunningJobs; + const staleCandidates = eligibleRunning.filter(isReclaimable); + const fencedJobIds = new Set(); + for (const staleCandidate of staleCandidates) { + const lockedStaleRow = await first( + transaction + .select() + .from(schema.jobs) + .where( + and( + eq(schema.jobs.id, staleCandidate.id), + eq(schema.jobs.status, "running"), + eq(schema.jobs.leaseVersion, staleCandidate.leaseVersion) + ) + ) + .for("update", { skipLocked: true }) + .limit(1) + ); + if (!lockedStaleRow) continue; + const stale = toJobRecord(lockedStaleRow); + const lockedOwnerHeartbeat = stale.lockedBy && stale.lockedBy !== this.workerId + ? await first( + transaction + .select() + .from(schema.workerHeartbeats) + .where(eq(schema.workerHeartbeats.workerId, stale.lockedBy)) + .for("update") + .limit(1) + ) + : undefined; + if (!this.isReclaimableWithHeartbeat(stale, lockedOwnerHeartbeat)) continue; + const interruptedOwn = stale.lockedBy === this.workerId; + const timestamp = nowIso(); + const ownerFilter = stale.lockedBy == null ? isNull(schema.jobs.lockedBy) : eq(schema.jobs.lockedBy, stale.lockedBy); + const fencedRow = await first( + transaction + .update(schema.jobs) + .set({ + status: "queued", + lockedBy: null, + lockedAt: null, + heartbeatAt: null, + leaseVersion: sql`${schema.jobs.leaseVersion} + 1` + }) + .where( + and( + eq(schema.jobs.id, stale.id), + eq(schema.jobs.status, "running"), + ownerFilter, + eq(schema.jobs.leaseVersion, stale.leaseVersion) + ) + ) + .returning() + ); + if (!fencedRow) continue; + fencedJobIds.add(stale.id); + const message = interruptedOwn ? "Interrupted job lease fenced and requeued" : "Stale running job lease fenced and requeued"; + await transaction.insert(schema.jobEvents).values({ + jobId: stale.id, + timestamp, + level: "warn", + message, + data: JSON.stringify({ workerId: this.workerId, leaseVersion: fencedRow.leaseVersion }) + }); + await transaction + .update(schema.scanRuns) + .set({ status: "failed", finishedAt: timestamp, errorMessage: message }) + .where(and(eq(schema.scanRuns.jobId, stale.id), eq(schema.scanRuns.status, "running"))); + await transaction + .update(schema.auditRuns) + .set({ status: "failed", finishedAt: timestamp }) + .where(and(eq(schema.auditRuns.jobId, stale.id), eq(schema.auditRuns.status, "running"))); } - } - if (!staleJob) return null; - - const timestamp = nowIso(); - const claimedRow = await first(this.db - .update(schema.jobs) - .set({ lockedBy: this.workerId, lockedAt: timestamp, heartbeatAt: timestamp }) - .where(and(eq(schema.jobs.id, staleJob.id), eq(schema.jobs.status, "running"))) - .returning()); - if (!claimedRow) return null; - const claimed = await this.queue.getJob(staleJob.id); - if (claimed?.status === "running" && claimed.lockedBy === this.workerId) { - await this.markExistingRunsStale(claimed.id); - await addEvent(this.db, claimed.id, "warn", "Stale running job reclaimed by worker", { workerId: this.workerId }); - return claimed; - } - return null; - } - private async claimInterruptedOwnJob(allowedType?: JobRecord["type"]): Promise { - const runningRows = await this.db - .select() - .from(schema.jobs) - .where(eq(schema.jobs.status, "running")) - .orderBy(asc(schema.jobs.id)); - let interruptedJob: JobRecord | null = null; - for (const job of runningRows.map(toJobRecord)) { - if (allowedType && job.type !== allowedType) continue; - if (job.lockedBy === this.workerId && isStaleRunningJob(job, this.reclaimOwnInterruptedAfterMs) && (await this.canStartJobType(job.type, job.id))) { - interruptedJob = job; - break; + const runningRows = await transaction.select().from(schema.jobs).where(eq(schema.jobs.status, "running")).orderBy(asc(schema.jobs.id)); + const runningJobs = runningRows.map(toJobRecord); + const queuedRows = await transaction.select().from(schema.jobs).where(eq(schema.jobs.status, "queued")).orderBy(asc(schema.jobs.id)).for("update"); + const eligibleQueued = pathConfigurationBlocked ? queuedRows.filter(allowedWhilePathsBlocked) : queuedRows; + const firstExclusive = eligibleQueued.find((job) => job.exclusive && this.limitForType(job.type as JobRecord["type"]) > 0); + const candidates = firstExclusive ? eligibleQueued.filter((job) => job.id <= firstExclusive.id) : eligibleQueued; + if (runningJobs.some((job) => job.exclusive)) return null; + + let candidate: JobRow | undefined; + for (const queuedJob of candidates) { + const jobType = queuedJob.type as JobRecord["type"]; + const typeLimit = this.limitForType(jobType); + if (typeLimit < 1) continue; + if (runningJobs.length >= this.concurrency.maxRunningJobs) continue; + if (runningJobs.filter((job) => job.type === jobType).length >= typeLimit) continue; + if (queuedJob.exclusive) { + if (runningJobs.length === 0) candidate = queuedJob; + break; + } + if (!(await this.hasClaimConflict(transaction, queuedJob.id, runningJobs.map((job) => job.id)))) { + candidate = queuedJob; + break; + } } - } - if (!interruptedJob) return null; - - const timestamp = nowIso(); - const claimedRow = await first(this.db - .update(schema.jobs) - .set({ lockedBy: this.workerId, lockedAt: timestamp, heartbeatAt: timestamp }) - .where(and(eq(schema.jobs.id, interruptedJob.id), eq(schema.jobs.status, "running"))) - .returning()); - if (!claimedRow) return null; - const claimed = await this.queue.getJob(interruptedJob.id); - if (claimed?.status === "running" && claimed.lockedBy === this.workerId) { - await this.markExistingRunsStale(claimed.id); - await addEvent(this.db, claimed.id, "warn", "Interrupted job reclaimed by replacement worker", { workerId: this.workerId }); - return claimed; - } - return null; + if (!candidate) return null; + const timestamp = nowIso(); + const claimedRow = await first( + transaction + .update(schema.jobs) + .set({ + status: "running", + startedAt: candidate.startedAt ?? timestamp, + lockedBy: this.workerId, + lockedAt: timestamp, + heartbeatAt: timestamp, + leaseVersion: sql`${schema.jobs.leaseVersion} + 1` + }) + .where(and(eq(schema.jobs.id, candidate.id), eq(schema.jobs.status, "queued"), eq(schema.jobs.leaseVersion, candidate.leaseVersion))) + .returning() + ); + return claimedRow ? { job: toJobRecord(claimedRow), reclaimed: fencedJobIds.has(claimedRow.id) } : null; + }); } - private async activeRunningJobs(excludedJobId?: number): Promise { - return (await this.db - .select() - .from(schema.jobs) - .where(eq(schema.jobs.status, "running")) - ) - .map(toJobRecord) - .filter((job) => job.id !== excludedJobId && !isStaleRunningJob(job, this.reclaimStaleAfterMs)); + private async hasClaimConflict(db: DbExecutor, candidateJobId: number, runningJobIds: number[]): Promise { + if (runningJobIds.length === 0) return false; + const conflict = await dbGet<{ value: number }>(db, sql` + WITH running_job_ids AS ( + SELECT value::integer AS job_id + FROM jsonb_array_elements_text(${JSON.stringify(runningJobIds)}::jsonb) + ) + SELECT 1 AS value + FROM job_resource_claims AS candidate + JOIN job_resource_claims AS active + ON active.resource_type = candidate.resource_type + AND active.resource_key = candidate.resource_key + AND (active.access = 'exclusive' OR candidate.access = 'exclusive') + JOIN running_job_ids ON running_job_ids.job_id = active.job_id + WHERE candidate.job_id = ${candidateJobId} + LIMIT 1 + `); + return Boolean(conflict); } private limitForType(type: JobRecord["type"]): number { @@ -1692,33 +2541,105 @@ export class JobWorker { return this.concurrency.maxRunningJobs; } - private async canStartJobType(type: JobRecord["type"], excludedJobId?: number): Promise { - const activeJobs = await this.activeRunningJobs(excludedJobId); - if (activeJobs.length >= this.concurrency.maxRunningJobs) return false; - return activeJobs.filter((job) => job.type === type).length < this.limitForType(type); - } - private async requeueInterruptedJobsForPathMigration(): Promise { - const rows = await this.db - .select() - .from(schema.jobs) - .where(and(eq(schema.jobs.status, "running"), ne(schema.jobs.type, "path_migration"))); - for (const job of rows.map(toJobRecord)) { - if (!isStaleRunningJob(job, this.reclaimOwnInterruptedAfterMs)) continue; - await this.requeueInterruptedJob(job.id, "Managed storage paths changed; interrupted job paused and requeued"); - } + await this.db.transaction(async (transaction) => { + await transaction.execute(sql`select pg_advisory_xact_lock(${schedulerLockKey})`); + const rows = await transaction + .select() + .from(schema.jobs) + .where(and(eq(schema.jobs.status, "running"), ne(schema.jobs.type, "path_migration"))) + .orderBy(asc(schema.jobs.id)); + const jobs = rows.map(toJobRecord); + const ownerIds = [...new Set(jobs.map((job) => job.lockedBy).filter((ownerId): ownerId is string => Boolean(ownerId)))]; + const ownerHeartbeats = ownerIds.length > 0 + ? await transaction.select().from(schema.workerHeartbeats).where(inArray(schema.workerHeartbeats.workerId, ownerIds)) + : []; + const ownerHeartbeatById = new Map(ownerHeartbeats.map((heartbeat) => [heartbeat.workerId, heartbeat])); + const timestamp = nowIso(); + for (const job of jobs) { + if (!this.isReclaimableWithHeartbeat(job, job.lockedBy ? ownerHeartbeatById.get(job.lockedBy) : undefined)) continue; + const lockedRow = await first( + transaction + .select() + .from(schema.jobs) + .where(and(eq(schema.jobs.id, job.id), eq(schema.jobs.status, "running"), eq(schema.jobs.leaseVersion, job.leaseVersion))) + .for("update", { skipLocked: true }) + .limit(1) + ); + if (!lockedRow) continue; + const lockedJob = toJobRecord(lockedRow); + const lockedOwnerHeartbeat = lockedJob.lockedBy && lockedJob.lockedBy !== this.workerId + ? await first( + transaction + .select() + .from(schema.workerHeartbeats) + .where(eq(schema.workerHeartbeats.workerId, lockedJob.lockedBy)) + .for("update") + .limit(1) + ) + : undefined; + if (!this.isReclaimableWithHeartbeat(lockedJob, lockedOwnerHeartbeat)) continue; + const requeued = await first( + transaction + .update(schema.jobs) + .set({ + status: "queued", + lockedBy: null, + lockedAt: null, + heartbeatAt: null, + leaseVersion: sql`${schema.jobs.leaseVersion} + 1` + }) + .where(and(eq(schema.jobs.id, job.id), eq(schema.jobs.status, "running"), eq(schema.jobs.leaseVersion, job.leaseVersion))) + .returning({ id: schema.jobs.id }) + ); + if (requeued) { + await transaction.insert(schema.jobEvents).values({ + jobId: job.id, + timestamp, + level: "warn", + message: "Managed storage paths changed; interrupted job paused and requeued", + data: JSON.stringify({ workerId: this.workerId }) + }); + } + } + }); } - private async runClaimedJob(job: JobRecord): Promise { + private async runClaimedJob(job: LeasedJob): Promise { const abortController = new AbortController(); this.activeAbortControllers.set(job.id, abortController); + let leaseLost = false; + const loseLease = (cause: unknown) => { + leaseLost = true; + const reason = cause instanceof LeaseLostError ? cause : new LeaseLostError(job.id, { cause }); + if (!abortController.signal.aborted) abortController.abort(reason); + return reason; + }; + let heartbeatInFlight = false; const heartbeat = setInterval(() => { - void this.heartbeat(job.id).catch((error: unknown) => { - this.logger.warn(`SRTL worker ${this.workerId} heartbeat failed for job #${job.id}: ${errorMessage(error)}`); - }); + if (heartbeatInFlight) return; + heartbeatInFlight = true; + void this.heartbeat(job) + .catch((error: unknown) => { + const leaseError = loseLease(error); + this.logger.warn(`SRTL worker ${this.workerId} heartbeat failed for job #${job.id}: ${leaseError.message}`); + }) + .finally(() => { + heartbeatInFlight = false; + }); }, this.heartbeatIntervalMs); + let heartbeatStopped = false; + const stopHeartbeat = () => { + if (heartbeatStopped) return; + clearInterval(heartbeat); + heartbeatStopped = true; + }; let pathMigrationPauseRequested = false; + let completedByHandler = false; + let cancellationWatchInFlight = false; const cancellationWatcher = setInterval(() => { + if (cancellationWatchInFlight) return; + cancellationWatchInFlight = true; void Promise.all([this.isCancellationRequested(job.id), job.type === "path_migration" ? Promise.resolve(false) : isPathConfigurationBlocked(this.db)]) .then(([cancelled, pathConfigurationBlocked]) => { if (pathConfigurationBlocked) pathMigrationPauseRequested = true; @@ -1726,6 +2647,9 @@ export class JobWorker { }) .catch((error: unknown) => { this.logger.warn(`SRTL worker ${this.workerId} cancellation watch failed for job #${job.id}: ${errorMessage(error)}`); + }) + .finally(() => { + cancellationWatchInFlight = false; }); }, 500); const isCancelled = async () => { @@ -1735,50 +2659,122 @@ export class JobWorker { if ((cancelled || pathConfigurationBlocked) && !abortController.signal.aborted) abortController.abort(); return cancelled || pathConfigurationBlocked; }; + const assertLease = async () => { + try { + await this.assertLease(job); + } catch (error: unknown) { + throw loseLease(error); + } + }; + const withLeaseDb = async (action: (db: DbExecutor) => Promise): Promise => { + let actionFailed = false; + let actionError: unknown; + try { + return await this.withLease(job, async (transaction) => { + try { + return await action(transaction); + } catch (error: unknown) { + actionFailed = true; + actionError = error; + throw error; + } + }); + } catch (error: unknown) { + if (actionFailed) throw actionError; + throw loseLease(error); + } + }; + const withLease = (action: () => Promise): Promise => withLeaseDb(() => action()); + const finishCompleted = async (action: (db: DbExecutor) => Promise): Promise => { + const completed = await this.finishCompletedJob(job, action); + if (completed) completedByHandler = true; + return completed; + }; + const finishCompletedIsolated = async (action: (db: DbExecutor) => Promise): Promise => { + const completed = await this.finishCompletedJob(job, action, false); + if (completed) completedByHandler = true; + return completed; + }; const ctx: JobContext = { jobId: job.id, signal: abortController.signal, - event: (level, message, data) => addEvent(this.db, job.id, level, message, data), - setProgress: (progress) => setProgress(this.db, job.id, progress), - isCancelled + event: async (level, message, data) => { + try { + await this.addLeasedEvent(job, level, message, data); + } catch (error: unknown) { + throw loseLease(error); + } + }, + setProgress: async (progress) => { + try { + await this.setLeasedProgress(job, progress); + } catch (error: unknown) { + throw loseLease(error); + } + }, + isCancelled, + assertLease, + withLease, + withLeaseDb, + finishCompleted, + finishCompletedIsolated }; - await addEvent(this.db, job.id, "info", "Worker started job", { workerId: this.workerId }); try { + await ctx.event("info", "Worker started job", { workerId: this.workerId, leaseVersion: job.leaseVersion }); await this.runHandler(job, ctx); + if (completedByHandler) return; if (job.type !== "path_migration" && (pathMigrationPauseRequested || (await isPathConfigurationBlocked(this.db)))) { - await this.requeueInterruptedJob(job.id, "Managed storage paths changed; job paused and requeued"); + stopHeartbeat(); + if (job.type === "copy") await this.settlePathInterruptedCopy(job); + else await this.requeueInterruptedJob(job, "Managed storage paths changed; job paused and requeued"); return; } const status: JobStatus = (await this.isCancellationRequested(job.id)) ? "cancelled" : "completed"; - await this.finishJob(job.id, status); - if (status === "completed" && job.type === "scan") await completeOnboardingScan(this.db, job.id); - await addEvent(this.db, job.id, status === "cancelled" ? "warn" : "info", status === "cancelled" ? "Job cancelled" : "Job completed"); + if (status === "completed" && job.type === "scan") { + await assertLease(); + await completeOnboardingScan(this.db, job.id); + } + stopHeartbeat(); + await this.finishJob(job, status, status === "cancelled" ? "warn" : "info", status === "cancelled" ? "Job cancelled" : "Job completed"); } catch (error: unknown) { + if (leaseLost || error instanceof LeaseLostError) { + this.logger.warn(`SRTL worker ${this.workerId} stopped job #${job.id} after losing its lease`); + return; + } if (pathMigrationPauseRequested) { - await this.requeueInterruptedJob(job.id, "Managed storage paths changed; job paused and requeued"); + stopHeartbeat(); + if (job.type === "copy") await this.settlePathInterruptedCopy(job, errorMessage(error)); + else await this.requeueInterruptedJob(job, "Managed storage paths changed; job paused and requeued"); return; } if (await this.shouldRequeueInterruptedJob(job.id, abortController)) { - await this.requeueInterruptedJob(job.id); + stopHeartbeat(); + await this.requeueInterruptedJob(job); + return; + } + if (error instanceof CopyReconciliationRequiredError) { + stopHeartbeat(); + await this.finishJob(job, "failed", "error", error.message); + this.logger.error(`SRTL worker ${this.workerId} stopped copy job #${job.id} for manual reconciliation: ${error.message}`); return; } if (await this.isCancellationRequested(job.id)) { - await this.finishJob(job.id, "cancelled"); - await addEvent(this.db, job.id, "warn", "Job cancelled"); + stopHeartbeat(); + await this.finishJob(job, "cancelled", "warn", "Job cancelled"); return; } if (error instanceof PartialJobFailureError) { - await this.finishJob(job.id, "partially_failed"); - await addEvent(this.db, job.id, "warn", error.message); + stopHeartbeat(); + await this.finishJob(job, "partially_failed", "warn", error.message); this.logger.warn(`SRTL worker ${this.workerId} partially failed job #${job.id}: ${error.message}`); return; } - await this.finishJob(job.id, "failed"); - await addEvent(this.db, job.id, "error", errorMessage(error)); + stopHeartbeat(); + await this.finishJob(job, "failed", "error", errorMessage(error)); this.logger.error(`SRTL worker ${this.workerId} failed job #${job.id}: ${errorMessage(error)}`); } finally { - clearInterval(heartbeat); + stopHeartbeat(); clearInterval(cancellationWatcher); this.activeAbortControllers.delete(job.id); } @@ -1808,47 +2804,237 @@ export class JobWorker { throw new Error(`No worker handler is registered for ${job.type} jobs`); } - private async heartbeat(jobId: number): Promise { - await this.db.update(schema.jobs).set({ heartbeatAt: nowIso() }).where(and(eq(schema.jobs.id, jobId), eq(schema.jobs.status, "running"))); + private async assertLease(job: LeasedJob): Promise { + const row = await first( + this.db + .select({ id: schema.jobs.id }) + .from(schema.jobs) + .where( + and( + eq(schema.jobs.id, job.id), + eq(schema.jobs.status, "running"), + eq(schema.jobs.lockedBy, this.workerId), + eq(schema.jobs.leaseVersion, job.leaseVersion) + ) + ) + .limit(1) + ); + if (!row) throw new LeaseLostError(job.id); + } + + private async withLease(job: LeasedJob, action: (db: DbExecutor) => Promise): Promise { + return this.db.transaction(async (transaction) => { + const row = await first( + transaction + .select({ id: schema.jobs.id }) + .from(schema.jobs) + .where( + and( + eq(schema.jobs.id, job.id), + eq(schema.jobs.status, "running"), + eq(schema.jobs.lockedBy, this.workerId), + eq(schema.jobs.leaseVersion, job.leaseVersion) + ) + ) + .for("update") + .limit(1) + ); + if (!row) throw new LeaseLostError(job.id); + return action(transaction); + }); + } + + private async heartbeat(job: LeasedJob): Promise { + const row = await first( + this.db + .update(schema.jobs) + .set({ heartbeatAt: nowIso() }) + .where( + and( + eq(schema.jobs.id, job.id), + eq(schema.jobs.status, "running"), + eq(schema.jobs.lockedBy, this.workerId), + eq(schema.jobs.leaseVersion, job.leaseVersion) + ) + ) + .returning({ id: schema.jobs.id }) + ); + if (!row) throw new LeaseLostError(job.id); + } + + private async setLeasedProgress(job: LeasedJob, progress: unknown): Promise { + const row = await first( + this.db + .update(schema.jobs) + .set({ progress: JSON.stringify(progress) }) + .where( + and( + eq(schema.jobs.id, job.id), + eq(schema.jobs.status, "running"), + eq(schema.jobs.lockedBy, this.workerId), + eq(schema.jobs.leaseVersion, job.leaseVersion) + ) + ) + .returning({ id: schema.jobs.id }) + ); + if (!row) throw new LeaseLostError(job.id); + } + + private async addLeasedEvent(job: LeasedJob, level: JobEventRecord["level"], message: string, data: unknown = {}): Promise { + const inserted = await this.db.transaction(async (transaction) => { + const lease = await first( + transaction + .select({ id: schema.jobs.id }) + .from(schema.jobs) + .where( + and( + eq(schema.jobs.id, job.id), + eq(schema.jobs.status, "running"), + eq(schema.jobs.lockedBy, this.workerId), + eq(schema.jobs.leaseVersion, job.leaseVersion) + ) + ) + .for("update") + .limit(1) + ); + if (!lease) return false; + await transaction.insert(schema.jobEvents).values({ jobId: job.id, timestamp: nowIso(), level, message, data: JSON.stringify(data) }); + return true; + }); + if (!inserted) throw new LeaseLostError(job.id); } private async shouldRequeueInterruptedJob(jobId: number, abortController: AbortController): Promise { - return this.stopped && abortController.signal.aborted && !(await this.isCancellationRequested(jobId)); + return this.stopRequested && abortController.signal.aborted && !(await this.isCancellationRequested(jobId)); + } + + private async settlePathInterruptedCopy(job: LeasedJob, failureMessage?: string): Promise { + const operations = await this.db + .select({ stage: schema.copyOperations.stage, errorMessage: schema.copyOperations.errorMessage }) + .from(schema.copyOperations) + .where(eq(schema.copyOperations.jobId, job.id)); + const manual = operations.find((operation) => operation.stage === "reconciliation_required"); + if (manual) { + await this.finishJob( + job, + "failed", + "error", + manual.errorMessage ?? failureMessage ?? "Copy recovery requires manual reconciliation before paths can be migrated" + ); + return; + } + const hasUnreconciledOperations = operations.some( + (operation) => operation.stage !== "rolled_back" && operation.stage !== "failed" + ); + if (hasUnreconciledOperations) { + await this.requeueInterruptedJob(job, "Managed storage paths changed; copy recovery remains queued"); + return; + } + await this.finishJob(job, "cancelled", "warn", "Copy cancelled after managed-path recovery"); } - private async requeueInterruptedJob(jobId: number, message = "Worker stopped; job requeued for resume"): Promise { - const requeued = await first(this.db - .update(schema.jobs) - .set({ status: "queued", lockedBy: null, lockedAt: null, heartbeatAt: null }) - .where(and(eq(schema.jobs.id, jobId), eq(schema.jobs.status, "running"))) - .returning({ id: schema.jobs.id })); - if (requeued) await addEvent(this.db, jobId, "warn", message, { workerId: this.workerId }); + private async requeueInterruptedJob(job: LeasedJob, message = "Worker stopped; job requeued for resume"): Promise { + const requeued = await this.db.transaction(async (transaction) => { + const timestamp = nowIso(); + const row = await first( + transaction + .update(schema.jobs) + .set({ status: "queued", lockedBy: null, lockedAt: null, heartbeatAt: null }) + .where( + and( + eq(schema.jobs.id, job.id), + eq(schema.jobs.status, "running"), + eq(schema.jobs.lockedBy, this.workerId), + eq(schema.jobs.leaseVersion, job.leaseVersion) + ) + ) + .returning({ id: schema.jobs.id }) + ); + if (!row) return false; + await transaction.insert(schema.jobEvents).values({ jobId: job.id, timestamp, level: "warn", message, data: JSON.stringify({ workerId: this.workerId }) }); + return true; + }); + if (!requeued) throw new LeaseLostError(job.id); } - private async markExistingRunsStale(jobId: number): Promise { - const timestamp = nowIso(); - await this.db - .update(schema.scanRuns) - .set({ status: "failed", finishedAt: timestamp, errorMessage: "Stale job reclaimed by worker" }) - .where(and(eq(schema.scanRuns.jobId, jobId), eq(schema.scanRuns.status, "running"))); - await this.db - .update(schema.auditRuns) - .set({ status: "failed", finishedAt: timestamp }) - .where(and(eq(schema.auditRuns.jobId, jobId), eq(schema.auditRuns.status, "running"))); + private async finishCompletedJob( + job: LeasedJob, + action: (db: DbExecutor) => Promise, + useSchedulerBarrier = true + ): Promise { + return this.db.transaction(async (transaction) => { + if (useSchedulerBarrier) await transaction.execute(sql`select pg_advisory_xact_lock(${schedulerLockKey})`); + const lease = await first( + transaction + .select({ id: schema.jobs.id, cancelRequestedAt: schema.jobs.cancelRequestedAt }) + .from(schema.jobs) + .where( + and( + eq(schema.jobs.id, job.id), + eq(schema.jobs.status, "running"), + eq(schema.jobs.lockedBy, this.workerId), + eq(schema.jobs.leaseVersion, job.leaseVersion) + ) + ) + .for("update") + .limit(1) + ); + if (!lease) throw new LeaseLostError(job.id); + if (job.type !== "path_migration" && (await isPathConfigurationBlocked(transaction))) return false; + if (lease.cancelRequestedAt) return false; + + await action(transaction); + const timestamp = nowIso(); + const completed = await first( + transaction + .update(schema.jobs) + .set({ status: "completed", finishedAt: timestamp, lockedBy: null, lockedAt: null, heartbeatAt: null, cancelRequestedAt: null }) + .where( + and( + eq(schema.jobs.id, job.id), + eq(schema.jobs.status, "running"), + eq(schema.jobs.lockedBy, this.workerId), + eq(schema.jobs.leaseVersion, job.leaseVersion), + isNull(schema.jobs.cancelRequestedAt) + ) + ) + .returning({ id: schema.jobs.id }) + ); + if (!completed) return false; + await transaction.insert(schema.jobEvents).values({ jobId: job.id, timestamp, level: "info", message: "Job completed", data: "{}" }); + return true; + }); } - private async finishJob(jobId: number, status: JobStatus): Promise { - await this.db - .update(schema.jobs) - .set({ - status, - finishedAt: nowIso(), - lockedBy: null, - lockedAt: null, - heartbeatAt: null, - cancelRequestedAt: status === "completed" ? null : undefined - }) - .where(eq(schema.jobs.id, jobId)); + private async finishJob(job: LeasedJob, status: JobStatus, level: JobEventRecord["level"], message: string): Promise { + const finished = await this.db.transaction(async (transaction) => { + const timestamp = nowIso(); + const row = await first( + transaction + .update(schema.jobs) + .set({ + status, + finishedAt: timestamp, + lockedBy: null, + lockedAt: null, + heartbeatAt: null, + cancelRequestedAt: status === "completed" ? null : undefined + }) + .where( + and( + eq(schema.jobs.id, job.id), + eq(schema.jobs.status, "running"), + eq(schema.jobs.lockedBy, this.workerId), + eq(schema.jobs.leaseVersion, job.leaseVersion) + ) + ) + .returning({ id: schema.jobs.id }) + ); + if (!row) return false; + await transaction.insert(schema.jobEvents).values({ jobId: job.id, timestamp, level, message, data: "{}" }); + return true; + }); + if (!finished) throw new LeaseLostError(job.id); } private async isCancellationRequested(jobId: number): Promise { @@ -1857,10 +3043,14 @@ export class JobWorker { } private async runScanJob(jobId: number, normalizedOptions: ScanOptions, ctx: JobContext): Promise { - const scanRun = await first(this.db - .insert(schema.scanRuns) - .values({ jobId, status: "running", startedAt: nowIso(), finishedAt: null, errorMessage: null, ...emptyScanTotals() }) - .returning({ id: schema.scanRuns.id })); + const scanRun = await ctx.withLeaseDb((leaseDb) => + first( + leaseDb + .insert(schema.scanRuns) + .values({ jobId, status: "running", startedAt: nowIso(), finishedAt: null, errorMessage: null, ...emptyScanTotals() }) + .returning({ id: schema.scanRuns.id }) + ) + ); if (!scanRun) throw new Error("Scan run was not created"); try { @@ -1887,7 +3077,7 @@ export class JobWorker { configuredSections, await getStoragePolicyMap(this.db), normalizedOptions, - ctx.isCancelled, + async () => ctx.signal.aborted || (await ctx.isCancelled()), async (activity) => { const liveTotals = emptyScanTotals(); liveTotals.totalLinks = activity.checkedLinks; @@ -1909,36 +3099,57 @@ export class JobWorker { }); } if (await ctx.isCancelled()) { - await this.db.update(schema.scanRuns).set({ status: "cancelled", finishedAt: nowIso(), errorMessage: "Job cancelled" }).where(eq(schema.scanRuns.id, scanRun.id)); + await ctx.withLeaseDb(async (leaseDb) => { + await leaseDb.update(schema.scanRuns).set({ status: "cancelled", finishedAt: nowIso(), errorMessage: "Job cancelled" }).where(eq(schema.scanRuns.id, scanRun.id)); + }); await ctx.setProgress(scanProgressPayload(normalizedOptions, "cancelled", "Scan cancelled before inventory results were written", result.inventory)); return; } - const persistedInventory = await this.db.transaction(async (transaction) => { - const inventory = await persistScanResult(transaction, result, jobId, ctx.isCancelled); - if (await ctx.isCancelled()) throw new Error("Scan indexing was cancelled"); - await transaction.update(schema.scanRuns).set({ status: "completed", finishedAt: nowIso(), errorMessage: null, ...inventory }).where(eq(schema.scanRuns.id, scanRun.id)); - return inventory; + let persistedInventory: InventorySummary | null = null; + const completionMessage = isTitleRescan + ? "Title rescan completed and symlink inventory was reconciled" + : "Scan completed and inventory counters were updated"; + const completionEvent = isTitleRescan + ? "Targeted title rescan reconciled symlinks" + : "Manual inventory scan indexed library links and storage files"; + const finalized = await ctx.finishCompleted(async (leaseDb) => { + if (ctx.signal.aborted) throw (ctx.signal.reason instanceof Error ? ctx.signal.reason : new Error("Scan indexing was cancelled")); + const inventory = await persistScanResult(leaseDb, result, jobId, async () => ctx.signal.aborted); + if (ctx.signal.aborted) throw (ctx.signal.reason instanceof Error ? ctx.signal.reason : new Error("Scan indexing was cancelled")); + await leaseDb.update(schema.scanRuns).set({ status: "completed", finishedAt: nowIso(), errorMessage: null, ...inventory }).where(eq(schema.scanRuns.id, scanRun.id)); + await completeOnboardingScan(leaseDb, jobId); + await leaseDb + .update(schema.jobs) + .set({ progress: JSON.stringify(scanProgressPayload(normalizedOptions, "completed", completionMessage, inventory)) }) + .where(eq(schema.jobs.id, jobId)); + await leaseDb.insert(schema.jobEvents).values({ + jobId, + timestamp: nowIso(), + level: "info", + message: completionEvent, + data: JSON.stringify({ options: normalizedOptions, ...inventory }) + }); + persistedInventory = inventory; }); - await ctx.setProgress( - scanProgressPayload( - normalizedOptions, - "completed", - isTitleRescan ? "Title rescan completed and symlink inventory was reconciled" : "Scan completed and inventory counters were updated", - persistedInventory - ) - ); - await ctx.event("info", isTitleRescan ? "Targeted title rescan reconciled symlinks" : "Manual inventory scan indexed library links and storage files", { - options: normalizedOptions, - ...persistedInventory + if (finalized && persistedInventory) return; + await ctx.withLeaseDb(async (leaseDb) => { + await leaseDb.update(schema.scanRuns).set({ status: "cancelled", finishedAt: nowIso(), errorMessage: "Job cancelled" }).where(eq(schema.scanRuns.id, scanRun.id)); }); + await ctx.setProgress(scanProgressPayload(normalizedOptions, "cancelled", "Scan cancelled before inventory results were written", result.inventory)); + await ctx.event("warn", "Scan cancelled before inventory results were written"); } catch (error: unknown) { + if (error instanceof LeaseLostError || (error instanceof Error && error.name === "LeaseLostError")) throw error; if (await ctx.isCancelled()) { - await this.db.update(schema.scanRuns).set({ status: "cancelled", finishedAt: nowIso(), errorMessage: "Job terminated" }).where(eq(schema.scanRuns.id, scanRun.id)); + await ctx.withLeaseDb(async (leaseDb) => { + await leaseDb.update(schema.scanRuns).set({ status: "cancelled", finishedAt: nowIso(), errorMessage: "Job terminated" }).where(eq(schema.scanRuns.id, scanRun.id)); + }); await ctx.setProgress(scanProgressPayload(normalizedOptions, "cancelled", "Scan terminated before inventory results were written")); await ctx.event("warn", "Scan terminated before inventory results were written"); return; } - await this.db.update(schema.scanRuns).set({ status: "failed", finishedAt: nowIso(), errorMessage: errorMessage(error) }).where(eq(schema.scanRuns.id, scanRun.id)); + await ctx.withLeaseDb(async (leaseDb) => { + await leaseDb.update(schema.scanRuns).set({ status: "failed", finishedAt: nowIso(), errorMessage: errorMessage(error) }).where(eq(schema.scanRuns.id, scanRun.id)); + }); await ctx.setProgress(scanProgressPayload(normalizedOptions, "failed", errorMessage(error))); throw error; } @@ -1947,25 +3158,29 @@ export class JobWorker { private async runAuditJob(jobId: number, normalizedOptions: AuditOptions, ctx: JobContext): Promise { const links = filterAuditLinks(await listMediaLinks(this.db), normalizedOptions); const startedAt = nowIso(); - const auditRun = await first(this.db - .insert(schema.auditRuns) - .values({ - jobId, - mode: normalizedOptions.mode, - status: "running", - startedAt, - finishedAt: null, - checked: 0, - passed: 0, - failed: 0, - sourceUnknown: 0, - sourceMissing: 0, - sourceCompareErrors: 0, - byteMismatches: 0, - targetValidationFailures: 0, - errorMessage: null - }) - .returning({ id: schema.auditRuns.id })); + const auditRun = await ctx.withLeaseDb((leaseDb) => + first( + leaseDb + .insert(schema.auditRuns) + .values({ + jobId, + mode: normalizedOptions.mode, + status: "running", + startedAt, + finishedAt: null, + checked: 0, + passed: 0, + failed: 0, + sourceUnknown: 0, + sourceMissing: 0, + sourceCompareErrors: 0, + byteMismatches: 0, + targetValidationFailures: 0, + errorMessage: null + }) + .returning({ id: schema.auditRuns.id }) + ) + ); if (!auditRun) throw new Error("Audit run was not created"); let checked = 0; let passed = 0; @@ -2032,13 +3247,13 @@ export class JobWorker { if (result.cmpStatus === "fail") byteMismatches += 1; if (result.ffmpegStatus === "fail") targetValidationFailures += 1; - await this.db.insert(schema.auditResults).values({ ...result, auditRunId: auditRun.id, createdAt: nowIso() }); + await ctx.withLeaseDb(async (leaseDb) => { + await leaseDb.insert(schema.auditResults).values({ ...result, auditRunId: auditRun.id, createdAt: nowIso() }); + }); await ctx.setProgress(auditProgress("auditing", "Recorded audit result", link)); } - const status = (await ctx.isCancelled()) ? "cancelled" : "completed"; - if (status === "cancelled") { - await this.db.delete(schema.auditResults).where(eq(schema.auditResults.auditRunId, auditRun.id)); + const cancelAudit = async () => { checked = 0; passed = 0; failed = 0; @@ -2047,31 +3262,72 @@ export class JobWorker { sourceCompareErrors = 0; byteMismatches = 0; targetValidationFailures = 0; + await ctx.withLeaseDb(async (leaseDb) => { + await leaseDb.delete(schema.auditResults).where(eq(schema.auditResults.auditRunId, auditRun.id)); + await leaseDb + .update(schema.auditRuns) + .set({ status: "cancelled", finishedAt: nowIso(), checked, passed, failed, sourceUnknown, sourceMissing, sourceCompareErrors, byteMismatches, targetValidationFailures, errorMessage: null }) + .where(eq(schema.auditRuns.id, auditRun.id)); + }); + await ctx.setProgress(auditProgress("cancelled", "Audit cancelled")); + await ctx.event("warn", `${normalizedOptions.mode} audit terminated; partial results discarded`, { + checked, + passed, + failed, + sourceUnknown, + sourceMissing, + sourceCompareErrors, + byteMismatches, + targetValidationFailures, + sections: normalizedOptions.sections, + section: normalizedOptions.section, + itemName: normalizedOptions.itemName + }); + }; + if (await ctx.isCancelled()) { + await cancelAudit(); + return; } - await this.db - .update(schema.auditRuns) - .set({ status, finishedAt: nowIso(), checked, passed, failed, sourceUnknown, sourceMissing, sourceCompareErrors, byteMismatches, targetValidationFailures, errorMessage: null }) - .where(eq(schema.auditRuns.id, auditRun.id)); - await ctx.setProgress(auditProgress(status, status === "cancelled" ? "Audit cancelled" : "Audit completed")); - await ctx.event(status === "cancelled" ? "warn" : "info", status === "cancelled" ? `${normalizedOptions.mode} audit terminated; partial results discarded` : `${normalizedOptions.mode} audit indexed results`, { - checked, - passed, - failed, - sourceUnknown, - sourceMissing, - sourceCompareErrors, - byteMismatches, - targetValidationFailures, - sections: normalizedOptions.sections, - section: normalizedOptions.section, - itemName: normalizedOptions.itemName + + const completedProgress = auditProgress("completed", "Audit completed"); + const finalized = await ctx.finishCompleted(async (leaseDb) => { + if (ctx.signal.aborted) throw (ctx.signal.reason instanceof Error ? ctx.signal.reason : new Error("Audit was interrupted")); + await leaseDb + .update(schema.auditRuns) + .set({ status: "completed", finishedAt: nowIso(), checked, passed, failed, sourceUnknown, sourceMissing, sourceCompareErrors, byteMismatches, targetValidationFailures, errorMessage: null }) + .where(eq(schema.auditRuns.id, auditRun.id)); + await leaseDb.update(schema.jobs).set({ progress: JSON.stringify(completedProgress) }).where(eq(schema.jobs.id, jobId)); + await leaseDb.insert(schema.jobEvents).values({ + jobId, + timestamp: nowIso(), + level: "info", + message: `${normalizedOptions.mode} audit indexed results`, + data: JSON.stringify({ + checked, + passed, + failed, + sourceUnknown, + sourceMissing, + sourceCompareErrors, + byteMismatches, + targetValidationFailures, + sections: normalizedOptions.sections, + section: normalizedOptions.section, + itemName: normalizedOptions.itemName + }) + }); }); + if (finalized) return; + await cancelAudit(); } catch (error: unknown) { + if (error instanceof LeaseLostError || (error instanceof Error && error.name === "LeaseLostError")) throw error; const message = errorMessage(error); - await this.db - .update(schema.auditRuns) - .set({ status: "failed", finishedAt: nowIso(), checked, passed, failed, sourceUnknown, sourceMissing, sourceCompareErrors, byteMismatches, targetValidationFailures, errorMessage: message }) - .where(eq(schema.auditRuns.id, auditRun.id)); + await ctx.withLeaseDb(async (leaseDb) => { + await leaseDb + .update(schema.auditRuns) + .set({ status: "failed", finishedAt: nowIso(), checked, passed, failed, sourceUnknown, sourceMissing, sourceCompareErrors, byteMismatches, targetValidationFailures, errorMessage: message }) + .where(eq(schema.auditRuns.id, auditRun.id)); + }); await ctx.setProgress(auditProgress("failed", message)); throw error; } @@ -2080,13 +3336,69 @@ export class JobWorker { private async runCopyJob(normalizedOptions: CopyOptions, ctx: JobContext): Promise { const paths = await getJsonSetting(this.db, "paths", { symlinkDir: "", localDir: "", remoteDir: "" }); if (!paths.symlinkDir || !paths.localDir || !paths.remoteDir) throw new Error("Path settings are incomplete"); + const durableClaims = await this.db.select().from(schema.jobResourceClaims).where(eq(schema.jobResourceClaims.jobId, ctx.jobId)); + const claimedPaths = new Set(durableClaims.filter((claim) => claim.resourceType === "path").map((claim) => path.resolve(claim.resourceKey))); + const claimedCopyPathBindings = new Map(); + const cleanupPathsByLink = new Map>(); + for (const claim of durableClaims) { + if (claim.resourceType === "copy_path_binding") { + const binding = parseCopyPathBindingResourceKey(claim.resourceKey); + if (binding) claimedCopyPathBindings.set(copyPathBindingMapKey(binding.linkId, binding.role), binding); + } + if (claim.resourceType !== "copy_cleanup") continue; + const marker = parseCopyCleanupMarker(claim.resourceKey); + if (!marker?.identity) continue; + const existing = cleanupPathsByLink.get(marker.linkId); + if (existing) existing.set(marker.filePath, marker.identity); + else cleanupPathsByLink.set(marker.linkId, new Map([[marker.filePath, marker.identity]])); + } + const copyPathsRemainClaimed = async (link: MediaLinkRow, destinationPath: string): Promise => { + if (claimedPaths.size === 0 && claimedCopyPathBindings.size === 0) return true; + const currentBindings = await copyPathBindingsForLink(link, paths, normalizedOptions.direction); + const currentDestination = currentBindings.find((binding) => binding.role === "destination"); + if (!currentDestination || currentDestination.lexicalPath !== path.resolve(destinationPath)) return false; + if ( + claimedPaths.size > 0 && + currentBindings.some( + (binding) => !claimedPaths.has(binding.lexicalPath) || !claimedPaths.has(binding.canonicalPath) + ) + ) { + return false; + } + if (claimedCopyPathBindings.size === 0) return true; + return currentBindings.every((binding) => { + const expected = claimedCopyPathBindings.get(copyPathBindingMapKey(binding.linkId, binding.role)); + return expected?.lexicalPath === binding.lexicalPath && expected.canonicalPath === binding.canonicalPath; + }); + }; const advancedSettings = normalizeAdvancedSettings(await getJsonSetting(this.db, "advancedSettings", {})); - await reconcileCopyOperationsForJob(this.db, ctx.jobId, paths, ctx.event); + await ctx.assertLease(); + // Path-blocked copy jobs are admitted only to reconcile and roll back their + // durable journal. Mark the context cancelled before replaying that journal. + await ctx.isCancelled(); + await reconcileCopyOperationsForJob(this.db, ctx.jobId, paths, ctx); const allLinks = await listMediaLinks(this.db, undefined, "current"); const links = filterCopyLinks(allLinks, normalizedOptions); - const hasDurableSelection = Boolean(normalizedOptions.linkIds?.length); + const hasDurableSelection = normalizedOptions.linkIds !== undefined; const selectedLinks = hasDurableSelection ? filterCopySelectedLinks(allLinks, normalizedOptions) : links; + const durableSelectedDestinations = [...claimedCopyPathBindings.values()] + .filter((binding) => binding.role === "destination") + .map((binding) => ({ + linkId: binding.linkId, + lexicalPath: binding.lexicalPath, + canonicalPath: binding.canonicalPath + })); + const durableDestinationLinkIds = new Set(durableSelectedDestinations.map((destination) => destination.linkId)); + const supplementalSelectedDestinations = await copySelectedDestinationsForLinks( + selectedLinks.filter((link) => !durableDestinationLinkIds.has(link.id)), + paths, + normalizedOptions.direction + ); + const selectedDestinations = indexCopySelectedDestinations([ + ...durableSelectedDestinations, + ...supplementalSelectedDestinations.entries + ]); const destinationKind = copyDestinationKind(normalizedOptions.direction); const resumeState = await readCopyResumeState(this.db, ctx.jobId, selectedLinks, destinationKind, hasDurableSelection); const selectedTotal = hasDurableSelection ? (normalizedOptions.linkIds?.length ?? 0) : selectedLinks.length; @@ -2103,26 +3415,60 @@ export class JobWorker { const resumedRepointed = repointed; let activeLink: MediaLinkRow | undefined; let activeUpdate: Partial | undefined; - const replacementCandidateEntries: Array<{ linkId: number; destinationPath: string; candidates: CopyLocalConflictCandidate[] }> = []; - const setCopyProgress = (stage: CopyProgressStage, message: string, link?: MediaLinkRow, update?: Partial) => - ctx.setProgress( - copyProgressPayload({ - options: normalizedOptions, - current, - total, - copied, - repointed, - skipped, - conflicts, - failed, - alreadyCompleted, - remaining: Math.max(0, total - current), - stage, - message, - link, - update - }) + const replacementCandidateEntries = new Map< + number, + { linkId: number; destinationPath: string; candidates: CopyLocalConflictCandidate[]; expectedIdentities: Map } + >(); + const committedReplacementOperations = await this.db + .select() + .from(schema.copyOperations) + .where( + and( + eq(schema.copyOperations.jobId, ctx.jobId), + eq(schema.copyOperations.stage, "committed"), + eq(schema.copyOperations.localConflictStrategy, "replace") + ) ); + for (const operation of committedReplacementOperations) { + const originalLink = copyOperationLink(operation); + const conflict = await copyLocalConflictForLink(this.db, originalLink, paths, selectedDestinations); + if (conflict) { + const expectedIdentities = cleanupPathsByLink.get(originalLink.id) ?? new Map(); + const allowedCandidates: CopyLocalConflictCandidate[] = []; + for (const candidate of conflict.candidates) { + const candidatePath = path.resolve(candidate.filePath); + if (sameCopyCleanupIdentity(expectedIdentities.get(candidatePath), await copyCleanupIdentity(candidatePath))) allowedCandidates.push(candidate); + } + if (allowedCandidates.length === 0) continue; + replacementCandidateEntries.set(originalLink.id, { + linkId: originalLink.id, + destinationPath: operation.destinationPath, + candidates: allowedCandidates, + expectedIdentities + }); + } + } + let progressWrite = Promise.resolve(); + const setCopyProgress = (stage: CopyProgressStage, message: string, link?: MediaLinkRow, update?: Partial) => { + const payload = copyProgressPayload({ + options: normalizedOptions, + current, + total, + copied, + repointed, + skipped, + conflicts, + failed, + alreadyCompleted, + remaining: Math.max(0, total - current), + stage, + message, + link, + update + }); + progressWrite = progressWrite.then(() => ctx.setProgress(payload)); + return progressWrite; + }; if (unavailable > 0) { await ctx.event("warn", "Selected copy media is no longer available", { total, unavailable }); @@ -2147,18 +3493,35 @@ export class JobWorker { await ctx.event("error", "Copy job failed processing media", { total, copied, repointed, skipped, conflicts, failed, unavailable }); throw new Error(failureMessage); } - await setCopyProgress("completed", alreadyCompleted > 0 ? "Copy job finished" : "No matching media found"); - await ctx.event("info", "Copy job finished processing media", { total, copied, repointed, skipped, conflicts, failed, unavailable }); - return; } - for (const link of links) { - if (await ctx.isCancelled()) break; + const withCopyMutationLease = (link: MediaLinkRow, destinationPath: string, mutation: () => Promise): Promise => + ctx.withLeaseDb(async (leaseDb) => { + if (await isPathConfigurationBlocked(leaseDb)) { + throw new Error("Managed storage paths changed before copy promotion"); + } + if (!(await copyPathsRemainClaimed(link, destinationPath))) { + throw new Error("Media paths changed after copy admission; queue the copy again"); + } + return mutation(); + }); + let cancellationReported = false; + const processLink = async (link: MediaLinkRow): Promise => { + if (await ctx.isCancelled()) return; activeLink = link; - activeUpdate = undefined; + let linkUpdate: Partial | undefined; + activeUpdate = linkUpdate; let activeOperationId: number | null = null; + let filesystemMutationCompleted = false; current += 1; await setCopyProgress("preparing", "Preparing media copy", link); + const destinationPath = copyDestinationPathForLink(link, paths, normalizedOptions.direction); + if (!(await copyPathsRemainClaimed(link, destinationPath))) { + conflicts += 1; + await setCopyProgress("conflict", "Media paths changed after copy admission; queue the copy again", link); + await ctx.event("warn", "Media paths changed after copy admission", { linkId: link.id, linkPath: link.linkPath, sourcePath: link.targetPath, destinationPath }); + return; + } const sourceTitleRisk = evaluateSourceTitleRisk({ expectedTitle: link.itemName, sourcePath: link.targetPath }); if (sourceTitleRisk.severity === "block") { conflicts += 1; @@ -2167,7 +3530,8 @@ export class JobWorker { linkPath: link.linkPath, sizeBytes: link.sizeBytes ?? undefined }; - await setCopyProgress("conflict", "Source title mismatch blocked copy", link, activeUpdate); + linkUpdate = activeUpdate; + await setCopyProgress("conflict", "Source title mismatch blocked copy", link, linkUpdate); await ctx.event("warn", "Source title mismatch blocked copy", { direction: normalizedOptions.direction, itemName: link.itemName, @@ -2175,7 +3539,7 @@ export class JobWorker { sourcePath: link.targetPath, risk: sourceTitleRisk }); - continue; + return; } if (sourceTitleRisk.severity === "warn") { await ctx.event("warn", "Source title risk warning", { @@ -2197,7 +3561,34 @@ export class JobWorker { try { let lastProgressEventKey: string | null = null; - const localConflict = normalizedOptions.direction === "to_local" ? await copyLocalConflictForLink(this.db, link, paths) : null; + const localConflict = + normalizedOptions.direction === "to_local" + ? await copyLocalConflictForLink(this.db, link, paths, selectedDestinations) + : null; + if (localConflict && normalizedOptions.localConflictStrategy === "replace") { + const expectedIdentities = cleanupPathsByLink.get(link.id) ?? new Map(); + const unclaimedCandidates: CopyLocalConflictCandidate[] = []; + for (const candidate of localConflict.candidates) { + const candidatePath = path.resolve(candidate.filePath); + if (!sameCopyCleanupIdentity(expectedIdentities.get(candidatePath), await copyCleanupIdentity(candidatePath))) unclaimedCandidates.push(candidate); + } + if (unclaimedCandidates.length > 0) { + conflicts += 1; + activeUpdate = { + sourcePath: link.targetPath, + destinationPath: localConflict.destinationPath, + linkPath: link.linkPath, + sizeBytes: link.sizeBytes ?? undefined + }; + linkUpdate = activeUpdate; + await setCopyProgress("conflict", "Local replacement candidates changed after copy admission; queue the copy again", link, linkUpdate); + await ctx.event("warn", "Local replacement candidates changed after copy admission", { + ...localConflict, + unclaimedPaths: unclaimedCandidates.map((candidate) => candidate.filePath) + }); + return; + } + } if (localConflict && !normalizedOptions.localConflictStrategy) { conflicts += 1; activeUpdate = { @@ -2206,105 +3597,145 @@ export class JobWorker { linkPath: link.linkPath, sizeBytes: link.sizeBytes ?? undefined }; - await setCopyProgress("conflict", "Existing local file requires copy resolution", link, activeUpdate); + linkUpdate = activeUpdate; + await setCopyProgress("conflict", "Existing local file requires copy resolution", link, linkUpdate); await ctx.event("warn", "Existing local file requires copy resolution", localConflict); - continue; + return; } - const destinationPath = copyDestinationPathForLink(link, paths, normalizedOptions.direction); const previousCopySource = (await first(this.db.select().from(schema.copySources).where(eq(schema.copySources.destinationPath, destinationPath)).limit(1))) ?? null; - const operation = await prepareCopyOperation( - this.db, - ctx.jobId, - link, - destinationPath, - previousCopySource, - normalizedOptions.localConflictStrategy + const operation = await ctx.withLeaseDb((leaseDb) => + prepareCopyOperation( + leaseDb, + ctx.jobId, + link, + destinationPath, + previousCopySource, + normalizedOptions.localConflictStrategy + ) ); activeOperationId = operation.id; - const result = await copyMediaLink( - link, - paths, - normalizedOptions.direction, - this.copyRunner, - async (update) => { - activeUpdate = update; - await setCopyProgress(update.stage, update.message, link, activeUpdate); - if (update.stage === "copying" || (update.stage === "preparing" && !/retry/i.test(update.message))) return; - const progressEventKey = `${update.stage}:${update.message}`; - if (progressEventKey === lastProgressEventKey) return; - lastProgressEventKey = progressEventKey; - await ctx.event( - "info", - update.message, - copyProgressPayload({ - options: normalizedOptions, - current, - total, - copied, - repointed, - skipped, - conflicts, - failed, - alreadyCompleted, - remaining: Math.max(0, total - current), - stage: update.stage, - message: update.message, - link, - update: activeUpdate - }) - ); - }, - advancedSettings.copy, - ctx.signal, - normalizedOptions.localConflictStrategy, - (update) => updateCopyOperation(this.db, operation.id, update) - ); + const releaseTransfer = await this.copyTransferLimiter.acquire(ctx.signal); + let result: CopyMediaResult; + try { + result = await copyMediaLink( + link, + paths, + normalizedOptions.direction, + this.copyRunner, + async (update) => { + linkUpdate = update; + activeUpdate = linkUpdate; + await setCopyProgress(update.stage, update.message, link, linkUpdate); + if (update.stage === "copying" || (update.stage === "preparing" && !/retry/i.test(update.message))) return; + const progressEventKey = `${update.stage}:${update.message}`; + if (progressEventKey === lastProgressEventKey) return; + lastProgressEventKey = progressEventKey; + await ctx.event( + "info", + update.message, + copyProgressPayload({ + options: normalizedOptions, + current, + total, + copied, + repointed, + skipped, + conflicts, + failed, + alreadyCompleted, + remaining: Math.max(0, total - current), + stage: update.stage, + message: update.message, + link, + update: linkUpdate + }) + ); + }, + advancedSettings.copy, + ctx.signal, + normalizedOptions.localConflictStrategy, + (update) => ctx.withLeaseDb((leaseDb) => updateCopyOperation(leaseDb, operation.id, update)), + (mutation) => withCopyMutationLease(link, destinationPath, mutation) + ); + } finally { + releaseTransfer(); + } + filesystemMutationCompleted = result.status === "copied" || result.status === "repointed"; if (result.status === "copied") { + await ctx.withLeaseDb((leaseDb) => commitCopyOperation(leaseDb, operation.id, link, result)); copied += 1; - await commitCopyOperation(this.db, operation.id, link, result); - activeUpdate = { ...(activeUpdate ?? {}), ...result }; - await setCopyProgress("done", result.message, link, activeUpdate); + linkUpdate = { ...(linkUpdate ?? {}), ...result }; + activeUpdate = linkUpdate; + await setCopyProgress("done", result.message, link, linkUpdate); await ctx.event("info", advancedSettings.copy.profile === "off" ? "Copy installed without verification" : "Verified copy installed", { ...result, itemName: link.itemName }); if (normalizedOptions.localConflictStrategy === "replace" && localConflict) { - replacementCandidateEntries.push({ linkId: link.id, destinationPath: result.destinationPath, candidates: localConflict.candidates }); + replacementCandidateEntries.set(link.id, { + linkId: link.id, + destinationPath: result.destinationPath, + candidates: localConflict.candidates, + expectedIdentities: cleanupPathsByLink.get(link.id) ?? new Map() + }); } } else if (result.status === "repointed") { + await ctx.withLeaseDb((leaseDb) => commitCopyOperation(leaseDb, operation.id, link, result)); repointed += 1; - await commitCopyOperation(this.db, operation.id, link, result); - activeUpdate = { ...(activeUpdate ?? {}), ...result }; - await setCopyProgress("done", result.message, link, activeUpdate); + linkUpdate = { ...(linkUpdate ?? {}), ...result }; + activeUpdate = linkUpdate; + await setCopyProgress("done", result.message, link, linkUpdate); await ctx.event("info", "Symlink repointed to existing verified file", { ...result, itemName: link.itemName }); if (normalizedOptions.localConflictStrategy === "replace" && localConflict) { - replacementCandidateEntries.push({ linkId: link.id, destinationPath: result.destinationPath, candidates: localConflict.candidates }); + replacementCandidateEntries.set(link.id, { + linkId: link.id, + destinationPath: result.destinationPath, + candidates: localConflict.candidates, + expectedIdentities: cleanupPathsByLink.get(link.id) ?? new Map() + }); } } else if (result.status === "conflict") { conflicts += 1; - await completeCopyOperationWithoutMutation(this.db, operation.id, result); - activeUpdate = { ...(activeUpdate ?? {}), ...result }; - await setCopyProgress("conflict", result.message, link, activeUpdate); + await ctx.withLeaseDb((leaseDb) => completeCopyOperationWithoutMutation(leaseDb, operation.id, result)); + linkUpdate = { ...(linkUpdate ?? {}), ...result }; + activeUpdate = linkUpdate; + await setCopyProgress("conflict", result.message, link, linkUpdate); await ctx.event("warn", "Destination conflict; file was not overwritten", result); } else { skipped += 1; - await completeCopyOperationWithoutMutation(this.db, operation.id, result); - activeUpdate = { ...(activeUpdate ?? {}), ...result }; - await setCopyProgress("skipped", result.message, link, activeUpdate); + await ctx.withLeaseDb((leaseDb) => completeCopyOperationWithoutMutation(leaseDb, operation.id, result)); + linkUpdate = { ...(linkUpdate ?? {}), ...result }; + activeUpdate = linkUpdate; + await setCopyProgress("skipped", result.message, link, linkUpdate); await ctx.event("info", "Copy skipped", result); } } catch (error: unknown) { + if (error instanceof CopyReconciliationRequiredError || filesystemMutationCompleted) { + const reconciliationError = + error instanceof CopyReconciliationRequiredError + ? error + : new CopyReconciliationRequiredError(`Copy operation could not be committed after filesystem promotion: ${errorMessage(error)}`, { cause: error }); + if (activeOperationId) { + await ctx.withLeaseDb((leaseDb) => requireCopyOperationReconciliation(leaseDb, activeOperationId!, reconciliationError.message)); + } + throw reconciliationError; + } if (await ctx.isCancelled()) { - await setCopyProgress("cancelled", "Copy job termination requested", link, activeUpdate); - await ctx.event("warn", "Copy job termination requested; active copy was stopped before promotion", { - direction: normalizedOptions.direction, - itemName: link.itemName, - linkPath: link.linkPath, - sourcePath: link.targetPath - }); - break; + if (!cancellationReported) { + cancellationReported = true; + await setCopyProgress("cancelled", "Copy job termination requested", link, linkUpdate); + await ctx.event("warn", "Copy job termination requested; active copies were stopped before promotion", { + direction: normalizedOptions.direction, + itemName: link.itemName, + linkPath: link.linkPath, + sourcePath: link.targetPath + }); + } + return; } - if (ctx.signal.aborted) throw error; + if (ctx.signal.aborted) throw (ctx.signal.reason instanceof Error ? ctx.signal.reason : error); failed += 1; - if (activeOperationId) await failCopyOperation(this.db, activeOperationId, errorMessage(error)); + if (activeOperationId && !filesystemMutationCompleted) { + await ctx.withLeaseDb((leaseDb) => failCopyOperation(leaseDb, activeOperationId!, errorMessage(error))); + } await setCopyProgress("failed", errorMessage(error), link); await ctx.event("error", errorMessage(error), { direction: normalizedOptions.direction, @@ -2313,19 +3744,31 @@ export class JobWorker { sourcePath: link.targetPath }); } - } + }; + + await runKeyedPool( + links, + this.concurrency.copyFileConcurrency, + (link) => String(link.id), + processLink, + () => !ctx.signal.aborted + ); const cancelled = await ctx.isCancelled(); - if (cancelled) { + if (!cancelled && ctx.signal.aborted) { + throw (ctx.signal.reason instanceof Error ? ctx.signal.reason : new WorkerShutdownError()); + } + const rollbackCancelledCopy = async () => { await setCopyProgress("cancelled", "Rolling back completed copy changes", activeLink, activeUpdate); - const { rolledBack, warnings } = await rollbackDurableCopyOperations(this.db, ctx.jobId, paths, ctx.event); + const { rolledBack, warnings } = await rollbackDurableCopyOperations(this.db, ctx.jobId, paths, ctx); copied = resumedCopied; repointed = resumedRepointed; for (const warning of warnings) { await ctx.event("warn", warning); } await ctx.event("warn", "Copy job terminated; completed copy changes rolled back", { rolledBack, warnings }); - } + }; + if (cancelled) await rollbackCancelledCopy(); if (!cancelled && failed > 0) { const itemLabel = total === 1 ? "media item" : "media items"; const completed = copied + repointed + skipped + conflicts; @@ -2344,14 +3787,72 @@ export class JobWorker { if (partialFailure) throw new PartialJobFailureError(failureMessage); throw new Error(failureMessage); } - if (!cancelled && replacementCandidateEntries.length > 0) { - await setCopyProgress("symlinking", "Removing previous local files", activeLink, activeUpdate); - for (const entry of replacementCandidateEntries) { - const removed = await removeLocalConflictCandidates(this.db, paths, entry.candidates, entry.destinationPath); - if (removed.length > 0) await ctx.event("info", "Replaced previous local files", { linkId: entry.linkId, removed }); + if (!cancelled) { + if (replacementCandidateEntries.size > 0) { + await setCopyProgress("symlinking", "Finalizing previous local-file replacements", activeLink, activeUpdate); } + await setCopyProgress("completed", links.length === 0 && alreadyCompleted === 0 ? "No matching media found" : "Copy job finished", activeLink, activeUpdate); + await ctx.event("info", "Copy job finished processing media", { total, copied, repointed, skipped, conflicts, failed, unavailable }); + const finalized = await ctx.finishCompletedIsolated(async (leaseDb) => { + const finalizationWarnings: string[] = []; + for (const entry of replacementCandidateEntries.values()) { + try { + const removed = await removeLocalConflictCandidates( + leaseDb, + paths, + entry.candidates, + entry.destinationPath, + entry.expectedIdentities, + entry.linkId, + selectedDestinations + ); + if (removed.length > 0) { + await leaseDb.insert(schema.jobEvents).values({ + jobId: ctx.jobId, + timestamp: nowIso(), + level: "info", + message: "Replaced previous local files", + data: JSON.stringify({ linkId: entry.linkId, removed }) + }); + } + } catch (error: unknown) { + finalizationWarnings.push(`Could not remove every previous local file for media #${entry.linkId}: ${errorMessage(error)}`); + } + } + + const operationsWithBackups = ( + await leaseDb + .select() + .from(schema.copyOperations) + .where(and(eq(schema.copyOperations.jobId, ctx.jobId), eq(schema.copyOperations.stage, "committed"))) + ).filter((operation) => Boolean(operation.displacedPath)); + for (const operation of operationsWithBackups) { + try { + if (operation.localConflictStrategy === "replace") { + const rootType = copyOperationDestinationRoot(operation, paths); + await removeJournalFile( + rootType === "local" ? paths.localDir : paths.remoteDir, + operation.displacedPath, + operation.displacedIdentity, + "Displaced destination backup" + ); + } + await leaseDb + .update(schema.copyOperations) + .set({ displacedPath: null, displacedIdentity: null, updatedAt: nowIso() }) + .where(eq(schema.copyOperations.id, operation.id)); + } catch (error: unknown) { + finalizationWarnings.push(`Could not finalize displaced backup for copy operation #${operation.id}: ${errorMessage(error)}`); + } + } + for (const warning of finalizationWarnings) { + await leaseDb.insert(schema.jobEvents).values({ jobId: ctx.jobId, timestamp: nowIso(), level: "warn", message: warning, data: "{}" }); + } + }); + if (finalized) return; + await rollbackCancelledCopy(); } - await setCopyProgress(cancelled ? "cancelled" : "completed", cancelled ? "Copy job terminated" : "Copy job finished", activeLink, activeUpdate); - await ctx.event(cancelled ? "warn" : "info", cancelled ? "Copy job terminated" : "Copy job finished processing media", { total, copied, repointed, skipped, conflicts, failed, unavailable }); + await setCopyProgress("cancelled", "Copy job terminated", activeLink, activeUpdate); + await ctx.event("warn", "Copy job terminated", { total, copied, repointed, skipped, conflicts, failed, unavailable }); } } diff --git a/src/server/jobs/resourceMutationGuard.ts b/src/server/jobs/resourceMutationGuard.ts new file mode 100644 index 0000000..92b5c2b --- /dev/null +++ b/src/server/jobs/resourceMutationGuard.ts @@ -0,0 +1,123 @@ +import { sql } from "drizzle-orm"; +import type { Db } from "../db/database"; +import * as schema from "../db/schema"; +import { canonicalTitleKey } from "../lib/storagePolicies"; +import { schedulerLockKey } from "./scheduling"; + +export interface MutationResource { + resourceType: string; + resourceKey: string; +} + +interface ActiveJobConflict extends Record { + jobId: number; + type: string; + status: string; +} + +interface PreparedResourceMutation { + resources: MutationResource[]; + mutate(): Promise; +} + +export class ActiveJobResourceConflictError extends Error { + readonly statusCode = 409; + + constructor(conflict: ActiveJobConflict, global: boolean) { + const scope = global ? "the library" : "the same media"; + if (conflict.status === "reconciliation_required") { + super(`Storage policy cannot be changed because copy data from job #${conflict.jobId} requires manual reconciliation for ${scope}.`); + this.name = "ActiveJobResourceConflictError"; + return; + } + super( + `Storage policy cannot be changed while ${conflict.type} job #${conflict.jobId} is ${conflict.status} for ${scope}. Wait for it to finish or terminate it before changing this policy.` + ); + this.name = "ActiveJobResourceConflictError"; + } +} + +function normalizeResources(resources: MutationResource[]): MutationResource[] { + const unique = new Map(); + for (const resource of resources) unique.set(`${resource.resourceType}\0${resource.resourceKey}`, resource); + return [...unique.values()]; +} + +export async function storagePolicyMutationResources(db: Db, titles: string[]): Promise { + const titleKeys = new Set(titles.map(canonicalTitleKey).filter(Boolean)); + if (titleKeys.size === 0) return []; + + const links = await db + .select({ id: schema.mediaLinks.id, section: schema.mediaLinks.section, itemName: schema.mediaLinks.itemName }) + .from(schema.mediaLinks); + + return normalizeResources( + links + .filter((link) => titleKeys.has(canonicalTitleKey(link.itemName))) + .flatMap((link) => [ + { resourceType: "media", resourceKey: String(link.id) }, + { resourceType: "title", resourceKey: JSON.stringify([link.section, link.itemName]) } + ]) + ); +} + +export async function withResourceMutationGuard( + db: Db, + prepare: (transaction: Db) => Promise> +): Promise { + return db.transaction(async (transaction) => { + await transaction.execute(sql`select pg_advisory_xact_lock(${schedulerLockKey})`); + const prepared = await prepare(transaction); + const resources = normalizeResources(prepared.resources); + + const globalConflict = ( + await transaction.execute(sql` + select id as "jobId", type, status + from jobs + where status in ('queued', 'running') + and exclusive = true + order by id + limit 1 + `) + ).rows[0]; + if (globalConflict) throw new ActiveJobResourceConflictError(globalConflict, true); + + if (resources.length > 0) { + const queryResources = resources.map((resource) => ({ resource_type: resource.resourceType, resource_key: resource.resourceKey })); + const conflict = ( + await transaction.execute(sql` + with requested_resources as ( + select resource_type, resource_key + from jsonb_to_recordset(${JSON.stringify(queryResources)}::jsonb) + as requested(resource_type text, resource_key text) + ), blocking_claims as ( + select active.job_id, active.resource_type, active.resource_key, jobs.type, jobs.status + from job_resource_claims as active + join jobs on jobs.id = active.job_id + where jobs.status in ('queued', 'running') + union + select active.job_id, active.resource_type, active.resource_key, 'copy'::text as type, 'reconciliation_required'::text as status + from job_resource_claims as active + join copy_operations as operation on operation.job_id = active.job_id + where operation.stage = 'reconciliation_required' + and active.resource_type <> 'title' + union + select operation.job_id, 'media'::text, operation.media_link_id::text, 'copy'::text as type, 'reconciliation_required'::text as status + from copy_operations as operation + where operation.stage = 'reconciliation_required' + ) + select active.job_id as "jobId", active.type, active.status + from requested_resources as requested + join blocking_claims as active + on active.resource_type = requested.resource_type + and active.resource_key = requested.resource_key + order by active.job_id + limit 1 + `) + ).rows[0]; + if (conflict) throw new ActiveJobResourceConflictError(conflict, false); + } + + return prepared.mutate(); + }); +} diff --git a/src/server/jobs/scheduling.ts b/src/server/jobs/scheduling.ts new file mode 100644 index 0000000..0860d1e --- /dev/null +++ b/src/server/jobs/scheduling.ts @@ -0,0 +1,3 @@ +// Serializes queue admission, worker claims, and path-configuration barriers. +// Keep this value stable so every process coordinates on the same advisory lock. +export const schedulerLockKey = 1_672_148_903; diff --git a/src/server/lib/copier.ts b/src/server/lib/copier.ts index 3662e46..990e647 100644 --- a/src/server/lib/copier.ts +++ b/src/server/lib/copier.ts @@ -54,16 +54,36 @@ export type CopyProgressReporter = (update: CopyProgressUpdate) => Promise export type CopyOperationStage = "planned" | "transferring" | "verified" | "destination_displaced" | "promoted" | "repointed"; +export interface CopyFileIdentity { + dev: string; + ino: string; + size: string; + mtimeNs: string; + ctimeNs: string; +} + export interface CopyOperationUpdate { stage: CopyOperationStage; tempPath?: string | null; displacedPath?: string | null; + tempIdentity?: string | null; + destinationIdentity?: string | null; + displacedIdentity?: string | null; sizeBytes?: number | null; resultStatus?: CopyMediaResult["status"] | null; } export type CopyOperationReporter = (update: CopyOperationUpdate) => Promise; +export type CopyMutationGuard = (mutation: () => Promise) => Promise; + +export class CopyReconciliationRequiredError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(message, options); + this.name = "CopyReconciliationRequiredError"; + } +} + export interface CopyMediaResult { status: "copied" | "repointed" | "skipped" | "conflict"; direction: CopyDirection; @@ -76,18 +96,18 @@ export interface CopyMediaResult { message: string; } -function abortError(): Error { - return new Error("Job terminated"); +function abortError(signal?: AbortSignal): Error { + return signal?.reason instanceof Error ? signal.reason : new Error("Job terminated"); } function throwIfAborted(signal?: AbortSignal): void { - if (signal?.aborted) throw abortError(); + if (signal?.aborted) throw abortError(signal); } function runCommand(command: string, args: string[], signal?: AbortSignal): Promise { return new Promise((resolve, reject) => { if (signal?.aborted) { - reject(abortError()); + reject(abortError(signal)); return; } const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] }); @@ -99,7 +119,7 @@ function runCommand(command: string, args: string[], signal?: AbortSignal): Prom settled = true; clearTimeout(deadline); killTimer = terminateChildProcess(child); - reject(abortError()); + reject(abortError(signal)); }; const deadline = setTimeout(() => { if (settled) return; @@ -171,7 +191,7 @@ async function copyFileWithProgress(sourcePath: string, tempPath: string, report await emitProgress(true); const sourceStream = createReadStream(sourcePath); - const targetStream = createWriteStream(tempPath, { mode: copyFileMode }); + const targetStream = createWriteStream(tempPath, { flags: "wx", mode: copyFileMode }); const configuredStallTimeout = Number(process.env.SRTL_COPY_STALL_TIMEOUT_MS); const stallTimeoutMs = Number.isInteger(configuredStallTimeout) && configuredStallTimeout >= 60_000 ? Math.min(configuredStallTimeout, 60 * 60_000) : 10 * 60_000; let stallTimer: ReturnType | null = null; @@ -188,7 +208,7 @@ async function copyFileWithProgress(sourcePath: string, tempPath: string, report const progressStream = new Transform({ transform(chunk: Buffer, _encoding, callback) { if (signal?.aborted) { - callback(abortError()); + callback(abortError(signal)); return; } bytesCopied += chunk.length; @@ -200,7 +220,7 @@ async function copyFileWithProgress(sourcePath: string, tempPath: string, report }); resetStallTimer(); const abort = () => { - const error = abortError(); + const error = abortError(signal); sourceStream.destroy(error); progressStream.destroy(error); targetStream.destroy(error); @@ -310,7 +330,7 @@ async function runFfmpegWithProgress(mode: AuditMode, targetPath: string, report return new Promise((resolve, reject) => { if (signal?.aborted) { - reject(abortError()); + reject(abortError(signal)); return; } const args = @@ -325,7 +345,7 @@ async function runFfmpegWithProgress(mode: AuditMode, targetPath: string, report settled = true; clearTimeout(deadline); killTimer = terminateChildProcess(child); - reject(abortError()); + reject(abortError(signal)); }; const deadline = setTimeout(() => { if (settled) return; @@ -455,6 +475,64 @@ async function destinationExists(destinationPath: string): Promise { } } +export async function readCopyFileIdentity(filePath: string): Promise { + try { + const stat = await fs.lstat(filePath, { bigint: true }); + if (!stat.isFile()) throw new Error(`Copy identity path is not a regular file: ${filePath}`); + return { + dev: stat.dev.toString(), + ino: stat.ino.toString(), + size: stat.size.toString(), + mtimeNs: stat.mtimeNs.toString(), + ctimeNs: stat.ctimeNs.toString() + }; + } catch (error) { + if (isMissingPathError(error)) return null; + throw error; + } +} + +export function copyFileIdentitiesMatch(left: CopyFileIdentity | null, right: CopyFileIdentity | null): boolean { + if (!left || !right) return left === right; + return left.dev === right.dev && left.ino === right.ino && left.size === right.size && left.mtimeNs === right.mtimeNs && left.ctimeNs === right.ctimeNs; +} + +export function serializeCopyFileIdentity(identity: CopyFileIdentity): string { + return JSON.stringify(identity); +} + +export function parseCopyFileIdentity(rawIdentity: string): CopyFileIdentity { + const identity = JSON.parse(rawIdentity) as Partial | null; + const keys: Array = ["dev", "ino", "size", "mtimeNs", "ctimeNs"]; + if (!identity || typeof identity !== "object" || !keys.every((key) => typeof identity[key] === "string" && /^\d+$/.test(identity[key]))) { + throw new Error("Copy file identity is invalid"); + } + return { + dev: identity.dev!, + ino: identity.ino!, + size: identity.size!, + mtimeNs: identity.mtimeNs!, + ctimeNs: identity.ctimeNs! + }; +} + +async function requireCopyFileIdentity(filePath: string, label: string): Promise { + const identity = await readCopyFileIdentity(filePath); + if (!identity) throw new Error(`${label} is missing or is not a regular file`); + return identity; +} + +async function destinationState(destinationPath: string): Promise { + return readCopyFileIdentity(destinationPath); +} + +async function assertDestinationState(destinationPath: string, expected: CopyFileIdentity | null): Promise { + const current = await destinationState(destinationPath); + if (!copyFileIdentitiesMatch(current, expected)) { + throw new Error("Destination changed before copy promotion; the copy was not installed"); + } +} + function ensureMediaCandidate(sourcePath: string, destinationPath: string, linkPath: string): void { if (!isMediaFile(sourcePath) && !isMediaFile(destinationPath) && !isMediaFile(linkPath)) { throw new Error("Source is not a recognized media file"); @@ -497,7 +575,7 @@ async function waitForSourceRetry(signal?: AbortSignal): Promise { const abort = () => { clearTimeout(timeout); signal?.removeEventListener("abort", abort); - reject(abortError()); + reject(abortError(signal)); }; signal?.addEventListener("abort", abort, { once: true }); }); @@ -720,9 +798,51 @@ export async function copyMediaLink( behavior: CopyJobBehaviorSettings = defaultCopyJobBehaviorSettings, signal?: AbortSignal, localConflictStrategy?: CopyLocalConflictStrategy, - reportOperation?: CopyOperationReporter + reportOperation?: CopyOperationReporter, + assertMutationAllowed?: CopyMutationGuard ): Promise { throwIfAborted(signal); + let mutationAuthorityLost = false; + const guardOwnedMutation = async (mutation: () => Promise): Promise => { + if (assertMutationAllowed) { + let mutationCompleted = false; + try { + return await assertMutationAllowed(async () => { + const result = await mutation(); + mutationCompleted = true; + return result; + }); + } catch (error) { + if (error instanceof Error && error.name === "LeaseLostError") mutationAuthorityLost = true; + if (mutationCompleted && !(error instanceof CopyReconciliationRequiredError)) { + throw new CopyReconciliationRequiredError( + "Filesystem mutation completed, but the worker could not confirm its mutation lease; durable copy reconciliation is required", + { cause: error } + ); + } + throw error; + } + } + return mutation(); + }; + const guardMutation = async (mutation: () => Promise): Promise => { + throwIfAborted(signal); + return guardOwnedMutation(async () => { + throwIfAborted(signal); + return mutation(); + }); + }; + const canRollbackSharedPaths = async () => { + if (mutationAuthorityLost || (signal?.reason instanceof Error && signal.reason.name === "LeaseLostError")) return false; + if (!assertMutationAllowed) return true; + try { + await guardOwnedMutation(async () => undefined); + return true; + } catch { + mutationAuthorityLost = true; + return false; + } + }; const verificationEnabled = behavior.byteCompare || behavior.mediaValidation !== "off"; const postTransferCheck = verificationEnabled ? "verification" : "transfer checks"; const destinationRootType = rootForDirection(direction); @@ -769,7 +889,8 @@ export async function copyMediaLink( const canResolveLocalDestination = direction === "to_local" && (localConflictStrategy === "replace" || localConflictStrategy === "keep_both"); await createDestinationParent(destinationRoot, destinationPath); - if (await destinationExists(destinationPath)) { + const initialDestinationState = await destinationState(destinationPath); + if (initialDestinationState) { await assertExistingPathInside(destinationRoot, destinationPath, "Destination path"); const cmp = await compareMediaBytes(runner, sourcePath, destinationPath, reportProgress, baseProgress, "Comparing existing destination file", signal); if (cmp.status === "pass") { @@ -779,8 +900,17 @@ export async function copyMediaLink( throwIfAborted(signal); const finalStat = await statRegularFile(destinationPath, "Destination file"); await reportCopyProgress(reportProgress, { stage: "symlinking", message: "Repointing symlink to existing verified file", sourcePath, destinationPath, linkPath: link.linkPath, sizeBytes: finalStat.size }); - await replaceSymlink(paths.symlinkDir, link.linkPath, destinationPath); - await reportOperation?.({ stage: "repointed", sizeBytes: finalStat.size, resultStatus: "repointed" }); + await reportOperation?.({ + stage: "repointed", + destinationIdentity: serializeCopyFileIdentity(initialDestinationState), + sizeBytes: finalStat.size, + resultStatus: "repointed" + }); + await guardMutation(async () => { + await validateLinkStillPointsTo(link, sourcePath); + await assertDestinationState(destinationPath, initialDestinationState); + await replaceSymlink(paths.symlinkDir, link.linkPath, destinationPath); + }); return { status: "repointed", direction, @@ -818,10 +948,21 @@ export async function copyMediaLink( const tempPath = tempFilePath(destinationPath); let displacedDestinationPath: string | null = null; + let destinationDisplaced = false; let promotedPath: string | null = null; - let promotedSize: number | null = null; + let tempIdentity: CopyFileIdentity | null = null; + let displacedIdentity: CopyFileIdentity | null = null; + let promotedIdentity: CopyFileIdentity | null = null; + let linkRepointed = false; try { - await reportOperation?.({ stage: "transferring", tempPath, sizeBytes: sourceStatBefore.size }); + await reportOperation?.({ + stage: "transferring", + tempPath, + tempIdentity: null, + destinationIdentity: null, + displacedIdentity: null, + sizeBytes: sourceStatBefore.size + }); const transferMessage = direction === "to_local" ? "Downloading source file to a temporary destination" : "Uploading source file to a temporary destination"; await reportCopyProgress(reportProgress, { stage: "copying", @@ -876,23 +1017,43 @@ export async function copyMediaLink( throw new Error(`Size mismatch after copy (${sourceStatBefore.size} != ${tempStat.size})`); } await verifyCopiedFile(runner, sourcePath, tempPath, reportProgress, baseProgress, behavior, signal); - await reportOperation?.({ stage: "verified", tempPath, sizeBytes: tempStat.size }); + tempIdentity = await requireCopyFileIdentity(tempPath, "Verified temporary copy"); + await reportOperation?.({ + stage: "verified", + tempPath, + tempIdentity: serializeCopyFileIdentity(tempIdentity), + sizeBytes: tempStat.size + }); throwIfAborted(signal); const sourceStatAfter = await assertReadableRegularFile(sourcePath, "Source file", { attempts: 3, retryDelayMs: 500, signal }); if (sourceStatAfter.size !== sourceStatBefore.size || sourceStatAfter.mtimeMs !== sourceStatBefore.mtimeMs) { throw new Error("Source file changed during copy; destination was not promoted"); } - if (await destinationExists(destinationPath)) { + const destinationStateBeforePromotion = await destinationState(destinationPath); + if (destinationStateBeforePromotion) { const cmp = await compareMediaBytes(runner, sourcePath, destinationPath, reportProgress, baseProgress, "Destination appeared during copy; comparing before promotion", signal); if (cmp.status === "pass") { if (behavior.mediaValidation !== "off") { await validateMediaStream(runner, behavior.mediaValidation, destinationPath, reportProgress, baseProgress, behavior.mediaValidation === "fast" ? "Fast validation of matching destination media" : "Deep validation of matching destination media", signal); } await fs.rm(tempPath, { force: true }); + tempIdentity = null; throwIfAborted(signal); await reportCopyProgress(reportProgress, { stage: "symlinking", message: "Repointing symlink to matching destination", sourcePath, destinationPath, linkPath: link.linkPath, sizeBytes: tempStat.size }); - await replaceSymlink(paths.symlinkDir, link.linkPath, destinationPath); - await reportOperation?.({ stage: "repointed", tempPath: null, sizeBytes: tempStat.size, resultStatus: "repointed" }); + await reportOperation?.({ + stage: "repointed", + tempPath: null, + tempIdentity: null, + destinationIdentity: serializeCopyFileIdentity(destinationStateBeforePromotion), + sizeBytes: tempStat.size, + resultStatus: "repointed" + }); + await guardMutation(async () => { + await validateLinkStillPointsTo(link, sourcePath); + await assertDestinationState(destinationPath, destinationStateBeforePromotion); + await replaceSymlink(paths.symlinkDir, link.linkPath, destinationPath); + }); + linkRepointed = true; return { status: "repointed", direction, @@ -915,8 +1076,29 @@ export async function copyMediaLink( sizeBytes: tempStat.size }); displacedDestinationPath = await destinationDisplacementPath(destinationPath, localConflictStrategy); - await fs.rename(destinationPath, displacedDestinationPath); - await reportOperation?.({ stage: "destination_displaced", tempPath, displacedPath: displacedDestinationPath, sizeBytes: tempStat.size }); + await reportOperation?.({ + stage: "destination_displaced", + tempPath, + displacedPath: displacedDestinationPath, + tempIdentity: serializeCopyFileIdentity(tempIdentity), + displacedIdentity: null, + sizeBytes: tempStat.size + }); + await guardMutation(async () => { + await assertDestinationState(destinationPath, destinationStateBeforePromotion); + await fs.rename(destinationPath, displacedDestinationPath!); + destinationDisplaced = true; + displacedIdentity = await requireCopyFileIdentity(displacedDestinationPath!, "Displaced destination"); + }); + if (!displacedIdentity) throw new Error("Displaced destination identity was not captured"); + await reportOperation?.({ + stage: "destination_displaced", + tempPath, + displacedPath: displacedDestinationPath, + tempIdentity: serializeCopyFileIdentity(tempIdentity), + displacedIdentity: serializeCopyFileIdentity(displacedIdentity), + sizeBytes: tempStat.size + }); } throwIfAborted(signal); await reportCopyProgress(reportProgress, { @@ -927,18 +1109,49 @@ export async function copyMediaLink( linkPath: link.linkPath, sizeBytes: tempStat.size }); - await fs.rename(tempPath, destinationPath); - promotedPath = destinationPath; - promotedSize = tempStat.size; - await reportOperation?.({ stage: "promoted", tempPath: null, displacedPath: displacedDestinationPath, sizeBytes: tempStat.size }); - await statRegularFile(destinationPath, "Promoted destination file"); - await replaceSymlink(paths.symlinkDir, link.linkPath, destinationPath); - await reportOperation?.({ stage: "repointed", tempPath: null, displacedPath: displacedDestinationPath, sizeBytes: tempStat.size, resultStatus: "copied" }); + await reportOperation?.({ + stage: "promoted", + tempPath, + displacedPath: displacedDestinationPath, + tempIdentity: serializeCopyFileIdentity(tempIdentity), + displacedIdentity: displacedIdentity ? serializeCopyFileIdentity(displacedIdentity) : null, + destinationIdentity: null, + sizeBytes: tempStat.size + }); + await guardMutation(async () => { + await assertDestinationState(destinationPath, null); + await fs.rename(tempPath, destinationPath); + promotedPath = destinationPath; + promotedIdentity = await requireCopyFileIdentity(destinationPath, "Promoted destination file"); + }); + if (!promotedIdentity) throw new Error("Promoted destination identity was not captured"); + const installedDestinationIdentity = promotedIdentity; + await reportOperation?.({ + stage: "promoted", + tempPath, + displacedPath: displacedDestinationPath, + tempIdentity: serializeCopyFileIdentity(tempIdentity), + destinationIdentity: serializeCopyFileIdentity(installedDestinationIdentity), + displacedIdentity: displacedIdentity ? serializeCopyFileIdentity(displacedIdentity) : null, + sizeBytes: tempStat.size + }); + await reportOperation?.({ + stage: "repointed", + tempPath: null, + displacedPath: displacedDestinationPath, + tempIdentity: null, + destinationIdentity: serializeCopyFileIdentity(installedDestinationIdentity), + displacedIdentity: displacedIdentity ? serializeCopyFileIdentity(displacedIdentity) : null, + sizeBytes: tempStat.size, + resultStatus: "copied" + }); + await guardMutation(async () => { + await validateLinkStillPointsTo(link, sourcePath); + await assertDestinationState(destinationPath, installedDestinationIdentity); + await replaceSymlink(paths.symlinkDir, link.linkPath, destinationPath); + }); + linkRepointed = true; promotedPath = null; - if (displacedDestinationPath && localConflictStrategy === "replace") { - await fs.rm(displacedDestinationPath, { force: true }); - displacedDestinationPath = null; - } return { status: "copied", direction, @@ -951,17 +1164,61 @@ export async function copyMediaLink( message: verificationEnabled ? "Verified copy installed and symlink repointed" : "Copy installed without verification and symlink repointed" }; } catch (error) { - await fs.rm(tempPath, { force: true }).catch(() => undefined); - if (promotedPath) { - const stat = await fs.stat(promotedPath).catch(() => null); - if (stat?.isFile() && stat.size === promotedSize) { - await fs.rm(promotedPath, { force: true }).catch(() => undefined); + const rollbackErrors: string[] = []; + try { + const currentTempIdentity = await readCopyFileIdentity(tempPath); + if (currentTempIdentity && tempIdentity && !copyFileIdentitiesMatch(currentTempIdentity, tempIdentity)) { + rollbackErrors.push("temporary copy changed before cleanup"); + } else if (currentTempIdentity) { + await fs.rm(tempPath, { force: true }); + } + } catch (rollbackError) { + rollbackErrors.push(`temporary copy cleanup failed: ${rollbackError instanceof Error ? rollbackError.message : String(rollbackError)}`); + } + const rollbackSharedPaths = await canRollbackSharedPaths(); + let rollbackDestination = rollbackSharedPaths; + if (rollbackSharedPaths && linkRepointed) { + try { + await guardOwnedMutation(async () => { + await validateLinkStillPointsTo(link, destinationPath); + await replaceSymlink(paths.symlinkDir, link.linkPath, sourcePath); + }); + } catch (rollbackError) { + rollbackDestination = false; + rollbackErrors.push(`symlink restore failed: ${rollbackError instanceof Error ? rollbackError.message : String(rollbackError)}`); + } + } + if (rollbackDestination && promotedPath) { + try { + await guardOwnedMutation(async () => { + const currentPromotedIdentity = await readCopyFileIdentity(promotedPath!); + if (!promotedIdentity || !copyFileIdentitiesMatch(currentPromotedIdentity, promotedIdentity)) { + throw new Error("promoted destination changed before rollback"); + } + await fs.rm(promotedPath!, { force: true }); + }); + } catch (rollbackError) { + rollbackDestination = false; + rollbackErrors.push(`promoted destination cleanup failed: ${rollbackError instanceof Error ? rollbackError.message : String(rollbackError)}`); + } + } + if (rollbackDestination && displacedDestinationPath && destinationDisplaced) { + try { + await guardOwnedMutation(async () => { + const currentDestinationIdentity = await destinationState(destinationPath); + if (currentDestinationIdentity) throw new Error("destination path became occupied before displaced destination rollback"); + const currentDisplacedIdentity = await readCopyFileIdentity(displacedDestinationPath!); + if (!displacedIdentity || !copyFileIdentitiesMatch(currentDisplacedIdentity, displacedIdentity)) { + throw new Error("displaced destination changed before rollback"); + } + await fs.rename(displacedDestinationPath!, destinationPath); + }); + } catch (rollbackError) { + rollbackErrors.push(`displaced destination restore failed: ${rollbackError instanceof Error ? rollbackError.message : String(rollbackError)}`); } } - if (displacedDestinationPath) { - const destinationStat = await fs.stat(destinationPath).catch(() => null); - if (destinationStat?.isFile()) await fs.rm(destinationPath, { force: true }).catch(() => undefined); - await fs.rename(displacedDestinationPath, destinationPath).catch(() => undefined); + if (rollbackErrors.length > 0) { + throw new CopyReconciliationRequiredError(`Copy rollback requires manual reconciliation (${rollbackErrors.join("; ")})`, { cause: error }); } throw error; } diff --git a/src/server/lib/env.ts b/src/server/lib/env.ts index 6e7c6e0..81fa474 100644 --- a/src/server/lib/env.ts +++ b/src/server/lib/env.ts @@ -15,6 +15,12 @@ export interface EnvSettings { SRTL_PORT?: string; SRTL_WEB_PORT?: string; SRTL_WORKER_COUNT?: string; + SRTL_MAX_RUNNING_JOBS?: string; + SRTL_MAX_RUNNING_SCANS?: string; + SRTL_MAX_RUNNING_AUDITS?: string; + SRTL_MAX_RUNNING_COPIES?: string; + SRTL_COPY_FILE_CONCURRENCY?: string; + SRTL_MAX_ACTIVE_COPY_FILES?: string; SRTL_ALLOWED_ORIGINS?: string; SRTL_COOKIE_SECURE?: string; SRTL_API_DOCS?: string; @@ -38,6 +44,12 @@ const supportedKeys = [ "SRTL_PORT", "SRTL_WEB_PORT", "SRTL_WORKER_COUNT", + "SRTL_MAX_RUNNING_JOBS", + "SRTL_MAX_RUNNING_SCANS", + "SRTL_MAX_RUNNING_AUDITS", + "SRTL_MAX_RUNNING_COPIES", + "SRTL_COPY_FILE_CONCURRENCY", + "SRTL_MAX_ACTIVE_COPY_FILES", "SRTL_ALLOWED_ORIGINS", "SRTL_COOKIE_SECURE", "SRTL_API_DOCS", diff --git a/src/server/lib/filesystemSafety.ts b/src/server/lib/filesystemSafety.ts index 3fb0ca9..4ab1330 100644 --- a/src/server/lib/filesystemSafety.ts +++ b/src/server/lib/filesystemSafety.ts @@ -115,6 +115,22 @@ async function nearestExistingAncestor(candidate: string, root: string): Promise throw new Error(`No existing parent for ${candidate} is inside configured root`); } +export async function canonicalPathForClaim( + root: string, + candidate: string, + label: string, + preserveLeaf = false +): Promise { + if (!isPathInside(root, candidate)) throw new Error(`${label} is outside configured root`); + const rootRealPath = await realPath(root, "Configured root"); + const claimBase = preserveLeaf ? path.dirname(candidate) : candidate; + const existingAncestor = await nearestExistingAncestor(claimBase, root); + const ancestorRealPath = await realPath(existingAncestor, `${label} claim path`); + const canonicalPath = path.resolve(ancestorRealPath, path.relative(existingAncestor, path.resolve(candidate))); + if (!isPathInside(rootRealPath, canonicalPath)) throw new Error(`${label} resolves outside configured root`); + return canonicalPath; +} + export async function assertExistingPathInside(root: string, candidate: string, label: string): Promise { if (!isPathInside(root, candidate)) throw new Error(`${label} is outside configured root`); const [rootRealPath, candidateRealPath] = await Promise.all([realPath(root, "Configured root"), realPath(candidate, label)]); diff --git a/src/server/lib/mountIdentity.ts b/src/server/lib/mountIdentity.ts new file mode 100644 index 0000000..d703181 --- /dev/null +++ b/src/server/lib/mountIdentity.ts @@ -0,0 +1,62 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import type { PathMountIdentity, PathRootIdentity } from "../../shared/types"; + +function decodeMountInfoField(value: string): string { + return value.replace(/\\([0-7]{3})/g, (_match, octal: string) => String.fromCharCode(Number.parseInt(octal, 8))); +} + +export function parseLinuxMountInfo(contents: string): PathMountIdentity[] { + const mounts: PathMountIdentity[] = []; + for (const line of contents.split("\n")) { + const separator = line.indexOf(" - "); + if (separator < 0) continue; + const mounted = line.slice(0, separator).split(" "); + const filesystem = line.slice(separator + 3).split(" "); + if (mounted.length < 5 || filesystem.length < 2) continue; + mounts.push({ + mountPoint: path.resolve(decodeMountInfoField(mounted[4])), + root: decodeMountInfoField(mounted[3]), + filesystemType: decodeMountInfoField(filesystem[0]), + source: decodeMountInfoField(filesystem[1]) + }); + } + return mounts; +} + +export function findMountIdentity(realPath: string, mounts: PathMountIdentity[]): PathMountIdentity | null { + const candidate = path.resolve(realPath); + let match: PathMountIdentity | null = null; + for (const mount of mounts) { + const relative = path.relative(mount.mountPoint, candidate); + const containsCandidate = relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); + if (containsCandidate && (!match || mount.mountPoint.length > match.mountPoint.length)) match = mount; + } + return match; +} + +function mountIdentitiesMatch(expected: PathMountIdentity, actual: PathMountIdentity): boolean { + return ( + expected.mountPoint === actual.mountPoint && + expected.root === actual.root && + expected.filesystemType === actual.filesystemType && + expected.source === actual.source + ); +} + +export function persistentRootIdentityMatch(expected: PathRootIdentity | null, actual: PathRootIdentity): boolean { + if (!expected?.available || !actual.available) return false; + if (!expected.realPath || !actual.realPath || expected.realPath !== actual.realPath) return false; + if (expected.mount) return Boolean(actual.mount && mountIdentitiesMatch(expected.mount, actual.mount)); + if (expected.device && actual.device && expected.device === actual.device) return true; + if (!actual.mount) return true; + return actual.mount.mountPoint !== path.parse(actual.realPath).root; +} + +export async function inspectMountIdentity(realPath: string): Promise { + if (process.platform !== "linux") return null; + const contents = await fs.readFile("/proc/self/mountinfo", "utf8"); + const mount = findMountIdentity(realPath, parseLinuxMountInfo(contents)); + if (!mount) throw new Error(`No Linux mount contains ${realPath}`); + return mount; +} diff --git a/src/server/lib/pathConfiguration.ts b/src/server/lib/pathConfiguration.ts index 2a3aea5..5de99ca 100644 --- a/src/server/lib/pathConfiguration.ts +++ b/src/server/lib/pathConfiguration.ts @@ -1,9 +1,11 @@ import fs from "node:fs/promises"; import path from "node:path"; -import { and, asc, count, desc, eq, inArray, isNull, ne, sql } from "drizzle-orm"; -import { first, nowIso, type Db } from "../db/database"; +import { and, asc, count, desc, eq, inArray, isNull, ne, notInArray, sql } from "drizzle-orm"; +import { first, nowIso, type Db, type DbExecutor } from "../db/database"; import * as schema from "../db/schema"; +import { schedulerLockKey } from "../jobs/scheduling"; import { assertPathParentInside } from "./filesystemSafety"; +import { inspectMountIdentity, persistentRootIdentityMatch } from "./mountIdentity"; import type { JobEventRecord, ManagedPathRoot, @@ -12,6 +14,7 @@ import type { PathMigrationRecord, PathMigrationStatus, PathMigrationSummary, + PathMountIdentity, PathRootChange, PathRootIdentity, PathsSettings @@ -20,17 +23,22 @@ import type { const pathConfigurationLockKey = 781_889_433; const rootInspectionTimeoutMs = 5_000; const filesystemReadTimeoutMs = 15_000; -const blockingMigrationStatuses: PathMigrationStatus[] = ["pending", "planning", "planned", "queued", "running", "failed"]; +const blockingMigrationStatuses: PathMigrationStatus[] = ["pending", "planning", "planned", "queued", "running", "rollback_pending", "failed"]; const emptyPaths: PathsSettings = { symlinkDir: "", localDir: "", remoteDir: "" }; type PathConfigurationRow = typeof schema.pathConfigurations.$inferSelect; type PathMigrationRow = typeof schema.pathMigrations.$inferSelect; export interface PathMigrationRunContext { + jobId?: number; signal: AbortSignal; event(level: JobEventRecord["level"], message: string, data?: unknown): Promise; setProgress(progress: unknown): Promise; isCancelled(): Promise; + assertLease(): Promise; + withLease(action: () => Promise): Promise; + withLeaseDb(action: (db: DbExecutor) => Promise): Promise; + finishCompleted(action: (db: DbExecutor) => Promise): Promise; } export interface ReconcileEnvironmentPathsOptions { @@ -43,6 +51,60 @@ function errorMessage(error: unknown): string { return cause instanceof Error && cause.message ? `${error.message}: ${cause.message}` : error.message; } +class PathMigrationLeaseLostError extends Error { + constructor(error: unknown) { + super(errorMessage(error), { cause: error }); + this.name = "PathMigrationLeaseLostError"; + } +} + +class PathMigrationRootIdentityError extends Error { + constructor(message: string) { + super(message); + this.name = "PathMigrationRootIdentityError"; + } +} + +async function assertMigrationLease(ctx: PathMigrationRunContext): Promise { + try { + await ctx.assertLease(); + } catch (error: unknown) { + throw new PathMigrationLeaseLostError(error); + } +} + +async function withMigrationLease(ctx: PathMigrationRunContext, action: () => Promise, allowAborted = false): Promise { + try { + return await ctx.withLease(async () => { + if (!allowAborted && ctx.signal.aborted) { + throw (ctx.signal.reason instanceof Error ? ctx.signal.reason : new Error("Path migration was terminated")); + } + return action(); + }); + } catch (error: unknown) { + if (error instanceof Error && error.name === "LeaseLostError") throw new PathMigrationLeaseLostError(error); + throw error; + } +} + +async function withMigrationLeaseDb( + ctx: PathMigrationRunContext, + action: (db: DbExecutor) => Promise, + allowAborted = false +): Promise { + try { + return await ctx.withLeaseDb(async (leaseDb) => { + if (!allowAborted && ctx.signal.aborted) { + throw (ctx.signal.reason instanceof Error ? ctx.signal.reason : new Error("Path migration was terminated")); + } + return action(leaseDb); + }); + } catch (error: unknown) { + if (error instanceof Error && error.name === "LeaseLostError") throw new PathMigrationLeaseLostError(error); + throw error; + } +} + async function withTimeout(operation: Promise, description: string, timeoutMs: number): Promise { let timeout: ReturnType | undefined; const timedOut = new Promise((_resolve, reject) => { @@ -84,8 +146,16 @@ export function validateManagedPaths(paths: PathsSettings): string[] { const configured = roots.filter(([, value]) => Boolean(value)); for (let left = 0; left < configured.length; left += 1) { for (let right = left + 1; right < configured.length; right += 1) { - if (path.resolve(configured[left][1]) === path.resolve(configured[right][1])) { - errors.push(`${configured[left][0]} and ${configured[right][0]} cannot use the same path.`); + const leftPath = path.resolve(configured[left][1]); + const rightPath = path.resolve(configured[right][1]); + const leftRelative = path.relative(leftPath, rightPath); + const rightRelative = path.relative(rightPath, leftPath); + const overlaps = + leftRelative === "" || + (!leftRelative.startsWith("..") && !path.isAbsolute(leftRelative)) || + (!rightRelative.startsWith("..") && !path.isAbsolute(rightRelative)); + if (overlaps) { + errors.push(`${configured[left][0]} and ${configured[right][0]} cannot use the same or overlapping path.`); } } } @@ -104,11 +174,30 @@ function parseIdentity(raw: string): PathRootIdentity | null { try { const value = JSON.parse(raw) as Partial; if (typeof value.available !== "boolean") return null; + let mount: PathMountIdentity | null = null; + if (value.mount != null) { + if ( + typeof value.mount !== "object" || + typeof value.mount.mountPoint !== "string" || + typeof value.mount.root !== "string" || + typeof value.mount.filesystemType !== "string" || + typeof value.mount.source !== "string" + ) { + return null; + } + mount = { + mountPoint: value.mount.mountPoint, + root: value.mount.root, + filesystemType: value.mount.filesystemType, + source: value.mount.source + }; + } return { available: value.available, realPath: typeof value.realPath === "string" ? value.realPath : null, device: typeof value.device === "string" ? value.device : null, inode: typeof value.inode === "string" ? value.inode : null, + mount, error: typeof value.error === "string" ? value.error : null }; } catch { @@ -117,21 +206,27 @@ function parseIdentity(raw: string): PathRootIdentity | null { } async function inspectRoot(root: string): Promise { - if (!root) return { available: false, realPath: null, device: null, inode: null, error: "Path is not configured" }; + if (!root) return { available: false, realPath: null, device: null, inode: null, mount: null, error: "Path is not configured" }; try { return await withTimeout( (async (): Promise => { - const [stat, realPath] = await Promise.all([fs.stat(root, { bigint: true }), fs.realpath(root)]); - if (!stat.isDirectory()) { - return { available: false, realPath, device: String(stat.dev), inode: String(stat.ino), error: "Path is not a directory" }; - } - return { available: true, realPath, device: String(stat.dev), inode: String(stat.ino), error: null }; + const realPath = await fs.realpath(root); + const stat = await fs.stat(realPath, { bigint: true }); + const verifiedRealPath = await fs.realpath(root); + if (verifiedRealPath !== realPath) { + return { available: false, realPath: verifiedRealPath, device: null, inode: null, mount: null, error: "Path changed during inspection" }; + } + if (!stat.isDirectory()) { + return { available: false, realPath, device: String(stat.dev), inode: String(stat.ino), mount: null, error: "Path is not a directory" }; + } + const mount = await inspectMountIdentity(realPath); + return { available: true, realPath, device: String(stat.dev), inode: String(stat.ino), mount, error: null }; })(), `Inspection of ${root}`, rootInspectionTimeoutMs ); } catch (error: unknown) { - return { available: false, realPath: null, device: null, inode: null, error: errorMessage(error) }; + return { available: false, realPath: null, device: null, inode: null, mount: null, error: errorMessage(error) }; } } @@ -140,11 +235,147 @@ async function inspectPaths(paths: PathsSettings): Promise<{ symlink: PathRootId return { symlink, local, remote }; } +type InspectedPaths = Awaited>; + +interface PathMigrationTargetIdentity { + dev: string; + ino: string; + size: string; + mtimeNs: string; + ctimeNs: string; +} + +function serializeTargetIdentity(identity: PathMigrationTargetIdentity): string { + return JSON.stringify(identity); +} + +function parseTargetIdentity(rawIdentity: string | null): PathMigrationTargetIdentity { + if (!rawIdentity) { + throw new Error("Mapped target has no exact file identity; analyze the path migration again before applying it"); + } + let value: unknown; + try { + value = JSON.parse(rawIdentity); + } catch (error: unknown) { + throw new Error("Mapped target has an invalid exact file identity; analyze the path migration again", { cause: error }); + } + if (!value || typeof value !== "object") { + throw new Error("Mapped target has an invalid exact file identity; analyze the path migration again"); + } + const identity = value as Partial; + const keys: Array = ["dev", "ino", "size", "mtimeNs", "ctimeNs"]; + if (!keys.every((key) => typeof identity[key] === "string" && /^\d+$/.test(identity[key]))) { + throw new Error("Mapped target has an invalid exact file identity; analyze the path migration again"); + } + return { + dev: identity.dev!, + ino: identity.ino!, + size: identity.size!, + mtimeNs: identity.mtimeNs!, + ctimeNs: identity.ctimeNs! + }; +} + +function targetIdentitiesMatch(left: PathMigrationTargetIdentity, right: PathMigrationTargetIdentity): boolean { + return left.dev === right.dev && left.ino === right.ino && left.size === right.size && left.mtimeNs === right.mtimeNs && left.ctimeNs === right.ctimeNs; +} + +async function readTargetIdentity(targetPath: string): Promise { + const stat = await withTimeout(fs.stat(targetPath, { bigint: true }), `Reading mapped target ${targetPath}`, filesystemReadTimeoutMs); + if (!stat.isFile()) throw new Error(`Mapped target is not a regular file: ${targetPath}`); + return { + dev: stat.dev.toString(), + ino: stat.ino.toString(), + size: stat.size.toString(), + mtimeNs: stat.mtimeNs.toString(), + ctimeNs: stat.ctimeNs.toString() + }; +} + +function physicalPathOverlap(left: string, right: string): boolean { + const leftRelative = path.relative(path.resolve(left), path.resolve(right)); + const rightRelative = path.relative(path.resolve(right), path.resolve(left)); + return ( + leftRelative === "" || + (!leftRelative.startsWith("..") && !path.isAbsolute(leftRelative)) || + (!rightRelative.startsWith("..") && !path.isAbsolute(rightRelative)) + ); +} + +function inspectedPathErrors(identities: InspectedPaths): string[] { + const roots: Array<[string, PathRootIdentity]> = [ + ["Symlink directory", identities.symlink], + ["Local directory", identities.local], + ["Remote directory", identities.remote] + ]; + const errors = roots + .filter(([, identity]) => !identity.available) + .map(([label, identity]) => `${label} is unavailable${identity.error ? `: ${identity.error}` : "."}`); + + for (let left = 0; left < roots.length; left += 1) { + for (let right = left + 1; right < roots.length; right += 1) { + const leftIdentity = roots[left][1]; + const rightIdentity = roots[right][1]; + if (!leftIdentity.available || !rightIdentity.available) continue; + const sameIdentity = + Boolean(leftIdentity.device && leftIdentity.inode) && + leftIdentity.device === rightIdentity.device && + leftIdentity.inode === rightIdentity.inode; + const overlappingRealPaths = Boolean( + leftIdentity.realPath && rightIdentity.realPath && physicalPathOverlap(leftIdentity.realPath, rightIdentity.realPath) + ); + if (sameIdentity || overlappingRealPaths) { + errors.push(`${roots[left][0]} and ${roots[right][0]} resolve to the same or overlapping physical path.`); + } + } + } + return errors; +} + +function strictRootIdentityMatch(expected: PathRootIdentity | null, actual: PathRootIdentity): boolean { + if (!expected || !persistentRootIdentityMatch(expected, actual)) return false; + if (expected.device && expected.inode && actual.device && actual.inode) { + return expected.device === actual.device && expected.inode === actual.inode; + } + return true; +} + +function configurationPersistentIdentityMatch(configuration: PathConfigurationRow, identities: InspectedPaths): boolean { + const expected = configurationIdentities(configuration); + return ( + persistentRootIdentityMatch(expected.symlink, identities.symlink) && + persistentRootIdentityMatch(expected.local, identities.local) && + persistentRootIdentityMatch(expected.remote, identities.remote) + ); +} + +function configurationStrictIdentityMatch(configuration: PathConfigurationRow, identities: InspectedPaths): boolean { + const expected = configurationIdentities(configuration); + return ( + strictRootIdentityMatch(expected.symlink, identities.symlink) && + strictRootIdentityMatch(expected.local, identities.local) && + strictRootIdentityMatch(expected.remote, identities.remote) + ); +} + +async function assertConfigurationRootIdentity(configuration: PathConfigurationRow, phase: string): Promise { + const paths = pathsFromConfiguration(configuration); + const lexicalErrors = validateManagedPaths(paths); + const identities = await inspectPaths(paths); + const errors = [...lexicalErrors, ...(lexicalErrors.length === 0 ? inspectedPathErrors(identities) : [])]; + if (errors.length > 0) { + throw new PathMigrationRootIdentityError(`${phase}: ${errors.join(" ")}`); + } + if (!configurationStrictIdentityMatch(configuration, identities)) { + throw new PathMigrationRootIdentityError( + `${phase}: a managed storage root no longer matches the physical directory recorded during migration analysis. Analyze the path change again before continuing.` + ); + } +} + function identityMatch(active: PathRootIdentity | null, detected: PathRootIdentity | null): PathRootChange["identityMatch"] { if (!active?.available || !detected?.available) return "unknown"; - if (active.realPath && active.realPath === detected.realPath) return "same"; - if (active.device && active.inode && active.device === detected.device && active.inode === detected.inode) return "same"; - return "different"; + return persistentRootIdentityMatch(active, detected) ? "same" : "different"; } function configurationIdentities(row: PathConfigurationRow | null): Record { @@ -168,16 +399,170 @@ async function latestBlockingMigration(db: Db): Promise ); } +async function cancelPathMigrations(db: DbExecutor, migrations: PathMigrationRow[], message: string): Promise { + for (const migration of migrations) { + const timestamp = nowIso(); + let job = migration.jobId == null + ? null + : await first(db.select().from(schema.jobs).where(eq(schema.jobs.id, migration.jobId)).for("update").limit(1)); + const needsRollback = + migration.status === "running" || + migration.status === "rollback_pending" || + (migration.status === "failed" && migration.startedAt != null); + + if (needsRollback) { + if (migration.status === "rollback_pending" && job && (job.status === "queued" || job.status === "running")) continue; + if (!job) { + job = await first( + db + .insert(schema.jobs) + .values({ + type: "path_migration", + status: "queued", + createdAt: timestamp, + startedAt: null, + finishedAt: null, + lockedBy: null, + lockedAt: null, + heartbeatAt: null, + leaseVersion: 0, + exclusive: true, + cancelRequestedAt: null, + progress: JSON.stringify({ migrationId: migration.id, stage: "rollback_pending", current: 0, total: 0, message: "Path migration rollback queued" }) + }) + .returning() + ); + } else { + job = await first( + db + .update(schema.jobs) + .set({ + status: "queued", + finishedAt: null, + lockedBy: null, + lockedAt: null, + heartbeatAt: null, + leaseVersion: sql`${schema.jobs.leaseVersion} + 1`, + cancelRequestedAt: null, + progress: JSON.stringify({ migrationId: migration.id, stage: "rollback_pending", current: 0, total: 0, message: "Path migration rollback queued" }) + }) + .where(eq(schema.jobs.id, job.id)) + .returning() + ); + } + if (!job) throw new Error(`Could not queue rollback recovery for path migration #${migration.id}`); + await db + .update(schema.pathMigrations) + .set({ status: "rollback_pending", jobId: job.id, finishedAt: null, errorMessage: message }) + .where(eq(schema.pathMigrations.id, migration.id)); + await db.update(schema.pathConfigurations).set({ status: "pending" }).where(eq(schema.pathConfigurations.id, migration.targetConfigId)); + await db.insert(schema.jobEvents).values({ + jobId: job.id, + timestamp, + level: "warn", + message: "Path migration rollback queued after environment paths changed", + data: JSON.stringify({ migrationId: migration.id, reason: message }) + }); + continue; + } + + await db + .update(schema.pathMigrations) + .set({ status: "cancelled", finishedAt: timestamp, errorMessage: message }) + .where(eq(schema.pathMigrations.id, migration.id)); + await db.update(schema.pathConfigurations).set({ status: "cancelled" }).where(eq(schema.pathConfigurations.id, migration.targetConfigId)); + if (job?.status === "queued") { + await db + .update(schema.jobs) + .set({ status: "cancelled", finishedAt: timestamp, cancelRequestedAt: timestamp }) + .where(and(eq(schema.jobs.id, job.id), eq(schema.jobs.status, "queued"))); + await db.insert(schema.jobEvents).values({ + jobId: job.id, + timestamp, + level: "warn", + message: "Queued path migration cancelled after environment paths changed", + data: JSON.stringify({ migrationId: migration.id, reason: message }) + }); + } + } +} + +async function legacyJournalPathExists(filePath: string | null): Promise { + if (!filePath) return false; + try { + await fs.lstat(filePath); + return true; + } catch (error) { + if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") return false; + return true; + } +} + +async function markUncertainLegacyFailedCopyOperations(db: DbExecutor): Promise { + const legacyOperations = ( + await db + .select() + .from(schema.copyOperations) + .where(eq(schema.copyOperations.stage, "failed")) + ).filter( + (operation) => + operation.tempIdentity == null && + operation.destinationIdentity == null && + operation.displacedIdentity == null + ); + + for (const operation of legacyOperations) { + const reasons: string[] = []; + try { + const target = await symlinkTarget(operation.linkPath); + if (path.resolve(target) !== path.resolve(operation.originalTargetPath)) { + reasons.push("the library symlink no longer points to its original target"); + } + } catch (error: unknown) { + reasons.push(`the library symlink cannot be verified: ${errorMessage(error)}`); + } + if (await legacyJournalPathExists(operation.tempPath)) reasons.push("a journaled temporary copy still exists"); + if (await legacyJournalPathExists(operation.displacedPath)) reasons.push("a journaled displaced destination still exists"); + if (await legacyJournalPathExists(operation.destinationPath)) { + reasons.push("the journaled destination exists but its ownership cannot be proven"); + } + if (reasons.length === 0) continue; + + await db + .update(schema.copyOperations) + .set({ + stage: "reconciliation_required", + errorMessage: `Legacy copy operation requires manual reconciliation because ${reasons.join(" and ")}`, + updatedAt: nowIso(), + completedAt: null + }) + .where(and(eq(schema.copyOperations.id, operation.id), eq(schema.copyOperations.stage, "failed"))); + } +} + export async function reconcileEnvironmentPaths( db: Db, environmentPaths: PathsSettings, options: ReconcileEnvironmentPathsOptions = {} ): Promise { const detectedPaths = normalizeManagedPaths(environmentPaths); - const environmentErrors = validateManagedPaths(detectedPaths); + const lexicalEnvironmentErrors = validateManagedPaths(detectedPaths); await db.transaction(async (transaction) => { + await transaction.execute(sql`select pg_advisory_xact_lock(${schedulerLockKey})`); await transaction.execute(sql`select pg_advisory_xact_lock(${pathConfigurationLockKey})`); + await markUncertainLegacyFailedCopyOperations(transaction); + const detectedIdentities = await inspectPaths(detectedPaths); + const environmentErrors = [ + ...lexicalEnvironmentErrors, + ...(lexicalEnvironmentErrors.length === 0 ? inspectedPathErrors(detectedIdentities) : []) + ]; + await transaction + .select({ id: schema.jobs.id }) + .from(schema.jobs) + .where(eq(schema.jobs.status, "running")) + .orderBy(asc(schema.jobs.id)) + .for("update"); let active = (await first(transaction.select().from(schema.pathConfigurations).where(eq(schema.pathConfigurations.status, "active")).limit(1))) ?? null; @@ -194,9 +579,20 @@ export async function reconcileEnvironmentPaths( } catch { legacyPaths = emptyPaths; } - const initialPaths = validateManagedPaths(legacyPaths).length === 0 ? legacyPaths : environmentErrors.length === 0 ? detectedPaths : null; - if (initialPaths) { - const identities = await inspectPaths(initialPaths); + let initialPaths: PathsSettings | null = null; + let identities: InspectedPaths | null = null; + if (validateManagedPaths(legacyPaths).length === 0) { + const legacyIdentities = await inspectPaths(legacyPaths); + if (inspectedPathErrors(legacyIdentities).length === 0) { + initialPaths = legacyPaths; + identities = legacyIdentities; + } + } + if (!initialPaths && environmentErrors.length === 0) { + initialPaths = detectedPaths; + identities = detectedIdentities; + } + if (initialPaths && identities) { active = (await first( transaction @@ -218,39 +614,37 @@ export async function reconcileEnvironmentPaths( } } - if (active && environmentErrors.length === 0 && pathsEqual(pathsFromConfiguration(active), detectedPaths)) { + if ( + active && + environmentErrors.length === 0 && + pathsEqual(pathsFromConfiguration(active), detectedPaths) && + configurationPersistentIdentityMatch(active, detectedIdentities) + ) { + const timestamp = nowIso(); const blocking = await transaction.select().from(schema.pathMigrations).where(inArray(schema.pathMigrations.status, blockingMigrationStatuses)); if (blocking.length > 0) { - const timestamp = nowIso(); - await transaction - .update(schema.pathMigrations) - .set({ status: "cancelled", finishedAt: timestamp, errorMessage: "Detected paths were restored before migration." }) - .where(inArray(schema.pathMigrations.id, blocking.map((row) => row.id))); - await transaction - .update(schema.pathConfigurations) - .set({ status: "cancelled" }) - .where(inArray(schema.pathConfigurations.id, blocking.map((row) => row.targetConfigId))); + await cancelPathMigrations(transaction, blocking, "Configured paths and storage mounts match the active configuration."); } + await transaction + .update(schema.pathConfigurations) + .set({ + symlinkIdentity: JSON.stringify(detectedIdentities.symlink), + localIdentity: JSON.stringify(detectedIdentities.local), + remoteIdentity: JSON.stringify(detectedIdentities.remote) + }) + .where(eq(schema.pathConfigurations.id, active.id)); await transaction .insert(schema.appSettings) - .values({ key: "paths", value: JSON.stringify(detectedPaths), updatedAt: nowIso() }) - .onConflictDoUpdate({ target: schema.appSettings.key, set: { value: JSON.stringify(detectedPaths), updatedAt: nowIso() } }); + .values({ key: "paths", value: JSON.stringify(detectedPaths), updatedAt: timestamp }) + .onConflictDoUpdate({ target: schema.appSettings.key, set: { value: JSON.stringify(detectedPaths), updatedAt: timestamp } }); return; } if (active && environmentErrors.length === 0 && options.allowDirectAdoptionBeforeInventory) { const timestamp = nowIso(); - const identities = await inspectPaths(detectedPaths); const blocking = await transaction.select().from(schema.pathMigrations).where(inArray(schema.pathMigrations.status, blockingMigrationStatuses)); if (blocking.length > 0) { - await transaction - .update(schema.pathMigrations) - .set({ status: "cancelled", finishedAt: timestamp, errorMessage: "Storage paths were corrected before the initial inventory scan." }) - .where(inArray(schema.pathMigrations.id, blocking.map((row) => row.id))); - await transaction - .update(schema.pathConfigurations) - .set({ status: "cancelled" }) - .where(inArray(schema.pathConfigurations.id, blocking.map((row) => row.targetConfigId))); + await cancelPathMigrations(transaction, blocking, "Storage paths were corrected before the initial inventory scan."); } await transaction .update(schema.pathConfigurations) @@ -258,9 +652,9 @@ export async function reconcileEnvironmentPaths( symlinkDir: detectedPaths.symlinkDir, localDir: detectedPaths.localDir, remoteDir: detectedPaths.remoteDir, - symlinkIdentity: JSON.stringify(identities.symlink), - localIdentity: JSON.stringify(identities.local), - remoteIdentity: JSON.stringify(identities.remote), + symlinkIdentity: JSON.stringify(detectedIdentities.symlink), + localIdentity: JSON.stringify(detectedIdentities.local), + remoteIdentity: JSON.stringify(detectedIdentities.remote), appliedAt: timestamp }) .where(eq(schema.pathConfigurations.id, active.id)); @@ -282,16 +676,14 @@ export async function reconcileEnvironmentPaths( )) ?? null; if (existing) { const target = await first(transaction.select().from(schema.pathConfigurations).where(eq(schema.pathConfigurations.id, existing.targetConfigId)).limit(1)); - if (target && pathsEqual(pathsFromConfiguration(target), detectedPaths)) return; - const timestamp = nowIso(); - await transaction - .update(schema.pathMigrations) - .set({ status: "cancelled", finishedAt: timestamp, errorMessage: "A newer environment path change replaced this migration." }) - .where(eq(schema.pathMigrations.id, existing.id)); - await transaction.update(schema.pathConfigurations).set({ status: "cancelled" }).where(eq(schema.pathConfigurations.id, existing.targetConfigId)); + if ( + target && + pathsEqual(pathsFromConfiguration(target), detectedPaths) && + configurationPersistentIdentityMatch(target, detectedIdentities) + ) return; + await cancelPathMigrations(transaction, [existing], "A newer environment path change replaced this migration."); } - const identities = await inspectPaths(detectedPaths); const target = await first( transaction .insert(schema.pathConfigurations) @@ -300,9 +692,9 @@ export async function reconcileEnvironmentPaths( symlinkDir: detectedPaths.symlinkDir, localDir: detectedPaths.localDir, remoteDir: detectedPaths.remoteDir, - symlinkIdentity: JSON.stringify(identities.symlink), - localIdentity: JSON.stringify(identities.local), - remoteIdentity: JSON.stringify(identities.remote), + symlinkIdentity: JSON.stringify(detectedIdentities.symlink), + localIdentity: JSON.stringify(detectedIdentities.local), + remoteIdentity: JSON.stringify(detectedIdentities.remote), createdAt: nowIso(), appliedAt: null }) @@ -323,7 +715,7 @@ export async function reconcileEnvironmentPaths( }); } -export async function isPathConfigurationBlocked(db: Db): Promise { +export async function isPathConfigurationBlocked(db: DbExecutor): Promise { const row = await first( db .select({ id: schema.pathMigrations.id }) @@ -335,13 +727,14 @@ export async function isPathConfigurationBlocked(db: Db): Promise { } function rootChange(root: ManagedPathRoot, label: string, activePath: string, detectedPath: string, activeIdentity: PathRootIdentity | null, detectedIdentity: PathRootIdentity | null): PathRootChange { + const rootIdentityMatch = identityMatch(activeIdentity, detectedIdentity); return { root, label, activePath, detectedPath, - changed: activePath !== detectedPath, - identityMatch: activePath === detectedPath ? "same" : identityMatch(activeIdentity, detectedIdentity), + changed: activePath !== detectedPath || rootIdentityMatch !== "same", + identityMatch: rootIdentityMatch, activeIdentity, detectedIdentity }; @@ -396,14 +789,19 @@ function stateStatus(migration: PathMigrationRow | null, environmentErrors: stri if (!migration) return "ready"; if (migration.status === "planning") return "planning"; if (migration.status === "planned") return "ready_to_apply"; - if (migration.status === "queued" || migration.status === "running") return "migrating"; + if (migration.status === "queued" || migration.status === "running" || migration.status === "rollback_pending") return "migrating"; if (migration.status === "failed") return "failed"; return "change_pending"; } export async function getPathConfigurationState(db: Db, environmentPaths: PathsSettings): Promise { const detectedPaths = normalizeManagedPaths(environmentPaths); - const environmentErrors = validateManagedPaths(detectedPaths); + const lexicalEnvironmentErrors = validateManagedPaths(detectedPaths); + const detectedIdentities = await inspectPaths(detectedPaths); + const environmentErrors = [ + ...lexicalEnvironmentErrors, + ...(lexicalEnvironmentErrors.length === 0 ? inspectedPathErrors(detectedIdentities) : []) + ]; const active = (await first(db.select().from(schema.pathConfigurations).where(eq(schema.pathConfigurations.status, "active")).limit(1))) ?? null; const migration = await latestBlockingMigration(db); @@ -413,7 +811,11 @@ export async function getPathConfigurationState(db: Db, environmentPaths: PathsS const activePaths = active ? pathsFromConfiguration(active) : null; const targetPaths = target ? pathsFromConfiguration(target) : detectedPaths; const activeIdentities = configurationIdentities(active); - const targetIdentities = configurationIdentities(target); + const targetIdentities = target + ? configurationIdentities(target) + : migration + ? configurationIdentities(null) + : detectedIdentities; const changes = [ rootChange("symlink", "Symlink directory", activePaths?.symlinkDir ?? "", targetPaths.symlinkDir, activeIdentities.symlink, targetIdentities.symlink), rootChange("local", "Local directory", activePaths?.localDir ?? "", targetPaths.localDir, activeIdentities.local, targetIdentities.local), @@ -510,6 +912,7 @@ async function validateMigrationItem(link: typeof schema.mediaLinks.$inferSelect targetPathAfter: targetPathForConfiguration(link.targetPath, source, target), targetChanged: false, expectedSizeBytes: link.sizeBytes, + targetIdentity: null, validationStatus: "blocked", message: `Symlink could not be validated at the detected path: ${errorMessage(error)}`, appliedAt: null, @@ -521,41 +924,38 @@ async function validateMigrationItem(link: typeof schema.mediaLinks.$inferSelect const targetChanged = path.resolve(actualTarget) !== path.resolve(desiredTarget); if (!linkPathChanged && !targetChanged) return null; - const managedTargetChanged = - (source.localDir !== target.localDir && (isPathInside(source.localDir, baselineTarget) || isPathInside(target.localDir, baselineTarget))) || - (source.remoteDir !== target.remoteDir && (isPathInside(source.remoteDir, baselineTarget) || isPathInside(target.remoteDir, baselineTarget))); let expectedSizeBytes = link.sizeBytes; if (targetChanged) { const oldTargetStat = await withTimeout(fs.stat(baselineTarget), `Reading source target ${baselineTarget}`, filesystemReadTimeoutMs).catch(() => null); if (oldTargetStat?.isFile()) expectedSizeBytes = Number(oldTargetStat.size); } - if (managedTargetChanged || targetChanged) { - try { - const destinationStat = await withTimeout(fs.stat(desiredTarget), `Reading mapped target ${desiredTarget}`, filesystemReadTimeoutMs); - if (!destinationStat.isFile()) throw new Error("Mapped target is not a regular file"); - if (expectedSizeBytes != null && Number(destinationStat.size) !== expectedSizeBytes) { - throw new Error(`Mapped target size differs from the indexed file (${Number(destinationStat.size)} bytes instead of ${expectedSizeBytes} bytes)`); - } - expectedSizeBytes = Number(destinationStat.size); - } catch (error: unknown) { - return { - migrationId: 0, - mediaLinkId: link.id, - itemName: link.itemName, - currentLinkPath: linkPathAfter, - linkPathBefore: link.linkPath, - linkPathAfter, - targetPathBefore: actualTarget, - targetPathAfter: desiredTarget, - targetChanged, - expectedSizeBytes, - validationStatus: "blocked", - message: `Mapped target could not be validated: ${errorMessage(error)}`, - appliedAt: null, - rolledBackAt: null - }; + let targetIdentity: PathMigrationTargetIdentity; + try { + targetIdentity = await readTargetIdentity(desiredTarget); + const destinationSize = Number(targetIdentity.size); + if (expectedSizeBytes != null && destinationSize !== expectedSizeBytes) { + throw new Error(`Mapped target size differs from the indexed file (${destinationSize} bytes instead of ${expectedSizeBytes} bytes)`); } + expectedSizeBytes = destinationSize; + } catch (error: unknown) { + return { + migrationId: 0, + mediaLinkId: link.id, + itemName: link.itemName, + currentLinkPath: linkPathAfter, + linkPathBefore: link.linkPath, + linkPathAfter, + targetPathBefore: actualTarget, + targetPathAfter: desiredTarget, + targetChanged, + expectedSizeBytes, + targetIdentity: null, + validationStatus: "blocked", + message: `Mapped target could not be validated: ${errorMessage(error)}`, + appliedAt: null, + rolledBackAt: null + }; } return { @@ -569,8 +969,9 @@ async function validateMigrationItem(link: typeof schema.mediaLinks.$inferSelect targetPathAfter: desiredTarget, targetChanged, expectedSizeBytes, + targetIdentity: serializeTargetIdentity(targetIdentity), validationStatus: "ready", - message: targetChanged ? "Mapped target exists and passed size validation." : "Symlink path is available under the detected root.", + message: targetChanged ? "Mapped target exists and passed exact identity validation." : "Symlink path and mapped target passed exact identity validation.", appliedAt: null, rolledBackAt: null }; @@ -603,6 +1004,54 @@ async function migrationContext(db: Db, migrationId: number): Promise<{ migratio return { migration, source, target }; } +async function assertCopyOperationsReconciledForPathMigration(db: DbExecutor): Promise { + const reconciliationOperation = await first( + db + .select({ id: schema.copyOperations.id }) + .from(schema.copyOperations) + .where(eq(schema.copyOperations.stage, "reconciliation_required")) + .limit(1) + ); + if (reconciliationOperation) { + throw new Error( + `Copy operation #${reconciliationOperation.id} requires manual reconciliation before managed paths can be migrated` + ); + } + + const incompleteOperation = await first( + db + .select({ id: schema.copyOperations.id, jobId: schema.copyOperations.jobId }) + .from(schema.copyOperations) + .where(notInArray(schema.copyOperations.stage, ["committed", "rolled_back", "failed"])) + .limit(1) + ); + if (incompleteOperation) { + throw new Error( + `Copy operation #${incompleteOperation.id} from job #${incompleteOperation.jobId} has unresolved filesystem changes; recover or manually reconcile it before managed paths can be migrated` + ); + } + + const activeCopyOperation = await first( + db + .select({ jobId: schema.jobs.id }) + .from(schema.copyOperations) + .innerJoin(schema.jobs, eq(schema.jobs.id, schema.copyOperations.jobId)) + .where( + and( + eq(schema.jobs.type, "copy"), + inArray(schema.jobs.status, ["queued", "running"]), + notInArray(schema.copyOperations.stage, ["rolled_back", "failed"]) + ) + ) + .limit(1) + ); + if (activeCopyOperation) { + throw new Error( + `Copy job #${activeCopyOperation.jobId} is still reconciling filesystem changes; wait for recovery before analyzing or applying the path migration` + ); + } +} + export async function planPathMigration(db: Db, migrationId: number): Promise { const { migration, source, target } = await migrationContext(db, migrationId); if (!["pending", "planned", "failed"].includes(migration.status)) throw new Error("Path migration cannot be analyzed in its current state"); @@ -610,34 +1059,78 @@ export async function planPathMigration(db: Db, migrationId: number): Promise 0) throw new Error(environmentErrors.join(" ")); - const claimed = await first( - db - .update(schema.pathMigrations) - .set({ status: "planning", errorMessage: null, plannedAt: null, finishedAt: null }) - .where(and(eq(schema.pathMigrations.id, migrationId), inArray(schema.pathMigrations.status, ["pending", "planned", "failed"]))) - .returning({ id: schema.pathMigrations.id }) - ); - if (!claimed) throw new Error("Path migration analysis is already running or the detected path change was replaced"); - await db.delete(schema.pathMigrationItems).where(eq(schema.pathMigrationItems.migrationId, migrationId)); + await db.transaction(async (transaction) => { + await transaction.execute(sql`select pg_advisory_xact_lock(${schedulerLockKey})`); + const currentMigration = await first( + transaction.select().from(schema.pathMigrations).where(eq(schema.pathMigrations.id, migrationId)).for("update").limit(1) + ); + if (!currentMigration || !["pending", "planned", "failed"].includes(currentMigration.status)) { + throw new Error("Path migration analysis is already running or the detected path change was replaced"); + } + const otherBlockingMigration = await first( + transaction + .select({ id: schema.pathMigrations.id }) + .from(schema.pathMigrations) + .where(and(ne(schema.pathMigrations.id, migrationId), inArray(schema.pathMigrations.status, blockingMigrationStatuses))) + .limit(1) + ); + if (otherBlockingMigration) { + throw new Error( + `Path migration #${otherBlockingMigration.id} is still being reconciled; wait for it to finish before analyzing this path change` + ); + } + const activePathJob = await first( + transaction + .select({ id: schema.jobs.id }) + .from(schema.jobs) + .where(and(eq(schema.jobs.type, "path_migration"), inArray(schema.jobs.status, ["queued", "running"]))) + .limit(1) + ); + if (activePathJob) { + throw new Error( + `Path migration job #${activePathJob.id} is still active; wait for it to finish before analyzing this path change` + ); + } + await assertCopyOperationsReconciledForPathMigration(transaction); + const runningJob = await first(transaction.select({ id: schema.jobs.id }).from(schema.jobs).where(eq(schema.jobs.status, "running")).limit(1)); + if (runningJob) throw new Error("Another job is still stopping for the path change; wait for it to pause before analyzing migration"); + if (currentMigration.status === "failed" && currentMigration.startedAt != null) { + const unresolvedItem = await first( + transaction + .select({ id: schema.pathMigrationItems.id }) + .from(schema.pathMigrationItems) + .where( + and( + eq(schema.pathMigrationItems.migrationId, migrationId), + ne(schema.pathMigrationItems.validationStatus, "rolled_back") + ) + ) + .limit(1) + ); + if (unresolvedItem) { + throw new Error( + "Path migration cannot be analyzed again until every previously applied symlink is rolled back or manually reconciled" + ); + } + } + const claimed = await first( + transaction + .update(schema.pathMigrations) + .set({ status: "planning", errorMessage: null, plannedAt: null, finishedAt: null }) + .where(and(eq(schema.pathMigrations.id, migrationId), inArray(schema.pathMigrations.status, ["pending", "planned", "failed"]))) + .returning({ id: schema.pathMigrations.id }) + ); + if (!claimed) throw new Error("Path migration analysis is already running or the detected path change was replaced"); + await transaction.delete(schema.pathMigrationItems).where(eq(schema.pathMigrationItems.migrationId, migrationId)); + }); try { const identities = await inspectPaths(targetPaths); await db .update(schema.pathConfigurations) .set({ symlinkIdentity: JSON.stringify(identities.symlink), localIdentity: JSON.stringify(identities.local), remoteIdentity: JSON.stringify(identities.remote) }) .where(eq(schema.pathConfigurations.id, target.id)); - const inspectedRoots: Array<[string, PathRootIdentity]> = [ - ["Symlink directory", identities.symlink], - ["Local directory", identities.local], - ["Remote directory", identities.remote] - ]; - const unavailableRoots = inspectedRoots.filter(([, identity]) => !identity.available); - if (unavailableRoots.length > 0) { - throw new Error( - unavailableRoots - .map(([label, identity]) => `${label} is unavailable${identity.error ? `: ${identity.error}` : ""}`) - .join(" ") - ); - } + const rootErrors = inspectedPathErrors(identities); + if (rootErrors.length > 0) throw new Error(rootErrors.join(" ")); const links = await db.select().from(schema.mediaLinks).where(isNull(schema.mediaLinks.missingSince)).orderBy(asc(schema.mediaLinks.id)); const planned = (await mapWithConcurrency(links, 8, (link) => validateMigrationItem(link, pathsFromConfiguration(source), targetPaths))).filter( (item): item is NonNullable => item !== null @@ -662,26 +1155,15 @@ export async function planPathMigration(db: Db, migrationId: number): Promise { - const migration = await first(db.select().from(schema.pathMigrations).where(eq(schema.pathMigrations.id, migrationId)).limit(1)); - if (!migration || migration.status !== "planned") throw new Error("Analyze the path change before starting migration"); - const blocked = await first( - db - .select({ value: count() }) - .from(schema.pathMigrationItems) - .where(and(eq(schema.pathMigrationItems.migrationId, migrationId), eq(schema.pathMigrationItems.validationStatus, "blocked"))) - ); - if (Number(blocked?.value ?? 0) > 0) throw new Error("Resolve every blocked symlink before starting migration"); - const running = await first( - db - .select({ value: count() }) - .from(schema.jobs) - .where(and(eq(schema.jobs.status, "running"), ne(schema.jobs.type, "path_migration"))) - ); - if (Number(running?.value ?? 0) > 0) throw new Error("Waiting for the active job to pause before path migration can start"); -} - -async function replaceSymlink(linkRoot: string, linkPath: string, targetPath: string): Promise { +async function replaceSymlink( + linkRoot: string, + linkPath: string, + expectedTargetPath: string, + targetPath: string, + ctx: PathMigrationRunContext, + allowAborted = false, + assertRootIdentity?: () => Promise +): Promise { await assertPathParentInside(linkRoot, linkPath, "Path migration symlink"); const tempPath = `${linkPath}.srtl-path-migration-${process.pid}-${Date.now()}-${Math.random().toString(16).slice(2)}`; try { @@ -689,38 +1171,83 @@ async function replaceSymlink(linkRoot: string, linkPath: string, targetPath: st const installedTarget = await symlinkTarget(tempPath); if (path.resolve(installedTarget) !== path.resolve(targetPath)) throw new Error("Temporary symlink target validation failed"); await assertPathParentInside(linkRoot, linkPath, "Path migration symlink"); - await fs.rename(tempPath, linkPath); + await withMigrationLease( + ctx, + async () => { + await assertRootIdentity?.(); + const currentTarget = await symlinkTarget(linkPath); + if (path.resolve(currentTarget) !== path.resolve(expectedTargetPath)) { + throw new Error(`Symlink changed while path migration waited for its lease: ${linkPath}`); + } + await fs.rename(tempPath, linkPath); + }, + allowAborted + ); } catch (error: unknown) { await fs.rm(tempPath, { force: true }).catch(() => undefined); throw error; } } -async function rollbackMigrationItems(db: Db, migrationId: number, linkRoot: string, ctx: PathMigrationRunContext): Promise { +async function rollbackMigrationItems( + db: Db, + migrationId: number, + linkRoot: string, + ctx: PathMigrationRunContext, + eligibleStatuses: string[] = ["ready", "applied"], + assertRootIdentity?: () => Promise +): Promise { const errors: string[] = []; const items = await db .select() .from(schema.pathMigrationItems) - .where(and(eq(schema.pathMigrationItems.migrationId, migrationId), eq(schema.pathMigrationItems.validationStatus, "applied"))) + .where(and(eq(schema.pathMigrationItems.migrationId, migrationId), inArray(schema.pathMigrationItems.validationStatus, eligibleStatuses))) .orderBy(desc(schema.pathMigrationItems.id)); for (const item of items) { - if (!item.targetChanged) continue; try { + if (!item.targetChanged) { + await withMigrationLeaseDb( + ctx, + async (leaseDb) => { + await leaseDb + .update(schema.pathMigrationItems) + .set({ validationStatus: "rolled_back", rolledBackAt: nowIso(), message: "Migration step remained at its original target." }) + .where(and(eq(schema.pathMigrationItems.id, item.id), inArray(schema.pathMigrationItems.validationStatus, eligibleStatuses))); + }, + true + ); + continue; + } const currentTarget = await symlinkTarget(item.currentLinkPath); - if (path.resolve(currentTarget) !== path.resolve(item.targetPathAfter)) { - const message = `Rollback stopped because the symlink target changed again. Expected ${item.targetPathAfter}, found ${currentTarget}. Manual review is required.`; - await db - .update(schema.pathMigrationItems) - .set({ validationStatus: "blocked", rolledBackAt: null, message }) - .where(eq(schema.pathMigrationItems.id, item.id)); + const resolvedCurrentTarget = path.resolve(currentTarget); + if (resolvedCurrentTarget === path.resolve(item.targetPathAfter)) { + await replaceSymlink(linkRoot, item.currentLinkPath, item.targetPathAfter, item.targetPathBefore, ctx, true, assertRootIdentity); + } else if (resolvedCurrentTarget !== path.resolve(item.targetPathBefore)) { + const message = `Rollback stopped because the symlink target changed again. Expected either ${item.targetPathBefore} or ${item.targetPathAfter}, found ${currentTarget}. Manual review is required.`; + await withMigrationLeaseDb( + ctx, + async (leaseDb) => { + await leaseDb + .update(schema.pathMigrationItems) + .set({ validationStatus: "blocked", rolledBackAt: null, message }) + .where(and(eq(schema.pathMigrationItems.id, item.id), inArray(schema.pathMigrationItems.validationStatus, eligibleStatuses))); + }, + true + ); throw new Error(message); } - await replaceSymlink(linkRoot, item.currentLinkPath, item.targetPathBefore); - await db - .update(schema.pathMigrationItems) - .set({ validationStatus: "rolled_back", rolledBackAt: nowIso(), message: "Repointing was rolled back after migration stopped." }) - .where(eq(schema.pathMigrationItems.id, item.id)); + await withMigrationLeaseDb( + ctx, + async (leaseDb) => { + await leaseDb + .update(schema.pathMigrationItems) + .set({ validationStatus: "rolled_back", rolledBackAt: nowIso(), message: "Repointing was rolled back after migration stopped." }) + .where(and(eq(schema.pathMigrationItems.id, item.id), inArray(schema.pathMigrationItems.validationStatus, eligibleStatuses))); + }, + true + ); } catch (error: unknown) { + if (error instanceof PathMigrationLeaseLostError) throw error; const message = `${item.currentLinkPath}: ${errorMessage(error)}`; errors.push(message); await ctx.event("error", "Path migration rollback failed", { linkPath: item.currentLinkPath, error: errorMessage(error) }); @@ -729,7 +1256,8 @@ async function rollbackMigrationItems(db: Db, migrationId: number, linkRoot: str return errors; } -async function cancelQueuedJobsForMigration(db: Db, migrationJobId: number): Promise { +async function cancelQueuedJobsForMigration(db: DbExecutor, migrationJobId: number): Promise { + await assertCopyOperationsReconciledForPathMigration(db); const queued = await db .select({ id: schema.jobs.id }) .from(schema.jobs) @@ -752,7 +1280,7 @@ async function cancelQueuedJobsForMigration(db: Db, migrationJobId: number): Pro return queued.length; } -async function rebaseStorageInventory(db: Pick, rootType: "local" | "remote", oldRoot: string, newRoot: string): Promise { +async function rebaseStorageInventory(db: Pick, rootType: "local" | "remote", oldRoot: string, newRoot: string): Promise { if (oldRoot === newRoot) return; await db.execute(sql` UPDATE storage_files @@ -764,7 +1292,7 @@ async function rebaseStorageInventory(db: Pick, rootType: "local" `); } -async function rebaseMissingMediaLinks(db: Pick, column: "link_path" | "target_path", oldRoot: string, newRoot: string): Promise { +async function rebaseMissingMediaLinks(db: Pick, column: "link_path" | "target_path", oldRoot: string, newRoot: string): Promise { if (oldRoot === newRoot) return; if (column === "link_path") { await db.execute(sql` @@ -801,6 +1329,8 @@ function rebaseCopySourceRow(row: typeof schema.copySources.$inferSelect, source export async function runPathMigration(db: Db, migrationId: number, ctx: PathMigrationRunContext): Promise { const { migration, source, target } = await migrationContext(db, migrationId); + const assertTargetRootIdentity = () => + assertConfigurationRootIdentity(target, "Managed storage root validation failed"); if (migration.status === "completed") { const itemCount = await first( db.select({ value: count() }).from(schema.pathMigrationItems).where(eq(schema.pathMigrationItems.migrationId, migrationId)) @@ -810,6 +1340,99 @@ export async function runPathMigration(db: Db, migrationId: number, ctx: PathMig await ctx.event("info", "Recovered completed managed path migration", { migrationId, total }); return; } + try { + await assertMigrationLease(ctx); + await assertTargetRootIdentity(); + await assertMigrationLease(ctx); + } catch (error: unknown) { + if (!(error instanceof PathMigrationRootIdentityError)) throw error; + await withMigrationLeaseDb( + ctx, + async (leaseDb) => { + const timestamp = nowIso(); + if (migration.status === "rollback_pending") { + await leaseDb + .update(schema.pathMigrations) + .set({ errorMessage: `${error.message} Automatic rollback was not attempted because the recorded target root is no longer trustworthy.` }) + .where(and(eq(schema.pathMigrations.id, migrationId), eq(schema.pathMigrations.status, "rollback_pending"))); + } else { + await leaseDb + .update(schema.pathMigrations) + .set({ status: "failed", errorMessage: error.message, finishedAt: timestamp }) + .where(and(eq(schema.pathMigrations.id, migrationId), inArray(schema.pathMigrations.status, ["queued", "running"]))); + } + }, + true + ); + throw error; + } + if (migration.status === "rollback_pending") { + const totalRow = await first( + db.select({ value: count() }).from(schema.pathMigrationItems).where(eq(schema.pathMigrationItems.migrationId, migrationId)) + ); + const total = Number(totalRow?.value ?? 0); + await ctx.event("warn", "Recovering interrupted path migration rollback", { migrationId, total }); + await ctx.setProgress({ migrationId, stage: "rollback_pending", current: 0, total, message: "Restoring symlinks to the active paths" }); + const rollbackErrors = await rollbackMigrationItems( + db, + migrationId, + target.symlinkDir, + ctx, + ["ready", "applied", "rolled_back", "blocked"], + assertTargetRootIdentity + ); + if (rollbackErrors.length > 0) { + const message = `Path migration rollback requires manual reconciliation for ${rollbackErrors.length} symlink(s).`; + await withMigrationLeaseDb( + ctx, + async (leaseDb) => { + await leaseDb.update(schema.pathMigrations).set({ errorMessage: message }).where(eq(schema.pathMigrations.id, migrationId)); + }, + true + ); + throw new Error(message); + } + const migrationJobId = ctx.jobId ?? migration.jobId; + if (migrationJobId == null) throw new Error("Rollback-pending path migration is not linked to a worker job"); + await withMigrationLeaseDb( + ctx, + async (leaseDb) => { + const remaining = await first( + leaseDb + .select({ value: count() }) + .from(schema.pathMigrationItems) + .where(and(eq(schema.pathMigrationItems.migrationId, migrationId), ne(schema.pathMigrationItems.validationStatus, "rolled_back"))) + ); + if (Number(remaining?.value ?? 0) > 0) throw new Error("Path migration rollback did not reconcile every item"); + const timestamp = nowIso(); + const cancelled = await first( + leaseDb + .update(schema.pathMigrations) + .set({ status: "cancelled", finishedAt: timestamp }) + .where(and(eq(schema.pathMigrations.id, migrationId), eq(schema.pathMigrations.status, "rollback_pending"))) + .returning({ id: schema.pathMigrations.id }) + ); + if (!cancelled) throw new Error("Path migration rollback state changed before recovery completed"); + await leaseDb.update(schema.pathConfigurations).set({ status: "cancelled" }).where(eq(schema.pathConfigurations.id, migration.targetConfigId)); + await leaseDb + .update(schema.jobs) + .set({ + cancelRequestedAt: timestamp, + progress: JSON.stringify({ migrationId, stage: "cancelled", current: total, total, message: "Path migration rollback completed" }) + }) + .where(eq(schema.jobs.id, migrationJobId)); + await leaseDb.insert(schema.jobEvents).values({ + jobId: migrationJobId, + timestamp, + level: "warn", + message: "Path migration rollback completed", + data: JSON.stringify({ migrationId, total }) + }); + }, + true + ); + return; + } if (!["queued", "running"].includes(migration.status)) throw new Error("Path migration is not queued"); const blocked = await first( db @@ -819,26 +1442,43 @@ export async function runPathMigration(db: Db, migrationId: number, ctx: PathMig ); if (Number(blocked?.value ?? 0) > 0) throw new Error("Path migration contains blocked symlinks"); - const runningJobs = await db - .select({ id: schema.jobs.id }) - .from(schema.jobs) - .where(and(eq(schema.jobs.status, "running"), ne(schema.jobs.id, migration.jobId ?? -1))); - if (runningJobs.length > 0) throw new Error("Another job is still running; wait for it to pause before migrating paths"); - - const cancelledJobs = await cancelQueuedJobsForMigration(db, migration.jobId ?? -1); - const runningMigration = await first( - db - .update(schema.pathMigrations) - .set({ status: "running", startedAt: migration.startedAt ?? nowIso(), errorMessage: null }) - .where(and(eq(schema.pathMigrations.id, migrationId), inArray(schema.pathMigrations.status, ["queued", "running"]))) - .returning({ id: schema.pathMigrations.id }) - ); - if (!runningMigration) throw new Error("Path migration was replaced before the worker started"); + const cancelledJobs = await withMigrationLeaseDb(ctx, async (leaseDb) => { + const runningJobs = await leaseDb + .select({ id: schema.jobs.id }) + .from(schema.jobs) + .where(and(eq(schema.jobs.status, "running"), ne(schema.jobs.id, migration.jobId ?? -1))); + if (runningJobs.length > 0) throw new Error("Another job is still running; wait for it to pause before migrating paths"); + const cancelled = await cancelQueuedJobsForMigration(leaseDb, migration.jobId ?? -1); + const runningMigration = await first( + leaseDb + .update(schema.pathMigrations) + .set({ status: "running", startedAt: migration.startedAt ?? nowIso(), errorMessage: null }) + .where(and(eq(schema.pathMigrations.id, migrationId), inArray(schema.pathMigrations.status, ["queued", "running"]))) + .returning({ id: schema.pathMigrations.id }) + ); + if (!runningMigration) throw new Error("Path migration was replaced before the worker started"); + return cancelled; + }); const items = await db.select().from(schema.pathMigrationItems).where(eq(schema.pathMigrationItems.migrationId, migrationId)).orderBy(asc(schema.pathMigrationItems.id)); const total = items.length; await ctx.event("warn", "Managed path migration started", { migrationId, total, cancelledJobs }); await ctx.setProgress({ migrationId, stage: "validating", current: 0, total, message: "Revalidating mapped symlinks" }); + const assertMappedTarget = async (item: typeof schema.pathMigrationItems.$inferSelect): Promise => { + const expectedIdentity = parseTargetIdentity(item.targetIdentity); + const currentIdentity = await readTargetIdentity(item.targetPathAfter); + if (!targetIdentitiesMatch(currentIdentity, expectedIdentity)) { + throw new Error(`Mapped target changed after analysis: ${item.targetPathAfter}`); + } + if (item.expectedSizeBytes != null && Number(currentIdentity.size) !== item.expectedSizeBytes) { + throw new Error(`Mapped target size no longer matches migration analysis: ${item.targetPathAfter}`); + } + }; + const assertMappedTargetAndRoot = async (item: typeof schema.pathMigrationItems.$inferSelect): Promise => { + await assertTargetRootIdentity(); + await assertMappedTarget(item); + }; + let completed = items.filter((item) => item.validationStatus === "applied").length; let committed = false; try { @@ -848,28 +1488,59 @@ export async function runPathMigration(db: Db, migrationId: number, ctx: PathMig const currentMigration = await first(db.select({ status: schema.pathMigrations.status }).from(schema.pathMigrations).where(eq(schema.pathMigrations.id, migrationId)).limit(1)); if (currentMigration?.status !== "running") throw new Error("Detected paths changed again while migration was running"); } - if (item.validationStatus === "applied") continue; + if (item.validationStatus === "applied") { + const currentTarget = await symlinkTarget(item.currentLinkPath); + await assertMappedTarget(item); + if (path.resolve(currentTarget) === path.resolve(item.targetPathAfter)) continue; + if (!item.targetChanged || path.resolve(currentTarget) !== path.resolve(item.targetPathBefore)) { + throw new Error(`Applied symlink changed after migration state was written: ${item.currentLinkPath}`); + } + await replaceSymlink( + target.symlinkDir, + item.currentLinkPath, + item.targetPathBefore, + item.targetPathAfter, + ctx, + false, + () => assertMappedTargetAndRoot(item) + ); + continue; + } if (item.validationStatus !== "ready") throw new Error(`Symlink is not ready for migration: ${item.currentLinkPath}`); if (item.targetChanged) { const currentTarget = await symlinkTarget(item.currentLinkPath); if (path.resolve(currentTarget) === path.resolve(item.targetPathBefore)) { - const destinationStat = await withTimeout(fs.stat(item.targetPathAfter), `Reading mapped target ${item.targetPathAfter}`, filesystemReadTimeoutMs); - if (!destinationStat.isFile()) throw new Error(`Mapped target is no longer a regular file: ${item.targetPathAfter}`); - if (item.expectedSizeBytes != null && Number(destinationStat.size) !== item.expectedSizeBytes) { - throw new Error(`Mapped target changed after analysis: ${item.targetPathAfter}`); - } - await replaceSymlink(target.symlinkDir, item.currentLinkPath, item.targetPathAfter); + await replaceSymlink( + target.symlinkDir, + item.currentLinkPath, + item.targetPathBefore, + item.targetPathAfter, + ctx, + false, + () => assertMappedTargetAndRoot(item) + ); } else if (path.resolve(currentTarget) !== path.resolve(item.targetPathAfter)) { throw new Error(`Symlink changed after analysis: ${item.currentLinkPath}`); } - } else { - await symlinkTarget(item.currentLinkPath); } completed += 1; - await db - .update(schema.pathMigrationItems) - .set({ validationStatus: "applied", appliedAt: nowIso(), rolledBackAt: null, message: "Migration step applied." }) - .where(eq(schema.pathMigrationItems.id, item.id)); + await withMigrationLeaseDb(ctx, async (leaseDb) => { + await assertMappedTarget(item); + if (!item.targetChanged) { + const currentTarget = await symlinkTarget(item.currentLinkPath); + if (path.resolve(currentTarget) !== path.resolve(item.targetPathAfter)) { + throw new Error(`Symlink changed after analysis: ${item.currentLinkPath}`); + } + } + const appliedItem = await first( + leaseDb + .update(schema.pathMigrationItems) + .set({ validationStatus: "applied", appliedAt: nowIso(), rolledBackAt: null, message: "Migration step applied." }) + .where(and(eq(schema.pathMigrationItems.id, item.id), eq(schema.pathMigrationItems.validationStatus, "ready"))) + .returning({ id: schema.pathMigrationItems.id }) + ); + if (!appliedItem) throw new Error(`Path migration item changed before it could be applied: ${item.currentLinkPath}`); + }); if (completed === total || completed % 25 === 0) { await ctx.setProgress({ migrationId, stage: "repointing", current: completed, total, message: `Validated and migrated ${completed} of ${total} symlinks` }); } @@ -877,14 +1548,19 @@ export async function runPathMigration(db: Db, migrationId: number, ctx: PathMig const sourcePaths = pathsFromConfiguration(source); const targetPaths = pathsFromConfiguration(target); - const copySources = await db.select().from(schema.copySources); - await db.transaction(async (transaction) => { - await transaction.execute(sql`select pg_advisory_xact_lock(${pathConfigurationLockKey})`); + const migrationJobId = ctx.jobId ?? migration.jobId; + if (ctx.signal.aborted) { + throw (ctx.signal.reason instanceof Error ? ctx.signal.reason : new Error("Path migration was terminated")); + } + const finalized = await ctx.finishCompleted(async (leaseDb) => { + await leaseDb.execute(sql`select pg_advisory_xact_lock(${pathConfigurationLockKey})`); const currentMigration = await first( - transaction.select({ status: schema.pathMigrations.status }).from(schema.pathMigrations).where(eq(schema.pathMigrations.id, migrationId)).limit(1) + leaseDb.select({ status: schema.pathMigrations.status }).from(schema.pathMigrations).where(eq(schema.pathMigrations.id, migrationId)).limit(1) ); if (currentMigration?.status !== "running") throw new Error("Detected paths changed again before migration could be committed"); - await transaction.execute(sql` + await assertConfigurationRootIdentity(target, "Managed storage roots changed before migration commit"); + await mapWithConcurrency(items, 8, assertMappedTarget); + await leaseDb.execute(sql` UPDATE media_links AS media SET link_path = item.link_path_after, target_path = item.target_path_after, @@ -895,44 +1571,79 @@ export async function runPathMigration(db: Db, migrationId: number, ctx: PathMig AND item.media_link_id = media.id AND item.validation_status = 'applied' `); - await rebaseStorageInventory(transaction, "local", sourcePaths.localDir, targetPaths.localDir); - await rebaseStorageInventory(transaction, "remote", sourcePaths.remoteDir, targetPaths.remoteDir); - await rebaseMissingMediaLinks(transaction, "link_path", sourcePaths.symlinkDir, targetPaths.symlinkDir); - await rebaseMissingMediaLinks(transaction, "target_path", sourcePaths.localDir, targetPaths.localDir); - await rebaseMissingMediaLinks(transaction, "target_path", sourcePaths.remoteDir, targetPaths.remoteDir); + await rebaseStorageInventory(leaseDb, "local", sourcePaths.localDir, targetPaths.localDir); + await rebaseStorageInventory(leaseDb, "remote", sourcePaths.remoteDir, targetPaths.remoteDir); + await rebaseMissingMediaLinks(leaseDb, "link_path", sourcePaths.symlinkDir, targetPaths.symlinkDir); + await rebaseMissingMediaLinks(leaseDb, "target_path", sourcePaths.localDir, targetPaths.localDir); + await rebaseMissingMediaLinks(leaseDb, "target_path", sourcePaths.remoteDir, targetPaths.remoteDir); + const copySources = await leaseDb.select().from(schema.copySources); for (const row of copySources) { const rebased = rebaseCopySourceRow(row, sourcePaths, targetPaths); - await transaction + await leaseDb .update(schema.copySources) .set({ sourcePath: rebased.sourcePath, destinationPath: rebased.destinationPath, linkPath: rebased.linkPath }) .where(eq(schema.copySources.id, row.id)); } const timestamp = nowIso(); - await transaction.update(schema.pathConfigurations).set({ status: "superseded" }).where(eq(schema.pathConfigurations.id, source.id)); - await transaction.update(schema.pathConfigurations).set({ status: "active", appliedAt: timestamp }).where(eq(schema.pathConfigurations.id, target.id)); - await transaction + await leaseDb.update(schema.pathConfigurations).set({ status: "superseded" }).where(eq(schema.pathConfigurations.id, source.id)); + await leaseDb.update(schema.pathConfigurations).set({ status: "active", appliedAt: timestamp }).where(eq(schema.pathConfigurations.id, target.id)); + await leaseDb .insert(schema.appSettings) .values({ key: "paths", value: JSON.stringify(targetPaths), updatedAt: timestamp }) .onConflictDoUpdate({ target: schema.appSettings.key, set: { value: JSON.stringify(targetPaths), updatedAt: timestamp } }); - await transaction + await leaseDb .update(schema.pathMigrations) .set({ status: "completed", finishedAt: timestamp, errorMessage: null }) .where(eq(schema.pathMigrations.id, migrationId)); + if (migrationJobId != null) { + await leaseDb + .update(schema.jobs) + .set({ progress: JSON.stringify({ migrationId, stage: "completed", current: total, total, message: "Path migration completed" }) }) + .where(eq(schema.jobs.id, migrationJobId)); + await leaseDb.insert(schema.jobEvents).values({ + jobId: migrationJobId, + timestamp, + level: "info", + message: "Managed path migration completed", + data: JSON.stringify({ migrationId, total }) + }); + } }); + if (!finalized) throw new Error("Path migration was terminated before its final commit"); committed = true; } catch (error: unknown) { if (!committed) { - const rollbackErrors = await rollbackMigrationItems(db, migrationId, target.symlinkDir, ctx); + if (error instanceof PathMigrationLeaseLostError) throw error; + await assertMigrationLease(ctx); + if (error instanceof PathMigrationRootIdentityError) { + const message = `${error.message} Automatic rollback was not attempted because the recorded target roots are no longer trustworthy.`; + await withMigrationLeaseDb( + ctx, + async (leaseDb) => { + await leaseDb + .update(schema.pathMigrations) + .set({ status: "failed", errorMessage: message, finishedAt: nowIso() }) + .where(eq(schema.pathMigrations.id, migrationId)); + }, + true + ); + throw new Error(message, { cause: error }); + } + const rollbackErrors = await rollbackMigrationItems(db, migrationId, target.symlinkDir, ctx, undefined, assertTargetRootIdentity); + await assertMigrationLease(ctx); const message = rollbackErrors.length > 0 ? `${errorMessage(error)} Rollback also failed for ${rollbackErrors.length} symlink(s).` : errorMessage(error); const failureFilter = rollbackErrors.length > 0 ? eq(schema.pathMigrations.id, migrationId) : and(eq(schema.pathMigrations.id, migrationId), eq(schema.pathMigrations.status, "running")); - await db.update(schema.pathMigrations).set({ status: "failed", errorMessage: message, finishedAt: nowIso() }).where(failureFilter); + await withMigrationLeaseDb( + ctx, + async (leaseDb) => { + await leaseDb.update(schema.pathMigrations).set({ status: "failed", errorMessage: message, finishedAt: nowIso() }).where(failureFilter); + }, + true + ); throw new Error(message, { cause: error }); } throw error; } - - await ctx.setProgress({ migrationId, stage: "completed", current: total, total, message: "Path migration completed" }).catch(() => undefined); - await ctx.event("info", "Managed path migration completed", { migrationId, total }).catch(() => undefined); } diff --git a/src/server/lib/scanner.ts b/src/server/lib/scanner.ts index 6862a82..991c59d 100644 --- a/src/server/lib/scanner.ts +++ b/src/server/lib/scanner.ts @@ -814,7 +814,7 @@ export async function persistScanResult(db: Db, result: ScanResult, jobId: numbe } } - await reconcileResolvedStorageFiles(db, timestamp); + await reconcileResolvedStorageFiles(db, timestamp, result.options.titleScopes?.length ? seenLinkPaths : undefined); await throwIfPersistenceCancelled(isCancelled); await applyPendingOnboardingPolicy(db, jobId); await throwIfPersistenceCancelled(isCancelled); @@ -879,11 +879,11 @@ export async function persistScanResult(db: Db, result: ScanResult, jobId: numbe }; } -async function reconcileResolvedStorageFiles(db: Db, timestamp: string): Promise { +async function reconcileResolvedStorageFiles(db: Db, timestamp: string, scopedLinkPaths?: ReadonlySet): Promise { const currentStorageFiles = (await db.select().from(schema.storageFiles)).filter((file) => !file.missingSince); const storageFileIdByPath = new Map(currentStorageFiles.map((file) => [file.filePath, file.id])); for (const link of await db.select().from(schema.mediaLinks)) { - if (link.missingSince) continue; + if (link.missingSince || (scopedLinkPaths && !scopedLinkPaths.has(link.linkPath))) continue; const resolvedStorageFileId = storageFileIdByPath.get(link.targetPath) ?? null; if (link.resolvedStorageFileId !== resolvedStorageFileId) { await db.update(schema.mediaLinks).set({ resolvedStorageFileId, updatedAt: timestamp }).where(eq(schema.mediaLinks.id, link.id)); diff --git a/src/server/lib/storagePolicies.ts b/src/server/lib/storagePolicies.ts index 0ab13b4..37e3ea0 100644 --- a/src/server/lib/storagePolicies.ts +++ b/src/server/lib/storagePolicies.ts @@ -324,13 +324,28 @@ async function assignLocalPolicy( }); } -async function syncStoragePolicyMediaLinksForTitleKeys(db: Db, titlePolicies: Map, timestamp: string): Promise { +async function syncStoragePolicyMediaLinksForTitleKeys( + db: Db, + titlePolicies: Map, + timestamp: string, + mediaLinkIds?: readonly number[] +): Promise { if (titlePolicies.size === 0) return; - for (const link of await db.select().from(schema.mediaLinks)) { - const policy = titlePolicies.get(canonicalTitleKey(link.itemName)); - if (!policy) continue; - if (link.storagePolicy === policy) continue; - await db.update(schema.mediaLinks).set({ storagePolicy: policy, updatedAt: timestamp }).where(eq(schema.mediaLinks.id, link.id)); + const policies = [...new Set(titlePolicies.values())]; + if (mediaLinkIds !== undefined && policies.length === 1) { + const uniqueIds = [...new Set(mediaLinkIds)]; + for (let offset = 0; offset < uniqueIds.length; offset += 500) { + await db + .update(schema.mediaLinks) + .set({ storagePolicy: policies[0]!, updatedAt: timestamp }) + .where(inArray(schema.mediaLinks.id, uniqueIds.slice(offset, offset + 500))); + } + } else { + for (const link of await db.select().from(schema.mediaLinks)) { + const policy = titlePolicies.get(canonicalTitleKey(link.itemName)); + if (!policy || link.storagePolicy === policy) continue; + await db.update(schema.mediaLinks).set({ storagePolicy: policy, updatedAt: timestamp }).where(eq(schema.mediaLinks.id, link.id)); + } } for (const file of await db.select().from(schema.storageFiles)) { const policy = titlePolicies.get(canonicalTitleKey(file.itemName)); @@ -470,7 +485,7 @@ export async function setStoragePolicyTitles( db: Db, titles: string[], policy: StoragePolicyKind, - options: { source?: string } = {} + options: { source?: string; mediaLinkIds?: readonly number[] } = {} ): Promise { const uniqueTitles = uniqueStoragePolicyTitles(titles); const timestamp = nowIso(); @@ -529,26 +544,8 @@ export async function setStoragePolicyTitles( } } - const policyRows = sql.join( - assignments.map((assignment) => sql`(${assignment.normalizedTitle}::text, ${policy}::text)`), - sql`, ` - ); - await transaction.execute(sql` - update media_links as ml - set storage_policy = policy_map.policy, - updated_at = ${timestamp} - from (values ${policyRows}) as policy_map(normalized_title, policy) - where lower(btrim(ml.item_name)) = policy_map.normalized_title - and ml.storage_policy <> policy_map.policy - `); - await transaction.execute(sql` - update storage_files as sf - set storage_policy = policy_map.policy, - updated_at = ${timestamp} - from (values ${policyRows}) as policy_map(normalized_title, policy) - where lower(btrim(sf.item_name)) = policy_map.normalized_title - and sf.storage_policy <> policy_map.policy - `); + const titlePolicies = new Map(assignments.map((assignment) => [canonicalTitleKey(assignment.title), policy])); + await syncStoragePolicyMediaLinksForTitleKeys(transaction, titlePolicies, timestamp, options.mediaLinkIds); }); } @@ -567,8 +564,13 @@ export async function setStoragePolicyTitles( return { updated: items.length, policy, items }; } -export async function setStoragePolicyTitle(db: Db, title: string, policy: StoragePolicyKind): Promise { - return (await setStoragePolicyTitles(db, [title], policy)).items[0] ?? { +export async function setStoragePolicyTitle( + db: Db, + title: string, + policy: StoragePolicyKind, + options: { source?: string; mediaLinkIds?: readonly number[] } = {} +): Promise { + return (await setStoragePolicyTitles(db, [title], policy, options)).items[0] ?? { id: null, title, normalizedTitle: normalizeTitle(title), diff --git a/src/server/lib/workerHeartbeats.ts b/src/server/lib/workerHeartbeats.ts new file mode 100644 index 0000000..cacf3e6 --- /dev/null +++ b/src/server/lib/workerHeartbeats.ts @@ -0,0 +1,44 @@ +import { and, eq, lt, sql } from "drizzle-orm"; +import type { Db } from "../db/database"; +import * as schema from "../db/schema"; + +export const workerHeartbeatRetentionMs = 24 * 60 * 60 * 1_000; + +export interface WorkerHeartbeat { + workerId: string; + startedAt: string; + heartbeatAt: string; + status: "running" | "stopped"; + capacity: number; +} + +export async function recordWorkerHeartbeat(db: Db, heartbeat: WorkerHeartbeat): Promise { + await db + .insert(schema.workerHeartbeats) + .values(heartbeat) + .onConflictDoUpdate({ + target: schema.workerHeartbeats.workerId, + set: { + heartbeatAt: heartbeat.heartbeatAt, + status: heartbeat.status, + capacity: heartbeat.capacity + } + }); +} + +export async function pruneWorkerHeartbeatHistory(db: Db, nowMs = Date.now()): Promise { + const cutoff = new Date(nowMs - workerHeartbeatRetentionMs).toISOString(); + await db.execute(sql` + delete from worker_heartbeats + where status = 'stopped' + and not exists ( + select 1 + from jobs + where jobs.status = 'running' + and jobs.locked_by = worker_heartbeats.worker_id + ) + `); + await db + .delete(schema.workerHeartbeats) + .where(and(eq(schema.workerHeartbeats.status, "running"), lt(schema.workerHeartbeats.heartbeatAt, cutoff))); +} diff --git a/src/server/routes/libraryRoutes.ts b/src/server/routes/libraryRoutes.ts index c868ad6..9587c42 100644 --- a/src/server/routes/libraryRoutes.ts +++ b/src/server/routes/libraryRoutes.ts @@ -4,6 +4,7 @@ import { desc, eq, inArray } from "drizzle-orm"; import { first, type Db } from "../db/database"; import * as schema from "../db/schema"; import type { JobRunner } from "../jobs/jobRunner"; +import { storagePolicyMutationResources, withResourceMutationGuard } from "../jobs/resourceMutationGuard"; import { findStoragePolicyCandidateTitle, findStoragePolicyCandidateTitles, @@ -56,6 +57,23 @@ const mediaLinkLookupInputSchema = z.object({ ids: z.array(z.coerce.number().int().positive()).max(1000) }); +class StoragePolicyRequestError extends Error { + constructor( + readonly statusCode: number, + message: string + ) { + super(message); + this.name = "StoragePolicyRequestError"; + } +} + +function mediaLinkIdsFromMutationResources(resources: Array<{ resourceType: string; resourceKey: string }>): number[] { + return resources + .filter((resource) => resource.resourceType === "media") + .map((resource) => Number(resource.resourceKey)) + .filter((id) => Number.isSafeInteger(id) && id > 0); +} + const copyInputSchema = z .object({ direction: z.enum(["to_local", "to_remote"]), @@ -285,31 +303,54 @@ export function registerLibraryRoutes(app: FastifyInstance, db: Db, jobs: JobRun return listStoragePolicyCandidates(db, query.q, query.limit); }); - app.post("/api/storage-policies", async (request, reply) => { + app.post("/api/storage-policies", async (request) => { const body = storagePolicyInputSchema.parse(request.body); - const candidateTitle = await findStoragePolicyCandidateTitle(db, body.title); - if (!candidateTitle) { - return reply.code(400).send({ error: "Choose a title from the scanned library." }); - } - return setStoragePolicyTitle(db, candidateTitle, body.policy as StoragePolicyKind); + return withResourceMutationGuard(db, async (transaction) => { + const candidateTitle = await findStoragePolicyCandidateTitle(transaction, body.title); + if (!candidateTitle) throw new StoragePolicyRequestError(400, "Choose a title from the scanned library."); + const resources = await storagePolicyMutationResources(transaction, [candidateTitle]); + return { + resources, + mutate: () => + setStoragePolicyTitle(transaction, candidateTitle, body.policy as StoragePolicyKind, { + mediaLinkIds: mediaLinkIdsFromMutationResources(resources) + }) + }; + }); }); - app.post("/api/storage-policies/bulk", async (request, reply) => { + app.post("/api/storage-policies/bulk", async (request) => { const body = storagePolicyBulkInputSchema.parse(request.body); - const { candidateTitles, invalidTitles } = await findStoragePolicyCandidateTitles(db, body.titles); - - if (invalidTitles.length > 0) { - return reply.code(400).send({ error: `Choose titles from the scanned library: ${invalidTitles.join(", ")}` }); - } - - return setStoragePolicyTitles(db, candidateTitles, body.policy as StoragePolicyKind); + return withResourceMutationGuard(db, async (transaction) => { + const { candidateTitles, invalidTitles } = await findStoragePolicyCandidateTitles(transaction, body.titles); + if (invalidTitles.length > 0) { + throw new StoragePolicyRequestError(400, `Choose titles from the scanned library: ${invalidTitles.join(", ")}`); + } + const resources = await storagePolicyMutationResources(transaction, candidateTitles); + return { + resources, + mutate: () => + setStoragePolicyTitles(transaction, candidateTitles, body.policy as StoragePolicyKind, { + mediaLinkIds: mediaLinkIdsFromMutationResources(resources) + }) + }; + }); }); - app.delete("/api/storage-policies/:id", async (request, reply) => { + app.delete("/api/storage-policies/:id", async (request) => { const params = z.object({ id: z.coerce.number().int().positive() }).parse(request.params); - const row = await removeStoragePolicyTitle(db, params.id); - if (!row) return reply.code(404).send({ error: "Storage policy item not found" }); - return row; + return withResourceMutationGuard(db, async (transaction) => { + const existing = await first(transaction.select().from(schema.storagePolicies).where(eq(schema.storagePolicies.id, params.id)).limit(1)); + if (!existing) throw new StoragePolicyRequestError(404, "Storage policy item not found"); + return { + resources: await storagePolicyMutationResources(transaction, [existing.normalizedTitle]), + mutate: async () => { + const row = await removeStoragePolicyTitle(transaction, params.id); + if (!row) throw new StoragePolicyRequestError(404, "Storage policy item not found"); + return row; + } + }; + }); }); } diff --git a/src/server/worker.ts b/src/server/worker.ts index b1b6e48..5d2a7a3 100644 --- a/src/server/worker.ts +++ b/src/server/worker.ts @@ -1,44 +1,42 @@ +import { randomUUID } from "node:crypto"; import os from "node:os"; import { loadConfig } from "./config"; import { nowIso, openDatabase } from "./db/database"; -import * as schema from "./db/schema"; +import { CopyTransferLimiter } from "./jobs/copyLimiter"; import { JobWorker } from "./jobs/jobRunner"; import { reconcileEnvironmentPaths } from "./lib/pathConfiguration"; +import { pruneWorkerHeartbeatHistory, recordWorkerHeartbeat } from "./lib/workerHeartbeats"; // Media outputs must remain readable by services running under a different account. process.umask(0o022); const config = loadConfig(); const database = await openDatabase({ databaseUrl: config.databaseUrl, migrate: config.autoMigrate }); +await pruneWorkerHeartbeatHistory(database.db); await reconcileEnvironmentPaths(database.db, config.paths); -const workerId = process.env.SRTL_WORKER_ID ?? `${os.hostname()}-${process.pid}`; +const workerBaseId = process.env.SRTL_WORKER_ID?.trim() || `${os.hostname()}-${process.pid}`; +const bootId = randomUUID(); +const workerId = `${workerBaseId}:${bootId}`; +const copyTransferLimiter = new CopyTransferLimiter(config.jobConcurrency.maxActiveCopyFiles); const startedAt = nowIso(); - -async function recordHeartbeat(status: "running" | "stopped"): Promise { - const heartbeatAt = nowIso(); - await database.db - .insert(schema.workerHeartbeats) - .values({ workerId, startedAt, heartbeatAt, status }) - .onConflictDoUpdate({ - target: schema.workerHeartbeats.workerId, - set: { heartbeatAt, status } - }); -} - -await recordHeartbeat("running"); -const heartbeatTimer = setInterval(() => { - void recordHeartbeat("running").catch((error: unknown) => { - console.error("Worker heartbeat failed", error); - }); -}, 5_000); -heartbeatTimer.unref(); - const worker = new JobWorker(database.db, { workerId, - concurrency: config.jobConcurrency + concurrency: config.jobConcurrency, + copyTransferLimiter, + dispatchConcurrency: config.jobConcurrency.maxRunningJobs }); +async function recordHeartbeat(status: "running" | "stopped"): Promise { + await recordWorkerHeartbeat(database.db, { + workerId, + startedAt, + heartbeatAt: nowIso(), + status, + capacity: config.jobConcurrency.maxRunningJobs + }); +} + let shuttingDown = false; function shutdown(): void { if (shuttingDown) return; @@ -49,10 +47,35 @@ function shutdown(): void { process.once("SIGINT", shutdown); process.once("SIGTERM", shutdown); +let heartbeatTimer: NodeJS.Timeout | null = null; +let heartbeatRun: Promise | null = null; +let workerRun: Promise | null = null; try { - await worker.start(); + await recordHeartbeat("running"); + // A shutdown signal may arrive while the initial database write is in flight. + // Starting afterward would clear JobWorker's stopped state and resume claims. + if (!shuttingDown) { + heartbeatTimer = setInterval(() => { + if (shuttingDown || heartbeatRun) return; + heartbeatRun = recordHeartbeat("running") + .catch((error: unknown) => { + console.error("Worker heartbeat failed", error); + }) + .finally(() => { + heartbeatRun = null; + }); + }, 5_000); + heartbeatTimer.unref(); + workerRun = worker.start(); + await workerRun; + } } finally { - clearInterval(heartbeatTimer); - await recordHeartbeat("stopped").catch(() => undefined); + shutdown(); + if (heartbeatTimer) clearInterval(heartbeatTimer); + if (workerRun) await Promise.allSettled([workerRun]); + await heartbeatRun; + await recordHeartbeat("stopped").catch((error: unknown) => { + console.error("Unable to record stopped worker status", error); + }); await database.close(); } diff --git a/src/shared/types.ts b/src/shared/types.ts index 7275ab3..5ed4a07 100644 --- a/src/shared/types.ts +++ b/src/shared/types.ts @@ -57,11 +57,19 @@ export type ManagedPathRoot = "symlink" | "local" | "remote"; export type PathRootIdentityMatch = "same" | "different" | "unknown"; +export interface PathMountIdentity { + mountPoint: string; + root: string; + filesystemType: string; + source: string; +} + export interface PathRootIdentity { available: boolean; realPath: string | null; device: string | null; inode: string | null; + mount: PathMountIdentity | null; error: string | null; } @@ -76,7 +84,7 @@ export interface PathRootChange { detectedIdentity: PathRootIdentity | null; } -export type PathMigrationStatus = "pending" | "planning" | "planned" | "queued" | "running" | "failed" | "completed" | "cancelled"; +export type PathMigrationStatus = "pending" | "planning" | "planned" | "queued" | "running" | "rollback_pending" | "failed" | "completed" | "cancelled"; export interface PathMigrationIssue { id: number; diff --git a/tests/app.test.ts b/tests/app.test.ts index caddb2c..927f396 100644 --- a/tests/app.test.ts +++ b/tests/app.test.ts @@ -2,13 +2,14 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { eq } from "drizzle-orm"; +import { eq, inArray, sql } from "drizzle-orm"; import { createApp, type AppContext } from "../src/server/app"; import { first, getJsonSetting, setSetting } from "../src/server/db/database"; import * as schema from "../src/server/db/schema"; +import { CopyTransferLimiter } from "../src/server/jobs/copyLimiter"; import { JobWorker } from "../src/server/jobs/jobRunner"; import type { AuditCommandRunner } from "../src/server/lib/auditor"; -import type { CopyCommandRunner, CopyFileProgressReporter } from "../src/server/lib/copier"; +import { readCopyFileIdentity, serializeCopyFileIdentity, type CopyCommandRunner, type CopyFileProgressReporter } from "../src/server/lib/copier"; import { reconcileEnvironmentPaths } from "../src/server/lib/pathConfiguration"; import { markOnboardingCompleteForExistingInstall } from "../src/server/lib/onboarding"; import { bootstrapLocalStoragePolicies, normalizeTitle } from "../src/server/lib/storagePolicies"; @@ -817,7 +818,7 @@ describe("api app", () => { expect(version.statusCode).toBe(200); expect(version.json()).toMatchObject({ - currentVersion: "0.1.2-beta.2", + currentVersion: "0.1.2-beta.3", currentChannel: "beta", currentChannelLabel: "Beta", latestVersion: null, @@ -884,7 +885,7 @@ describe("api app", () => { expect(version.statusCode).toBe(200); expect(version.json()).toMatchObject({ - currentVersion: "0.1.2-beta.2", + currentVersion: "0.1.2-beta.3", currentChannel: "beta", currentChannelLabel: "Beta", latestVersion: "0.2.0-beta.1", @@ -1512,6 +1513,64 @@ describe("api app", () => { expect(results[0]?.linkPath).toContain(path.join("Scoped Show", "Season 01", "episode-1.mkv")); }); + it("freezes scoped audit media when the job is queued", async () => { + const firstFixture = await insertCopySymlink({ + itemName: "Frozen Audit Show", + kind: "remote", + storagePolicy: "unassigned", + section: "shows", + relativePath: path.join("Frozen Audit Show", "Season 01", "episode-1.mkv") + }); + const jobId = await ctx.jobs.startAudit({ mode: "fast", section: "shows", itemName: "Frozen Audit Show" }); + const secondFixture = await insertCopySymlink({ + itemName: "Frozen Audit Show", + kind: "remote", + storagePolicy: "unassigned", + section: "shows", + relativePath: path.join("Frozen Audit Show", "Season 01", "episode-2.mkv") + }); + const auditedPaths: string[] = []; + const auditRunner: AuditCommandRunner = { + runFfmpeg: async (_mode, targetPath) => { + auditedPaths.push(targetPath); + return { status: "pass", output: "" }; + }, + runCmp: async () => ({ status: "pass", output: "" }) + }; + + await expect(runQueuedJob(jobId, { auditRunner })).resolves.toMatchObject({ + status: "completed", + progress: expect.objectContaining({ options: expect.objectContaining({ linkIds: [firstFixture.id] }), checked: 1, total: 1 }) + }); + expect(auditedPaths).toEqual([firstFixture.sourcePath]); + expect(auditedPaths).not.toContain(secondFixture.sourcePath); + }); + + it("keeps an empty scoped audit empty when matching inventory appears later", async () => { + const jobId = await ctx.jobs.startAudit({ mode: "fast", section: "shows", itemName: "Later Audit Show" }); + await insertCopySymlink({ + itemName: "Later Audit Show", + kind: "remote", + storagePolicy: "unassigned", + section: "shows", + relativePath: path.join("Later Audit Show", "Season 01", "episode-1.mkv") + }); + let auditCalls = 0; + const auditRunner: AuditCommandRunner = { + runFfmpeg: async () => { + auditCalls += 1; + return { status: "pass", output: "" }; + }, + runCmp: async () => ({ status: "pass", output: "" }) + }; + + await expect(runQueuedJob(jobId, { auditRunner })).resolves.toMatchObject({ + status: "completed", + progress: expect.objectContaining({ options: expect.objectContaining({ linkIds: [] }), checked: 0, total: 0 }) + }); + expect(auditCalls).toBe(0); + }); + it("copies assign-local remote symlinks to local storage with verification", async () => { const cookie = await createAdminSession(); const fixture = await insertCopySymlink({ itemName: "Copy Local Movie", kind: "remote", storagePolicy: "location_1", content: "copy me local" }); @@ -1616,6 +1675,8 @@ describe("api app", () => { await fs.rm(fixture.linkPath); await fs.symlink(fixture.destinationPath, fixture.linkPath); const sizeBytes = (await fs.stat(fixture.destinationPath)).size; + const destinationIdentity = await readCopyFileIdentity(fixture.destinationPath); + if (!destinationIdentity) throw new Error("Interrupted copy destination identity was not readable"); const timestamp = new Date().toISOString(); await ctx.database.db.insert(schema.copyOperations).values({ jobId, @@ -1628,6 +1689,9 @@ describe("api app", () => { previousCopySource: null, tempPath: null, displacedPath: null, + tempIdentity: null, + destinationIdentity: serializeCopyFileIdentity(destinationIdentity), + displacedIdentity: null, stage: "repointed", resultStatus: "copied", localConflictStrategy: null, @@ -1658,6 +1722,151 @@ describe("api app", () => { ); }); + it("preserves an unowned replacement without locking a newly scanned link from the same title", async () => { + const cookie = await createAdminSession(); + const fixture = await insertCopySymlink({ itemName: "Interrupted Identity Copy", kind: "remote", storagePolicy: "location_1", content: "durable copy" }); + const relatedFixture = await insertCopySymlink({ + itemName: "Interrupted Identity Copy", + kind: "remote", + storagePolicy: "location_1", + relativePath: path.join("Interrupted Identity Copy", "related.mkv"), + content: "related durable copy" + }); + const jobId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: [fixture.id] }); + const originalLink = await first(ctx.database.db.select().from(schema.mediaLinks).where(eq(schema.mediaLinks.id, fixture.id)).limit(1)); + if (!originalLink) throw new Error("Copy fixture was not found"); + + await fs.mkdir(path.dirname(fixture.destinationPath), { recursive: true }); + await fs.copyFile(fixture.sourcePath, fixture.destinationPath); + await fs.rm(fixture.linkPath); + await fs.symlink(fixture.destinationPath, fixture.linkPath); + const journaledIdentity = await readCopyFileIdentity(fixture.destinationPath); + if (!journaledIdentity) throw new Error("Interrupted copy destination identity was not readable"); + const timestamp = new Date().toISOString(); + await ctx.database.db.insert(schema.copyOperations).values({ + jobId, + mediaLinkId: fixture.id, + linkPath: fixture.linkPath, + sourcePath: fixture.sourcePath, + destinationPath: fixture.destinationPath, + originalTargetPath: fixture.sourcePath, + originalLinkState: JSON.stringify(originalLink), + previousCopySource: null, + tempPath: null, + displacedPath: null, + tempIdentity: null, + destinationIdentity: serializeCopyFileIdentity(journaledIdentity), + displacedIdentity: null, + stage: "repointed", + resultStatus: "copied", + localConflictStrategy: null, + sizeBytes: Buffer.byteLength("durable copy"), + errorMessage: null, + createdAt: timestamp, + updatedAt: timestamp, + completedAt: null + }); + + const replacementPath = `${fixture.destinationPath}.replacement`; + await fs.writeFile(replacementPath, "foreign file"); + await fs.rename(replacementPath, fixture.destinationPath); + + await expect(runQueuedJob(jobId, { copyRunner: testCopyRunner })).resolves.toMatchObject({ status: "failed" }); + await expect(fs.readFile(fixture.destinationPath, "utf8")).resolves.toBe("foreign file"); + await expect(fs.readlink(fixture.linkPath)).resolves.toBe(fixture.destinationPath); + await expect(first(ctx.database.db.select().from(schema.copyOperations).where(eq(schema.copyOperations.jobId, jobId)).limit(1))).resolves.toMatchObject({ + stage: "reconciliation_required", + errorMessage: expect.stringContaining("changed after its file identity was journaled") + }); + await expect(ctx.jobs.startCopy({ direction: "to_local", linkIds: [fixture.id] })).rejects.toThrow( + `Copy data from job #${jobId} requires manual reconciliation` + ); + await expect(ctx.jobs.startAudit({ mode: "fast", linkIds: [fixture.id], byteCompare: false })).rejects.toThrow( + `Copy data from job #${jobId} requires manual reconciliation` + ); + const relatedJobId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: [relatedFixture.id] }); + await expect(ctx.jobs.getJob(relatedJobId)).resolves.toMatchObject({ status: "queued" }); + await expect(ctx.jobs.terminate(relatedJobId)).resolves.toBe(true); + const policyMutation = await ctx.app.inject({ + method: "POST", + url: "/api/storage-policies", + headers: { cookie }, + payload: { title: "Interrupted Identity Copy", policy: "location_2" } + }); + expect(policyMutation.statusCode).toBe(409); + expect(policyMutation.json()).toMatchObject({ error: expect.stringContaining(`copy data from job #${jobId} requires manual reconciliation`) }); + const scanResponse = await ctx.app.inject({ + method: "POST", + url: "/api/scans", + headers: { cookie }, + payload: { scanSymlinks: true, scanLocal: false, scanRemote: false } + }); + expect(scanResponse.statusCode).toBe(200); + const scanJobId = Number(scanResponse.json().jobId); + await expect(ctx.jobs.getJob(scanJobId)).resolves.toMatchObject({ status: "queued", exclusive: true }); + await expect(ctx.jobs.terminate(scanJobId)).resolves.toBe(true); + const auditJobId = await ctx.jobs.startAudit("fast"); + await expect(ctx.jobs.getJob(auditJobId)).resolves.toMatchObject({ status: "queued", exclusive: true }); + await expect(ctx.jobs.terminate(auditJobId)).resolves.toBe(true); + }); + + it("requires reconciliation when an already-restored displaced destination has been replaced", async () => { + const fixture = await insertCopySymlink({ + itemName: "Interrupted Displaced Restore", + kind: "remote", + storagePolicy: "location_1", + content: "copy source payload" + }); + const jobId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: [fixture.id] }); + const originalLink = await first(ctx.database.db.select().from(schema.mediaLinks).where(eq(schema.mediaLinks.id, fixture.id)).limit(1)); + if (!originalLink) throw new Error("Displaced-restore copy fixture was not found"); + + await fs.mkdir(path.dirname(fixture.destinationPath), { recursive: true }); + const displacedContents = "original local data"; + await fs.writeFile(fixture.destinationPath, displacedContents); + const displacedIdentity = await readCopyFileIdentity(fixture.destinationPath); + if (!displacedIdentity) throw new Error("Displaced destination identity was not readable"); + const displacedPath = `${fixture.destinationPath}.srtl-displaced`; + const replacementPath = `${fixture.destinationPath}.replacement`; + const replacementContents = "foreign local data!"; + expect(Buffer.byteLength(replacementContents)).toBe(Buffer.byteLength(displacedContents)); + await fs.writeFile(replacementPath, replacementContents); + await fs.rename(replacementPath, fixture.destinationPath); + + const timestamp = new Date().toISOString(); + await ctx.database.db.insert(schema.copyOperations).values({ + jobId, + mediaLinkId: fixture.id, + linkPath: fixture.linkPath, + sourcePath: fixture.sourcePath, + destinationPath: fixture.destinationPath, + originalTargetPath: fixture.sourcePath, + originalLinkState: JSON.stringify(originalLink), + previousCopySource: null, + tempPath: null, + displacedPath, + tempIdentity: null, + destinationIdentity: null, + displacedIdentity: serializeCopyFileIdentity(displacedIdentity), + stage: "destination_displaced", + resultStatus: null, + localConflictStrategy: "replace", + sizeBytes: Buffer.byteLength("copy source payload"), + errorMessage: null, + createdAt: timestamp, + updatedAt: timestamp, + completedAt: null + }); + + await expect(runQueuedJob(jobId, { copyRunner: testCopyRunner })).resolves.toMatchObject({ status: "failed" }); + await expect(fs.readFile(fixture.destinationPath, "utf8")).resolves.toBe(replacementContents); + await expect(fs.readlink(fixture.linkPath)).resolves.toBe(fixture.sourcePath); + await expect(first(ctx.database.db.select().from(schema.copyOperations).where(eq(schema.copyOperations.jobId, jobId)).limit(1))).resolves.toMatchObject({ + stage: "reconciliation_required", + errorMessage: expect.stringContaining("Restored displaced destination changed after its file identity was journaled") + }); + }); + it("rejects duplicate copy jobs while matching media is already queued", async () => { const cookie = await createAdminSession(); const fixture = await insertCopySymlink({ itemName: "Duplicate Queue Movie", kind: "remote", storagePolicy: "location_1", content: "copy once" }); @@ -1682,6 +1891,34 @@ describe("api app", () => { expect((await ctx.jobs.listJobs()).filter((job) => job.type === "copy")).toHaveLength(1); }); + it("serializes concurrent duplicate job admission", async () => { + const fixture = await insertCopySymlink({ itemName: "Atomic Duplicate Movie", kind: "remote", storagePolicy: "location_1", content: "copy once atomically" }); + + const results = await Promise.allSettled([ + ctx.jobs.startCopy({ direction: "to_local", linkIds: [fixture.id] }), + ctx.jobs.startCopy({ direction: "to_local", linkIds: [fixture.id] }) + ]); + + expect(results.filter((result) => result.status === "fulfilled")).toHaveLength(1); + const rejected = results.find((result): result is PromiseRejectedResult => result.status === "rejected"); + expect(rejected?.reason).toBeInstanceOf(Error); + expect(rejected?.reason).toMatchObject({ message: expect.stringContaining("already queued") }); + expect((await ctx.jobs.listJobs()).filter((job) => job.type === "copy")).toHaveLength(1); + expect(await ctx.database.db.select().from(schema.jobResourceClaims)).not.toHaveLength(0); + }); + + it("keeps active copy claims immutable when the inventory row changes", async () => { + const fixture = await insertCopySymlink({ itemName: "Immutable Claim Movie", kind: "remote", storagePolicy: "location_1", content: "claim snapshot" }); + const jobId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: [fixture.id] }); + await ctx.database.db + .update(schema.mediaLinks) + .set({ kind: "local", targetPath: fixture.destinationPath, updatedAt: new Date().toISOString() }) + .where(eq(schema.mediaLinks.id, fixture.id)); + + await expect(ctx.jobs.startCopy({ direction: "to_local", linkIds: [fixture.id] })).rejects.toThrow(`Job #${jobId} is already queued`); + expect((await ctx.jobs.listJobs()).filter((job) => job.type === "copy")).toHaveLength(1); + }); + it("queues targeted title rescans and blocks overlapping title actions", async () => { const cookie = await createAdminSession(); const fixture = await insertCopySymlink({ itemName: "Targeted Rescan Movie", kind: "remote", storagePolicy: "location_1", content: "rescan me" }); @@ -1761,6 +1998,22 @@ describe("api app", () => { expect((await ctx.jobs.listJobs()).filter((job) => job.type === "audit")).toHaveLength(0); }); + it("allows overlapping scoped audits to share read-only claims", async () => { + const fixture = await insertCopySymlink({ itemName: "Shared Audit Movie", kind: "remote", storagePolicy: "location_1", content: "audit together" }); + + const jobIds = await Promise.all([ + ctx.jobs.startAudit({ mode: "fast", linkIds: [fixture.id] }), + ctx.jobs.startAudit({ mode: "deep", linkIds: [fixture.id] }) + ]); + + expect(jobIds[0]).not.toBe(jobIds[1]); + const jobs = await Promise.all(jobIds.map((jobId) => ctx.jobs.getJob(jobId))); + expect(jobs).toEqual([expect.objectContaining({ status: "queued", exclusive: false }), expect.objectContaining({ status: "queued", exclusive: false })]); + const claims = await ctx.database.db.select().from(schema.jobResourceClaims).where(inArray(schema.jobResourceClaims.jobId, jobIds)); + expect(claims).not.toHaveLength(0); + expect(claims.every((claim) => claim.access === "shared")).toBe(true); + }); + it("uses advanced copy verification settings when copying media", async () => { const cookie = await createAdminSession(); const fixture = await insertCopySymlink({ itemName: "Deep Verify Movie", kind: "remote", storagePolicy: "location_1", content: "copy me deeply" }); @@ -2005,10 +2258,11 @@ describe("api app", () => { status: "completed", progress: expect.objectContaining({ options: expect.objectContaining({ direction: "to_local", section: "shows", itemName: "Scoped Copy Show", relativePathPrefix: "Scoped Copy Show/Season 01" }), - current: 1, - total: 1, + current: 2, + total: 2, copied: 1, - skipped: 0, + skipped: 1, + alreadyCompleted: 1, conflicts: 0, failed: 0 }) @@ -2034,6 +2288,30 @@ describe("api app", () => { ); }); + it("keeps an empty scoped copy empty when matching inventory appears later", async () => { + const jobId = await ctx.jobs.startCopy({ direction: "to_local", section: "movies", itemName: "Later Copy Movie" }); + const laterFixture = await insertCopySymlink({ + itemName: "Later Copy Movie", + kind: "remote", + storagePolicy: "location_1", + content: "must wait for a newly queued copy" + }); + + await expect(runQueuedJob(jobId, { copyRunner: testCopyRunner })).resolves.toMatchObject({ + status: "completed", + progress: expect.objectContaining({ + options: expect.objectContaining({ linkIds: [] }), + current: 0, + total: 0, + copied: 0, + failed: 0, + message: "No matching media found" + }) + }); + await expect(fs.stat(laterFixture.destinationPath)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(fs.readlink(laterFixture.linkPath)).resolves.toBe(laterFixture.sourcePath); + }); + it("resumes stale copy jobs without shrinking the original selected total", async () => { const fixtures = await Promise.all([ insertCopySymlink({ itemName: "Resume Copy One", kind: "remote", storagePolicy: "location_1", content: "resume one" }), @@ -2105,7 +2383,7 @@ describe("api app", () => { const events = await ctx.jobs.listEvents(jobId); expect(events).toEqual( expect.arrayContaining([ - expect.objectContaining({ message: "Stale running job reclaimed by worker" }), + expect.objectContaining({ message: "Stale running job lease fenced and requeued" }), expect.objectContaining({ message: "Copy job resumed", data: expect.objectContaining({ total: 3, remaining: 2, copied: 1, alreadyCompleted: 1 }) @@ -2232,6 +2510,111 @@ describe("api app", () => { await expect(fs.readlink(fixture.linkPath)).resolves.toBe(fixture.sourcePath); }); + it("blocks replacement when a claimed local candidate changes before the worker starts", async () => { + const fixture = await insertCopySymlink({ itemName: "Changed Candidate Movie", kind: "remote", storagePolicy: "location_1", content: "new candidate version" }); + const oldRelativePath = path.join("movies", "Changed Candidate Movie", "old-version.mkv"); + const oldPath = path.join(tmpDir, "local", oldRelativePath); + await fs.mkdir(path.dirname(oldPath), { recursive: true }); + await fs.writeFile(oldPath, "old candidate version"); + await insertStorageFile("local", oldRelativePath, Buffer.byteLength("old candidate version")); + const jobId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: [fixture.id], localConflictStrategy: "replace" }); + await fs.writeFile(oldPath, "externally changed candidate version"); + + await expect(runQueuedJob(jobId, { copyRunner: testCopyRunner })).resolves.toMatchObject({ + status: "completed", + progress: expect.objectContaining({ copied: 0, conflicts: 1, failed: 0 }) + }); + await expect(fs.readFile(oldPath, "utf8")).resolves.toBe("externally changed candidate version"); + await expect(fs.stat(fixture.destinationPath)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(fs.readlink(fixture.linkPath)).resolves.toBe(fixture.sourcePath); + await expect(ctx.jobs.listEvents(jobId)).resolves.toEqual( + expect.arrayContaining([expect.objectContaining({ message: "Local replacement candidates changed after copy admission" })]) + ); + }); + + it("blocks replacement when a new unclaimed local candidate appears after admission", async () => { + const fixture = await insertCopySymlink({ itemName: "Late Candidate Movie", kind: "remote", storagePolicy: "location_1", content: "new late candidate version" }); + const jobId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: [fixture.id], localConflictStrategy: "replace" }); + const latePath = path.join(tmpDir, "local", "movies", "Late Candidate Movie", "late-version.mkv"); + await fs.mkdir(path.dirname(latePath), { recursive: true }); + await fs.writeFile(latePath, "late external candidate"); + + await expect(runQueuedJob(jobId, { copyRunner: testCopyRunner })).resolves.toMatchObject({ + status: "completed", + progress: expect.objectContaining({ copied: 0, conflicts: 1, failed: 0 }) + }); + await expect(fs.readFile(latePath, "utf8")).resolves.toBe("late external candidate"); + await expect(fs.stat(fixture.destinationPath)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(fs.readlink(fixture.linkPath)).resolves.toBe(fixture.sourcePath); + }); + + it("keeps another selected destination out of replacement cleanup while replacing the current destination", async () => { + const first = await insertCopySymlink({ + itemName: "Shared Replacement Movie", + kind: "remote", + storagePolicy: "location_1", + relativePath: path.join("Shared Replacement Movie", "first.mkv"), + content: "first replacement" + }); + const second = await insertCopySymlink({ + itemName: "Shared Replacement Movie", + kind: "remote", + storagePolicy: "location_1", + relativePath: path.join("Shared Replacement Movie", "second.mkv"), + content: "second replacement" + }); + await fs.mkdir(path.dirname(second.destinationPath), { recursive: true }); + await fs.writeFile(second.destinationPath, "existing second destination"); + + await expect(ctx.jobs.previewCopyConflicts({ direction: "to_local", linkIds: [first.id, second.id] })).resolves.toMatchObject({ + totalConflicts: 1, + totalCandidates: 1, + conflicts: [ + expect.objectContaining({ + linkId: second.id, + candidates: [expect.objectContaining({ filePath: second.destinationPath, source: "destination" })] + }) + ] + }); + const jobId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: [first.id, second.id], localConflictStrategy: "replace" }); + + await expect(runQueuedJob(jobId, { copyRunner: testCopyRunner })).resolves.toMatchObject({ + status: "completed", + progress: expect.objectContaining({ copied: 2, conflicts: 0, failed: 0 }) + }); + await expect(fs.readFile(first.destinationPath, "utf8")).resolves.toBe("first replacement"); + await expect(fs.readFile(second.destinationPath, "utf8")).resolves.toBe("second replacement"); + await expect(fs.readlink(first.linkPath)).resolves.toBe(first.destinationPath); + await expect(fs.readlink(second.linkPath)).resolves.toBe(second.destinationPath); + }); + + it("keeps replacement cleanup paths exclusive across queued copy jobs", async () => { + const cleanupOwner = await insertCopySymlink({ + itemName: "Cleanup Claim Movie", + kind: "remote", + storagePolicy: "location_1", + relativePath: path.join("Cleanup Claim Movie", "new-version.mkv"), + content: "new cleanup owner" + }); + const claimedRelativePath = path.join("Cleanup Claim Movie", "old-version.mkv"); + const claimedPath = path.join(tmpDir, "local", "movies", claimedRelativePath); + await fs.mkdir(path.dirname(claimedPath), { recursive: true }); + await fs.writeFile(claimedPath, "old claimed cleanup"); + await insertStorageFile("local", path.join("movies", claimedRelativePath), Buffer.byteLength("old claimed cleanup")); + const firstJobId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: [cleanupOwner.id], localConflictStrategy: "replace" }); + const destinationOwner = await insertCopySymlink({ + itemName: "Different Destination Owner", + kind: "remote", + storagePolicy: "location_1", + relativePath: claimedRelativePath, + sourceRelativePath: path.join("Different Destination Owner", "source.mkv"), + content: "different destination owner" + }); + + await expect(ctx.jobs.startCopy({ direction: "to_local", linkIds: [destinationOwner.id] })).rejects.toThrow(`Job #${firstJobId} is already queued`); + await expect(fs.readFile(claimedPath, "utf8")).resolves.toBe("old claimed cleanup"); + }); + it("keeps existing local files when copy to local is started with keep both resolution", async () => { const cookie = await createAdminSession(); const fixture = await insertCopySymlink({ itemName: "Keep Both Movie", kind: "remote", storagePolicy: "location_1", content: "new keep both" }); @@ -2299,6 +2682,110 @@ describe("api app", () => { await expect(fs.readFile(path.join(path.dirname(fixture.destinationPath), keptFiles[0]), "utf8")).resolves.toBe("old exact keep both"); }); + it("rolls back an exact-path replacement when cancellation wins atomic completion", async () => { + const fixture = await insertCopySymlink({ itemName: "Cancelled Exact Replace", kind: "remote", storagePolicy: "location_1", content: "new cancelled replace" }); + await fs.mkdir(path.dirname(fixture.destinationPath), { recursive: true }); + await fs.writeFile(fixture.destinationPath, "old cancelled replace"); + const storageFileId = await insertStorageFile("local", path.join("movies", fixture.relativePath), Buffer.byteLength("old cancelled replace")); + const jobId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: [fixture.id], localConflictStrategy: "replace" }); + if (!Number.isSafeInteger(jobId)) throw new Error("Copy job ID is invalid"); + await ctx.database.db.execute(sql.raw(` + CREATE FUNCTION srtl_test_cancel_copy_before_finish() RETURNS trigger + LANGUAGE plpgsql AS $function$ + BEGIN + UPDATE jobs SET cancel_requested_at = clock_timestamp()::text WHERE id = NEW.job_id; + RETURN NEW; + END; + $function$ + `)); + await ctx.database.db.execute(sql.raw(` + CREATE TRIGGER srtl_test_cancel_copy_before_finish + AFTER INSERT ON job_events + FOR EACH ROW + WHEN (NEW.job_id = ${jobId} AND NEW.message = 'Copy job finished processing media') + EXECUTE FUNCTION srtl_test_cancel_copy_before_finish() + `)); + + await expect(runQueuedJob(jobId, { copyRunner: testCopyRunner })).resolves.toMatchObject({ + status: "cancelled", + finishedAt: expect.any(String), + progress: expect.objectContaining({ stage: "cancelled", copied: 0, repointed: 0, message: "Copy job terminated" }) + }); + + await expect(fs.readFile(fixture.destinationPath, "utf8")).resolves.toBe("old cancelled replace"); + await expect(fs.readlink(fixture.linkPath)).resolves.toBe(fixture.sourcePath); + expect((await fs.readdir(path.dirname(fixture.destinationPath))).filter((file) => file.includes(".srtl-replace-"))).toEqual([]); + await expect(first(ctx.database.db.select().from(schema.mediaLinks).where(eq(schema.mediaLinks.id, fixture.id)).limit(1))).resolves.toMatchObject({ + kind: "remote", + targetPath: fixture.sourcePath, + targetExists: true + }); + await expect(first(ctx.database.db.select().from(schema.storageFiles).where(eq(schema.storageFiles.id, storageFileId)).limit(1))).resolves.toMatchObject({ + filePath: fixture.destinationPath, + missingSince: null + }); + await expect(first(ctx.database.db.select().from(schema.copyOperations).where(eq(schema.copyOperations.jobId, jobId)).limit(1))).resolves.toMatchObject({ + stage: "rolled_back", + localConflictStrategy: "replace" + }); + expect(await ctx.database.db.select().from(schema.copySources).where(eq(schema.copySources.destinationPath, fixture.destinationPath))).toEqual([]); + await expect(ctx.jobs.listEvents(jobId)).resolves.toEqual( + expect.arrayContaining([expect.objectContaining({ level: "warn", message: "Copy job terminated; completed copy changes rolled back" })]) + ); + }); + + it("finalizes exact-path conflict backups only after successful completion", async () => { + const replaceFixture = await insertCopySymlink({ itemName: "Successful Exact Replace", kind: "remote", storagePolicy: "location_1", content: "new successful replace" }); + const keepBothFixture = await insertCopySymlink({ itemName: "Successful Exact Keep Both", kind: "remote", storagePolicy: "location_1", content: "new successful keep both" }); + await Promise.all([ + fs.mkdir(path.dirname(replaceFixture.destinationPath), { recursive: true }), + fs.mkdir(path.dirname(keepBothFixture.destinationPath), { recursive: true }) + ]); + await Promise.all([ + fs.writeFile(replaceFixture.destinationPath, "old successful replace"), + fs.writeFile(keepBothFixture.destinationPath, "old successful keep both") + ]); + await Promise.all([ + insertStorageFile("local", path.join("movies", replaceFixture.relativePath), Buffer.byteLength("old successful replace")), + insertStorageFile("local", path.join("movies", keepBothFixture.relativePath), Buffer.byteLength("old successful keep both")) + ]); + const replaceJobId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: [replaceFixture.id], localConflictStrategy: "replace" }); + await expect(runQueuedJob(replaceJobId, { copyRunner: testCopyRunner })).resolves.toMatchObject({ + status: "completed", + finishedAt: expect.any(String), + progress: expect.objectContaining({ copied: 1, conflicts: 0, failed: 0 }) + }); + const keepBothJobId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: [keepBothFixture.id], localConflictStrategy: "keep_both" }); + await expect(runQueuedJob(keepBothJobId, { copyRunner: testCopyRunner })).resolves.toMatchObject({ + status: "completed", + finishedAt: expect.any(String), + progress: expect.objectContaining({ copied: 1, conflicts: 0, failed: 0 }) + }); + + await expect(fs.readFile(replaceFixture.destinationPath, "utf8")).resolves.toBe("new successful replace"); + await expect(fs.readFile(keepBothFixture.destinationPath, "utf8")).resolves.toBe("new successful keep both"); + await expect(fs.readlink(replaceFixture.linkPath)).resolves.toBe(replaceFixture.destinationPath); + await expect(fs.readlink(keepBothFixture.linkPath)).resolves.toBe(keepBothFixture.destinationPath); + expect((await fs.readdir(path.dirname(replaceFixture.destinationPath))).filter((file) => file.includes(".srtl-replace-"))).toEqual([]); + const keptFiles = (await fs.readdir(path.dirname(keepBothFixture.destinationPath))).filter((file) => file.includes(".srtl-kept-")); + expect(keptFiles).toHaveLength(1); + await expect(fs.readFile(path.join(path.dirname(keepBothFixture.destinationPath), keptFiles[0]), "utf8")).resolves.toBe("old successful keep both"); + const mediaLinks = await ctx.database.db.select().from(schema.mediaLinks).where(inArray(schema.mediaLinks.id, [replaceFixture.id, keepBothFixture.id])); + expect(mediaLinks).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: replaceFixture.id, kind: "local", targetPath: replaceFixture.destinationPath, targetExists: true }), + expect.objectContaining({ id: keepBothFixture.id, kind: "local", targetPath: keepBothFixture.destinationPath, targetExists: true }) + ]) + ); + const operations = await ctx.database.db.select().from(schema.copyOperations).where(inArray(schema.copyOperations.jobId, [replaceJobId, keepBothJobId])); + expect(operations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ jobId: replaceJobId, stage: "committed", localConflictStrategy: "replace", displacedPath: null }), + expect.objectContaining({ jobId: keepBothJobId, stage: "committed", localConflictStrategy: "keep_both", displacedPath: null }) + ]) + ); + }); + it("replaces existing local files only after a verified copy is installed", async () => { const cookie = await createAdminSession(); const fixture = await insertCopySymlink({ itemName: "Replace Movie", kind: "remote", storagePolicy: "location_1", content: "new replace" }); @@ -2497,7 +2984,7 @@ describe("api app", () => { expect(await ctx.jobs.listEvents(jobId)).toEqual(expect.arrayContaining([expect.objectContaining({ message: "Worker stopped; job requeued for resume" })])); }); - it("requeues an active copy when a managed path change is detected", async () => { + it("cancels an active copy after safely rolling it back for a managed path change", async () => { const fixture = await insertCopySymlink({ itemName: "Path Change Copy Movie", kind: "remote", storagePolicy: "location_1", content: "path change copy" }); const jobId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: [fixture.id] }); let resolveCopyStarted: (() => void) | null = null; @@ -2535,10 +3022,12 @@ describe("api app", () => { }); await expect(run).resolves.toBe(true); - expect(await ctx.jobs.getJob(jobId)).toMatchObject({ status: "queued", lockedBy: null, heartbeatAt: null }); + expect(await ctx.jobs.getJob(jobId)).toMatchObject({ status: "cancelled", lockedBy: null, heartbeatAt: null }); await expect(fs.readlink(fixture.linkPath)).resolves.toBe(fixture.sourcePath); await expect(fs.stat(fixture.destinationPath)).rejects.toMatchObject({ code: "ENOENT" }); - expect(await ctx.jobs.listEvents(jobId)).toEqual(expect.arrayContaining([expect.objectContaining({ message: "Managed storage paths changed; job paused and requeued" })])); + expect(await ctx.jobs.listEvents(jobId)).toEqual( + expect.arrayContaining([expect.objectContaining({ message: "Copy cancelled after managed-path recovery" })]) + ); }); it("records failed scans in scan history with the failure message", async () => { @@ -2616,6 +3105,486 @@ describe("api app", () => { expect(await ctx.jobs.getJob(queuedJobId)).toMatchObject({ status: "queued", startedAt: null, lockedBy: null, heartbeatAt: null }); }); + it("claims unrelated work without waiting on a running job's lease lock", async () => { + const activeStartedAt = new Date().toISOString(); + const activeJobId = await insertRunningJob(activeStartedAt, "audit"); + await ctx.database.db + .update(schema.jobs) + .set({ exclusive: false, lockedBy: "lease-holder", lockedAt: activeStartedAt, heartbeatAt: activeStartedAt }) + .where(eq(schema.jobs.id, activeJobId)); + const queuedJobId = await ctx.jobs.startAudit({ mode: "fast", itemName: "Unrelated Empty Audit" }); + const lockClient = await ctx.database.pool.connect(); + await lockClient.query("BEGIN"); + await lockClient.query("SELECT id FROM jobs WHERE id = $1 FOR UPDATE", [activeJobId]); + const worker = new JobWorker(ctx.database.db, { + workerId: "skip-locked-claim-worker", + pollIntervalMs: 1, + heartbeatIntervalMs: 10, + logger: silentLogger, + concurrency: { + workerCount: 2, + maxRunningJobs: 2, + maxRunningScans: 1, + maxRunningAudits: 2, + maxRunningCopies: 1, + copyFileConcurrency: 1, + maxActiveCopyFiles: 1 + } + }); + + try { + await expect( + Promise.race([ + worker.runOnce(), + new Promise((_resolve, reject) => setTimeout(() => reject(new Error("claim blocked on unrelated lease")), 1_000)) + ]) + ).resolves.toBe(true); + } finally { + await lockClient.query("ROLLBACK"); + lockClient.release(); + } + expect(await ctx.jobs.getJob(queuedJobId)).toMatchObject({ status: "completed" }); + expect(await ctx.jobs.getJob(activeJobId)).toMatchObject({ status: "running", lockedBy: "lease-holder" }); + }); + + it("claims disjoint copy jobs concurrently up to the configured limits", async () => { + const fixtures = await Promise.all([ + insertCopySymlink({ itemName: "Parallel Claim One", kind: "remote", storagePolicy: "location_1", content: "parallel one" }), + insertCopySymlink({ itemName: "Parallel Claim Two", kind: "remote", storagePolicy: "location_1", content: "parallel two" }) + ]); + const jobIds = await Promise.all(fixtures.map((fixture) => ctx.jobs.startCopy({ direction: "to_local", linkIds: [fixture.id] }))); + let releaseCopy: () => void = () => undefined; + const copyReleased = new Promise((resolve) => { + releaseCopy = resolve; + }); + const blockingRunner: CopyCommandRunner = { + ...testCopyRunner, + async copyFile(sourcePath, tempPath, reportProgress, signal) { + await copyReleased; + if (signal?.aborted) throw new Error("copy interrupted"); + await testCopyRunner.copyFile(sourcePath, tempPath, reportProgress, signal); + } + }; + const concurrency = { + workerCount: 2, + maxRunningJobs: 2, + maxRunningScans: 2, + maxRunningAudits: 2, + maxRunningCopies: 2, + copyFileConcurrency: 1, + maxActiveCopyFiles: 2 + }; + const workers = ["parallel-worker-1", "parallel-worker-2"].map( + (workerId) => new JobWorker(ctx.database.db, { workerId, pollIntervalMs: 1, heartbeatIntervalMs: 10, logger: silentLogger, copyRunner: blockingRunner, concurrency }) + ); + const runs = workers.map((worker) => worker.runOnce()); + + try { + let runningIds: number[] = []; + for (let attempt = 0; attempt < 100; attempt += 1) { + const jobs = await Promise.all(jobIds.map((jobId) => ctx.jobs.getJob(jobId))); + runningIds = jobs.filter((job) => job?.status === "running").map((job) => job?.id ?? 0); + if (runningIds.length === 2) break; + await new Promise((resolve) => setTimeout(resolve, 5)); + } + expect(runningIds.sort((firstId, secondId) => firstId - secondId)).toEqual([...jobIds].sort((firstId, secondId) => firstId - secondId)); + } finally { + releaseCopy(); + await Promise.allSettled(runs); + } + await expect(Promise.all(runs)).resolves.toEqual([true, true]); + }); + + it("completes a targeted rescan while a disjoint copy is still transferring", async () => { + const copyFixture = await insertCopySymlink({ itemName: "Copy During Rescan", kind: "remote", storagePolicy: "location_1", content: "copy stays active" }); + await insertCopySymlink({ itemName: "Rescan During Copy", kind: "remote", storagePolicy: "location_1", content: "rescan independently" }); + const copyJobId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: [copyFixture.id] }); + const scanJobId = await ctx.jobs.startScan({ + scanSymlinks: true, + scanLocal: false, + scanRemote: false, + symlinkSections: ["movies"], + localSections: [], + titleScopes: [{ section: "movies", itemName: "Rescan During Copy" }] + }); + let releaseCopy!: () => void; + let markCopyStarted!: () => void; + const copyReleased = new Promise((resolve) => { + releaseCopy = resolve; + }); + const copyStarted = new Promise((resolve) => { + markCopyStarted = resolve; + }); + const blockingRunner: CopyCommandRunner = { + ...testCopyRunner, + async copyFile(sourcePath, tempPath, reportProgress, signal) { + markCopyStarted(); + await copyReleased; + await testCopyRunner.copyFile(sourcePath, tempPath, reportProgress, signal); + } + }; + const concurrency = { + workerCount: 2, + maxRunningJobs: 2, + maxRunningScans: 1, + maxRunningAudits: 1, + maxRunningCopies: 1, + copyFileConcurrency: 1, + maxActiveCopyFiles: 1 + }; + const workers = ["copy-rescan-worker-1", "copy-rescan-worker-2"].map( + (workerId) => new JobWorker(ctx.database.db, { workerId, pollIntervalMs: 1, heartbeatIntervalMs: 10, logger: silentLogger, copyRunner: blockingRunner, concurrency }) + ); + const runs = workers.map((worker) => worker.runOnce()); + + try { + await copyStarted; + let scanStatus: string | undefined; + for (let attempt = 0; attempt < 200; attempt += 1) { + scanStatus = (await ctx.jobs.getJob(scanJobId))?.status; + if (scanStatus === "completed") break; + await new Promise((resolve) => setTimeout(resolve, 5)); + } + expect(scanStatus).toBe("completed"); + expect(await ctx.jobs.getJob(copyJobId)).toMatchObject({ status: "running" }); + } finally { + releaseCopy(); + await Promise.allSettled(runs); + } + await expect(Promise.all(runs)).resolves.toEqual([true, true]); + expect(await ctx.jobs.getJob(copyJobId)).toMatchObject({ status: "completed" }); + }); + + it("copies independent titles concurrently within one copy job", async () => { + const fixtures = await Promise.all([ + insertCopySymlink({ itemName: "Concurrent Title One", kind: "remote", storagePolicy: "location_1", content: "concurrent title one" }), + insertCopySymlink({ itemName: "Concurrent Title Two", kind: "remote", storagePolicy: "location_1", content: "concurrent title two" }) + ]); + const jobId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: fixtures.map((fixture) => fixture.id) }); + let activeTransfers = 0; + let maximumActiveTransfers = 0; + const trackingRunner: CopyCommandRunner = { + ...testCopyRunner, + async copyFile(sourcePath, tempPath, reportProgress, signal) { + activeTransfers += 1; + maximumActiveTransfers = Math.max(maximumActiveTransfers, activeTransfers); + try { + await new Promise((resolve) => setTimeout(resolve, 40)); + await testCopyRunner.copyFile(sourcePath, tempPath, reportProgress, signal); + } finally { + activeTransfers -= 1; + } + } + }; + const worker = new JobWorker(ctx.database.db, { + workerId: "within-job-parallel-worker", + pollIntervalMs: 1, + heartbeatIntervalMs: 10, + logger: silentLogger, + copyRunner: trackingRunner, + concurrency: { + workerCount: 1, + maxRunningJobs: 1, + maxRunningScans: 1, + maxRunningAudits: 1, + maxRunningCopies: 1, + copyFileConcurrency: 2, + maxActiveCopyFiles: 2 + } + }); + + expect(await worker.runOnce()).toBe(true); + expect(maximumActiveTransfers).toBe(2); + expect(await ctx.jobs.getJob(jobId)).toMatchObject({ status: "completed", progress: expect.objectContaining({ copied: 2, failed: 0 }) }); + }); + + it("copies links for the same title concurrently within one copy job", async () => { + const fixtures = await Promise.all([ + insertCopySymlink({ + itemName: "Serialized Title", + kind: "local", + storagePolicy: "location_2", + relativePath: path.join("Serialized Title", "serialized-title-cd1.mkv"), + content: "serialized title part one" + }), + insertCopySymlink({ + itemName: "Serialized Title", + kind: "local", + storagePolicy: "location_2", + relativePath: path.join("Serialized Title", "serialized-title-cd2.mkv"), + content: "serialized title part two" + }) + ]); + const jobId = await ctx.jobs.startCopy({ direction: "to_remote", linkIds: fixtures.map((fixture) => fixture.id) }); + let activeTransfers = 0; + let maximumActiveTransfers = 0; + const trackingRunner: CopyCommandRunner = { + ...testCopyRunner, + async copyFile(sourcePath, tempPath, reportProgress, signal) { + activeTransfers += 1; + maximumActiveTransfers = Math.max(maximumActiveTransfers, activeTransfers); + try { + await new Promise((resolve) => setTimeout(resolve, 40)); + await testCopyRunner.copyFile(sourcePath, tempPath, reportProgress, signal); + } finally { + activeTransfers -= 1; + } + } + }; + const worker = new JobWorker(ctx.database.db, { + workerId: "same-title-parallel-worker", + pollIntervalMs: 1, + heartbeatIntervalMs: 10, + logger: silentLogger, + copyRunner: trackingRunner, + concurrency: { + workerCount: 1, + maxRunningJobs: 1, + maxRunningScans: 1, + maxRunningAudits: 1, + maxRunningCopies: 1, + copyFileConcurrency: 2, + maxActiveCopyFiles: 2 + } + }); + + expect(await worker.runOnce()).toBe(true); + expect(maximumActiveTransfers).toBe(2); + expect(await ctx.jobs.getJob(jobId)).toMatchObject({ status: "completed", progress: expect.objectContaining({ copied: 2, failed: 0 }) }); + }); + + it("does not treat a same-title sibling destination that appears during replacement copy as a conflict or cleanup candidate", async () => { + const fixtures = await Promise.all( + ["first", "second", "third"].map((part) => + insertCopySymlink({ + itemName: "Parallel Replacement Title", + kind: "remote", + storagePolicy: "location_1", + relativePath: path.join("Parallel Replacement Title", `${part}.mkv`), + content: `${part} parallel replacement` + }) + ) + ); + const jobId = await ctx.jobs.startCopy({ + direction: "to_local", + linkIds: fixtures.map((fixture) => fixture.id), + localConflictStrategy: "replace" + }); + const transferGates = new Map; release: () => void }>(); + for (const fixture of fixtures) { + let release: (() => void) | null = null; + const wait = new Promise((resolve) => { + release = resolve; + }); + if (!release) throw new Error(`Transfer gate was not initialized for ${fixture.sourcePath}`); + transferGates.set(fixture.sourcePath, { wait, release }); + } + const startedSources: string[] = []; + let activeTransfers = 0; + let maximumActiveTransfers = 0; + const blockingRunner: CopyCommandRunner = { + ...testCopyRunner, + async copyFile(sourcePath, tempPath, reportProgress, signal) { + activeTransfers += 1; + maximumActiveTransfers = Math.max(maximumActiveTransfers, activeTransfers); + startedSources.push(sourcePath); + try { + const gate = transferGates.get(sourcePath); + if (!gate) throw new Error(`Missing transfer gate for ${sourcePath}`); + await gate.wait; + await testCopyRunner.copyFile(sourcePath, tempPath, reportProgress, signal); + } finally { + activeTransfers -= 1; + } + } + }; + const worker = new JobWorker(ctx.database.db, { + workerId: "same-title-replacement-parallel-worker", + pollIntervalMs: 1, + heartbeatIntervalMs: 10, + logger: silentLogger, + copyRunner: blockingRunner, + concurrency: { + workerCount: 1, + maxRunningJobs: 1, + maxRunningScans: 1, + maxRunningAudits: 1, + maxRunningCopies: 1, + copyFileConcurrency: 2, + maxActiveCopyFiles: 2 + } + }); + const run = worker.runOnce(); + + try { + await vi.waitFor(() => expect(startedSources).toHaveLength(2)); + const completedSource = startedSources[0]!; + transferGates.get(completedSource)!.release(); + await vi.waitFor(() => expect(startedSources).toHaveLength(3)); + const completedFixture = fixtures.find((fixture) => fixture.sourcePath === completedSource); + if (!completedFixture) throw new Error(`Missing completed fixture for ${completedSource}`); + expect(activeTransfers).toBe(2); + await expect(fs.readFile(completedFixture.destinationPath, "utf8")).resolves.toBe( + `${path.basename(completedFixture.sourcePath, ".mkv")} parallel replacement` + ); + for (const gate of transferGates.values()) gate.release(); + expect(await run).toBe(true); + } finally { + for (const gate of transferGates.values()) gate.release(); + await Promise.allSettled([run]); + } + + expect(maximumActiveTransfers).toBe(2); + expect(await ctx.jobs.getJob(jobId)).toMatchObject({ + status: "completed", + progress: expect.objectContaining({ copied: 3, conflicts: 0, failed: 0 }) + }); + for (const fixture of fixtures) { + await expect(fs.readFile(fixture.destinationPath, "utf8")).resolves.toBe( + `${path.basename(fixture.sourcePath, ".mkv")} parallel replacement` + ); + await expect(fs.readlink(fixture.linkPath)).resolves.toBe(fixture.destinationPath); + } + const events = await ctx.jobs.listEvents(jobId); + expect(events.some((event) => event.message === "Local replacement candidates changed after copy admission")).toBe(false); + expect(events.some((event) => event.message === "Replaced previous local files")).toBe(false); + expect((await fs.readdir(path.dirname(fixtures[0].destinationPath))).filter((file) => file.includes(".srtl-replace-"))).toEqual([]); + }); + + it("shares the active-copy-file limit across workers in one process", async () => { + const fixtures = await Promise.all([ + insertCopySymlink({ itemName: "Global Limit One", kind: "remote", storagePolicy: "location_1", content: "global limit one" }), + insertCopySymlink({ itemName: "Global Limit Two", kind: "remote", storagePolicy: "location_1", content: "global limit two" }) + ]); + const jobIds = await Promise.all(fixtures.map((fixture) => ctx.jobs.startCopy({ direction: "to_local", linkIds: [fixture.id] }))); + let activeTransfers = 0; + let maximumActiveTransfers = 0; + let observedBothJobsRunning = false; + const trackingRunner: CopyCommandRunner = { + ...testCopyRunner, + async copyFile(sourcePath, tempPath, reportProgress, signal) { + activeTransfers += 1; + maximumActiveTransfers = Math.max(maximumActiveTransfers, activeTransfers); + try { + for (let attempt = 0; attempt < 100; attempt += 1) { + const jobs = await Promise.all(jobIds.map((jobId) => ctx.jobs.getJob(jobId))); + if (jobs.every((job) => job?.status === "running")) { + observedBothJobsRunning = true; + break; + } + await new Promise((resolve) => setTimeout(resolve, 5)); + } + await testCopyRunner.copyFile(sourcePath, tempPath, reportProgress, signal); + } finally { + activeTransfers -= 1; + } + } + }; + const concurrency = { + workerCount: 2, + maxRunningJobs: 2, + maxRunningScans: 2, + maxRunningAudits: 2, + maxRunningCopies: 2, + copyFileConcurrency: 2, + maxActiveCopyFiles: 1 + }; + const copyTransferLimiter = new CopyTransferLimiter(1); + const workers = ["global-limit-worker-1", "global-limit-worker-2"].map( + (workerId) => + new JobWorker(ctx.database.db, { + workerId, + pollIntervalMs: 1, + heartbeatIntervalMs: 10, + logger: silentLogger, + copyRunner: trackingRunner, + concurrency, + copyTransferLimiter + }) + ); + + await expect(Promise.all(workers.map((worker) => worker.runOnce()))).resolves.toEqual([true, true]); + expect(observedBothJobsRunning).toBe(true); + expect(maximumActiveTransfers).toBe(1); + await expect(Promise.all(jobIds.map((jobId) => ctx.jobs.getJob(jobId)))).resolves.toEqual([ + expect.objectContaining({ status: "completed" }), + expect.objectContaining({ status: "completed" }) + ]); + }); + + it("enforces a per-type limit independently of the global running-job limit", async () => { + const timestamp = new Date().toISOString(); + const active = await first( + ctx.database.db + .insert(schema.jobs) + .values({ + type: "copy", + status: "running", + createdAt: timestamp, + startedAt: timestamp, + finishedAt: null, + lockedBy: "active-copy-worker", + lockedAt: timestamp, + heartbeatAt: timestamp, + leaseVersion: 1, + exclusive: false, + progress: "{}" + }) + .returning({ id: schema.jobs.id }) + ); + if (!active) throw new Error("Active copy job was not inserted"); + const fixture = await insertCopySymlink({ itemName: "Type Limited Copy", kind: "remote", storagePolicy: "location_1", content: "queued by type" }); + const queuedJobId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: [fixture.id] }); + const worker = new JobWorker(ctx.database.db, { + workerId: "type-limited-worker", + pollIntervalMs: 1, + heartbeatIntervalMs: 10, + logger: silentLogger, + concurrency: { + workerCount: 2, + maxRunningJobs: 2, + maxRunningScans: 2, + maxRunningAudits: 2, + maxRunningCopies: 1, + copyFileConcurrency: 1, + maxActiveCopyFiles: 2 + } + }); + + expect(await worker.runOnce()).toBe(false); + expect(await ctx.jobs.getJob(queuedJobId)).toMatchObject({ status: "queued", startedAt: null, lockedBy: null, leaseVersion: 0 }); + }); + + it("does not claim work past a queued exclusive barrier", async () => { + const fixtures = await Promise.all([ + insertCopySymlink({ itemName: "Before Barrier Copy", kind: "remote", storagePolicy: "location_1", content: "before barrier" }), + insertCopySymlink({ itemName: "After Barrier Copy", kind: "remote", storagePolicy: "location_1", content: "after barrier" }) + ]); + const firstCopyId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: [fixtures[0].id] }); + const barrierId = await ctx.jobs.startScan({ scanSymlinks: false, scanLocal: false, scanRemote: true, symlinkSections: [], localSections: [] }); + const secondCopyId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: [fixtures[1].id] }); + const concurrency = { + workerCount: 3, + maxRunningJobs: 3, + maxRunningScans: 3, + maxRunningAudits: 3, + maxRunningCopies: 3, + copyFileConcurrency: 1, + maxActiveCopyFiles: 3 + }; + const firstWorker = new JobWorker(ctx.database.db, { workerId: "barrier-worker-1", logger: silentLogger, concurrency }) as unknown as { + claimNextJob(): Promise<{ job: { id: number } } | null>; + }; + const secondWorker = new JobWorker(ctx.database.db, { workerId: "barrier-worker-2", logger: silentLogger, concurrency }) as unknown as { + claimNextJob(): Promise<{ job: { id: number } } | null>; + }; + + await expect(firstWorker.claimNextJob()).resolves.toMatchObject({ job: { id: firstCopyId } }); + await expect(secondWorker.claimNextJob()).resolves.toBeNull(); + expect(await ctx.jobs.getJob(barrierId)).toMatchObject({ status: "queued", exclusive: true }); + expect(await ctx.jobs.getJob(secondCopyId)).toMatchObject({ status: "queued", exclusive: false }); + }); + it("reclaims stale running scan jobs in the worker process", async () => { const staleStartedAt = new Date(Date.now() - 30 * 60_000).toISOString(); await fs.mkdir(path.join(tmpDir, "remote", "Recovered Release"), { recursive: true }); @@ -2644,7 +3613,7 @@ describe("api app", () => { const scanRuns = await ctx.database.db.select().from(schema.scanRuns).where(eq(schema.scanRuns.jobId, job.id)); expect(scanRuns).toEqual( expect.arrayContaining([ - expect.objectContaining({ status: "failed", errorMessage: "Stale job reclaimed by worker" }), + expect.objectContaining({ status: "failed", errorMessage: "Stale running job lease fenced and requeued" }), expect.objectContaining({ status: "completed", remoteFiles: 1 }) ]) ); @@ -2685,7 +3654,97 @@ describe("api app", () => { expect(await worker.runOnce()).toBe(true); expect(await ctx.jobs.getJob(job.id)).toMatchObject({ status: "completed", lockedBy: null, heartbeatAt: null }); - expect(await ctx.jobs.listEvents(job.id)).toEqual(expect.arrayContaining([expect.objectContaining({ message: "Interrupted job reclaimed by replacement worker" })])); + expect(await ctx.jobs.listEvents(job.id)).toEqual(expect.arrayContaining([expect.objectContaining({ message: "Interrupted job lease fenced and requeued" })])); + }); + + it("allows exactly one worker to reclaim a stale lease and increments its version", async () => { + const staleStartedAt = new Date(Date.now() - 30 * 60_000).toISOString(); + const job = await first( + ctx.database.db + .insert(schema.jobs) + .values({ + type: "scan", + status: "running", + createdAt: staleStartedAt, + startedAt: staleStartedAt, + finishedAt: null, + lockedBy: "dead-worker", + lockedAt: staleStartedAt, + heartbeatAt: staleStartedAt, + leaseVersion: 7, + exclusive: true, + progress: JSON.stringify({ + options: { scanSymlinks: false, scanLocal: false, scanRemote: true, symlinkSections: [], localSections: [] } + }) + }) + .returning({ id: schema.jobs.id }) + ); + if (!job) throw new Error("Stale lease job was not inserted"); + await insertRunningScanRun(job.id, staleStartedAt); + const workers = ["stale-contender-1", "stale-contender-2"].map( + (workerId) => + new JobWorker(ctx.database.db, { + workerId, + reclaimStaleAfterMs: 60_000, + pollIntervalMs: 1, + heartbeatIntervalMs: 10, + logger: silentLogger + }) + ); + + const results = await Promise.all(workers.map((worker) => worker.runOnce())); + + expect(results.filter(Boolean)).toHaveLength(1); + expect(await ctx.jobs.getJob(job.id)).toMatchObject({ status: "completed", leaseVersion: 9, lockedBy: null }); + expect((await ctx.jobs.listEvents(job.id)).filter((event) => event.message === "Stale running job lease fenced and requeued")).toHaveLength(1); + }); + + it("rejects progress, finish, and requeue writes from an old lease", async () => { + const timestamp = new Date().toISOString(); + const inserted = await first( + ctx.database.db + .insert(schema.jobs) + .values({ + type: "copy", + status: "running", + createdAt: timestamp, + startedAt: timestamp, + finishedAt: null, + lockedBy: "old-lease-worker", + lockedAt: timestamp, + heartbeatAt: timestamp, + leaseVersion: 3, + exclusive: false, + progress: JSON.stringify({ stage: "original" }) + }) + .returning() + ); + if (!inserted) throw new Error("Old lease job was not inserted"); + const worker = new JobWorker(ctx.database.db, { workerId: "old-lease-worker", logger: silentLogger }); + const fencedWorker = worker as unknown as { + setLeasedProgress(job: unknown, progress: unknown): Promise; + addLeasedEvent(job: unknown, level: "info", message: string, data?: unknown): Promise; + heartbeat(job: unknown): Promise; + finishJob(job: unknown, status: "completed", level: "info", message: string): Promise; + requeueInterruptedJob(job: unknown, message?: string): Promise; + }; + await ctx.database.db + .update(schema.jobs) + .set({ lockedBy: "replacement-worker", leaseVersion: 4, heartbeatAt: new Date().toISOString() }) + .where(eq(schema.jobs.id, inserted.id)); + + await expect(fencedWorker.setLeasedProgress(inserted, { stage: "stale" })).rejects.toMatchObject({ name: "LeaseLostError" }); + await expect(fencedWorker.addLeasedEvent(inserted, "info", "stale event")).rejects.toMatchObject({ name: "LeaseLostError" }); + await expect(fencedWorker.heartbeat(inserted)).rejects.toMatchObject({ name: "LeaseLostError" }); + await expect(fencedWorker.finishJob(inserted, "completed", "info", "stale finish")).rejects.toMatchObject({ name: "LeaseLostError" }); + await expect(fencedWorker.requeueInterruptedJob(inserted)).rejects.toMatchObject({ name: "LeaseLostError" }); + expect(await ctx.jobs.getJob(inserted.id)).toMatchObject({ + status: "running", + lockedBy: "replacement-worker", + leaseVersion: 4, + progress: { stage: "original" } + }); + expect(await ctx.jobs.listEvents(inserted.id)).toEqual([]); }); it("saves section display titles and types separately from symlink directory names", async () => { @@ -2730,6 +3789,111 @@ describe("api app", () => { ); }); + it("rejects storage policy changes overlapping a queued copy while allowing a disjoint title", async () => { + const cookie = await createAdminSession(); + const copyFixture = await insertCopySymlink({ + itemName: "Policy Copy Movie", + kind: "remote", + storagePolicy: "location_1", + content: "copy remains isolated" + }); + const disjointLinkId = await insertMediaLink("Disjoint Policy Movie"); + const copyJobId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: [copyFixture.id] }); + + const overlapping = await ctx.app.inject({ + method: "POST", + url: "/api/storage-policies", + headers: { cookie }, + payload: { title: "Policy Copy Movie", policy: "location_2" } + }); + expect(overlapping.statusCode).toBe(409); + expect(overlapping.json()).toMatchObject({ error: expect.stringContaining(`copy job #${copyJobId} is queued for the same media`) }); + expect(await first(ctx.database.db.select().from(schema.mediaLinks).where(eq(schema.mediaLinks.id, copyFixture.id)).limit(1))).toMatchObject({ + storagePolicy: "location_1" + }); + + const disjoint = await ctx.app.inject({ + method: "POST", + url: "/api/storage-policies", + headers: { cookie }, + payload: { title: "Disjoint Policy Movie", policy: "location_2" } + }); + expect(disjoint.statusCode).toBe(200); + expect(disjoint.json()).toMatchObject({ title: "Disjoint Policy Movie", policy: "location_2" }); + expect(await first(ctx.database.db.select().from(schema.mediaLinks).where(eq(schema.mediaLinks.id, disjointLinkId)).limit(1))).toMatchObject({ + storagePolicy: "location_2" + }); + }); + + it("rejects bulk and delete policy mutations overlapping a running audit", async () => { + const cookie = await createAdminSession(); + const linkId = await insertMediaLink("Policy Audit Movie"); + const assigned = await ctx.app.inject({ + method: "POST", + url: "/api/storage-policies", + headers: { cookie }, + payload: { title: "Policy Audit Movie", policy: "location_2" } + }); + expect(assigned.statusCode).toBe(200); + const policyId = assigned.json<{ id: number }>().id; + + const auditJobId = await ctx.jobs.startAudit({ mode: "fast", linkIds: [linkId] }); + const timestamp = new Date().toISOString(); + await ctx.database.db + .update(schema.jobs) + .set({ status: "running", startedAt: timestamp, lockedBy: "policy-test-worker", lockedAt: timestamp, heartbeatAt: timestamp }) + .where(eq(schema.jobs.id, auditJobId)); + + const bulk = await ctx.app.inject({ + method: "POST", + url: "/api/storage-policies/bulk", + headers: { cookie }, + payload: { titles: ["Policy Audit Movie"], policy: "location_1" } + }); + expect(bulk.statusCode).toBe(409); + expect(bulk.json()).toMatchObject({ error: expect.stringContaining(`audit job #${auditJobId} is running for the same media`) }); + + const remove = await ctx.app.inject({ method: "DELETE", url: `/api/storage-policies/${policyId}`, headers: { cookie } }); + expect(remove.statusCode).toBe(409); + expect(remove.json()).toMatchObject({ error: expect.stringContaining(`audit job #${auditJobId} is running for the same media`) }); + expect(await first(ctx.database.db.select().from(schema.mediaLinks).where(eq(schema.mediaLinks.id, linkId)).limit(1))).toMatchObject({ + storagePolicy: "location_2" + }); + expect(await first(ctx.database.db.select().from(schema.storagePolicies).where(eq(schema.storagePolicies.id, policyId)).limit(1))).toMatchObject({ + policy: "location_2" + }); + }); + + it("applies storage policies to canonical-equivalent media and storage titles", async () => { + const cookie = await createAdminSession(); + const ampersandLinkId = await insertMediaLink("Rock & Roll Movie"); + const wordLinkId = await insertMediaLink("Rock and Roll Movie"); + await insertStorageFile("remote", path.join("movies", "Rock & Roll Movie", "ampersand.mkv")); + await insertStorageFile("remote", path.join("movies", "Rock and Roll Movie", "word.mkv")); + + const response = await ctx.app.inject({ + method: "POST", + url: "/api/storage-policies", + headers: { cookie }, + payload: { title: "Rock and Roll Movie", policy: "location_2" } + }); + + expect(response.statusCode).toBe(200); + expect(response.json()).toMatchObject({ policy: "location_2", linkCount: 2, fileCount: 2 }); + const links = await ctx.database.db + .select({ id: schema.mediaLinks.id, storagePolicy: schema.mediaLinks.storagePolicy }) + .from(schema.mediaLinks) + .where(inArray(schema.mediaLinks.id, [ampersandLinkId, wordLinkId])); + expect(links).toHaveLength(2); + expect(links.every((link) => link.storagePolicy === "location_2")).toBe(true); + const files = await ctx.database.db + .select({ itemName: schema.storageFiles.itemName, storagePolicy: schema.storageFiles.storagePolicy }) + .from(schema.storageFiles) + .where(inArray(schema.storageFiles.itemName, ["Rock & Roll Movie", "Rock and Roll Movie"])); + expect(files).toHaveLength(2); + expect(files.every((file) => file.storagePolicy === "location_2")).toBe(true); + }); + it("manages storage policies and bulk assignments", async () => { const cookie = await createAdminSession(); await ensureSection("anime", null, "shows"); diff --git a/tests/config.test.ts b/tests/config.test.ts index f73e4ee..843c684 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -29,23 +29,89 @@ describe("config", () => { expect(config.paths).toEqual({ symlinkDir: "/mnt/links", localDir: "/mnt/local", remoteDir: "/mnt/remote" }); }); - it("keeps worker concurrency hard-capped while the worker count setting is a placeholder", async () => { + it("loads an arbitrary worker count and its bounded concurrency safeguards", async () => { await fs.writeFile( path.join(tmpDir, ".env"), - ["SRTL_WORKER_COUNT=4", "SRTL_MAX_RUNNING_JOBS=8", "SRTL_MAX_RUNNING_SCANS=2", "SRTL_MAX_RUNNING_AUDITS=3", "SRTL_MAX_RUNNING_COPIES=5"].join("\n") + [ + "SRTL_WORKER_COUNT=128", + "SRTL_MAX_RUNNING_JOBS=64", + "SRTL_MAX_RUNNING_SCANS=2", + "SRTL_MAX_RUNNING_AUDITS=3", + "SRTL_MAX_RUNNING_COPIES=60", + "SRTL_COPY_FILE_CONCURRENCY=8", + "SRTL_MAX_ACTIVE_COPY_FILES=32" + ].join("\n") ); const config = loadConfig({ rootDir: tmpDir }); expect(config.jobConcurrency).toEqual({ + workerCount: 128, + maxRunningJobs: 64, + maxRunningScans: 2, + maxRunningAudits: 3, + maxRunningCopies: 60, + copyFileConcurrency: 8, + maxActiveCopyFiles: 32 + }); + }); + + it("uses the serial worker default from the checked-in example environment", async () => { + const example = await fs.readFile(new URL("../.env.example", import.meta.url), "utf8"); + const workerCountSetting = example + .split(/\r?\n/) + .find((line) => line.startsWith("SRTL_WORKER_COUNT=")); + + expect(workerCountSetting).toBe("SRTL_WORKER_COUNT=1"); + await fs.writeFile(path.join(tmpDir, ".env"), `${workerCountSetting}\n`); + + expect(loadConfig({ rootDir: tmpDir }).jobConcurrency).toEqual({ workerCount: 1, maxRunningJobs: 1, maxRunningScans: 1, maxRunningAudits: 1, - maxRunningCopies: 1 + maxRunningCopies: 1, + copyFileConcurrency: 1, + maxActiveCopyFiles: 1 + }); + }); + + it("derives safe limits from the worker count while keeping each copy job serial by default", async () => { + await fs.writeFile(path.join(tmpDir, ".env"), "SRTL_WORKER_COUNT=4\n"); + + expect(loadConfig({ rootDir: tmpDir }).jobConcurrency).toEqual({ + workerCount: 4, + maxRunningJobs: 4, + maxRunningScans: 4, + maxRunningAudits: 4, + maxRunningCopies: 4, + copyFileConcurrency: 1, + maxActiveCopyFiles: 4 }); }); + it("allows a job type to be paused with a zero per-type limit", async () => { + await fs.writeFile(path.join(tmpDir, ".env"), ["SRTL_WORKER_COUNT=3", "SRTL_MAX_RUNNING_SCANS=0"].join("\n")); + + expect(loadConfig({ rootDir: tmpDir }).jobConcurrency.maxRunningScans).toBe(0); + }); + + it("rejects malformed and contradictory worker concurrency settings", async () => { + const invalidSettings = [ + ["SRTL_WORKER_COUNT=0", "SRTL_WORKER_COUNT must be a positive safe integer"], + ["SRTL_WORKER_COUNT=2.5", "SRTL_WORKER_COUNT must be a positive safe integer"], + ["SRTL_WORKER_COUNT=9007199254740992", "SRTL_WORKER_COUNT must be a positive safe integer"], + [["SRTL_WORKER_COUNT=2", "SRTL_MAX_RUNNING_JOBS=3"].join("\n"), "SRTL_MAX_RUNNING_JOBS must not exceed SRTL_WORKER_COUNT"], + [["SRTL_WORKER_COUNT=4", "SRTL_MAX_RUNNING_JOBS=2", "SRTL_MAX_RUNNING_COPIES=3"].join("\n"), "SRTL_MAX_RUNNING_COPIES must not exceed SRTL_MAX_RUNNING_JOBS"], + [["SRTL_COPY_FILE_CONCURRENCY=3", "SRTL_MAX_ACTIVE_COPY_FILES=2"].join("\n"), "SRTL_COPY_FILE_CONCURRENCY must not exceed SRTL_MAX_ACTIVE_COPY_FILES"] + ] as const; + + for (const [contents, message] of invalidSettings) { + await fs.writeFile(path.join(tmpDir, ".env"), contents); + expect(() => loadConfig({ rootDir: tmpDir })).toThrow(message); + } + }); + it("constructs the database URL from a single password setting", async () => { await fs.writeFile( path.join(tmpDir, ".env"), diff --git a/tests/copier.test.ts b/tests/copier.test.ts index 3271ed8..5ea72f6 100644 --- a/tests/copier.test.ts +++ b/tests/copier.test.ts @@ -1,10 +1,43 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { describe, expect, it } from "vitest"; -import { copyMediaLink, defaultCopyRunner, type CopyFileProgress } from "../src/server/lib/copier"; +import { describe, expect, it, vi } from "vitest"; +import { CopyReconciliationRequiredError, copyMediaLink, defaultCopyRunner, type CopyFileProgress, type CopyOperationUpdate } from "../src/server/lib/copier"; +import type { MediaLinkRow } from "../src/shared/types"; + +function remoteCopyLink(itemName: string, relativePath: string, linkPath: string, sourcePath: string, sizeBytes: number): MediaLinkRow { + const timestamp = new Date().toISOString(); + return { + id: 1, + section: "items", + itemName, + relativePath, + linkPath, + targetPath: sourcePath, + kind: "remote", + targetExists: true, + isMedia: true, + storagePolicy: "location_1", + resolvedStorageFileId: null, + sizeBytes, + firstSeenAt: timestamp, + lastSeenAt: timestamp, + lastChangedAt: timestamp, + missingSince: null, + updatedAt: timestamp + }; +} describe("copy runner", () => { + it("preserves a lease-loss abort reason", async () => { + const controller = new AbortController(); + const leaseLost = new Error("lease lost while copying"); + leaseLost.name = "LeaseLostError"; + controller.abort(leaseLost); + + await expect(defaultCopyRunner.copyFile("unused-source", "unused-target", undefined, controller.signal)).rejects.toBe(leaseLost); + }); + it("reports byte progress while comparing files", async () => { const directory = await fs.mkdtemp(path.join(os.tmpdir(), "srtl-copier-")); try { @@ -210,4 +243,558 @@ describe("copy runner", () => { await fs.rm(directory, { recursive: true, force: true }); } }); + + it("checks the job lease before promoting a transferred file", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "srtl-copier-lease-")); + try { + const symlinkDir = path.join(directory, "symlinks"); + const localDir = path.join(directory, "local"); + const remoteDir = path.join(directory, "remote"); + const relativePath = path.join("Lease Title", "lease.mkv"); + const sourcePath = path.join(remoteDir, "items", relativePath); + const linkPath = path.join(symlinkDir, "items", relativePath); + const destinationPath = path.join(localDir, "items", relativePath); + await Promise.all([ + fs.mkdir(path.dirname(sourcePath), { recursive: true }), + fs.mkdir(path.dirname(linkPath), { recursive: true }), + fs.mkdir(path.join(localDir, "items"), { recursive: true }) + ]); + await fs.writeFile(sourcePath, "lease source"); + await fs.symlink(sourcePath, linkPath); + const timestamp = new Date().toISOString(); + const leaseLost = new Error("Job lease is no longer owned by this worker"); + leaseLost.name = "LeaseLostError"; + + await expect( + copyMediaLink( + { + id: 1, + section: "items", + itemName: "Lease Title", + relativePath, + linkPath, + targetPath: sourcePath, + kind: "remote", + targetExists: true, + isMedia: true, + storagePolicy: "location_1", + resolvedStorageFileId: null, + sizeBytes: 12, + firstSeenAt: timestamp, + lastSeenAt: timestamp, + lastChangedAt: timestamp, + missingSince: null, + updatedAt: timestamp + }, + { symlinkDir, localDir, remoteDir }, + "to_local", + defaultCopyRunner, + undefined, + { profile: "off", byteCompare: false, mediaValidation: "off" }, + undefined, + undefined, + undefined, + async () => { + throw leaseLost; + } + ) + ).rejects.toThrow("Job lease is no longer owned"); + + await expect(fs.readlink(linkPath)).resolves.toBe(sourcePath); + await expect(fs.stat(destinationPath)).rejects.toMatchObject({ code: "ENOENT" }); + expect((await fs.readdir(path.dirname(destinationPath))).filter((name) => name.includes("srtl-copy"))).toEqual([]); + } finally { + await fs.rm(directory, { recursive: true, force: true }); + } + }); + + it("restores the original symlink when journaling fails after repointing", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "srtl-copier-journal-")); + try { + const symlinkDir = path.join(directory, "symlinks"); + const localDir = path.join(directory, "local"); + const remoteDir = path.join(directory, "remote"); + const relativePath = path.join("Journal Title", "journal.mkv"); + const sourcePath = path.join(remoteDir, "items", relativePath); + const linkPath = path.join(symlinkDir, "items", relativePath); + const destinationPath = path.join(localDir, "items", relativePath); + await Promise.all([ + fs.mkdir(path.dirname(sourcePath), { recursive: true }), + fs.mkdir(path.dirname(linkPath), { recursive: true }), + fs.mkdir(path.join(localDir, "items"), { recursive: true }) + ]); + await fs.writeFile(sourcePath, "journal source"); + await fs.symlink(sourcePath, linkPath); + const timestamp = new Date().toISOString(); + + await expect( + copyMediaLink( + { + id: 1, + section: "items", + itemName: "Journal Title", + relativePath, + linkPath, + targetPath: sourcePath, + kind: "remote", + targetExists: true, + isMedia: true, + storagePolicy: "location_1", + resolvedStorageFileId: null, + sizeBytes: 14, + firstSeenAt: timestamp, + lastSeenAt: timestamp, + lastChangedAt: timestamp, + missingSince: null, + updatedAt: timestamp + }, + { symlinkDir, localDir, remoteDir }, + "to_local", + defaultCopyRunner, + undefined, + { profile: "off", byteCompare: false, mediaValidation: "off" }, + undefined, + undefined, + async (update) => { + if (update.stage === "repointed") throw new Error("journal unavailable"); + } + ) + ).rejects.toThrow("journal unavailable"); + + await expect(fs.readlink(linkPath)).resolves.toBe(sourcePath); + await expect(fs.stat(destinationPath)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + await fs.rm(directory, { recursive: true, force: true }); + } + }); + + it("leaves the original destination intact when displacement fails", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "srtl-copier-displace-")); + try { + const symlinkDir = path.join(directory, "symlinks"); + const localDir = path.join(directory, "local"); + const remoteDir = path.join(directory, "remote"); + const relativePath = path.join("Displacement Title", "movie.mkv"); + const sourcePath = path.join(remoteDir, "items", relativePath); + const linkPath = path.join(symlinkDir, "items", relativePath); + const destinationPath = path.join(localDir, "items", relativePath); + await Promise.all([ + fs.mkdir(path.dirname(sourcePath), { recursive: true }), + fs.mkdir(path.dirname(linkPath), { recursive: true }), + fs.mkdir(path.dirname(destinationPath), { recursive: true }) + ]); + await fs.writeFile(sourcePath, "new destination contents"); + await fs.writeFile(destinationPath, "original destination contents"); + await fs.symlink(sourcePath, linkPath); + let displacementBlocker: string | null = null; + + await expect( + copyMediaLink( + remoteCopyLink("Displacement Title", relativePath, linkPath, sourcePath, Buffer.byteLength("new destination contents")), + { symlinkDir, localDir, remoteDir }, + "to_local", + defaultCopyRunner, + undefined, + { profile: "off", byteCompare: false, mediaValidation: "off" }, + undefined, + "replace", + async (update) => { + if (update.stage !== "destination_displaced" || !update.displacedPath) return; + displacementBlocker = update.displacedPath; + await fs.mkdir(displacementBlocker); + await fs.writeFile(path.join(displacementBlocker, "sentinel"), "block file rename"); + } + ) + ).rejects.toMatchObject({ code: expect.stringMatching(/EISDIR|ENOTDIR|ENOTEMPTY/) }); + + await expect(fs.readFile(destinationPath, "utf8")).resolves.toBe("original destination contents"); + await expect(fs.readlink(linkPath)).resolves.toBe(sourcePath); + if (displacementBlocker) await fs.rm(displacementBlocker, { recursive: true, force: true }); + expect((await fs.readdir(path.dirname(destinationPath))).filter((name) => name.includes(".srtl-replace-") || name.includes(".srtl-copy-"))).toEqual([]); + } finally { + await fs.rm(directory, { recursive: true, force: true }); + } + }); + + it("surfaces a reconciliation error when promoted-copy rollback fails", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "srtl-copier-rollback-")); + let removeSpy: ReturnType | undefined; + try { + const symlinkDir = path.join(directory, "symlinks"); + const localDir = path.join(directory, "local"); + const remoteDir = path.join(directory, "remote"); + const relativePath = path.join("Rollback Title", "movie.mkv"); + const sourcePath = path.join(remoteDir, "items", relativePath); + const linkPath = path.join(symlinkDir, "items", relativePath); + const destinationPath = path.join(localDir, "items", relativePath); + await Promise.all([ + fs.mkdir(path.dirname(sourcePath), { recursive: true }), + fs.mkdir(path.dirname(linkPath), { recursive: true }), + fs.mkdir(path.dirname(destinationPath), { recursive: true }) + ]); + await fs.writeFile(sourcePath, "rollback contents"); + await fs.symlink(sourcePath, linkPath); + + const originalRemove = fs.rm.bind(fs); + removeSpy = vi.spyOn(fs, "rm").mockImplementation(async (targetPath, options) => { + if (path.resolve(String(targetPath)) === destinationPath) { + throw Object.assign(new Error("injected rollback removal failure"), { code: "EIO" }); + } + return originalRemove(targetPath, options); + }); + + await expect( + copyMediaLink( + remoteCopyLink("Rollback Title", relativePath, linkPath, sourcePath, Buffer.byteLength("rollback contents")), + { symlinkDir, localDir, remoteDir }, + "to_local", + defaultCopyRunner, + undefined, + { profile: "off", byteCompare: false, mediaValidation: "off" }, + undefined, + undefined, + async (update) => { + if (update.stage === "repointed") throw new Error("injected journal failure after promotion"); + }, + (mutation) => mutation() + ) + ).rejects.toBeInstanceOf(CopyReconciliationRequiredError); + + await expect(fs.readFile(destinationPath, "utf8")).resolves.toBe("rollback contents"); + await expect(fs.readlink(linkPath)).resolves.toBe(sourcePath); + } finally { + removeSpy?.mockRestore(); + await fs.rm(directory, { recursive: true, force: true }); + } + }); + + it("does not overwrite a destination that appears while promotion waits for the lease guard", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "srtl-copier-promotion-race-")); + try { + const symlinkDir = path.join(directory, "symlinks"); + const localDir = path.join(directory, "local"); + const remoteDir = path.join(directory, "remote"); + const relativePath = path.join("Promotion Race Title", "movie.mkv"); + const sourcePath = path.join(remoteDir, "items", relativePath); + const linkPath = path.join(symlinkDir, "items", relativePath); + const destinationPath = path.join(localDir, "items", relativePath); + await Promise.all([ + fs.mkdir(path.dirname(sourcePath), { recursive: true }), + fs.mkdir(path.dirname(linkPath), { recursive: true }), + fs.mkdir(path.dirname(destinationPath), { recursive: true }) + ]); + await fs.writeFile(sourcePath, "copied contents"); + await fs.symlink(sourcePath, linkPath); + let guardCalls = 0; + + await expect( + copyMediaLink( + remoteCopyLink("Promotion Race Title", relativePath, linkPath, sourcePath, Buffer.byteLength("copied contents")), + { symlinkDir, localDir, remoteDir }, + "to_local", + defaultCopyRunner, + undefined, + { profile: "off", byteCompare: false, mediaValidation: "off" }, + undefined, + undefined, + undefined, + async (mutation) => { + guardCalls += 1; + if (guardCalls === 1) await fs.writeFile(destinationPath, "concurrent destination"); + return mutation(); + } + ) + ).rejects.toThrow("Destination changed before copy promotion"); + + await expect(fs.readFile(destinationPath, "utf8")).resolves.toBe("concurrent destination"); + await expect(fs.readlink(linkPath)).resolves.toBe(sourcePath); + expect((await fs.readdir(path.dirname(destinationPath))).filter((name) => name.includes(".srtl-copy-"))).toEqual([]); + } finally { + await fs.rm(directory, { recursive: true, force: true }); + } + }); + + it("does not displace a destination replaced while conflict handling waits for the lease guard", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "srtl-copier-replacement-race-")); + try { + const symlinkDir = path.join(directory, "symlinks"); + const localDir = path.join(directory, "local"); + const remoteDir = path.join(directory, "remote"); + const relativePath = path.join("Replacement Race Title", "movie.mkv"); + const sourcePath = path.join(remoteDir, "items", relativePath); + const linkPath = path.join(symlinkDir, "items", relativePath); + const destinationPath = path.join(localDir, "items", relativePath); + const concurrentPath = path.join(path.dirname(destinationPath), "concurrent.mkv"); + await Promise.all([ + fs.mkdir(path.dirname(sourcePath), { recursive: true }), + fs.mkdir(path.dirname(linkPath), { recursive: true }), + fs.mkdir(path.dirname(destinationPath), { recursive: true }) + ]); + await fs.writeFile(sourcePath, "new copied contents"); + await fs.writeFile(destinationPath, "original destination"); + await fs.symlink(sourcePath, linkPath); + let guardCalls = 0; + + await expect( + copyMediaLink( + remoteCopyLink("Replacement Race Title", relativePath, linkPath, sourcePath, Buffer.byteLength("new copied contents")), + { symlinkDir, localDir, remoteDir }, + "to_local", + defaultCopyRunner, + undefined, + { profile: "off", byteCompare: false, mediaValidation: "off" }, + undefined, + "replace", + undefined, + async (mutation) => { + guardCalls += 1; + if (guardCalls === 1) { + await fs.writeFile(concurrentPath, "concurrent destination"); + await fs.rename(concurrentPath, destinationPath); + } + return mutation(); + } + ) + ).rejects.toThrow("Destination changed before copy promotion"); + + await expect(fs.readFile(destinationPath, "utf8")).resolves.toBe("concurrent destination"); + await expect(fs.readlink(linkPath)).resolves.toBe(sourcePath); + expect((await fs.readdir(path.dirname(destinationPath))).filter((name) => name.includes(".srtl-replace-") || name.includes(".srtl-copy-"))).toEqual([]); + } finally { + await fs.rm(directory, { recursive: true, force: true }); + } + }); + + it("does not repoint a source symlink retargeted by another process during transfer", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "srtl-copier-link-retarget-")); + try { + const symlinkDir = path.join(directory, "symlinks"); + const localDir = path.join(directory, "local"); + const remoteDir = path.join(directory, "remote"); + const relativePath = path.join("Retargeted Title", "movie.mkv"); + const sourcePath = path.join(remoteDir, "items", relativePath); + const alternateSourcePath = path.join(remoteDir, "items", "Retargeted Title", "alternate.mkv"); + const linkPath = path.join(symlinkDir, "items", relativePath); + const destinationPath = path.join(localDir, "items", relativePath); + await Promise.all([ + fs.mkdir(path.dirname(sourcePath), { recursive: true }), + fs.mkdir(path.dirname(linkPath), { recursive: true }), + fs.mkdir(path.dirname(destinationPath), { recursive: true }) + ]); + await Promise.all([fs.writeFile(sourcePath, "admitted source"), fs.writeFile(alternateSourcePath, "external retarget")]); + await fs.symlink(sourcePath, linkPath); + + const runner = { + ...defaultCopyRunner, + async copyFile(...args: Parameters) { + await defaultCopyRunner.copyFile(...args); + await fs.rm(linkPath); + await fs.symlink(alternateSourcePath, linkPath); + } + }; + + await expect( + copyMediaLink( + remoteCopyLink("Retargeted Title", relativePath, linkPath, sourcePath, Buffer.byteLength("admitted source")), + { symlinkDir, localDir, remoteDir }, + "to_local", + runner, + undefined, + { profile: "off", byteCompare: false, mediaValidation: "off" }, + undefined, + undefined, + undefined, + (mutation) => mutation() + ) + ).rejects.toThrow("Symlink target changed since the last inventory scan"); + + await expect(fs.readlink(linkPath)).resolves.toBe(alternateSourcePath); + await expect(fs.stat(destinationPath)).rejects.toMatchObject({ code: "ENOENT" }); + expect((await fs.readdir(path.dirname(destinationPath))).filter((name) => name.includes(".srtl-copy-"))).toEqual([]); + } finally { + await fs.rm(directory, { recursive: true, force: true }); + } + }); + + it("preserves a same-size replacement when promoted-copy rollback detects a different identity", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "srtl-copier-identity-rollback-")); + try { + const symlinkDir = path.join(directory, "symlinks"); + const localDir = path.join(directory, "local"); + const remoteDir = path.join(directory, "remote"); + const relativePath = path.join("Identity Rollback Title", "movie.mkv"); + const sourcePath = path.join(remoteDir, "items", relativePath); + const linkPath = path.join(symlinkDir, "items", relativePath); + const destinationPath = path.join(localDir, "items", relativePath); + const replacementPath = path.join(path.dirname(destinationPath), "replacement.mkv"); + const sourceContents = Buffer.alloc(32, 0x61); + const originalDestinationContents = Buffer.alloc(32, 0x63); + const replacementContents = Buffer.alloc(32, 0x62); + await Promise.all([ + fs.mkdir(path.dirname(sourcePath), { recursive: true }), + fs.mkdir(path.dirname(linkPath), { recursive: true }), + fs.mkdir(path.dirname(destinationPath), { recursive: true }) + ]); + await Promise.all([fs.writeFile(sourcePath, sourceContents), fs.writeFile(destinationPath, originalDestinationContents)]); + await fs.symlink(sourcePath, linkPath); + const operationUpdates: CopyOperationUpdate[] = []; + + await expect( + copyMediaLink( + remoteCopyLink("Identity Rollback Title", relativePath, linkPath, sourcePath, sourceContents.length), + { symlinkDir, localDir, remoteDir }, + "to_local", + defaultCopyRunner, + undefined, + { profile: "off", byteCompare: false, mediaValidation: "off" }, + undefined, + "replace", + async (update) => { + operationUpdates.push({ ...update }); + if (update.stage !== "repointed" || update.resultStatus !== "copied") return; + await fs.writeFile(replacementPath, replacementContents); + await fs.rename(replacementPath, destinationPath); + throw new Error("injected journal failure after external destination replacement"); + }, + (mutation) => mutation() + ) + ).rejects.toBeInstanceOf(CopyReconciliationRequiredError); + + await expect(fs.readFile(destinationPath)).resolves.toEqual(replacementContents); + await expect(fs.readlink(linkPath)).resolves.toBe(sourcePath); + const identityUpdates = [ + operationUpdates.find((update) => update.stage === "verified" && update.tempIdentity)?.tempIdentity, + operationUpdates.find((update) => update.stage === "destination_displaced" && update.displacedIdentity)?.displacedIdentity, + operationUpdates.find((update) => update.stage === "promoted" && update.destinationIdentity)?.destinationIdentity + ]; + for (const rawIdentity of identityUpdates) { + expect(rawIdentity).toBeTypeOf("string"); + expect(JSON.parse(rawIdentity!)).toEqual({ + dev: expect.stringMatching(/^\d+$/), + ino: expect.stringMatching(/^\d+$/), + size: sourceContents.length.toString(), + mtimeNs: expect.stringMatching(/^\d+$/), + ctimeNs: expect.stringMatching(/^\d+$/) + }); + } + const displacedUpdate = operationUpdates.find((update) => update.stage === "destination_displaced" && update.displacedIdentity); + expect(displacedUpdate?.displacedPath).toBeTypeOf("string"); + await expect(fs.readFile(displacedUpdate!.displacedPath!)).resolves.toEqual(originalDestinationContents); + } finally { + await fs.rm(directory, { recursive: true, force: true }); + } + }); + + it("requires durable reconciliation when the lease wrapper fails after a filesystem mutation", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "srtl-copier-post-mutation-lease-")); + try { + const symlinkDir = path.join(directory, "symlinks"); + const localDir = path.join(directory, "local"); + const remoteDir = path.join(directory, "remote"); + const relativePath = path.join("Post Mutation Lease", "movie.mkv"); + const sourcePath = path.join(remoteDir, "items", relativePath); + const linkPath = path.join(symlinkDir, "items", relativePath); + const destinationPath = path.join(localDir, "items", relativePath); + await Promise.all([ + fs.mkdir(path.dirname(sourcePath), { recursive: true }), + fs.mkdir(path.dirname(linkPath), { recursive: true }), + fs.mkdir(path.dirname(destinationPath), { recursive: true }) + ]); + await Promise.all([fs.writeFile(sourcePath, "matching contents"), fs.writeFile(destinationPath, "matching contents")]); + await fs.symlink(sourcePath, linkPath); + const operationUpdates: CopyOperationUpdate[] = []; + + await expect( + copyMediaLink( + remoteCopyLink("Post Mutation Lease", relativePath, linkPath, sourcePath, Buffer.byteLength("matching contents")), + { symlinkDir, localDir, remoteDir }, + "to_local", + defaultCopyRunner, + undefined, + { profile: "off", byteCompare: false, mediaValidation: "off" }, + undefined, + undefined, + async (update) => { + operationUpdates.push(update); + }, + async (mutation) => { + await mutation(); + throw new Error("injected lease transaction commit failure"); + } + ) + ).rejects.toMatchObject({ + name: "CopyReconciliationRequiredError", + message: expect.stringContaining("Filesystem mutation completed") + }); + + await expect(fs.readlink(linkPath)).resolves.toBe(destinationPath); + expect(operationUpdates).toEqual( + expect.arrayContaining([expect.objectContaining({ stage: "repointed", destinationIdentity: expect.any(String), resultStatus: "repointed" })]) + ); + } finally { + await fs.rm(directory, { recursive: true, force: true }); + } + }); + + it("rechecks cancellation after the lease guard finishes waiting", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "srtl-copier-cancel-lease-")); + try { + const symlinkDir = path.join(directory, "symlinks"); + const localDir = path.join(directory, "local"); + const remoteDir = path.join(directory, "remote"); + const relativePath = path.join("Cancellation Title", "movie.mkv"); + const sourcePath = path.join(remoteDir, "items", relativePath); + const linkPath = path.join(symlinkDir, "items", relativePath); + const destinationPath = path.join(localDir, "items", relativePath); + await Promise.all([ + fs.mkdir(path.dirname(sourcePath), { recursive: true }), + fs.mkdir(path.dirname(linkPath), { recursive: true }), + fs.mkdir(path.dirname(destinationPath), { recursive: true }) + ]); + await fs.writeFile(sourcePath, "cancelled contents"); + await fs.symlink(sourcePath, linkPath); + + const controller = new AbortController(); + const cancellation = new Error("cancelled while waiting for the lease"); + let releaseLease!: () => void; + const leaseReleased = new Promise((resolve) => { + releaseLease = resolve; + }); + let reportLeaseWait!: () => void; + const leaseWaitStarted = new Promise((resolve) => { + reportLeaseWait = resolve; + }); + let guardCalls = 0; + const copy = copyMediaLink( + remoteCopyLink("Cancellation Title", relativePath, linkPath, sourcePath, Buffer.byteLength("cancelled contents")), + { symlinkDir, localDir, remoteDir }, + "to_local", + defaultCopyRunner, + undefined, + { profile: "off", byteCompare: false, mediaValidation: "off" }, + controller.signal, + undefined, + undefined, + async (mutation) => { + guardCalls += 1; + if (guardCalls === 1) { + reportLeaseWait(); + await leaseReleased; + } + return mutation(); + } + ); + + await leaseWaitStarted; + controller.abort(cancellation); + releaseLease(); + + await expect(copy).rejects.toBe(cancellation); + await expect(fs.stat(destinationPath)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(fs.readlink(linkPath)).resolves.toBe(sourcePath); + expect((await fs.readdir(path.dirname(destinationPath))).filter((name) => name.includes(".srtl-copy-"))).toEqual([]); + } finally { + await fs.rm(directory, { recursive: true, force: true }); + } + }); }); diff --git a/tests/copyLimiter.test.ts b/tests/copyLimiter.test.ts new file mode 100644 index 0000000..1c36ee7 --- /dev/null +++ b/tests/copyLimiter.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest"; +import { CopyTransferLimiter } from "../src/server/jobs/copyLimiter"; + +describe("copy transfer limiter", () => { + it("admits up to the configured limit and releases waiters in order", async () => { + const limiter = new CopyTransferLimiter(2); + const signal = new AbortController().signal; + const first = await limiter.acquire(signal); + const second = await limiter.acquire(signal); + let thirdStarted = false; + const thirdPromise = limiter.acquire(signal).then((release) => { + thirdStarted = true; + return release; + }); + + await Promise.resolve(); + expect(limiter.activeCount).toBe(2); + expect(limiter.waitingCount).toBe(1); + expect(thirdStarted).toBe(false); + + first(); + const third = await thirdPromise; + expect(thirdStarted).toBe(true); + expect(limiter.activeCount).toBe(2); + + second(); + third(); + expect(limiter.activeCount).toBe(0); + }); + + it("removes an aborted waiter without consuming a transfer slot", async () => { + const limiter = new CopyTransferLimiter(1); + const active = await limiter.acquire(new AbortController().signal); + const waitingController = new AbortController(); + const waiting = limiter.acquire(waitingController.signal); + + waitingController.abort(new Error("job lease lost")); + await expect(waiting).rejects.toThrow("job lease lost"); + expect(limiter.waitingCount).toBe(0); + + active(); + expect(limiter.activeCount).toBe(0); + }); + + it("does not impose an arbitrary upper limit", () => { + expect(new CopyTransferLimiter(128).maximum).toBe(128); + }); +}); diff --git a/tests/copyPool.test.ts b/tests/copyPool.test.ts new file mode 100644 index 0000000..346c6a4 --- /dev/null +++ b/tests/copyPool.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it, vi } from "vitest"; +import { runKeyedPool } from "../src/server/jobs/copyPool"; + +describe("keyed copy pool", () => { + it("runs independent lanes concurrently without overlapping one lane", async () => { + const activeKeys = new Set(); + let active = 0; + let maximumActive = 0; + const release: Array<() => void> = []; + const started: string[] = []; + + const run = runKeyedPool( + ["title-a/1", "title-a/2", "title-b/1", "title-c/1"], + 3, + (item) => item.split("/")[0]!, + async (item) => { + const key = item.split("/")[0]!; + expect(activeKeys.has(key)).toBe(false); + activeKeys.add(key); + active += 1; + maximumActive = Math.max(maximumActive, active); + started.push(item); + await new Promise((resolve) => release.push(resolve)); + active -= 1; + activeKeys.delete(key); + } + ); + + await vi.waitFor(() => expect(started).toEqual(["title-a/1", "title-b/1", "title-c/1"])); + expect(maximumActive).toBe(3); + release.shift()?.(); + await vi.waitFor(() => expect(started).toContain("title-a/2")); + while (release.length > 0) release.shift()?.(); + await run; + }); + + it("waits for active lanes to settle after one fails", async () => { + let releaseSecond!: () => void; + let markSecondStarted!: () => void; + const secondStarted = new Promise((resolve) => { + markSecondStarted = resolve; + }); + let secondSettled = false; + const run = runKeyedPool( + ["first", "second"], + 2, + (item) => item, + async (item) => { + if (item === "first") throw new Error("lease lost"); + await new Promise((resolve) => { + releaseSecond = resolve; + markSecondStarted(); + }); + secondSettled = true; + } + ); + + await secondStarted; + let rejected = false; + void run.catch(() => { + rejected = true; + }); + await Promise.resolve(); + expect(rejected).toBe(false); + releaseSecond(); + await expect(run).rejects.toThrow("lease lost"); + expect(secondSettled).toBe(true); + }); +}); diff --git a/tests/database.test.ts b/tests/database.test.ts index a1e7073..f8f7c0b 100644 --- a/tests/database.test.ts +++ b/tests/database.test.ts @@ -23,7 +23,9 @@ describe("database bootstrap", () => { from information_schema.columns where table_name = 'jobs' `); - expect(columns.rows.map((column) => column.column_name)).toEqual(expect.arrayContaining(["locked_by", "locked_at", "heartbeat_at", "cancel_requested_at"])); + expect(columns.rows.map((column) => column.column_name)).toEqual( + expect.arrayContaining(["locked_by", "locked_at", "heartbeat_at", "lease_version", "exclusive", "cancel_requested_at"]) + ); const indexes = await database.pool.query<{ indexname: string }>(` select indexname @@ -33,7 +35,7 @@ describe("database bootstrap", () => { expect(indexes.rows.map((index) => index.indexname)).toEqual(expect.arrayContaining(["jobs_status_idx", "jobs_heartbeat_idx"])); const migrations = await database.pool.query<{ version: number }>("select version from schema_migrations order by version"); - expect(migrations.rows).toEqual([{ version: 1 }, { version: 2 }, { version: 3 }, { version: 4 }]); + expect(migrations.rows).toEqual([{ version: 1 }, { version: 2 }, { version: 3 }, { version: 4 }, { version: 5 }, { version: 6 }, { version: 7 }]); const workerColumns = await database.pool.query<{ column_name: string }>(` select column_name @@ -41,8 +43,61 @@ describe("database bootstrap", () => { where table_name = 'worker_heartbeats' `); expect(workerColumns.rows.map((column) => column.column_name)).toEqual( - expect.arrayContaining(["worker_id", "started_at", "heartbeat_at", "status"]) + expect.arrayContaining(["worker_id", "started_at", "heartbeat_at", "status", "capacity"]) ); + const workerIndexes = await database.pool.query<{ indexname: string }>(` + SELECT indexname FROM pg_indexes WHERE tablename = 'worker_heartbeats' + `); + expect(workerIndexes.rows.map((index) => index.indexname)).toContain("worker_heartbeats_status_heartbeat_idx"); + const copyOperationColumns = await database.pool.query<{ column_name: string; is_nullable: string }>(` + SELECT column_name, is_nullable FROM information_schema.columns WHERE table_name = 'copy_operations' + `); + for (const columnName of ["temp_identity", "destination_identity", "displaced_identity"]) { + expect(copyOperationColumns.rows).toContainEqual({ column_name: columnName, is_nullable: "YES" }); + } + const pathMigrationColumns = await database.pool.query<{ column_name: string; is_nullable: string }>(` + SELECT column_name, is_nullable FROM information_schema.columns WHERE table_name = 'path_migration_items' + `); + expect(pathMigrationColumns.rows).toContainEqual({ column_name: "target_identity", is_nullable: "YES" }); + const insertedHeartbeat = await database.pool.query<{ capacity: string }>(` + INSERT INTO worker_heartbeats (worker_id, started_at, heartbeat_at, status) + VALUES ('default-capacity-worker', '2026-07-29T00:00:00.000Z', '2026-07-29T00:00:00.000Z', 'running') + RETURNING capacity + `); + expect(insertedHeartbeat.rows).toEqual([{ capacity: "1" }]); + + const insertedJob = await database.pool.query<{ id: number; lease_version: number; exclusive: boolean }>(` + INSERT INTO jobs (type, status, created_at, progress) + VALUES ('copy', 'queued', '2026-07-29T00:00:00.000Z', '{}') + RETURNING id, lease_version, exclusive + `); + expect(insertedJob.rows[0]).toMatchObject({ lease_version: 0, exclusive: true }); + + const jobId = insertedJob.rows[0]!.id; + const insertedClaim = await database.pool.query<{ access: string }>(` + INSERT INTO job_resource_claims (job_id, resource_type, resource_key, created_at) + VALUES ($1, 'path', '/media/title', '2026-07-29T00:00:00.000Z') + RETURNING access + `, [jobId]); + expect(insertedClaim.rows).toEqual([{ access: "exclusive" }]); + await expect( + database.pool.query( + `INSERT INTO job_resource_claims (job_id, resource_type, resource_key, access, created_at) VALUES ($1, 'path', '/media/other', 'invalid', '2026-07-29T00:00:00.000Z')`, + [jobId] + ) + ).rejects.toThrow(); + + const claimIndexes = await database.pool.query<{ indexname: string }>(` + SELECT indexname FROM pg_indexes WHERE tablename = 'job_resource_claims' + `); + expect(claimIndexes.rows.map((index) => index.indexname)).toEqual( + expect.arrayContaining(["job_resource_claims_job_resource_idx", "job_resource_claims_lookup_idx"]) + ); + + await database.pool.query(`DELETE FROM jobs WHERE id = $1`, [jobId]); + expect((await database.pool.query<{ count: string }>(`SELECT count(*) FROM job_resource_claims WHERE job_id = $1`, [jobId])).rows).toEqual([ + { count: "0" } + ]); } finally { await secondDatabase.close(); await database.close(); @@ -63,6 +118,9 @@ describe("database bootstrap", () => { await legacyPool.query(`CREATE TABLE storage_policies (id SERIAL PRIMARY KEY, policy TEXT NOT NULL)`); await legacyPool.query(`ALTER TABLE storage_policies ADD CONSTRAINT storage_policies_policy_check CHECK (policy IN ('assign_local', 'assign_remote'))`); await legacyPool.query(`CREATE TABLE copy_operations (id SERIAL PRIMARY KEY, original_link_state TEXT NOT NULL)`); + await legacyPool.query(`CREATE TABLE jobs (id SERIAL PRIMARY KEY)`); + await legacyPool.query(`CREATE TABLE worker_heartbeats (worker_id TEXT PRIMARY KEY, started_at TEXT NOT NULL, heartbeat_at TEXT NOT NULL, status TEXT NOT NULL)`); + await legacyPool.query(`CREATE TABLE path_migration_items (id SERIAL PRIMARY KEY)`); await legacyPool.query(`INSERT INTO media_links (storage_policy, is_assigned_remote) VALUES ('assign_local', false), ('assign_remote', true), ('unassigned', false)`); await legacyPool.query(`INSERT INTO storage_files (storage_policy) VALUES ('assign_local'), ('assign_remote'), ('unassigned')`); await legacyPool.query(`INSERT INTO storage_policies (policy) VALUES ('assign_local'), ('assign_remote')`); @@ -102,7 +160,134 @@ describe("database bootstrap", () => { { version: 1 }, { version: 2 }, { version: 3 }, - { version: 4 } + { version: 4 }, + { version: 5 }, + { version: 6 }, + { version: 7 } + ]); + } finally { + await database.close(); + await testDatabase.cleanup(); + } + }); + + it("upgrades active version-four jobs as conservative exclusive leases", async () => { + const testDatabase = await createTestDatabase(); + const legacyPool = new Pool({ connectionString: testDatabase.databaseUrl }); + try { + await legacyPool.query(`CREATE TABLE schema_migrations (version INTEGER PRIMARY KEY, name TEXT NOT NULL, applied_at TEXT NOT NULL)`); + await legacyPool.query(` + INSERT INTO schema_migrations (version, name, applied_at) + VALUES + (1, 'initial', now()::text), + (2, 'hardening', now()::text), + (3, 'cleanup', now()::text), + (4, 'location_identity', now()::text) + `); + await legacyPool.query(` + CREATE TABLE jobs ( + id SERIAL PRIMARY KEY, + type TEXT NOT NULL, + status TEXT NOT NULL, + created_at TEXT NOT NULL, + started_at TEXT, + finished_at TEXT, + locked_by TEXT, + locked_at TEXT, + heartbeat_at TEXT, + cancel_requested_at TEXT, + progress TEXT NOT NULL + ) + `); + await legacyPool.query(`CREATE TABLE worker_heartbeats (worker_id TEXT PRIMARY KEY, started_at TEXT NOT NULL, heartbeat_at TEXT NOT NULL, status TEXT NOT NULL)`); + await legacyPool.query(`CREATE TABLE copy_operations (id SERIAL PRIMARY KEY)`); + await legacyPool.query(`CREATE TABLE path_migration_items (id SERIAL PRIMARY KEY)`); + await legacyPool.query(` + INSERT INTO worker_heartbeats (worker_id, started_at, heartbeat_at, status) + VALUES ('legacy-worker-process', now()::text, now()::text, 'running') + `); + await legacyPool.query(` + INSERT INTO jobs (type, status, created_at, started_at, locked_by, locked_at, heartbeat_at, progress) + VALUES + ('copy', 'queued', now()::text, NULL, NULL, NULL, NULL, '{}'), + ('scan', 'running', now()::text, now()::text, 'legacy-worker', now()::text, now()::text, '{}') + `); + } finally { + await legacyPool.end(); + } + + const database = await openDatabase(testDatabase.databaseUrl); + try { + const rows = await database.pool.query<{ status: string; lease_version: number; exclusive: boolean }>(` + SELECT status, lease_version, exclusive FROM jobs ORDER BY id + `); + expect(rows.rows).toEqual([ + { status: "queued", lease_version: 0, exclusive: true }, + { status: "running", lease_version: 0, exclusive: true } + ]); + expect( + (await database.pool.query<{ worker_id: string; capacity: string }>(`SELECT worker_id, capacity FROM worker_heartbeats`)).rows + ).toEqual([{ worker_id: "legacy-worker-process", capacity: "1" }]); + const identityColumns = await database.pool.query<{ column_name: string; is_nullable: string }>(` + SELECT column_name, is_nullable + FROM information_schema.columns + WHERE table_name = 'copy_operations' AND column_name IN ('temp_identity', 'destination_identity', 'displaced_identity') + ORDER BY column_name + `); + expect(identityColumns.rows).toEqual([ + { column_name: "destination_identity", is_nullable: "YES" }, + { column_name: "displaced_identity", is_nullable: "YES" }, + { column_name: "temp_identity", is_nullable: "YES" } + ]); + } finally { + await database.close(); + await testDatabase.cleanup(); + } + }); + + it("repairs version-five schemas missing copy and path-migration file identities", async () => { + const testDatabase = await createTestDatabase(); + const legacyPool = new Pool({ connectionString: testDatabase.databaseUrl }); + try { + await legacyPool.query(`CREATE TABLE schema_migrations (version INTEGER PRIMARY KEY, name TEXT NOT NULL, applied_at TEXT NOT NULL)`); + await legacyPool.query(` + INSERT INTO schema_migrations (version, name, applied_at) + VALUES + (1, 'initial_postgres_schema', now()::text), + (2, 'beta_security_and_recovery', now()::text), + (3, 'beta_runtime_health_and_cleanup', now()::text), + (4, 'location_identity_storage_policies', now()::text), + (5, 'multi_worker_job_claims', now()::text) + `); + await legacyPool.query(`CREATE TABLE copy_operations (id SERIAL PRIMARY KEY)`); + await legacyPool.query(`CREATE TABLE path_migration_items (id SERIAL PRIMARY KEY)`); + await legacyPool.query(`INSERT INTO path_migration_items DEFAULT VALUES`); + } finally { + await legacyPool.end(); + } + + const database = await openDatabase(testDatabase.databaseUrl); + try { + const identityColumns = await database.pool.query<{ column_name: string; is_nullable: string }>(` + SELECT column_name, is_nullable + FROM information_schema.columns + WHERE table_name = 'copy_operations' AND column_name IN ('temp_identity', 'destination_identity', 'displaced_identity') + ORDER BY column_name + `); + expect(identityColumns.rows).toEqual([ + { column_name: "destination_identity", is_nullable: "YES" }, + { column_name: "displaced_identity", is_nullable: "YES" }, + { column_name: "temp_identity", is_nullable: "YES" } + ]); + expect((await database.pool.query<{ target_identity: string | null }>(`SELECT target_identity FROM path_migration_items`)).rows).toEqual([ + { target_identity: null } + ]); + expect( + (await database.pool.query<{ version: number; name: string }>(`SELECT version, name FROM schema_migrations WHERE version >= 5 ORDER BY version`)).rows + ).toEqual([ + { version: 5, name: "multi_worker_job_claims" }, + { version: 6, name: "copy_operation_file_identities" }, + { version: 7, name: "path_migration_target_identities" } ]); } finally { await database.close(); diff --git a/tests/e2e/app-smoke.spec.ts b/tests/e2e/app-smoke.spec.ts index e0b3f75..3eeb649 100644 --- a/tests/e2e/app-smoke.spec.ts +++ b/tests/e2e/app-smoke.spec.ts @@ -89,7 +89,7 @@ test("dashboard task notifications overlay without shifting content", async ({ p else if (url.pathname === "/api/settings/storage-locations") body = { locations: [{ key: "location_1", rootType: "local", displayName: "Local", path: "/mnt/local" }, { key: "location_2", rootType: "remote", displayName: "Remote", path: "/mnt/remote" }] }; else if (url.pathname === "/api/system/version") { const unavailableRelease = { latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, releaseNotes: null, message: "Unavailable" }; - body = { currentVersion: "0.1.2-beta.2", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; + body = { currentVersion: "0.1.2-beta.3", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; } else if (url.pathname === "/api/settings/sections") body = { sections: ["shows"], sectionTitles: { shows: "Shows" }, sectionTypes: { shows: "shows" } }; else if (url.pathname === "/api/settings/paths") body = { symlinkDir: "/mnt/links", localDir: "/mnt/local", remoteDir: "/mnt/remote" }; else if (url.pathname === "/api/settings/scan") body = { scanSymlinks: true, scanLocal: false, scanRemote: false, symlinkSections: ["shows"], localSections: [] }; @@ -220,7 +220,7 @@ test("refreshes an open work list when an inventory job finishes", async ({ page else if (url.pathname === "/api/settings/storage-locations") body = { locations: [{ key: "location_1", rootType: "local", displayName: "Local", path: "/mnt/local" }, { key: "location_2", rootType: "remote", displayName: "Remote", path: "/mnt/remote" }] }; else if (url.pathname === "/api/system/version") { const unavailableRelease = { latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, releaseNotes: null, message: "Unavailable" }; - body = { currentVersion: "0.1.2-beta.2", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; + body = { currentVersion: "0.1.2-beta.3", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; } else if (url.pathname === "/api/settings/sections") body = { sections: ["shows"], sectionTitles: { shows: "Shows" }, sectionTypes: { shows: "shows" } }; else if (url.pathname === "/api/settings/paths") body = { symlinkDir: "/mnt/links", localDir: "/mnt/local", remoteDir: "/mnt/remote" }; @@ -250,6 +250,108 @@ test("refreshes an open work list when an inventory job finishes", async ({ page expect(workListRequests).toBeGreaterThanOrEqual(2); }); +test("loads every work-list page and scopes show copies beyond the first page", async ({ page }) => { + const timestamp = "2026-07-29T21:20:00.000Z"; + const requestedOffsets: number[] = []; + let conflictPayload: Record | null = null; + let copyPayload: Record | null = null; + const mediaRow = (id: number, itemName: string, episode: string) => ({ + id, + section: "shows", + itemName, + relativePath: `${itemName}/Season 01/${itemName} - ${episode}.mkv`, + linkPath: `/mnt/links/shows/${itemName}/Season 01/${itemName} - ${episode}.mkv`, + targetPath: `/mnt/remote/${itemName} - ${episode}.mkv`, + kind: "remote", + targetExists: true, + isMedia: true, + storagePolicy: "location_1", + resolvedStorageFileId: null, + sizeBytes: 100, + firstSeenAt: timestamp, + lastSeenAt: timestamp, + lastChangedAt: timestamp, + missingSince: null, + updatedAt: timestamp + }); + const firstPage = [mediaRow(3, "Page Split Show", "S01E01"), mediaRow(2, "Another Show", "S01E01")]; + const secondPage = [mediaRow(1, "Page Split Show", "S01E02")]; + const inventory = { + totalLinks: 3, + remoteLinks: 3, + localLinks: 0, + brokenLinks: 0, + otherLinks: 0, + nonMediaLinks: 0, + actionableRemoteLinks: 3, + actionableLocalLinks: 0, + assignedRemoteLinks: 0, + unassignedRemoteLinks: 0, + unassignedLocalLinks: 0, + localFiles: 0, + remoteFiles: 0, + actionableRemoteFiles: 0, + actionableLocalFiles: 0, + assignedRemoteFiles: 0, + unassignedRemoteFiles: 0, + unassignedLocalFiles: 0, + localOrphanFiles: 0, + remoteOrphanFiles: 0, + missingLinks: 0, + missingLocalFiles: 0, + missingRemoteFiles: 0 + }; + + await page.route("**/api/**", async (route) => { + const url = new URL(route.request().url()); + let body: unknown = {}; + + if (url.pathname === "/api/auth/me") body = { setupRequired: false, authenticated: true, user: { id: 1, username: "admin" } }; + else if (url.pathname === "/api/system/path-migration") body = { status: "ready", blocking: false, activePaths: {}, detectedPaths: {}, environmentErrors: [], changes: [], migration: null }; + else if (url.pathname === "/api/onboarding") body = { required: false, phase: "completed" }; + else if (url.pathname === "/api/settings/user-preferences") body = { timeFormat: "12h", autoOpenTaskStatus: false, recentJobsCompletedWindowMinutes: 1440 }; + else if (url.pathname === "/api/settings/storage-locations") body = { locations: [{ key: "location_1", rootType: "local", displayName: "Local", path: "/mnt/local" }, { key: "location_2", rootType: "remote", displayName: "Remote", path: "/mnt/remote" }] }; + else if (url.pathname === "/api/system/version") { + const unavailableRelease = { latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, releaseNotes: null, message: "Unavailable" }; + body = { currentVersion: "0.1.2-beta.3", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; + } + else if (url.pathname === "/api/settings/sections") body = { sections: ["shows"], sectionTitles: { shows: "Shows" }, sectionTypes: { shows: "shows" } }; + else if (url.pathname === "/api/settings/paths") body = { symlinkDir: "/mnt/links", localDir: "/mnt/local", remoteDir: "/mnt/remote" }; + else if (url.pathname === "/api/settings/scan") body = { scanSymlinks: true, scanLocal: false, scanRemote: false, symlinkSections: ["shows"], localSections: [] }; + else if (url.pathname === "/api/settings/audit") body = { sections: ["shows"], targets: ["local", "remote"] }; + else if (url.pathname === "/api/sections") body = [{ section: "shows", title: "Shows", type: "shows", totalLinks: 3, itemCount: 2, seasonCount: 2, episodeCount: 3, remoteLinks: 3, localLinks: 0, brokenLinks: 0, otherLinks: 0, nonMediaLinks: 0, actionableRemoteLinks: 3, actionableLocalLinks: 0, assignedRemoteLinks: 0, unassignedRemoteLinks: 0, unassignedLocalLinks: 0 }]; + else if (url.pathname === "/api/inventory/summary") body = inventory; + else if (url.pathname === "/api/inventory/scan-timestamps") body = { symlinkSections: { shows: timestamp }, localSections: { shows: null }, remoteRoot: null }; + else if (url.pathname === "/api/media-links/page") { + const offset = Number(url.searchParams.get("offset") ?? 0); + requestedOffsets.push(offset); + const rows = offset === 0 ? firstPage : offset === 2 ? secondPage : []; + body = { rows, total: 3, limit: 250, offset, hasMore: offset === 0 }; + } else if (url.pathname === "/api/copies/conflicts") { + conflictPayload = route.request().postDataJSON() as Record; + body = { conflicts: [], totalConflicts: 0, totalCandidates: 0 }; + } else if (url.pathname === "/api/copies") { + copyPayload = route.request().postDataJSON() as Record; + body = { jobId: 555 }; + } else if (url.pathname === "/api/jobs") body = []; + + await route.fulfill({ status: 200, contentType: "application/json", body: JSON.stringify(body) }); + }); + + await page.goto(baseUrl!); + await page.getByRole("button", { name: "Shows: 3 episodes need copy to Local", exact: true }).click(); + await expect(page.getByText("Showing all 3 links across 2 shows", { exact: true })).toBeVisible(); + const splitShow = page.locator(".actionable-show-group").filter({ hasText: "Page Split Show" }); + await expect(splitShow).toHaveCount(1); + await expect(splitShow.locator(".actionable-show-counts")).toContainText("2Episodes"); + await splitShow.getByRole("button", { name: "Copy Show to Local", exact: true }).click(); + await expect.poll(() => copyPayload).not.toBeNull(); + + expect(requestedOffsets.slice(0, 2)).toEqual([0, 2]); + expect(conflictPayload).toEqual({ direction: "to_local", section: "shows", itemName: "Page Split Show" }); + expect(copyPayload).toEqual(conflictPayload); +}); + test("renders every authenticated route without page errors or global overflow", async ({ page }) => { test.skip(!sessionToken, "Set SRTL_E2E_SESSION_TOKEN to exercise authenticated pages."); test.setTimeout(90_000); @@ -385,8 +487,8 @@ test("applies and persists theme changes across the sidebar and content shell", }); test("a detected path change blocks the app behind a responsive migration gate", async ({ page }) => { - const unavailableIdentity = { available: false, realPath: null, device: null, inode: null, error: "Identity unavailable in browser fixture" }; - const sameIdentity = { available: true, realPath: "/storage/local", device: "1", inode: "2", error: null }; + const unavailableIdentity = { available: false, realPath: null, device: null, inode: null, mount: null, error: "Identity unavailable in browser fixture" }; + const sameIdentity = { available: true, realPath: "/storage/local", device: "1", inode: "2", mount: null, error: null }; const state = { status: "ready_to_apply", blocking: true, @@ -462,7 +564,7 @@ test("a detected path change blocks the app behind a responsive migration gate", await expect(page.getByRole("heading", { name: "Storage paths require attention", exact: true })).toBeVisible(); await expect(page.getByText("Maintenance mode", { exact: true })).toBeVisible(); await expect(page.locator(".path-change-row")).toHaveCount(3); - await expect(page.getByText("Same root detected", { exact: true })).toBeVisible(); + await expect(page.getByText("Same storage mount", { exact: true })).toBeVisible(); const applyButton = page.getByRole("button", { name: "Apply validated migration", exact: true }); await expect(applyButton).toBeDisabled(); await page.getByRole("checkbox", { name: "I confirm the detected paths expose the same storage content.", exact: true }).check(); @@ -760,6 +862,73 @@ test("title rescan controls explain their scope and lock sibling actions while q }); }); +test("failed copy admission does not display a waiting job", async ({ page }) => { + const timestamp = "2026-07-29T20:42:08.000Z"; + const title = "Newly Scanned Title (2026)"; + const item = { + id: 42, + title, + normalizedTitle: "newly scanned title (2026)", + policy: "location_1", + category: "other", + sections: ["shows"], + linkCount: 1, + remoteLinkCount: 1, + localLinkCount: 0, + fileCount: 1, + remoteFileCount: 1, + localFileCount: 0, + sectionCount: 1, + source: "scan", + updatedAt: timestamp + }; + const admissionError = "Copy data from job #199 requires manual reconciliation before another action can touch the same media item or managed path."; + + await page.route("**/api/**", async (route) => { + const url = new URL(route.request().url()); + let body: unknown = {}; + let status = 200; + + if (url.pathname === "/api/auth/me") body = { setupRequired: false, authenticated: true, user: { id: 1, username: "admin" } }; + else if (url.pathname === "/api/system/path-migration") body = { status: "ready", blocking: false, activePaths: {}, detectedPaths: {}, environmentErrors: [], changes: [], migration: null }; + else if (url.pathname === "/api/onboarding") body = { required: false, phase: "completed" }; + else if (url.pathname === "/api/settings/user-preferences") body = { timeFormat: "12h", autoOpenTaskStatus: true, recentJobsCompletedWindowMinutes: 1440 }; + else if (url.pathname === "/api/settings/storage-locations") body = { locations: [{ key: "location_1", rootType: "local", displayName: "Local", path: "/mnt/local" }, { key: "location_2", rootType: "remote", displayName: "Remote", path: "/mnt/remote" }] }; + else if (url.pathname === "/api/system/version") { + const unavailableRelease = { latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, releaseNotes: null, message: "Unavailable" }; + body = { currentVersion: "0.1.2-beta.3", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; + } + else if (url.pathname === "/api/settings/sections") body = { sections: ["shows"], sectionTitles: { shows: "Shows" }, sectionTypes: { shows: "shows" } }; + else if (url.pathname === "/api/settings/paths") body = { symlinkDir: "/mnt/links", localDir: "/mnt/local", remoteDir: "/mnt/remote" }; + else if (url.pathname === "/api/sections") body = []; + else if (url.pathname === "/api/inventory/summary") body = {}; + else if (url.pathname === "/api/inventory/scan-timestamps") body = { symlinkSections: { shows: timestamp }, localSections: { shows: null }, remoteRoot: null }; + else if (url.pathname === "/api/jobs") body = []; + else if (url.pathname === "/api/storage-policies") body = url.searchParams.get("policy") === "location_1" ? [item] : []; + else if (url.pathname === "/api/copies/conflicts") body = { conflicts: [], totalConflicts: 0, totalCandidates: 0 }; + else if (url.pathname === "/api/copies") { + status = 409; + body = { error: admissionError }; + } + + await route.fulfill({ status, contentType: "application/json", body: JSON.stringify(body) }); + }); + + await page.goto(`${baseUrl}/library`); + await page.getByRole("button", { name: "Storage Policies", exact: true }).click(); + await page.locator(".policy-tabs button").filter({ hasText: "Local" }).click(); + await page.getByPlaceholder("Filter scanned titles", { exact: true }).fill(title); + const row = page.locator("tbody tr").filter({ hasText: title }); + await expect(row).toHaveCount(1); + await row.getByRole("button", { name: "Copy to Local", exact: true }).click(); + + const dialog = page.locator(".copy-dialog"); + await expect(dialog.locator(".action-error")).toHaveText(admissionError); + await expect(dialog.getByText("The copy job was not queued. No files were changed.", { exact: true })).toBeVisible(); + await expect(dialog.locator(".copy-progress-panel")).toHaveCount(0); + await expect(dialog.locator(".audit-dialog-events")).toHaveCount(0); +}); + test("recent jobs identifies a targeted scan by title instead of only its parent folder", async ({ page }) => { test.skip(!sessionToken, "Set SRTL_E2E_SESSION_TOKEN to exercise authenticated pages."); const jobId = 999997; @@ -1184,7 +1353,7 @@ test("copy progress opens a persistent, scrollable completed item summary", asyn else if (url.pathname === "/api/settings/storage-locations") body = { locations: [{ key: "location_1", rootType: "local", displayName: "Local", path: "/mnt/local" }, { key: "location_2", rootType: "remote", displayName: "Remote", path: "/mnt/remote" }] }; else if (url.pathname === "/api/system/version") { const unavailableRelease = { latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, releaseNotes: null, message: "Unavailable" }; - body = { currentVersion: "0.1.2-beta.2", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; + body = { currentVersion: "0.1.2-beta.3", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; } else if (url.pathname === `/api/jobs/${jobId}/events/page`) body = { events, total: events.length, hasOlder: false }; else if (url.pathname === `/api/jobs/${jobId}`) body = job; else if (url.pathname === "/api/jobs") body = [job]; diff --git a/tests/env.test.ts b/tests/env.test.ts index 0865339..a8fb9f3 100644 --- a/tests/env.test.ts +++ b/tests/env.test.ts @@ -8,14 +8,25 @@ SYMLINK_DIR="/symlinks" SRTL_LOCATION_1_PATH='/local' SRTL_LOCATION_2_PATH=/remote SRTL_WORKER_COUNT=4 -SRTL_MAX_RUNNING_JOBS=8 +SRTL_MAX_RUNNING_JOBS=4 +SRTL_MAX_RUNNING_SCANS=1 +SRTL_MAX_RUNNING_AUDITS=3 +SRTL_MAX_RUNNING_COPIES=2 +SRTL_COPY_FILE_CONCURRENCY=2 +SRTL_MAX_ACTIVE_COPY_FILES=4 IGNORED=value `); expect(env).toEqual({ SYMLINK_DIR: "/symlinks", SRTL_LOCATION_1_PATH: "/local", SRTL_LOCATION_2_PATH: "/remote", - SRTL_WORKER_COUNT: "4" + SRTL_WORKER_COUNT: "4", + SRTL_MAX_RUNNING_JOBS: "4", + SRTL_MAX_RUNNING_SCANS: "1", + SRTL_MAX_RUNNING_AUDITS: "3", + SRTL_MAX_RUNNING_COPIES: "2", + SRTL_COPY_FILE_CONCURRENCY: "2", + SRTL_MAX_ACTIVE_COPY_FILES: "4" }); }); @@ -30,13 +41,18 @@ SRTL_LOCATION_2_PATH=/old/remote SRTL_LOCATION_1_PATH: "/mnt/local/nas/local", SRTL_LOCATION_2_PATH: "/mnt/remote", SRTL_WORKER_COUNT: "2", - SRTL_MAX_RUNNING_COPIES: "1" + SRTL_MAX_RUNNING_COPIES: "1", + SRTL_COPY_FILE_CONCURRENCY: "2", + SRTL_MAX_ACTIVE_COPY_FILES: "3" }); expect(mergeEnvSettings(fileDefaults, runtime)).toEqual({ SYMLINK_DIR: "/mnt/local/nas/symlinks", SRTL_LOCATION_1_PATH: "/mnt/local/nas/local", SRTL_LOCATION_2_PATH: "/mnt/remote", - SRTL_WORKER_COUNT: "2" + SRTL_WORKER_COUNT: "2", + SRTL_MAX_RUNNING_COPIES: "1", + SRTL_COPY_FILE_CONCURRENCY: "2", + SRTL_MAX_ACTIVE_COPY_FILES: "3" }); }); }); diff --git a/tests/jobScheduler.test.ts b/tests/jobScheduler.test.ts new file mode 100644 index 0000000..f04df2e --- /dev/null +++ b/tests/jobScheduler.test.ts @@ -0,0 +1,943 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { count, eq } from "drizzle-orm"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { createApp, type AppContext } from "../src/server/app"; +import type { JobConcurrencySettings } from "../src/server/config"; +import { first, setSetting } from "../src/server/db/database"; +import * as schema from "../src/server/db/schema"; +import { JobWorker } from "../src/server/jobs/jobRunner"; +import { schedulerLockKey } from "../src/server/jobs/scheduling"; +import type { AuditCommandRunner } from "../src/server/lib/auditor"; +import type { CopyCommandRunner } from "../src/server/lib/copier"; +import { createTestDatabase, type TestDatabaseHandle } from "./testDb"; + +const silentLogger = { + info: () => undefined, + warn: () => undefined, + error: () => undefined +}; + +const twoJobConcurrency: JobConcurrencySettings = { + workerCount: 2, + maxRunningJobs: 2, + maxRunningScans: 2, + maxRunningAudits: 2, + maxRunningCopies: 2, + copyFileConcurrency: 1, + maxActiveCopyFiles: 2 +}; + +let tmpDir: string; +let ctx: AppContext; +let testDatabase: TestDatabaseHandle; + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function waitUntil(check: () => boolean | Promise, message: string, timeoutMs = 5_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (await check()) return; + await delay(5); + } + throw new Error(message); +} + +async function withTimeout(promise: Promise, message: string, timeoutMs = 5_000): Promise { + let timeout: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + promise, + new Promise((_resolve, reject) => { + timeout = setTimeout(() => reject(new Error(message)), timeoutMs); + }) + ]); + } finally { + if (timeout) clearTimeout(timeout); + } +} + +const copyRunner: CopyCommandRunner = { + async copyFile(sourcePath, tempPath, reportProgress) { + const source = await fs.readFile(sourcePath); + await reportProgress?.({ bytesCopied: 0, bytesProcessed: 0, totalBytes: source.length, bytesPerSecond: 0, remainingSeconds: null }); + await fs.writeFile(tempPath, source); + await reportProgress?.({ + bytesCopied: source.length, + bytesProcessed: source.length, + totalBytes: source.length, + bytesPerSecond: source.length, + remainingSeconds: 0 + }); + }, + async runCmp(sourcePath, targetPath, reportProgress) { + const [source, target] = await Promise.all([fs.readFile(sourcePath), fs.readFile(targetPath)]); + await reportProgress?.({ bytesProcessed: source.length, totalBytes: source.length, bytesPerSecond: source.length, remainingSeconds: 0 }); + return { status: source.equals(target) ? "pass" : "fail", output: source.equals(target) ? "" : "test byte mismatch" }; + }, + async runFfmpeg(_mode, _targetPath, reportProgress) { + await reportProgress?.({ bytesProcessed: 1, totalBytes: 1, bytesPerSecond: 1, remainingSeconds: 0 }); + return { status: "pass", output: "" }; + } +}; + +async function insertCopySymlink(itemName: string, content: string, relativePath?: string): Promise { + const timestamp = new Date().toISOString(); + const mediaRelativePath = relativePath ?? path.join(itemName, `${itemName.toLowerCase().replace(/[^a-z0-9]+/g, "-")}.mkv`); + const sourcePath = path.join(tmpDir, "remote", "movies", mediaRelativePath); + const linkPath = path.join(tmpDir, "symlinks", "movies", mediaRelativePath); + await Promise.all([fs.mkdir(path.dirname(sourcePath), { recursive: true }), fs.mkdir(path.dirname(linkPath), { recursive: true })]); + await fs.writeFile(sourcePath, content); + await fs.symlink(sourcePath, linkPath); + const row = await first( + ctx.database.db + .insert(schema.mediaLinks) + .values({ + section: "movies", + itemName, + relativePath: mediaRelativePath, + linkPath, + targetPath: sourcePath, + kind: "remote", + targetExists: true, + isMedia: true, + storagePolicy: "location_1", + resolvedStorageFileId: null, + sizeBytes: Buffer.byteLength(content), + firstSeenAt: timestamp, + lastSeenAt: timestamp, + lastChangedAt: timestamp, + missingSince: null, + lastSeenJobId: 1, + updatedAt: timestamp + }) + .returning({ id: schema.mediaLinks.id }) + ); + if (!row) throw new Error("Copy fixture was not inserted"); + return row.id; +} + +async function waitForTerminalJobs(jobIds: number[]): Promise { + await waitUntil( + async () => { + const jobs = await Promise.all(jobIds.map((jobId) => ctx.jobs.getJob(jobId))); + return jobs.every((job) => job && ["completed", "partially_failed", "failed", "cancelled"].includes(job.status)); + }, + `Timed out waiting for jobs ${jobIds.join(", ")}`, + 10_000 + ); +} + +describe("job scheduler", () => { + beforeEach(async () => { + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "srtl-job-scheduler-")); + const paths = { + symlinkDir: path.join(tmpDir, "symlinks"), + localDir: path.join(tmpDir, "local"), + remoteDir: path.join(tmpDir, "remote") + }; + await Promise.all(Object.values(paths).map((directory) => fs.mkdir(directory, { recursive: true }))); + testDatabase = await createTestDatabase(); + ctx = await createApp({ + rootDir: tmpDir, + dataDir: path.join(tmpDir, "data"), + databaseUrl: testDatabase.databaseUrl, + apiDocsEnabled: false, + autoMigrate: true, + paths, + jobConcurrency: { + workerCount: 1, + maxRunningJobs: 1, + maxRunningScans: 1, + maxRunningAudits: 1, + maxRunningCopies: 1, + copyFileConcurrency: 1, + maxActiveCopyFiles: 1 + } + }); + const timestamp = new Date().toISOString(); + await setSetting(ctx.database.db, "sections", { + sections: ["movies"], + sectionTitles: {}, + sectionTypes: { movies: "movies" } + }); + await ctx.database.db + .insert(schema.sections) + .values({ name: "movies", displayName: null, contentType: "movies", createdAt: timestamp, updatedAt: timestamp }) + .onConflictDoNothing(); + }); + + afterEach(async () => { + if (typeof ctx !== "undefined") await ctx.app.close(); + if (typeof testDatabase !== "undefined") await testDatabase.cleanup(); + if (typeof tmpDir !== "undefined") await fs.rm(tmpDir, { recursive: true, force: true }); + }); + + it("dispatches four disjoint copy jobs concurrently from one worker loop and stops cleanly", async () => { + const linkIds = await Promise.all([ + insertCopySymlink("Dispatcher Copy One", "dispatcher one"), + insertCopySymlink("Dispatcher Copy Two", "dispatcher two"), + insertCopySymlink("Dispatcher Copy Three", "dispatcher three"), + insertCopySymlink("Dispatcher Copy Four", "dispatcher four") + ]); + const jobIds = await Promise.all(linkIds.map((linkId) => ctx.jobs.startCopy({ direction: "to_local", linkIds: [linkId] }))); + let activeTransfers = 0; + let maximumActiveTransfers = 0; + let startedTransfers = 0; + let releaseTransfers: () => void = () => undefined; + let markAllStarted: () => void = () => undefined; + const transfersReleased = new Promise((resolve) => { + releaseTransfers = resolve; + }); + const allStarted = new Promise((resolve) => { + markAllStarted = resolve; + }); + const blockingCopyRunner: CopyCommandRunner = { + ...copyRunner, + async copyFile(sourcePath, tempPath, reportProgress, signal) { + activeTransfers += 1; + maximumActiveTransfers = Math.max(maximumActiveTransfers, activeTransfers); + startedTransfers += 1; + if (startedTransfers === 4) markAllStarted(); + try { + await transfersReleased; + if (signal?.aborted) throw (signal.reason instanceof Error ? signal.reason : new Error("Copy interrupted")); + await copyRunner.copyFile(sourcePath, tempPath, reportProgress, signal); + } finally { + activeTransfers -= 1; + } + } + }; + const worker = new JobWorker(ctx.database.db, { + workerId: "single-dispatcher", + dispatchConcurrency: 4, + pollIntervalMs: 2, + heartbeatIntervalMs: 20, + logger: silentLogger, + copyRunner: blockingCopyRunner, + concurrency: { + workerCount: 4, + maxRunningJobs: 4, + maxRunningScans: 4, + maxRunningAudits: 4, + maxRunningCopies: 4, + copyFileConcurrency: 1, + maxActiveCopyFiles: 4 + } + }); + const workerRun = worker.start(); + + try { + await withTimeout(allStarted, "The dispatcher did not start four disjoint copies concurrently"); + await expect(Promise.all(jobIds.map((jobId) => ctx.jobs.getJob(jobId)))).resolves.toEqual( + jobIds.map(() => expect.objectContaining({ status: "running", lockedBy: "single-dispatcher" })) + ); + expect(maximumActiveTransfers).toBe(4); + releaseTransfers(); + await waitForTerminalJobs(jobIds); + await expect(Promise.all(jobIds.map((jobId) => ctx.jobs.getJob(jobId)))).resolves.toEqual( + jobIds.map(() => expect.objectContaining({ status: "completed" })) + ); + worker.stop(); + await withTimeout(workerRun, "The dispatcher did not stop cleanly"); + } finally { + releaseTransfers(); + worker.stop(); + await Promise.allSettled([workerRun]); + } + }); + + it("rejects copy destinations that are distinct lexically but share a physical directory", async () => { + const firstLinkId = await insertCopySymlink("Physical Alias One", "first alias", path.join("Physical Alias One", "shared.mkv")); + const secondLinkId = await insertCopySymlink("Physical Alias Two", "second alias", path.join("Physical Alias Two", "shared.mkv")); + const physicalDirectory = path.join(tmpDir, "local", "movies", "Physical Alias One"); + const aliasDirectory = path.join(tmpDir, "local", "movies", "Physical Alias Two"); + await fs.mkdir(physicalDirectory, { recursive: true }); + await fs.symlink(physicalDirectory, aliasDirectory, "dir"); + + await expect(ctx.jobs.startCopy({ direction: "to_local", linkIds: [firstLinkId] })).resolves.toEqual(expect.any(Number)); + await expect(ctx.jobs.startCopy({ direction: "to_local", linkIds: [secondLinkId] })).rejects.toThrow("already queued"); + }); + + it("rejects physical destination aliases selected within one copy job", async () => { + const firstLinkId = await insertCopySymlink("Same Job Alias One", "first alias", path.join("Same Job Alias One", "shared.mkv")); + const secondLinkId = await insertCopySymlink("Same Job Alias Two", "second alias", path.join("Same Job Alias Two", "shared.mkv")); + const physicalDirectory = path.join(tmpDir, "local", "movies", "Same Job Alias One"); + const aliasDirectory = path.join(tmpDir, "local", "movies", "Same Job Alias Two"); + await fs.mkdir(physicalDirectory, { recursive: true }); + await fs.symlink(physicalDirectory, aliasDirectory, "dir"); + + await expect(ctx.jobs.startCopy({ direction: "to_local", linkIds: [firstLinkId, secondLinkId] })).rejects.toThrow( + "resolve to the same copy destination" + ); + expect(await ctx.database.db.select({ id: schema.jobs.id }).from(schema.jobs).where(eq(schema.jobs.type, "copy"))).toEqual([]); + }); + + it("fails closed when a claimed destination ancestor is retargeted after admission", async () => { + const linkId = await insertCopySymlink("Retargeted Alias", "retargeted alias source"); + const fileName = "retargeted-alias.mkv"; + const aliasDirectory = path.join(tmpDir, "local", "movies", "Retargeted Alias"); + const admittedPhysicalDirectory = path.join(tmpDir, "local", "admitted-physical-directory"); + const unclaimedPhysicalDirectory = path.join(tmpDir, "local", "unclaimed-physical-directory"); + await Promise.all([ + fs.mkdir(path.dirname(aliasDirectory), { recursive: true }), + fs.mkdir(admittedPhysicalDirectory, { recursive: true }), + fs.mkdir(unclaimedPhysicalDirectory, { recursive: true }) + ]); + await fs.symlink(admittedPhysicalDirectory, aliasDirectory, "dir"); + const jobId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: [linkId] }); + + await fs.unlink(aliasDirectory); + await fs.symlink(unclaimedPhysicalDirectory, aliasDirectory, "dir"); + let transferAttempts = 0; + const observingCopyRunner: CopyCommandRunner = { + ...copyRunner, + async copyFile(sourcePath, tempPath, reportProgress, signal) { + transferAttempts += 1; + await copyRunner.copyFile(sourcePath, tempPath, reportProgress, signal); + } + }; + const worker = new JobWorker(ctx.database.db, { + workerId: "retargeted-alias-worker", + logger: silentLogger, + copyRunner: observingCopyRunner + }); + + await expect(worker.runOnce()).resolves.toBe(true); + expect(await ctx.jobs.getJob(jobId)).toMatchObject({ + status: "completed", + progress: expect.objectContaining({ copied: 0, conflicts: 1, failed: 0, stage: "completed" }) + }); + expect(transferAttempts).toBe(0); + await expect(fs.stat(path.join(admittedPhysicalDirectory, fileName))).rejects.toMatchObject({ code: "ENOENT" }); + await expect(fs.stat(path.join(unclaimedPhysicalDirectory, fileName))).rejects.toMatchObject({ code: "ENOENT" }); + await expect(fs.readlink(path.join(tmpDir, "symlinks", "movies", "Retargeted Alias", fileName))).resolves.toBe( + path.join(tmpDir, "remote", "movies", "Retargeted Alias", fileName) + ); + }); + + it("does not accept another selected link's canonical destination binding", async () => { + const firstLinkId = await insertCopySymlink("Binding Alias One", "first binding", path.join("Binding Alias One", "shared.mkv")); + const secondLinkId = await insertCopySymlink("Binding Alias Two", "second binding", path.join("Binding Alias Two", "shared.mkv")); + const firstAliasDirectory = path.join(tmpDir, "local", "movies", "Binding Alias One"); + const firstPhysicalDirectory = path.join(tmpDir, "local", "binding-alias-one-physical"); + const secondPhysicalDirectory = path.join(tmpDir, "local", "movies", "Binding Alias Two"); + await Promise.all([ + fs.mkdir(path.dirname(firstAliasDirectory), { recursive: true }), + fs.mkdir(firstPhysicalDirectory, { recursive: true }), + fs.mkdir(secondPhysicalDirectory, { recursive: true }) + ]); + await fs.symlink(firstPhysicalDirectory, firstAliasDirectory, "dir"); + const jobId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: [firstLinkId, secondLinkId] }); + + await fs.unlink(firstAliasDirectory); + await fs.symlink(secondPhysicalDirectory, firstAliasDirectory, "dir"); + const worker = new JobWorker(ctx.database.db, { + workerId: "binding-alias-worker", + logger: silentLogger, + copyRunner + }); + + await expect(worker.runOnce()).resolves.toBe(true); + expect(await ctx.jobs.getJob(jobId)).toMatchObject({ + status: "completed", + progress: expect.objectContaining({ copied: 1, conflicts: 1, failed: 0, stage: "completed" }) + }); + await expect(fs.stat(path.join(firstPhysicalDirectory, "shared.mkv"))).rejects.toMatchObject({ code: "ENOENT" }); + await expect(fs.readFile(path.join(secondPhysicalDirectory, "shared.mkv"), "utf8")).resolves.toBe("second binding"); + }); + + it("rechecks canonical destination bindings inside the copy mutation lease", async () => { + const linkId = await insertCopySymlink("Mutation Retarget", "mutation retarget source"); + const fileName = "mutation-retarget.mkv"; + const aliasDirectory = path.join(tmpDir, "local", "movies", "Mutation Retarget"); + const admittedPhysicalDirectory = path.join(tmpDir, "local", "mutation-admitted-physical"); + const unclaimedPhysicalDirectory = path.join(tmpDir, "local", "mutation-unclaimed-physical"); + await Promise.all([ + fs.mkdir(path.dirname(aliasDirectory), { recursive: true }), + fs.mkdir(admittedPhysicalDirectory, { recursive: true }), + fs.mkdir(unclaimedPhysicalDirectory, { recursive: true }) + ]); + await fs.symlink(admittedPhysicalDirectory, aliasDirectory, "dir"); + const jobId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: [linkId] }); + let releaseTransfer: () => void = () => undefined; + let markTransferStarted: () => void = () => undefined; + const transferGate = new Promise((resolve) => { + releaseTransfer = resolve; + }); + const transferStarted = new Promise((resolve) => { + markTransferStarted = resolve; + }); + const blockingCopyRunner: CopyCommandRunner = { + ...copyRunner, + async copyFile(sourcePath, tempPath, reportProgress, signal) { + markTransferStarted(); + await transferGate; + await copyRunner.copyFile(sourcePath, tempPath, reportProgress, signal); + } + }; + const worker = new JobWorker(ctx.database.db, { + workerId: "mutation-retarget-worker", + logger: silentLogger, + copyRunner: blockingCopyRunner + }); + const run = worker.runOnce(); + + await withTimeout(transferStarted, "The mutation-retarget copy did not start"); + try { + await fs.unlink(aliasDirectory); + await fs.symlink(unclaimedPhysicalDirectory, aliasDirectory, "dir"); + } finally { + releaseTransfer(); + } + await expect(run).resolves.toBe(true); + + expect(await ctx.jobs.getJob(jobId)).toMatchObject({ + status: "failed", + progress: expect.objectContaining({ copied: 0, repointed: 0, conflicts: 0, failed: 1, stage: "failed" }) + }); + await expect(ctx.database.db.select().from(schema.copyOperations).where(eq(schema.copyOperations.jobId, jobId))).resolves.toEqual([ + expect.objectContaining({ + stage: "failed", + resultStatus: null, + errorMessage: "Media paths changed after copy admission; queue the copy again" + }) + ]); + await expect(fs.readdir(admittedPhysicalDirectory)).resolves.toEqual([]); + await expect(fs.readdir(unclaimedPhysicalDirectory)).resolves.toEqual([]); + await expect(fs.readlink(path.join(tmpDir, "symlinks", "movies", "Mutation Retarget", fileName))).resolves.toBe( + path.join(tmpDir, "remote", "movies", "Mutation Retarget", fileName) + ); + }); + + it("does not reclaim or path-requeue an active same-worker handler while another slot remains available", async () => { + const firstLinkId = await insertCopySymlink("Active Lease One", "first active lease"); + const secondLinkId = await insertCopySymlink("Active Lease Two", "second active lease"); + let releaseTransfers: () => void = () => undefined; + const transferGate = new Promise((resolve) => { + releaseTransfers = resolve; + }); + const startedSources: string[] = []; + const blockingCopyRunner: CopyCommandRunner = { + ...copyRunner, + async copyFile(sourcePath, tempPath) { + startedSources.push(sourcePath); + await transferGate; + await fs.copyFile(sourcePath, tempPath); + } + }; + const firstJobId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: [firstLinkId] }); + const worker = new JobWorker(ctx.database.db, { + workerId: "active-same-worker", + pollIntervalMs: 2, + heartbeatIntervalMs: 60_000, + reclaimOwnInterruptedAfterMs: 20, + dispatchConcurrency: 2, + logger: silentLogger, + copyRunner: blockingCopyRunner, + concurrency: twoJobConcurrency + }); + const workerRun = worker.start(); + + try { + await waitUntil(() => startedSources.length === 1, "The first copy handler did not start"); + const staleTimestamp = new Date(Date.now() - 60_000).toISOString(); + await ctx.database.db.update(schema.jobs).set({ heartbeatAt: staleTimestamp }).where(eq(schema.jobs.id, firstJobId)); + const probe = worker as unknown as { requeueInterruptedJobsForPathMigration(): Promise }; + await probe.requeueInterruptedJobsForPathMigration(); + await delay(40); + + expect(await ctx.jobs.getJob(firstJobId)).toMatchObject({ status: "running", lockedBy: "active-same-worker", leaseVersion: 1 }); + expect(startedSources).toHaveLength(1); + + const secondJobId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: [secondLinkId] }); + await waitUntil(() => startedSources.length === 2, "The second available dispatcher slot was not used"); + expect(new Set(startedSources).size).toBe(2); + releaseTransfers(); + await waitForTerminalJobs([firstJobId, secondJobId]); + await expect(Promise.all([ctx.jobs.getJob(firstJobId), ctx.jobs.getJob(secondJobId)])).resolves.toEqual([ + expect.objectContaining({ status: "completed", leaseVersion: 1 }), + expect.objectContaining({ status: "completed", leaseVersion: 1 }) + ]); + } finally { + releaseTransfers(); + worker.stop(); + await Promise.allSettled([workerRun]); + } + }); + + it("requeues a job claimed after stop was requested while the scheduler lock was held", async () => { + const linkId = await insertCopySymlink("Stop During Claim", "stop during claim"); + const jobId = await ctx.jobs.startAudit({ mode: "fast", linkIds: [linkId], byteCompare: false }); + let auditCalls = 0; + const auditRunner: AuditCommandRunner = { + async runFfmpeg() { + auditCalls += 1; + return { status: "pass", output: "" }; + }, + async runCmp() { + auditCalls += 1; + return { status: "pass", output: "" }; + } + }; + const lockClient = await ctx.database.pool.connect(); + let lockHeld = false; + const worker = new JobWorker(ctx.database.db, { + workerId: "stopping-dispatcher", + pollIntervalMs: 2, + heartbeatIntervalMs: 20, + logger: silentLogger, + auditRunner + }); + let workerRun: Promise | null = null; + + try { + await lockClient.query("SELECT pg_advisory_lock($1)", [schedulerLockKey]); + lockHeld = true; + workerRun = worker.start(); + await waitUntil(async () => { + const result = await ctx.database.pool.query<{ count: number }>(` + SELECT count(*)::integer AS count + FROM pg_locks + WHERE locktype = 'advisory' + AND database = (SELECT oid FROM pg_database WHERE datname = current_database()) + AND classid = 0 + AND objid = $1 + AND NOT granted + `, [schedulerLockKey]); + return (result.rows[0]?.count ?? 0) > 0; + }, "The worker did not wait for the scheduler lock"); + + worker.stop(); + await lockClient.query("SELECT pg_advisory_unlock($1)", [schedulerLockKey]); + lockHeld = false; + await withTimeout(workerRun, "The stopped dispatcher did not settle after its blocked claim resumed"); + } finally { + worker.stop(); + if (lockHeld) await lockClient.query("SELECT pg_advisory_unlock($1)", [schedulerLockKey]); + lockClient.release(); + if (workerRun) await Promise.allSettled([workerRun]); + } + + expect(auditCalls).toBe(0); + expect(await ctx.jobs.getJob(jobId)).toMatchObject({ status: "queued", lockedBy: null, heartbeatAt: null, leaseVersion: 1 }); + expect(await ctx.database.db.select().from(schema.auditRuns).where(eq(schema.auditRuns.jobId, jobId))).toHaveLength(0); + expect((await ctx.jobs.listEvents(jobId)).map((event) => event.message)).not.toContain("Worker started job"); + }); + + it("rejects copy admission when the selected media changes while waiting for the scheduler lock", async () => { + const linkId = await insertCopySymlink("Changed Admission Copy", "initial admission snapshot"); + const lockClient = await ctx.database.pool.connect(); + let lockHeld = false; + let admission: Promise | null = null; + + try { + await lockClient.query("SELECT pg_advisory_lock($1)", [schedulerLockKey]); + lockHeld = true; + admission = ctx.jobs.startCopy({ direction: "to_local", linkIds: [linkId] }); + void admission.catch(() => undefined); + + await waitUntil(async () => { + const result = await ctx.database.pool.query<{ count: number }>(` + SELECT count(*)::integer AS count + FROM pg_locks + WHERE locktype = 'advisory' + AND database = (SELECT oid FROM pg_database WHERE datname = current_database()) + AND classid = 0 + AND objid = $1 + AND NOT granted + `, [schedulerLockKey]); + return (result.rows[0]?.count ?? 0) > 0; + }, "Copy admission did not wait for the scheduler lock after taking its initial snapshot"); + + await ctx.database.db + .update(schema.mediaLinks) + .set({ storagePolicy: "location_2", updatedAt: new Date().toISOString() }) + .where(eq(schema.mediaLinks.id, linkId)); + + await lockClient.query("SELECT pg_advisory_unlock($1)", [schedulerLockKey]); + lockHeld = false; + await expect(withTimeout(admission, "Copy admission did not settle after the scheduler lock was released")).rejects.toThrow( + "Copy selection changed while the job was being prepared. Review the current inventory and queue it again." + ); + } finally { + if (lockHeld) await lockClient.query("SELECT pg_advisory_unlock($1)", [schedulerLockKey]); + lockClient.release(); + if (admission) await Promise.allSettled([admission]); + } + + expect(await ctx.database.db.select({ id: schema.jobs.id }).from(schema.jobs).where(eq(schema.jobs.type, "copy"))).toEqual([]); + expect(await ctx.database.db.select({ jobId: schema.jobResourceClaims.jobId }).from(schema.jobResourceClaims)).toEqual([]); + }); + + it("leaves exclusive scans and audits queued when their per-type limits are zero", async () => { + const scanJobId = await ctx.jobs.startScan({ + scanSymlinks: false, + scanLocal: false, + scanRemote: true, + symlinkSections: [], + localSections: [] + }); + const auditJobId = await ctx.jobs.startAudit("fast"); + const worker = new JobWorker(ctx.database.db, { + workerId: "paused-types-worker", + logger: silentLogger, + concurrency: { + workerCount: 2, + maxRunningJobs: 2, + maxRunningScans: 0, + maxRunningAudits: 0, + maxRunningCopies: 2, + copyFileConcurrency: 1, + maxActiveCopyFiles: 2 + } + }); + + expect(await worker.runOnce()).toBe(false); + await expect(Promise.all([ctx.jobs.getJob(scanJobId), ctx.jobs.getJob(auditJobId)])).resolves.toEqual([ + expect.objectContaining({ status: "queued", exclusive: true, startedAt: null, leaseVersion: 0 }), + expect.objectContaining({ status: "queued", exclusive: true, startedAt: null, leaseVersion: 0 }) + ]); + }); + + it("fences a reclaimable paused job before admitting exclusive work", async () => { + const oldTimestamp = new Date(Date.now() - 60_000).toISOString(); + await ctx.database.db.insert(schema.workerHeartbeats).values({ + workerId: "paused-stale-owner", + startedAt: oldTimestamp, + heartbeatAt: oldTimestamp, + status: "stopped", + capacity: 1 + }); + const staleJob = await first( + ctx.database.db + .insert(schema.jobs) + .values({ + type: "copy", + status: "running", + createdAt: oldTimestamp, + startedAt: oldTimestamp, + finishedAt: null, + lockedBy: "paused-stale-owner", + lockedAt: oldTimestamp, + heartbeatAt: oldTimestamp, + leaseVersion: 4, + exclusive: false, + cancelRequestedAt: null, + progress: "{}" + }) + .returning({ id: schema.jobs.id }) + ); + if (!staleJob) throw new Error("Paused stale job was not inserted"); + const auditJobId = await ctx.jobs.startAudit("fast"); + const worker = new JobWorker(ctx.database.db, { + workerId: "paused-stale-reclaimer", + reclaimOwnInterruptedAfterMs: 1_000, + logger: silentLogger, + concurrency: { + ...twoJobConcurrency, + maxRunningCopies: 0 + } + }) as unknown as { claimNextJob(): Promise }; + + await expect(worker.claimNextJob()).resolves.toMatchObject({ job: { id: auditJobId, status: "running", lockedBy: "paused-stale-reclaimer" } }); + expect(await ctx.jobs.getJob(staleJob.id)).toMatchObject({ status: "queued", lockedBy: null, leaseVersion: 5 }); + expect(await ctx.jobs.getJob(auditJobId)).toMatchObject({ status: "running", lockedBy: "paused-stale-reclaimer", leaseVersion: 1 }); + }); + + it("keeps a fenced stale overlap queued behind an unfenced locked row", async () => { + const oldTimestamp = new Date(Date.now() - 60_000).toISOString(); + await ctx.database.db.insert(schema.workerHeartbeats).values( + ["locked-stale-owner", "overlapping-stale-owner"].map((workerId) => ({ + workerId, + startedAt: oldTimestamp, + heartbeatAt: oldTimestamp, + status: "stopped", + capacity: 1 + })) + ); + const staleJobs = await ctx.database.db + .insert(schema.jobs) + .values( + ["locked-stale-owner", "overlapping-stale-owner"].map((lockedBy) => ({ + type: "copy", + status: "running", + createdAt: oldTimestamp, + startedAt: oldTimestamp, + finishedAt: null, + lockedBy, + lockedAt: oldTimestamp, + heartbeatAt: oldTimestamp, + leaseVersion: 6, + exclusive: false, + cancelRequestedAt: null, + progress: "{}" + })) + ) + .returning({ id: schema.jobs.id }); + expect(staleJobs).toHaveLength(2); + await ctx.database.db.insert(schema.jobResourceClaims).values( + staleJobs.map((job) => ({ + jobId: job.id, + resourceType: "media", + resourceKey: "shared-stale-media", + access: "exclusive", + createdAt: oldTimestamp + })) + ); + const lockClient = await ctx.database.pool.connect(); + const worker = new JobWorker(ctx.database.db, { + workerId: "skip-locked-reclaimer", + reclaimOwnInterruptedAfterMs: 1_000, + logger: silentLogger, + concurrency: twoJobConcurrency + }) as unknown as { claimNextJob(): Promise }; + + try { + await lockClient.query("BEGIN"); + await lockClient.query("SELECT id FROM jobs WHERE id = $1 FOR UPDATE", [staleJobs[0]!.id]); + await expect(worker.claimNextJob()).resolves.toBeNull(); + } finally { + await lockClient.query("ROLLBACK"); + lockClient.release(); + } + + await expect(Promise.all(staleJobs.map((job) => ctx.jobs.getJob(job.id)))).resolves.toEqual([ + expect.objectContaining({ status: "running", lockedBy: "locked-stale-owner", leaseVersion: 6 }), + expect.objectContaining({ status: "queued", lockedBy: null, leaseVersion: 7 }) + ]); + }); + + it("does not reclaim a fresh job lease solely because its process heartbeat is stale", async () => { + const oldTimestamp = new Date(Date.now() - 60_000).toISOString(); + const freshTimestamp = new Date().toISOString(); + await ctx.database.db.insert(schema.workerHeartbeats).values({ + workerId: "stale-process-live-job", + startedAt: oldTimestamp, + heartbeatAt: oldTimestamp, + status: "running", + capacity: 1 + }); + const liveJob = await first( + ctx.database.db + .insert(schema.jobs) + .values({ + type: "copy", + status: "running", + createdAt: oldTimestamp, + startedAt: oldTimestamp, + finishedAt: null, + lockedBy: "stale-process-live-job", + lockedAt: oldTimestamp, + heartbeatAt: freshTimestamp, + leaseVersion: 2, + exclusive: false, + cancelRequestedAt: null, + progress: "{}" + }) + .returning({ id: schema.jobs.id }) + ); + if (!liveJob) throw new Error("Fresh live job was not inserted"); + const auditJobId = await ctx.jobs.startAudit("fast"); + const worker = new JobWorker(ctx.database.db, { + workerId: "stale-process-contender", + reclaimOwnInterruptedAfterMs: 1_000, + logger: silentLogger, + concurrency: twoJobConcurrency + }) as unknown as { claimNextJob(): Promise }; + + await expect(worker.claimNextJob()).resolves.toBeNull(); + expect(await ctx.jobs.getJob(liveJob.id)).toMatchObject({ status: "running", lockedBy: "stale-process-live-job", leaseVersion: 2 }); + expect(await ctx.jobs.getJob(auditJobId)).toMatchObject({ status: "queued", startedAt: null, lockedBy: null }); + }); + + it("does not path-requeue a stale job lease while its owning process heartbeat is fresh", async () => { + const oldTimestamp = new Date(Date.now() - 60_000).toISOString(); + const freshTimestamp = new Date().toISOString(); + await ctx.database.db.insert(schema.workerHeartbeats).values({ + workerId: "live-path-owner", + startedAt: oldTimestamp, + heartbeatAt: freshTimestamp, + status: "running", + capacity: 1 + }); + const liveJob = await first( + ctx.database.db + .insert(schema.jobs) + .values({ + type: "audit", + status: "running", + createdAt: oldTimestamp, + startedAt: oldTimestamp, + finishedAt: null, + lockedBy: "live-path-owner", + lockedAt: oldTimestamp, + heartbeatAt: oldTimestamp, + leaseVersion: 5, + exclusive: false, + cancelRequestedAt: null, + progress: "{}" + }) + .returning({ id: schema.jobs.id }) + ); + if (!liveJob) throw new Error("Live path-owner job was not inserted"); + const worker = new JobWorker(ctx.database.db, { + workerId: "path-requeue-contender", + reclaimOwnInterruptedAfterMs: 1_000, + logger: silentLogger, + concurrency: twoJobConcurrency + }) as unknown as { requeueInterruptedJobsForPathMigration(): Promise }; + + await worker.requeueInterruptedJobsForPathMigration(); + expect(await ctx.jobs.getJob(liveJob.id)).toMatchObject({ status: "running", lockedBy: "live-path-owner", leaseVersion: 5 }); + }); + + it("applies global and per-type caps while quickly reclaiming jobs from known stopped or stale owners", async () => { + const oldTimestamp = new Date(Date.now() - 60_000).toISOString(); + const currentTimestamp = new Date().toISOString(); + await ctx.database.db.insert(schema.workerHeartbeats).values([ + { workerId: "stopped-owner-one", startedAt: oldTimestamp, heartbeatAt: currentTimestamp, status: "stopped", capacity: 1 }, + { workerId: "stale-owner", startedAt: oldTimestamp, heartbeatAt: oldTimestamp, status: "running", capacity: 1 }, + { workerId: "stopped-owner-two", startedAt: oldTimestamp, heartbeatAt: currentTimestamp, status: "stopped", capacity: 1 } + ]); + const jobs = await ctx.database.db + .insert(schema.jobs) + .values(["stopped-owner-one", "stale-owner", "stopped-owner-two"].map((lockedBy) => ({ + type: "copy", + status: "running", + createdAt: oldTimestamp, + startedAt: oldTimestamp, + finishedAt: null, + lockedBy, + lockedAt: oldTimestamp, + heartbeatAt: oldTimestamp, + leaseVersion: 3, + exclusive: false, + cancelRequestedAt: null, + progress: "{}" + }))) + .returning({ id: schema.jobs.id }); + expect(jobs).toHaveLength(3); + + type ClaimProbe = { + claimNextJob(): Promise<{ job: { id: number; leaseVersion: number; lockedBy: string | null } } | null>; + }; + const globalWorker = new JobWorker(ctx.database.db, { + workerId: "global-reclaimer", + reclaimStaleAfterMs: 60 * 60_000, + reclaimOwnInterruptedAfterMs: 1_000, + logger: silentLogger, + concurrency: { + ...twoJobConcurrency, + maxRunningJobs: 1, + maxRunningCopies: 2 + } + }) as unknown as ClaimProbe; + + await expect(globalWorker.claimNextJob()).resolves.toMatchObject({ job: { id: jobs[0]!.id, leaseVersion: 5, lockedBy: "global-reclaimer" } }); + await expect(globalWorker.claimNextJob()).resolves.toBeNull(); + expect(await ctx.jobs.getJob(jobs[1]!.id)).toMatchObject({ status: "queued", lockedBy: null, leaseVersion: 4 }); + expect(await ctx.jobs.getJob(jobs[2]!.id)).toMatchObject({ status: "queued", lockedBy: null, leaseVersion: 4 }); + + await ctx.database.db + .update(schema.jobs) + .set({ status: "completed", finishedAt: currentTimestamp, lockedBy: null, lockedAt: null, heartbeatAt: null }) + .where(eq(schema.jobs.id, jobs[0]!.id)); + await expect(globalWorker.claimNextJob()).resolves.toMatchObject({ job: { id: jobs[1]!.id, leaseVersion: 5, lockedBy: "global-reclaimer" } }); + + const typeWorker = new JobWorker(ctx.database.db, { + workerId: "type-reclaimer", + reclaimStaleAfterMs: 60 * 60_000, + reclaimOwnInterruptedAfterMs: 1_000, + logger: silentLogger, + concurrency: { + ...twoJobConcurrency, + maxRunningJobs: 2, + maxRunningCopies: 1 + } + }) as unknown as ClaimProbe; + await expect(typeWorker.claimNextJob()).resolves.toBeNull(); + expect(await ctx.jobs.getJob(jobs[2]!.id)).toMatchObject({ status: "queued", lockedBy: null, leaseVersion: 4 }); + + await ctx.database.db + .update(schema.jobs) + .set({ status: "completed", finishedAt: currentTimestamp, lockedBy: null, lockedAt: null, heartbeatAt: null }) + .where(eq(schema.jobs.id, jobs[1]!.id)); + await expect(typeWorker.claimNextJob()).resolves.toMatchObject({ job: { id: jobs[2]!.id, leaseVersion: 5, lockedBy: "type-reclaimer" } }); + await ctx.database.db + .update(schema.jobs) + .set({ status: "completed", finishedAt: currentTimestamp, lockedBy: null, lockedAt: null, heartbeatAt: null }) + .where(eq(schema.jobs.id, jobs[2]!.id)); + const unknownOwnerJob = await first( + ctx.database.db + .insert(schema.jobs) + .values({ + type: "copy", + status: "running", + createdAt: oldTimestamp, + startedAt: oldTimestamp, + finishedAt: null, + lockedBy: "unknown-owner", + lockedAt: oldTimestamp, + heartbeatAt: oldTimestamp, + leaseVersion: 3, + exclusive: false, + cancelRequestedAt: null, + progress: "{}" + }) + .returning({ id: schema.jobs.id }) + ); + if (!unknownOwnerJob) throw new Error("Unknown-owner job was not inserted"); + await expect(globalWorker.claimNextJob()).resolves.toBeNull(); + expect(await ctx.jobs.getJob(unknownOwnerJob.id)).toMatchObject({ status: "running", lockedBy: "unknown-owner", leaseVersion: 3 }); + }); + + it("queues claims for more than 3300 scoped audit links without exceeding PostgreSQL bind limits", async () => { + const linkCount = 3_301; + const timestamp = new Date().toISOString(); + const linkIds: number[] = []; + for (let offset = 0; offset < linkCount; offset += 250) { + const values = Array.from({ length: Math.min(250, linkCount - offset) }, (_unused, index) => { + const id = offset + index; + const itemName = `Large Audit ${id}`; + const relativePath = path.join(itemName, `large-audit-${id}.mkv`); + return { + section: "movies", + itemName, + relativePath, + linkPath: path.join(tmpDir, "symlinks", "movies", relativePath), + targetPath: path.join(tmpDir, "remote", "movies", relativePath), + kind: "remote", + targetExists: true, + isMedia: true, + storagePolicy: "unassigned", + resolvedStorageFileId: null, + sizeBytes: 1, + firstSeenAt: timestamp, + lastSeenAt: timestamp, + lastChangedAt: timestamp, + missingSince: null, + lastSeenJobId: 1, + updatedAt: timestamp + }; + }); + const inserted = await ctx.database.db.insert(schema.mediaLinks).values(values).returning({ id: schema.mediaLinks.id }); + linkIds.push(...inserted.map((row) => row.id)); + } + + expect(linkIds).toHaveLength(linkCount); + const jobId = await ctx.jobs.startAudit({ mode: "fast", linkIds, byteCompare: false }); + expect(await ctx.jobs.getJob(jobId)).toMatchObject({ status: "queued", exclusive: false }); + const claimCount = await first( + ctx.database.db + .select({ value: count() }) + .from(schema.jobResourceClaims) + .where(eq(schema.jobResourceClaims.jobId, jobId)) + ); + expect(Number(claimCount?.value ?? 0)).toBeGreaterThanOrEqual(linkCount * 4); + expect(Number(claimCount?.value ?? 0)).toBeLessThanOrEqual(linkCount * 6); + }, 60_000); +}); diff --git a/tests/mountIdentity.test.ts b/tests/mountIdentity.test.ts new file mode 100644 index 0000000..17798f0 --- /dev/null +++ b/tests/mountIdentity.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "vitest"; +import { findMountIdentity, parseLinuxMountInfo, persistentRootIdentityMatch } from "../src/server/lib/mountIdentity"; +import type { PathRootIdentity } from "../src/shared/types"; + +const decypharrMount = { + mountPoint: "/mnt/remote/realdebrid", + root: "/", + filesystemType: "fuse.decypharr", + source: "decypharr" +}; + +function rootIdentity(device: string, mount = decypharrMount): PathRootIdentity { + return { + available: true, + realPath: "/mnt/remote/realdebrid/__all__", + device, + inode: "1320498586805139757", + mount, + error: null + }; +} + +describe("managed root mount identity", () => { + it("finds the most specific Linux mount and decodes escaped fields", () => { + const mounts = parseLinuxMountInfo( + [ + "36 25 8:2 / / rw,relatime - ext4 /dev/sda2 rw", + "51 36 0:51 / /mnt/local/nas rw,relatime - nfs4 10.0.0.204:/mnt/Exos-Pool/media rw", + "65 36 0:65 / /mnt/remote/realdebrid rw,nosuid,nodev - fuse.decypharr decypharr rw", + "66 36 0:66 / /mnt/media\\040pool rw - fuse.example source\\040name rw" + ].join("\n") + ); + + expect(findMountIdentity("/mnt/remote/realdebrid/__all__", mounts)).toEqual(decypharrMount); + expect(findMountIdentity("/mnt/media pool/library", mounts)).toMatchObject({ mountPoint: "/mnt/media pool", source: "source name" }); + }); + + it("accepts a new device number for the same canonical path and mount source", () => { + expect(persistentRootIdentityMatch(rootIdentity("79"), rootIdentity("65"))).toBe(true); + }); + + it("rejects the same path when it falls back to a different mounted filesystem", () => { + expect( + persistentRootIdentityMatch(rootIdentity("79"), rootIdentity("2050", { mountPoint: "/", root: "/", filesystemType: "ext4", source: "/dev/sda2" })) + ).toBe(false); + }); + + it("upgrades a legacy identity without mount metadata using its canonical path", () => { + expect(persistentRootIdentityMatch({ ...rootIdentity("79"), mount: null }, rootIdentity("65"))).toBe(true); + }); + + it("does not upgrade a legacy mounted identity onto the host root filesystem", () => { + expect( + persistentRootIdentityMatch( + { ...rootIdentity("79"), mount: null }, + rootIdentity("2050", { mountPoint: "/", root: "/", filesystemType: "ext4", source: "/dev/sda2" }) + ) + ).toBe(false); + }); +}); diff --git a/tests/pathConfiguration.test.ts b/tests/pathConfiguration.test.ts index c1714b4..1f0152e 100644 --- a/tests/pathConfiguration.test.ts +++ b/tests/pathConfiguration.test.ts @@ -8,7 +8,7 @@ import { first } from "../src/server/db/database"; import * as schema from "../src/server/db/schema"; import { JobWorker } from "../src/server/jobs/jobRunner"; import { markOnboardingCompleteForExistingInstall } from "../src/server/lib/onboarding"; -import { reconcileEnvironmentPaths, runPathMigration } from "../src/server/lib/pathConfiguration"; +import { planPathMigration, reconcileEnvironmentPaths, runPathMigration } from "../src/server/lib/pathConfiguration"; import type { OnboardingState, PathConfigurationState } from "../src/shared/types"; import { createTestDatabase, type TestDatabaseHandle } from "./testDb"; @@ -157,6 +157,281 @@ describe("managed path configuration", () => { expect(await secondApp.database.db.select().from(schema.pathMigrations)).toEqual([]); }); + it("rejects managed roots that resolve to the same physical directory", async () => { + const firstApp = await openApp(); + const cookie = await createAdminSession(firstApp); + await insertIndexedSymlink(firstApp, "Physical Root Alias", "physical-root.bin"); + await fs.mkdir(newSymlinkDir, { recursive: true }); + await fs.symlink(remoteDir, newLocalDir, "dir"); + + const secondApp = await restartWithPaths(newSymlinkDir, newLocalDir); + const state = (await secondApp.app.inject({ + method: "GET", + url: "/api/system/path-migration", + headers: { cookie } + })).json(); + + expect(state).toMatchObject({ blocking: true, status: "invalid_environment" }); + expect(state.environmentErrors).toEqual([ + expect.stringContaining("resolve to the same or overlapping physical path") + ]); + const plan = await secondApp.app.inject({ + method: "POST", + url: "/api/system/path-migration/plan", + headers: { cookie }, + payload: { migrationId: state.migration?.id } + }); + expect(plan.statusCode).toBe(409); + expect(plan.json()).toMatchObject({ error: expect.stringContaining("same or overlapping physical path") }); + }); + + it("accepts a remounted root at the same configured path and refreshes its transient identity", async () => { + const firstApp = await openApp(); + const cookie = await createAdminSession(firstApp); + const fixture = await insertIndexedSymlink(firstApp, "Same Path Replacement", "same-path.bin"); + const activeBefore = await first(firstApp.database.db.select().from(schema.pathConfigurations).where(eq(schema.pathConfigurations.status, "active")).limit(1)); + const displacedRoot = path.join(tmpDir, "local-displaced"); + const displacedTarget = path.join(displacedRoot, path.relative(oldLocalDir, fixture.targetPath)); + const replacementContent = await fs.readFile(fixture.targetPath); + + await fs.rename(oldLocalDir, displacedRoot); + await fs.mkdir(path.dirname(fixture.targetPath), { recursive: true }); + await fs.writeFile(fixture.targetPath, replacementContent); + expect(displacedTarget).not.toBe(fixture.targetPath); + + const secondApp = await restartWithPaths(oldSymlinkDir, oldLocalDir); + const state = (await secondApp.app.inject({ + method: "GET", + url: "/api/system/path-migration", + headers: { cookie } + })).json(); + const localChange = state.changes.find((change) => change.root === "local"); + const activeAfter = await first(secondApp.database.db.select().from(schema.pathConfigurations).where(eq(schema.pathConfigurations.status, "active")).limit(1)); + const refreshedIdentity = JSON.parse(activeAfter?.localIdentity ?? "null") as { device?: string; inode?: string } | null; + const currentStat = await fs.stat(oldLocalDir, { bigint: true }); + + expect(state).toMatchObject({ blocking: false, status: "ready", migration: null }); + expect(localChange).toMatchObject({ changed: false, identityMatch: "same", activePath: oldLocalDir, detectedPath: oldLocalDir }); + expect(activeAfter?.localIdentity).not.toBe(activeBefore?.localIdentity); + expect(refreshedIdentity).toMatchObject({ device: String(currentStat.dev), inode: String(currentStat.ino) }); + expect(await secondApp.database.db.select().from(schema.pathMigrations)).toEqual([]); + }); + + it("fails closed when a planned target root is replaced before apply", async () => { + const firstApp = await openApp(); + const cookie = await createAdminSession(firstApp); + const fixture = await insertIndexedSymlink(firstApp, "Apply Root Replacement", "apply-root.bin"); + await fs.symlink(oldSymlinkDir, newSymlinkDir, "dir"); + await fs.symlink(oldLocalDir, newLocalDir, "dir"); + const secondApp = await restartWithPaths(newSymlinkDir, newLocalDir); + const state = (await secondApp.app.inject({ + method: "GET", + url: "/api/system/path-migration", + headers: { cookie } + })).json(); + const migrationId = state.migration?.id ?? 0; + const plan = await secondApp.app.inject({ + method: "POST", + url: "/api/system/path-migration/plan", + headers: { cookie }, + payload: { migrationId } + }); + expect(plan.statusCode).toBe(200); + const apply = await secondApp.app.inject({ + method: "POST", + url: "/api/system/path-migration/apply", + headers: { cookie }, + payload: { migrationId, confirmSameStorage: true } + }); + expect(apply.statusCode).toBe(200); + const migrationJobId = apply.json<{ jobId: number }>().jobId; + + await fs.rm(newLocalDir); + await fs.mkdir(newLocalDir, { recursive: true }); + const worker = new JobWorker(secondApp.database.db, { + workerId: "root-replacement-worker", + pollIntervalMs: 1, + heartbeatIntervalMs: 10, + logger: silentLogger + }); + await expect(worker.runOnce()).resolves.toBe(true); + + await expect(secondApp.jobs.getJob(migrationJobId)).resolves.toMatchObject({ status: "failed" }); + await expect( + first(secondApp.database.db.select().from(schema.pathMigrations).where(eq(schema.pathMigrations.id, migrationId)).limit(1)) + ).resolves.toMatchObject({ + status: "failed", + errorMessage: expect.stringContaining("no longer matches the physical directory recorded during migration analysis") + }); + expect(path.resolve(await fs.readlink(fixture.linkPath))).toBe(path.resolve(fixture.targetPath)); + }); + + it("fails closed when an upgraded planned migration has no exact target identity", async () => { + const firstApp = await openApp(); + const cookie = await createAdminSession(firstApp); + const fixture = await insertIndexedSymlink(firstApp, "Legacy Planned Identity", "legacy-planned.bin"); + await fs.symlink(oldSymlinkDir, newSymlinkDir, "dir"); + await fs.symlink(oldLocalDir, newLocalDir, "dir"); + const secondApp = await restartWithPaths(newSymlinkDir, newLocalDir); + const state = (await secondApp.app.inject({ + method: "GET", + url: "/api/system/path-migration", + headers: { cookie } + })).json(); + const migrationId = state.migration?.id ?? 0; + const plan = await secondApp.app.inject({ + method: "POST", + url: "/api/system/path-migration/plan", + headers: { cookie }, + payload: { migrationId } + }); + expect(plan.statusCode).toBe(200); + await secondApp.database.db + .update(schema.pathMigrationItems) + .set({ targetIdentity: null }) + .where(eq(schema.pathMigrationItems.migrationId, migrationId)); + const apply = await secondApp.app.inject({ + method: "POST", + url: "/api/system/path-migration/apply", + headers: { cookie }, + payload: { migrationId, confirmSameStorage: true } + }); + expect(apply.statusCode).toBe(200); + const jobId = apply.json<{ jobId: number }>().jobId; + const worker = new JobWorker(secondApp.database.db, { + workerId: "legacy-planned-identity-worker", + pollIntervalMs: 1, + heartbeatIntervalMs: 10, + logger: silentLogger + }); + + await expect(worker.runOnce()).resolves.toBe(true); + await expect(secondApp.jobs.getJob(jobId)).resolves.toMatchObject({ status: "failed" }); + await expect( + first(secondApp.database.db.select().from(schema.pathMigrations).where(eq(schema.pathMigrations.id, migrationId)).limit(1)) + ).resolves.toMatchObject({ + status: "failed", + errorMessage: expect.stringContaining("has no exact file identity") + }); + expect(path.resolve(await fs.readlink(fixture.linkPath))).toBe(path.resolve(fixture.targetPath)); + }); + + it("marks an ambiguous legacy failed copy journal for manual reconciliation", async () => { + const app = await openApp(); + const fixture = await insertIndexedSymlink(app, "Legacy Failed Copy", "legacy-failed.bin"); + const copyJobId = await app.jobs.startCopy({ direction: "to_remote", linkIds: [fixture.linkId] }); + const originalLink = await first(app.database.db.select().from(schema.mediaLinks).where(eq(schema.mediaLinks.id, fixture.linkId)).limit(1)); + if (!originalLink) throw new Error("Legacy failed copy fixture was not found"); + const destinationPath = path.join(remoteDir, path.relative(oldLocalDir, fixture.targetPath)); + const timestamp = new Date().toISOString(); + await fs.mkdir(path.dirname(destinationPath), { recursive: true }); + await fs.copyFile(fixture.targetPath, destinationPath); + await fs.rm(fixture.linkPath); + await fs.symlink(destinationPath, fixture.linkPath); + await app.database.db.update(schema.jobs).set({ status: "failed", finishedAt: timestamp }).where(eq(schema.jobs.id, copyJobId)); + const operation = await first( + app.database.db + .insert(schema.copyOperations) + .values({ + jobId: copyJobId, + mediaLinkId: fixture.linkId, + linkPath: fixture.linkPath, + sourcePath: fixture.targetPath, + destinationPath, + originalTargetPath: fixture.targetPath, + originalLinkState: JSON.stringify(originalLink), + previousCopySource: null, + tempPath: null, + displacedPath: null, + tempIdentity: null, + destinationIdentity: null, + displacedIdentity: null, + stage: "failed", + resultStatus: null, + localConflictStrategy: null, + sizeBytes: fixture.fileSize, + errorMessage: "Legacy worker stopped after repointing", + createdAt: timestamp, + updatedAt: timestamp, + completedAt: timestamp + }) + .returning({ id: schema.copyOperations.id }) + ); + if (!operation) throw new Error("Legacy failed copy operation was not inserted"); + + await reconcileEnvironmentPaths(app.database.db, { + symlinkDir: oldSymlinkDir, + localDir: oldLocalDir, + remoteDir + }); + + await expect( + first(app.database.db.select().from(schema.copyOperations).where(eq(schema.copyOperations.id, operation.id)).limit(1)) + ).resolves.toMatchObject({ + stage: "reconciliation_required", + completedAt: null, + errorMessage: expect.stringContaining("library symlink no longer points to its original target") + }); + }); + + it("fences a legacy failed copy journal when an unowned destination artifact remains", async () => { + const app = await openApp(); + const fixture = await insertIndexedSymlink(app, "Legacy Destination Artifact", "legacy-destination.bin"); + const copyJobId = await app.jobs.startCopy({ direction: "to_remote", linkIds: [fixture.linkId] }); + const originalLink = await first(app.database.db.select().from(schema.mediaLinks).where(eq(schema.mediaLinks.id, fixture.linkId)).limit(1)); + if (!originalLink) throw new Error("Legacy destination fixture was not found"); + const destinationPath = path.join(remoteDir, path.relative(oldLocalDir, fixture.targetPath)); + const timestamp = new Date().toISOString(); + await fs.mkdir(path.dirname(destinationPath), { recursive: true }); + await fs.copyFile(fixture.targetPath, destinationPath); + await app.database.db.update(schema.jobs).set({ status: "failed", finishedAt: timestamp }).where(eq(schema.jobs.id, copyJobId)); + const operation = await first( + app.database.db + .insert(schema.copyOperations) + .values({ + jobId: copyJobId, + mediaLinkId: fixture.linkId, + linkPath: fixture.linkPath, + sourcePath: fixture.targetPath, + destinationPath, + originalTargetPath: fixture.targetPath, + originalLinkState: JSON.stringify(originalLink), + previousCopySource: null, + tempPath: null, + displacedPath: null, + tempIdentity: null, + destinationIdentity: null, + displacedIdentity: null, + stage: "failed", + resultStatus: null, + localConflictStrategy: null, + sizeBytes: fixture.fileSize, + errorMessage: "Legacy worker could not remove a promoted destination", + createdAt: timestamp, + updatedAt: timestamp, + completedAt: timestamp + }) + .returning({ id: schema.copyOperations.id }) + ); + if (!operation) throw new Error("Legacy destination copy operation was not inserted"); + + await reconcileEnvironmentPaths(app.database.db, { + symlinkDir: oldSymlinkDir, + localDir: oldLocalDir, + remoteDir + }); + + await expect(fs.readlink(fixture.linkPath)).resolves.toBe(fixture.targetPath); + await expect(fs.readFile(destinationPath)).resolves.toEqual(await fs.readFile(fixture.targetPath)); + await expect( + first(app.database.db.select().from(schema.copyOperations).where(eq(schema.copyOperations.id, operation.id)).limit(1)) + ).resolves.toMatchObject({ + stage: "reconciliation_required", + completedAt: null, + errorMessage: expect.stringContaining("journaled destination exists but its ownership cannot be proven") + }); + }); + it("blocks normal mutations and safely rebases validated symlinks after a restart", async () => { const firstApp = await openApp(); const cookie = await createAdminSession(firstApp); @@ -230,12 +505,245 @@ describe("managed path configuration", () => { setProgress: async (progress) => { recoveredProgress.push(progress); }, - isCancelled: async () => false + isCancelled: async () => false, + assertLease: async () => undefined, + withLease: (action) => action(), + withLeaseDb: (action) => action(secondApp.database.db), + finishCompleted: async (action) => { + await action(secondApp.database.db); + return true; + } }) ).resolves.toBeUndefined(); expect(recoveredProgress.at(-1)).toMatchObject({ stage: "completed", current: 1, total: 1 }); }); + it("rolls back when cancellation wins the atomic path-migration completion boundary", async () => { + const firstApp = await openApp(); + const cookie = await createAdminSession(firstApp); + const fixture = await insertIndexedSymlink(firstApp, "Cancelled Migration"); + await fs.symlink(oldSymlinkDir, newSymlinkDir, "dir"); + await fs.symlink(oldLocalDir, newLocalDir, "dir"); + const secondApp = await restartWithPaths(newSymlinkDir, newLocalDir); + const state = (await secondApp.app.inject({ method: "GET", url: "/api/system/path-migration", headers: { cookie } })).json(); + const migrationId = state.migration?.id ?? 0; + const plan = await secondApp.app.inject({ + method: "POST", + url: "/api/system/path-migration/plan", + headers: { cookie }, + payload: { migrationId } + }); + expect(plan.statusCode).toBe(200); + const apply = await secondApp.app.inject({ + method: "POST", + url: "/api/system/path-migration/apply", + headers: { cookie }, + payload: { migrationId, confirmSameStorage: true } + }); + expect(apply.statusCode).toBe(200); + const { jobId } = apply.json<{ jobId: number }>(); + if (!Number.isSafeInteger(jobId)) throw new Error("Path migration job ID is invalid"); + await secondApp.database.pool.query(` + CREATE FUNCTION srtl_test_cancel_path_before_finish() RETURNS trigger + LANGUAGE plpgsql AS $function$ + BEGIN + IF NEW.id = ${jobId} + AND NEW.status = 'running' + AND NEW.progress::jsonb ->> 'stage' = 'repointing' + AND NEW.progress::jsonb ->> 'current' = NEW.progress::jsonb ->> 'total' + THEN + NEW.cancel_requested_at = clock_timestamp()::text; + END IF; + RETURN NEW; + END; + $function$ + `); + await secondApp.database.pool.query(` + CREATE TRIGGER srtl_test_cancel_path_before_finish + BEFORE UPDATE OF progress ON jobs + FOR EACH ROW + EXECUTE FUNCTION srtl_test_cancel_path_before_finish() + `); + + const worker = new JobWorker(secondApp.database.db, { workerId: "path-cancel-test-worker", pollIntervalMs: 1, heartbeatIntervalMs: 10, logger: silentLogger }); + await expect(worker.runOnce()).resolves.toBe(true); + + await expect(secondApp.jobs.getJob(jobId)).resolves.toMatchObject({ status: "cancelled", finishedAt: expect.any(String) }); + const migratedLinkPath = rebaseFixturePath(fixture.linkPath, oldSymlinkDir, newSymlinkDir); + expect(path.resolve(await fs.readlink(migratedLinkPath))).toBe(path.resolve(fixture.targetPath)); + await expect(first(secondApp.database.db.select().from(schema.pathMigrations).where(eq(schema.pathMigrations.id, migrationId)).limit(1))).resolves.toMatchObject({ + status: "failed", + errorMessage: expect.stringContaining("terminated before its final commit") + }); + await expect(first(secondApp.database.db.select().from(schema.mediaLinks).where(eq(schema.mediaLinks.id, fixture.linkId)).limit(1))).resolves.toMatchObject({ + linkPath: fixture.linkPath, + targetPath: fixture.targetPath + }); + expect((await secondApp.app.inject({ method: "GET", url: "/api/system/path-migration", headers: { cookie } })).json()).toMatchObject({ + blocking: true + }); + }); + + it("serializes path reconciliation behind the final migration commit without deadlocking", async () => { + const firstApp = await openApp(); + const cookie = await createAdminSession(firstApp); + await insertIndexedSymlink(firstApp, "Final Commit Lock Order", "lock-order.bin"); + await fs.symlink(oldSymlinkDir, newSymlinkDir, "dir"); + await fs.symlink(oldLocalDir, newLocalDir, "dir"); + const secondApp = await restartWithPaths(newSymlinkDir, newLocalDir); + const state = (await secondApp.app.inject({ method: "GET", url: "/api/system/path-migration", headers: { cookie } })).json(); + const migrationId = state.migration?.id ?? 0; + const plan = await secondApp.app.inject({ method: "POST", url: "/api/system/path-migration/plan", headers: { cookie }, payload: { migrationId } }); + expect(plan.statusCode).toBe(200); + const apply = await secondApp.app.inject({ + method: "POST", + url: "/api/system/path-migration/apply", + headers: { cookie }, + payload: { migrationId, confirmSameStorage: true } + }); + expect(apply.statusCode).toBe(200); + const jobId = apply.json<{ jobId: number }>().jobId; + + // Mirrors the private process-wide lock used by pathConfiguration.ts so this test can hold the final-commit boundary open. + const pathConfigurationAdvisoryLockKey = 781_889_433; + const gateClient = await secondApp.database.pool.connect(); + let gateHeld = false; + let workerRun: Promise | null = null; + let reconciliation: Promise | null = null; + try { + await gateClient.query("BEGIN"); + await gateClient.query("SELECT pg_advisory_xact_lock($1)", [pathConfigurationAdvisoryLockKey]); + gateHeld = true; + + const worker = new JobWorker(secondApp.database.db, { + workerId: "path-final-lock-order-worker", + pollIntervalMs: 1, + heartbeatIntervalMs: 10, + logger: silentLogger + }); + workerRun = worker.runOnce(); + await expect + .poll( + async () => { + const result = await secondApp.database.pool.query<{ waiting: string }>(` + SELECT count(*)::text AS waiting + FROM pg_locks + WHERE locktype = 'advisory' + AND database = (SELECT oid FROM pg_database WHERE datname = current_database()) + AND NOT granted + `); + return Number(result.rows[0]?.waiting ?? 0); + }, + { interval: 10, timeout: 2_000 } + ) + .toBeGreaterThanOrEqual(1); + + reconciliation = reconcileEnvironmentPaths(secondApp.database.db, { symlinkDir: newSymlinkDir, localDir: newLocalDir, remoteDir }); + await expect + .poll( + async () => { + const result = await secondApp.database.pool.query<{ waiting: string }>(` + SELECT count(*)::text AS waiting + FROM pg_locks + WHERE locktype = 'advisory' + AND database = (SELECT oid FROM pg_database WHERE datname = current_database()) + AND NOT granted + `); + return Number(result.rows[0]?.waiting ?? 0); + }, + { interval: 10, timeout: 2_000 } + ) + .toBeGreaterThanOrEqual(2); + + await gateClient.query("COMMIT"); + gateHeld = false; + let timeout: ReturnType | undefined; + try { + const [didWork] = await Promise.race([ + Promise.all([workerRun, reconciliation]), + new Promise((_resolve, reject) => { + timeout = setTimeout(() => reject(new Error("Path finalization and reconciliation deadlocked")), 3_000); + }) + ]); + expect(didWork).toBe(true); + } finally { + if (timeout) clearTimeout(timeout); + } + } finally { + if (gateHeld) await gateClient.query("ROLLBACK").catch(() => undefined); + gateClient.release(); + await Promise.allSettled([...(workerRun ? [workerRun] : []), ...(reconciliation ? [reconciliation] : [])]); + } + + await expect(secondApp.jobs.getJob(jobId)).resolves.toMatchObject({ status: "completed" }); + await expect(first(secondApp.database.db.select().from(schema.pathMigrations).where(eq(schema.pathMigrations.id, migrationId)).limit(1))).resolves.toMatchObject({ + status: "completed" + }); + expect((await secondApp.app.inject({ method: "GET", url: "/api/system/path-migration", headers: { cookie } })).json()).toMatchObject({ + blocking: false, + status: "ready" + }); + }); + + it("waits for running-job mutation leases before recording a path change", async () => { + const app = await openApp(); + await fs.symlink(oldSymlinkDir, newSymlinkDir, "dir"); + await fs.symlink(oldLocalDir, newLocalDir, "dir"); + const timestamp = new Date().toISOString(); + const runningJob = await first( + app.database.db + .insert(schema.jobs) + .values({ + type: "copy", + status: "running", + createdAt: timestamp, + startedAt: timestamp, + finishedAt: null, + lockedBy: "path-reconcile-mutation-holder", + lockedAt: timestamp, + heartbeatAt: timestamp, + exclusive: false, + cancelRequestedAt: null, + progress: "{}" + }) + .returning({ id: schema.jobs.id }) + ); + if (!runningJob) throw new Error("Running path-reconciliation fixture job was not inserted"); + + const gateClient = await app.database.pool.connect(); + let gateHeld = false; + let reconciliation: Promise | null = null; + try { + await gateClient.query("BEGIN"); + await gateClient.query("SELECT id FROM jobs WHERE id = $1 FOR UPDATE", [runningJob.id]); + gateHeld = true; + + reconciliation = reconcileEnvironmentPaths(app.database.db, { + symlinkDir: newSymlinkDir, + localDir: newLocalDir, + remoteDir + }); + const boundary = await Promise.race([ + reconciliation.then(() => "completed" as const), + new Promise<"waiting">((resolve) => setTimeout(() => resolve("waiting"), 100)) + ]); + expect(boundary).toBe("waiting"); + expect(await app.database.db.select().from(schema.pathMigrations)).toEqual([]); + + await gateClient.query("COMMIT"); + gateHeld = false; + await reconciliation; + } finally { + if (gateHeld) await gateClient.query("ROLLBACK").catch(() => undefined); + gateClient.release(); + if (reconciliation) await reconciliation.catch(() => undefined); + } + + await expect(first(app.database.db.select().from(schema.pathMigrations).orderBy(desc(schema.pathMigrations.id)).limit(1))).resolves.toMatchObject({ + status: "pending" + }); + }); + it("keeps migration blocked when mapped files do not exist", async () => { const firstApp = await openApp(); const cookie = await createAdminSession(firstApp); @@ -278,6 +786,358 @@ describe("managed path configuration", () => { expect(plannedState).toMatchObject({ blocking: true, status: "ready_to_apply" }); }); + it("preserves unresolved started-migration journals while allowing safe reanalysis", async () => { + const firstApp = await openApp(); + const cookie = await createAdminSession(firstApp); + await insertIndexedSymlink(firstApp, "Reanalysis Journal", "journal.bin"); + await fs.symlink(oldSymlinkDir, newSymlinkDir, "dir"); + await fs.symlink(oldLocalDir, newLocalDir, "dir"); + const secondApp = await restartWithPaths(newSymlinkDir, newLocalDir); + const state = (await secondApp.app.inject({ method: "GET", url: "/api/system/path-migration", headers: { cookie } })).json(); + const migrationId = state.migration?.id ?? 0; + const initialPlan = await secondApp.app.inject({ + method: "POST", + url: "/api/system/path-migration/plan", + headers: { cookie }, + payload: { migrationId } + }); + expect(initialPlan.statusCode).toBe(200); + + const timestamp = new Date().toISOString(); + await secondApp.database.db + .update(schema.pathMigrations) + .set({ status: "failed", startedAt: null, finishedAt: timestamp, errorMessage: "Analysis failed before migration started" }) + .where(eq(schema.pathMigrations.id, migrationId)); + const beforeStartReplan = await secondApp.app.inject({ + method: "POST", + url: "/api/system/path-migration/plan", + headers: { cookie }, + payload: { migrationId } + }); + expect(beforeStartReplan.statusCode).toBe(200); + + const unresolvedItem = await first( + secondApp.database.db.select().from(schema.pathMigrationItems).where(eq(schema.pathMigrationItems.migrationId, migrationId)).limit(1) + ); + if (!unresolvedItem) throw new Error("Reanalysis journal fixture item was not created"); + const manualMessage = "Manual review is required before this symlink can be migrated again."; + await secondApp.database.db + .update(schema.pathMigrations) + .set({ status: "failed", startedAt: timestamp, finishedAt: timestamp, errorMessage: manualMessage }) + .where(eq(schema.pathMigrations.id, migrationId)); + await secondApp.database.db + .update(schema.pathMigrationItems) + .set({ validationStatus: "blocked", rolledBackAt: null, message: manualMessage }) + .where(eq(schema.pathMigrationItems.id, unresolvedItem.id)); + + const blockedReplan = await secondApp.app.inject({ + method: "POST", + url: "/api/system/path-migration/plan", + headers: { cookie }, + payload: { migrationId } + }); + expect(blockedReplan.statusCode).toBe(409); + expect(blockedReplan.json()).toMatchObject({ error: expect.stringContaining("manually reconciled") }); + await expect(first(secondApp.database.db.select().from(schema.pathMigrationItems).where(eq(schema.pathMigrationItems.id, unresolvedItem.id)).limit(1))).resolves.toMatchObject({ + validationStatus: "blocked", + message: manualMessage, + rolledBackAt: null + }); + + await secondApp.database.db + .update(schema.pathMigrationItems) + .set({ validationStatus: "rolled_back", rolledBackAt: timestamp }) + .where(eq(schema.pathMigrationItems.id, unresolvedItem.id)); + const reconciledReplan = await secondApp.app.inject({ + method: "POST", + url: "/api/system/path-migration/plan", + headers: { cookie }, + payload: { migrationId } + }); + expect(reconciledReplan.statusCode).toBe(200); + await expect(first(secondApp.database.db.select().from(schema.pathMigrationItems).where(eq(schema.pathMigrationItems.migrationId, migrationId)).limit(1))).resolves.toMatchObject({ + validationStatus: "ready" + }); + }); + + it("waits for predecessor rollback and its path job before planning a successor", async () => { + const firstApp = await openApp(); + const cookie = await createAdminSession(firstApp); + await insertIndexedSymlink(firstApp, "Successor Barrier", "successor.bin"); + await fs.symlink(oldSymlinkDir, newSymlinkDir, "dir"); + await fs.symlink(oldLocalDir, newLocalDir, "dir"); + const secondApp = await restartWithPaths(newSymlinkDir, newLocalDir); + const initialState = (await secondApp.app.inject({ method: "GET", url: "/api/system/path-migration", headers: { cookie } })).json(); + const predecessorId = initialState.migration?.id ?? 0; + await planPathMigration(secondApp.database.db, predecessorId); + const predecessorJobId = await secondApp.jobs.startPathMigration(predecessorId); + const timestamp = new Date().toISOString(); + await secondApp.database.db + .update(schema.pathMigrations) + .set({ status: "running", startedAt: timestamp }) + .where(eq(schema.pathMigrations.id, predecessorId)); + + const thirdSymlinkDir = path.join(tmpDir, "symlinks-third"); + const thirdLocalDir = path.join(tmpDir, "local-third"); + await fs.symlink(oldSymlinkDir, thirdSymlinkDir, "dir"); + await fs.symlink(oldLocalDir, thirdLocalDir, "dir"); + await reconcileEnvironmentPaths(secondApp.database.db, { + symlinkDir: thirdSymlinkDir, + localDir: thirdLocalDir, + remoteDir + }); + + const migrations = await secondApp.database.db.select().from(schema.pathMigrations).orderBy(desc(schema.pathMigrations.id)); + const successor = migrations[0]; + const predecessor = migrations.find((migration) => migration.id === predecessorId); + if (!successor || successor.id === predecessorId) throw new Error("Successor path migration was not created"); + expect(predecessor).toMatchObject({ status: "rollback_pending", jobId: predecessorJobId }); + + await expect(planPathMigration(secondApp.database.db, successor.id)).rejects.toThrow(`Path migration #${predecessorId} is still being reconciled`); + await secondApp.database.db.update(schema.pathMigrations).set({ status: "cancelled", finishedAt: timestamp }).where(eq(schema.pathMigrations.id, predecessorId)); + await expect(planPathMigration(secondApp.database.db, successor.id)).rejects.toThrow(`Path migration job #${predecessorJobId} is still active`); + + await secondApp.database.db + .update(schema.jobs) + .set({ status: "cancelled", finishedAt: timestamp, cancelRequestedAt: timestamp }) + .where(eq(schema.jobs.id, predecessorJobId)); + await expect(planPathMigration(secondApp.database.db, successor.id)).resolves.toBeUndefined(); + await expect(first(secondApp.database.db.select().from(schema.pathMigrations).where(eq(schema.pathMigrations.id, successor.id)).limit(1))).resolves.toMatchObject({ + status: "planned" + }); + }); + + it("waits for active and manually blocked copy journals before planning or applying paths", async () => { + const firstApp = await openApp(); + const cookie = await createAdminSession(firstApp); + const fixture = await insertIndexedSymlink(firstApp, "Copy Recovery Barrier", "copy-recovery.bin"); + const originalLink = await first(firstApp.database.db.select().from(schema.mediaLinks).where(eq(schema.mediaLinks.id, fixture.linkId)).limit(1)); + if (!originalLink) throw new Error("Copy-recovery fixture link was not found"); + const timestamp = new Date().toISOString(); + const copyJob = await first( + firstApp.database.db + .insert(schema.jobs) + .values({ + type: "copy", + status: "queued", + createdAt: timestamp, + startedAt: timestamp, + finishedAt: null, + lockedBy: null, + lockedAt: null, + heartbeatAt: null, + exclusive: false, + cancelRequestedAt: null, + progress: "{}" + }) + .returning({ id: schema.jobs.id }) + ); + if (!copyJob) throw new Error("Copy-recovery fixture job was not inserted"); + const copyOperation = await first( + firstApp.database.db + .insert(schema.copyOperations) + .values({ + jobId: copyJob.id, + mediaLinkId: fixture.linkId, + linkPath: fixture.linkPath, + sourcePath: fixture.targetPath, + destinationPath: path.join(remoteDir, "files", "Copy Recovery Barrier", "copy-recovery.bin"), + originalTargetPath: fixture.targetPath, + originalLinkState: JSON.stringify(originalLink), + previousCopySource: null, + tempPath: null, + displacedPath: null, + stage: "committed", + resultStatus: "copied", + localConflictStrategy: null, + sizeBytes: fixture.fileSize, + errorMessage: null, + createdAt: timestamp, + updatedAt: timestamp, + completedAt: timestamp + }) + .returning({ id: schema.copyOperations.id }) + ); + if (!copyOperation) throw new Error("Copy-recovery fixture operation was not inserted"); + + await fs.symlink(oldSymlinkDir, newSymlinkDir, "dir"); + await fs.symlink(oldLocalDir, newLocalDir, "dir"); + const secondApp = await restartWithPaths(newSymlinkDir, newLocalDir); + const state = (await secondApp.app.inject({ method: "GET", url: "/api/system/path-migration", headers: { cookie } })).json(); + const migrationId = state.migration?.id ?? 0; + + await expect(planPathMigration(secondApp.database.db, migrationId)).rejects.toThrow(`Copy job #${copyJob.id} is still reconciling filesystem changes`); + await secondApp.database.db.update(schema.pathMigrations).set({ status: "queued" }).where(eq(schema.pathMigrations.id, migrationId)); + await expect( + runPathMigration(secondApp.database.db, migrationId, { + signal: new AbortController().signal, + event: async () => undefined, + setProgress: async () => undefined, + isCancelled: async () => false, + assertLease: async () => undefined, + withLease: (action) => action(), + withLeaseDb: (action) => action(secondApp.database.db), + finishCompleted: async (action) => { + await action(secondApp.database.db); + return true; + } + }) + ).rejects.toThrow(`Copy job #${copyJob.id} is still reconciling filesystem changes`); + await expect(secondApp.jobs.getJob(copyJob.id)).resolves.toMatchObject({ status: "queued" }); + + await secondApp.database.db.update(schema.pathMigrations).set({ status: "pending" }).where(eq(schema.pathMigrations.id, migrationId)); + await secondApp.database.db + .update(schema.jobs) + .set({ status: "failed", finishedAt: timestamp }) + .where(eq(schema.jobs.id, copyJob.id)); + await secondApp.database.db + .update(schema.copyOperations) + .set({ stage: "promoted", completedAt: null }) + .where(eq(schema.copyOperations.id, copyOperation.id)); + await expect(planPathMigration(secondApp.database.db, migrationId)).rejects.toThrow( + `Copy operation #${copyOperation.id} from job #${copyJob.id} has unresolved filesystem changes` + ); + await secondApp.database.db + .update(schema.copyOperations) + .set({ stage: "committed", completedAt: timestamp }) + .where(eq(schema.copyOperations.id, copyOperation.id)); + await expect(planPathMigration(secondApp.database.db, migrationId)).resolves.toBeUndefined(); + const plannedItems = await secondApp.database.db.select().from(schema.pathMigrationItems).where(eq(schema.pathMigrationItems.migrationId, migrationId)); + + await secondApp.database.db + .update(schema.copyOperations) + .set({ stage: "reconciliation_required", errorMessage: "Manual copy reconciliation is required" }) + .where(eq(schema.copyOperations.id, copyOperation.id)); + await expect(planPathMigration(secondApp.database.db, migrationId)).rejects.toThrow(`Copy operation #${copyOperation.id} requires manual reconciliation`); + expect(await secondApp.database.db.select().from(schema.pathMigrationItems).where(eq(schema.pathMigrationItems.migrationId, migrationId))).toEqual(plannedItems); + + await secondApp.database.db + .update(schema.copyOperations) + .set({ stage: "rolled_back", errorMessage: null }) + .where(eq(schema.copyOperations.id, copyOperation.id)); + await expect(planPathMigration(secondApp.database.db, migrationId)).resolves.toBeUndefined(); + }); + + it("waits for a running job to requeue before taking the migration analysis snapshot", async () => { + const firstApp = await openApp(); + const cookie = await createAdminSession(firstApp); + const fixture = await insertIndexedSymlink(firstApp, "Snapshot Barrier Title", "snapshot.bin"); + await fs.symlink(oldSymlinkDir, newSymlinkDir, "dir"); + await fs.symlink(oldLocalDir, newLocalDir, "dir"); + const secondApp = await restartWithPaths(newSymlinkDir, newLocalDir); + const state = (await secondApp.app.inject({ method: "GET", url: "/api/system/path-migration", headers: { cookie } })).json(); + const migrationId = state.migration?.id ?? 0; + const timestamp = new Date().toISOString(); + const runningJob = await first( + secondApp.database.db + .insert(schema.jobs) + .values({ + type: "scan", + status: "running", + createdAt: timestamp, + startedAt: timestamp, + finishedAt: null, + lockedBy: "snapshot-draining-worker", + lockedAt: timestamp, + heartbeatAt: timestamp, + exclusive: false, + cancelRequestedAt: null, + progress: "{}" + }) + .returning({ id: schema.jobs.id }) + ); + if (!runningJob) throw new Error("Running snapshot-barrier job was not inserted"); + + const blockedPlan = await secondApp.app.inject({ + method: "POST", + url: "/api/system/path-migration/plan", + headers: { cookie }, + payload: { migrationId } + }); + expect(blockedPlan.statusCode).toBe(409); + expect(blockedPlan.json()).toMatchObject({ error: expect.stringContaining("wait for it to pause") }); + await expect(first(secondApp.database.db.select().from(schema.pathMigrations).where(eq(schema.pathMigrations.id, migrationId)).limit(1))).resolves.toMatchObject({ + status: "pending" + }); + await expect(secondApp.database.db.select().from(schema.pathMigrationItems).where(eq(schema.pathMigrationItems.migrationId, migrationId))).resolves.toEqual([]); + + await secondApp.database.db + .update(schema.jobs) + .set({ status: "queued", lockedBy: null, lockedAt: null, heartbeatAt: null }) + .where(eq(schema.jobs.id, runningJob.id)); + + const planned = await secondApp.app.inject({ + method: "POST", + url: "/api/system/path-migration/plan", + headers: { cookie }, + payload: { migrationId } + }); + expect(planned.statusCode).toBe(200); + expect(planned.json().migration?.summary).toMatchObject({ affectedLinks: 1, readyLinks: 1, blockedLinks: 0 }); + expect(await secondApp.database.db.select().from(schema.pathMigrationItems).where(eq(schema.pathMigrationItems.migrationId, migrationId))).toEqual([ + expect.objectContaining({ mediaLinkId: fixture.linkId, validationStatus: "ready" }) + ]); + expect(await secondApp.jobs.getJob(runningJob.id)).toMatchObject({ status: "queued", lockedBy: null, heartbeatAt: null }); + }); + + it("queues a validated migration while another job drains", async () => { + const firstApp = await openApp(); + const cookie = await createAdminSession(firstApp); + await insertIndexedSymlink(firstApp); + await fs.symlink(oldSymlinkDir, newSymlinkDir, "dir"); + await fs.symlink(oldLocalDir, newLocalDir, "dir"); + const secondApp = await restartWithPaths(newSymlinkDir, newLocalDir); + const state = (await secondApp.app.inject({ method: "GET", url: "/api/system/path-migration", headers: { cookie } })).json(); + const plan = await secondApp.app.inject({ method: "POST", url: "/api/system/path-migration/plan", headers: { cookie }, payload: { migrationId: state.migration?.id } }); + expect(plan.statusCode).toBe(200); + + const timestamp = new Date().toISOString(); + const runningJob = await first( + secondApp.database.db + .insert(schema.jobs) + .values({ + type: "scan", + status: "running", + createdAt: timestamp, + startedAt: timestamp, + finishedAt: null, + lockedBy: "draining-worker", + lockedAt: timestamp, + heartbeatAt: timestamp, + exclusive: false, + cancelRequestedAt: null, + progress: "{}" + }) + .returning({ id: schema.jobs.id }) + ); + expect(runningJob).toBeTruthy(); + + const apply = await secondApp.app.inject({ + method: "POST", + url: "/api/system/path-migration/apply", + headers: { cookie }, + payload: { migrationId: state.migration?.id, confirmSameStorage: true } + }); + expect(apply.statusCode).toBe(200); + expect(await secondApp.jobs.getJob(apply.json<{ jobId: number }>().jobId)).toMatchObject({ type: "path_migration", status: "queued" }); + expect(await secondApp.jobs.getJob(runningJob?.id ?? 0)).toMatchObject({ type: "scan", status: "running" }); + await expect( + runPathMigration(secondApp.database.db, state.migration?.id ?? 0, { + signal: new AbortController().signal, + event: async () => undefined, + setProgress: async () => undefined, + isCancelled: async () => false, + assertLease: async () => undefined, + withLease: (action) => action(), + withLeaseDb: (action) => action(secondApp.database.db), + finishCompleted: async (action) => { + await action(secondApp.database.db); + return true; + } + }) + ).rejects.toThrow("Another job is still running"); + }); + it("fails analysis early when a managed root is unavailable", async () => { const firstApp = await openApp(); const cookie = await createAdminSession(firstApp); @@ -292,7 +1152,7 @@ describe("managed path configuration", () => { expect(plan.statusCode).toBe(409); expect(plan.json()).toMatchObject({ error: expect.stringContaining("Remote directory is unavailable") }); const failedState = (await secondApp.app.inject({ method: "GET", url: "/api/system/path-migration", headers: { cookie } })).json(); - expect(failedState).toMatchObject({ blocking: true, status: "failed" }); + expect(failedState).toMatchObject({ blocking: true, status: "invalid_environment" }); expect(failedState.migration?.errorMessage).toContain("Remote directory is unavailable"); }); @@ -373,7 +1233,7 @@ describe("managed path configuration", () => { expect(migration?.status).toBe("cancelled"); }); - it("rolls back symlinks already repointed when a later target changes after analysis", async () => { + it("rolls back symlinks already repointed when a later target is replaced by a same-size file after analysis", async () => { const firstApp = await openApp(); const cookie = await createAdminSession(firstApp); const firstFixture = await insertIndexedSymlink(firstApp, "First Title", "first.bin"); @@ -385,7 +1245,10 @@ describe("managed path configuration", () => { const plan = await secondApp.app.inject({ method: "POST", url: "/api/system/path-migration/plan", headers: { cookie }, payload: { migrationId: state.migration?.id } }); expect(plan.json().migration?.summary).toMatchObject({ readyLinks: 2, blockedLinks: 0 }); - await fs.writeFile(secondFixture.targetPath, "changed after migration analysis"); + const originalTarget = await fs.readFile(secondFixture.targetPath); + const sameSizeReplacement = `${secondFixture.targetPath}.replacement`; + await fs.writeFile(sameSizeReplacement, Buffer.alloc(originalTarget.length, 0x7a)); + await fs.rename(sameSizeReplacement, secondFixture.targetPath); const apply = await secondApp.app.inject({ method: "POST", url: "/api/system/path-migration/apply", @@ -406,6 +1269,407 @@ describe("managed path configuration", () => { expect(failedState.migration?.errorMessage).toContain("Mapped target changed after analysis"); }); + it("rolls back an applied symlink when cancellation is requested under the owned lease", async () => { + const firstApp = await openApp(); + const cookie = await createAdminSession(firstApp); + const firstFixture = await insertIndexedSymlink(firstApp, "Cancelled Title One", "first.bin"); + const secondFixture = await insertIndexedSymlink(firstApp, "Cancelled Title Two", "second.bin"); + await fs.symlink(oldSymlinkDir, newSymlinkDir, "dir"); + await fs.symlink(oldLocalDir, newLocalDir, "dir"); + const secondApp = await restartWithPaths(newSymlinkDir, newLocalDir); + const state = (await secondApp.app.inject({ method: "GET", url: "/api/system/path-migration", headers: { cookie } })).json(); + await secondApp.app.inject({ method: "POST", url: "/api/system/path-migration/plan", headers: { cookie }, payload: { migrationId: state.migration?.id } }); + await secondApp.app.inject({ + method: "POST", + url: "/api/system/path-migration/apply", + headers: { cookie }, + payload: { migrationId: state.migration?.id, confirmSameStorage: true } + }); + + const abortController = new AbortController(); + let cancellationChecks = 0; + let leaseChecks = 0; + await expect( + runPathMigration(secondApp.database.db, state.migration?.id ?? 0, { + signal: abortController.signal, + event: async () => undefined, + setProgress: async () => undefined, + isCancelled: async () => { + cancellationChecks += 1; + if (cancellationChecks > 1) abortController.abort(); + return cancellationChecks > 1; + }, + assertLease: async () => { + leaseChecks += 1; + }, + withLease: async (action) => { + leaseChecks += 1; + return action(); + }, + withLeaseDb: async (action) => { + leaseChecks += 1; + return action(secondApp.database.db); + }, + finishCompleted: async (action) => { + await action(secondApp.database.db); + return true; + } + }) + ).rejects.toThrow("Path migration was terminated"); + + const firstNewLinkPath = rebaseFixturePath(firstFixture.linkPath, oldSymlinkDir, newSymlinkDir); + const secondNewLinkPath = rebaseFixturePath(secondFixture.linkPath, oldSymlinkDir, newSymlinkDir); + expect(path.resolve(await fs.readlink(firstNewLinkPath))).toBe(path.resolve(firstFixture.targetPath)); + expect(path.resolve(await fs.readlink(secondNewLinkPath))).toBe(path.resolve(secondFixture.targetPath)); + expect(leaseChecks).toBeGreaterThanOrEqual(3); + const firstItem = await first( + secondApp.database.db.select().from(schema.pathMigrationItems).where(eq(schema.pathMigrationItems.mediaLinkId, firstFixture.linkId)).limit(1) + ); + expect(firstItem).toMatchObject({ validationStatus: "rolled_back" }); + }); + + it("replays a symlink rename after losing its lease before the item state write", async () => { + const firstApp = await openApp(); + const cookie = await createAdminSession(firstApp); + const fixture = await insertIndexedSymlink(firstApp, "Lease Lost Title", "lease-lost.bin"); + await fs.symlink(oldSymlinkDir, newSymlinkDir, "dir"); + await fs.symlink(oldLocalDir, newLocalDir, "dir"); + const secondApp = await restartWithPaths(newSymlinkDir, newLocalDir); + const state = (await secondApp.app.inject({ method: "GET", url: "/api/system/path-migration", headers: { cookie } })).json(); + await secondApp.app.inject({ method: "POST", url: "/api/system/path-migration/plan", headers: { cookie }, payload: { migrationId: state.migration?.id } }); + await secondApp.app.inject({ + method: "POST", + url: "/api/system/path-migration/apply", + headers: { cookie }, + payload: { migrationId: state.migration?.id, confirmSameStorage: true } + }); + + let leaseOwned = true; + const leaseLostError = () => { + const error = new Error("Job lease was superseded"); + error.name = "LeaseLostError"; + return error; + }; + await expect( + runPathMigration(secondApp.database.db, state.migration?.id ?? 0, { + signal: new AbortController().signal, + event: async () => undefined, + setProgress: async () => undefined, + isCancelled: async () => false, + assertLease: async () => { + if (!leaseOwned) throw leaseLostError(); + }, + withLease: async (action) => { + if (!leaseOwned) throw leaseLostError(); + const result = await action(); + leaseOwned = false; + return result; + }, + withLeaseDb: async (action) => { + if (!leaseOwned) throw leaseLostError(); + return action(secondApp.database.db); + }, + finishCompleted: async (action) => { + if (!leaseOwned) throw leaseLostError(); + await action(secondApp.database.db); + return true; + } + }) + ).rejects.toThrow("Job lease was superseded"); + + const newLinkPath = rebaseFixturePath(fixture.linkPath, oldSymlinkDir, newSymlinkDir); + const newTargetPath = rebaseFixturePath(fixture.targetPath, oldLocalDir, newLocalDir); + expect(path.resolve(await fs.readlink(newLinkPath))).toBe(path.resolve(newTargetPath)); + const mediaLink = await first(secondApp.database.db.select().from(schema.mediaLinks).where(eq(schema.mediaLinks.id, fixture.linkId)).limit(1)); + expect(mediaLink).toMatchObject({ linkPath: fixture.linkPath, targetPath: fixture.targetPath }); + const migration = await first(secondApp.database.db.select().from(schema.pathMigrations).where(eq(schema.pathMigrations.id, state.migration?.id ?? 0)).limit(1)); + expect(migration).toMatchObject({ status: "running" }); + const item = await first( + secondApp.database.db.select().from(schema.pathMigrationItems).where(eq(schema.pathMigrationItems.mediaLinkId, fixture.linkId)).limit(1) + ); + expect(item).toMatchObject({ validationStatus: "ready", rolledBackAt: null }); + + await expect( + runPathMigration(secondApp.database.db, state.migration?.id ?? 0, { + signal: new AbortController().signal, + event: async () => undefined, + setProgress: async () => undefined, + isCancelled: async () => false, + assertLease: async () => undefined, + withLease: (action) => action(), + withLeaseDb: (action) => action(secondApp.database.db), + finishCompleted: async (action) => { + await action(secondApp.database.db); + return true; + } + }) + ).resolves.toBeUndefined(); + const recoveredMediaLink = await first(secondApp.database.db.select().from(schema.mediaLinks).where(eq(schema.mediaLinks.id, fixture.linkId)).limit(1)); + expect(recoveredMediaLink).toMatchObject({ linkPath: newLinkPath, targetPath: newTargetPath }); + const recoveredMigration = await first( + secondApp.database.db.select().from(schema.pathMigrations).where(eq(schema.pathMigrations.id, state.migration?.id ?? 0)).limit(1) + ); + expect(recoveredMigration).toMatchObject({ status: "completed" }); + const recoveredItem = await first( + secondApp.database.db.select().from(schema.pathMigrationItems).where(eq(schema.pathMigrationItems.mediaLinkId, fixture.linkId)).limit(1) + ); + expect(recoveredItem).toMatchObject({ validationStatus: "applied", rolledBackAt: null }); + }); + + it("preserves a symlink changed while migration waits for its mutation lease", async () => { + const firstApp = await openApp(); + const cookie = await createAdminSession(firstApp); + const fixture = await insertIndexedSymlink(firstApp, "Lease Wait Mutation", "lease-wait.bin"); + await fs.symlink(oldSymlinkDir, newSymlinkDir, "dir"); + await fs.symlink(oldLocalDir, newLocalDir, "dir"); + const secondApp = await restartWithPaths(newSymlinkDir, newLocalDir); + const state = (await secondApp.app.inject({ method: "GET", url: "/api/system/path-migration", headers: { cookie } })).json(); + const migrationId = state.migration?.id ?? 0; + const plan = await secondApp.app.inject({ method: "POST", url: "/api/system/path-migration/plan", headers: { cookie }, payload: { migrationId } }); + expect(plan.statusCode).toBe(200); + const apply = await secondApp.app.inject({ + method: "POST", + url: "/api/system/path-migration/apply", + headers: { cookie }, + payload: { migrationId, confirmSameStorage: true } + }); + expect(apply.statusCode).toBe(200); + + const migratedLinkPath = rebaseFixturePath(fixture.linkPath, oldSymlinkDir, newSymlinkDir); + const externalTarget = path.join(tmpDir, "lease-wait-external-target.bin"); + await fs.writeFile(externalTarget, "external target installed while waiting for lease"); + let changedWhileWaiting = false; + + await expect( + runPathMigration(secondApp.database.db, migrationId, { + signal: new AbortController().signal, + event: async () => undefined, + setProgress: async () => undefined, + isCancelled: async () => false, + assertLease: async () => undefined, + withLease: async (action) => { + if (!changedWhileWaiting) { + changedWhileWaiting = true; + await fs.rm(migratedLinkPath); + await fs.symlink(externalTarget, migratedLinkPath); + } + return action(); + }, + withLeaseDb: (action) => action(secondApp.database.db), + finishCompleted: async (action) => { + await action(secondApp.database.db); + return true; + } + }) + ).rejects.toThrow("Symlink changed while path migration waited for its lease"); + + expect(changedWhileWaiting).toBe(true); + expect(path.resolve(await fs.readlink(migratedLinkPath))).toBe(path.resolve(externalTarget)); + await expect(first(secondApp.database.db.select().from(schema.pathMigrationItems).where(eq(schema.pathMigrationItems.mediaLinkId, fixture.linkId)).limit(1))).resolves.toMatchObject({ + validationStatus: "blocked", + rolledBackAt: null, + message: expect.stringContaining("Manual review is required") + }); + await expect(first(secondApp.database.db.select().from(schema.pathMigrations).where(eq(schema.pathMigrations.id, migrationId)).limit(1))).resolves.toMatchObject({ + status: "failed", + errorMessage: expect.stringContaining("Rollback also failed for 1 symlink(s)") + }); + }); + + it("revalidates an unchanged target while waiting to record the migration step", async () => { + const firstApp = await openApp(); + const cookie = await createAdminSession(firstApp); + const fixture = await insertIndexedSymlink(firstApp, "Unchanged Target Lease Wait", "unchanged-target.bin"); + await fs.symlink(oldSymlinkDir, newSymlinkDir, "dir"); + const secondApp = await restartWithPaths(newSymlinkDir, oldLocalDir); + const state = (await secondApp.app.inject({ method: "GET", url: "/api/system/path-migration", headers: { cookie } })).json(); + const migrationId = state.migration?.id ?? 0; + const plan = await secondApp.app.inject({ method: "POST", url: "/api/system/path-migration/plan", headers: { cookie }, payload: { migrationId } }); + expect(plan.statusCode).toBe(200); + await expect( + first(secondApp.database.db.select().from(schema.pathMigrationItems).where(eq(schema.pathMigrationItems.mediaLinkId, fixture.linkId)).limit(1)) + ).resolves.toMatchObject({ targetChanged: false, validationStatus: "ready" }); + const apply = await secondApp.app.inject({ + method: "POST", + url: "/api/system/path-migration/apply", + headers: { cookie }, + payload: { migrationId, confirmSameStorage: true } + }); + expect(apply.statusCode).toBe(200); + + const migratedLinkPath = rebaseFixturePath(fixture.linkPath, oldSymlinkDir, newSymlinkDir); + const externalTarget = path.join(tmpDir, "unchanged-target-external.bin"); + await fs.writeFile(externalTarget, "external target installed before migration state was recorded"); + let leaseDbCalls = 0; + + await expect( + runPathMigration(secondApp.database.db, migrationId, { + signal: new AbortController().signal, + event: async () => undefined, + setProgress: async () => undefined, + isCancelled: async () => false, + assertLease: async () => undefined, + withLease: (action) => action(), + withLeaseDb: async (action) => { + leaseDbCalls += 1; + if (leaseDbCalls === 2) { + await fs.rm(migratedLinkPath); + await fs.symlink(externalTarget, migratedLinkPath); + } + return action(secondApp.database.db); + }, + finishCompleted: async (action) => { + await action(secondApp.database.db); + return true; + } + }) + ).rejects.toThrow("Symlink changed after analysis"); + + expect(leaseDbCalls).toBeGreaterThanOrEqual(4); + expect(path.resolve(await fs.readlink(migratedLinkPath))).toBe(path.resolve(externalTarget)); + await expect(first(secondApp.database.db.select().from(schema.pathMigrationItems).where(eq(schema.pathMigrationItems.mediaLinkId, fixture.linkId)).limit(1))).resolves.toMatchObject({ + validationStatus: "rolled_back", + message: "Migration step remained at its original target." + }); + await expect(first(secondApp.database.db.select().from(schema.pathMigrations).where(eq(schema.pathMigrations.id, migrationId)).limit(1))).resolves.toMatchObject({ + status: "failed", + errorMessage: expect.stringContaining("Symlink changed after analysis") + }); + }); + + it("reapplies an applied item after lease loss between rollback rename and state write", async () => { + const firstApp = await openApp(); + const cookie = await createAdminSession(firstApp); + const fixture = await insertIndexedSymlink(firstApp, "Rollback Lease Lost", "rollback-lease-lost.bin"); + await fs.symlink(oldSymlinkDir, newSymlinkDir, "dir"); + await fs.symlink(oldLocalDir, newLocalDir, "dir"); + const secondApp = await restartWithPaths(newSymlinkDir, newLocalDir); + const state = (await secondApp.app.inject({ method: "GET", url: "/api/system/path-migration", headers: { cookie } })).json(); + await secondApp.app.inject({ method: "POST", url: "/api/system/path-migration/plan", headers: { cookie }, payload: { migrationId: state.migration?.id } }); + const apply = await secondApp.app.inject({ + method: "POST", + url: "/api/system/path-migration/apply", + headers: { cookie }, + payload: { migrationId: state.migration?.id, confirmSameStorage: true } + }); + expect(apply.statusCode).toBe(200); + + let leaseOwned = true; + let filesystemMutations = 0; + const leaseLostError = () => { + const error = new Error("Job lease was superseded during rollback"); + error.name = "LeaseLostError"; + return error; + }; + await expect( + runPathMigration(secondApp.database.db, state.migration?.id ?? 0, { + signal: new AbortController().signal, + event: async () => undefined, + setProgress: async () => undefined, + isCancelled: async () => false, + assertLease: async () => { + if (!leaseOwned) throw leaseLostError(); + }, + withLease: async (action) => { + if (!leaseOwned) throw leaseLostError(); + const result = await action(); + filesystemMutations += 1; + if (filesystemMutations === 2) leaseOwned = false; + return result; + }, + withLeaseDb: async (action) => { + if (!leaseOwned) throw leaseLostError(); + return action(secondApp.database.db); + }, + finishCompleted: async () => { + throw new Error("Injected final commit failure"); + } + }) + ).rejects.toThrow("Job lease was superseded during rollback"); + + const newLinkPath = rebaseFixturePath(fixture.linkPath, oldSymlinkDir, newSymlinkDir); + const newTargetPath = rebaseFixturePath(fixture.targetPath, oldLocalDir, newLocalDir); + expect(path.resolve(await fs.readlink(newLinkPath))).toBe(path.resolve(fixture.targetPath)); + await expect( + first(secondApp.database.db.select().from(schema.pathMigrationItems).where(eq(schema.pathMigrationItems.mediaLinkId, fixture.linkId)).limit(1)) + ).resolves.toMatchObject({ validationStatus: "applied", rolledBackAt: null }); + + await expect( + runPathMigration(secondApp.database.db, state.migration?.id ?? 0, { + signal: new AbortController().signal, + event: async () => undefined, + setProgress: async () => undefined, + isCancelled: async () => false, + assertLease: async () => undefined, + withLease: (action) => action(), + withLeaseDb: (action) => action(secondApp.database.db), + finishCompleted: async (action) => { + await action(secondApp.database.db); + return true; + } + }) + ).resolves.toBeUndefined(); + + expect(path.resolve(await fs.readlink(newLinkPath))).toBe(path.resolve(newTargetPath)); + await expect(first(secondApp.database.db.select().from(schema.mediaLinks).where(eq(schema.mediaLinks.id, fixture.linkId)).limit(1))).resolves.toMatchObject({ + linkPath: newLinkPath, + targetPath: newTargetPath + }); + await expect(first(secondApp.database.db.select().from(schema.pathMigrations).where(eq(schema.pathMigrations.id, state.migration?.id ?? 0)).limit(1))).resolves.toMatchObject({ + status: "completed" + }); + }); + + it("restores a renamed symlink when the durable item state write fails", async () => { + const firstApp = await openApp(); + const cookie = await createAdminSession(firstApp); + const fixture = await insertIndexedSymlink(firstApp, "State Write Failure", "state-write-failure.bin"); + await fs.symlink(oldSymlinkDir, newSymlinkDir, "dir"); + await fs.symlink(oldLocalDir, newLocalDir, "dir"); + const secondApp = await restartWithPaths(newSymlinkDir, newLocalDir); + const state = (await secondApp.app.inject({ method: "GET", url: "/api/system/path-migration", headers: { cookie } })).json(); + await secondApp.app.inject({ method: "POST", url: "/api/system/path-migration/plan", headers: { cookie }, payload: { migrationId: state.migration?.id } }); + const apply = await secondApp.app.inject({ + method: "POST", + url: "/api/system/path-migration/apply", + headers: { cookie }, + payload: { migrationId: state.migration?.id, confirmSameStorage: true } + }); + expect(apply.statusCode).toBe(200); + + let leaseDbCalls = 0; + const injectedError = "Injected item state write failure"; + await expect( + runPathMigration(secondApp.database.db, state.migration?.id ?? 0, { + signal: new AbortController().signal, + event: async () => undefined, + setProgress: async () => undefined, + isCancelled: async () => false, + assertLease: async () => undefined, + withLease: (action) => action(), + withLeaseDb: async (action) => { + leaseDbCalls += 1; + if (leaseDbCalls === 2) throw new Error(injectedError); + return action(secondApp.database.db); + }, + finishCompleted: async (action) => { + await action(secondApp.database.db); + return true; + } + }) + ).rejects.toThrow(injectedError); + + const newLinkPath = rebaseFixturePath(fixture.linkPath, oldSymlinkDir, newSymlinkDir); + expect(path.resolve(await fs.readlink(newLinkPath))).toBe(path.resolve(fixture.targetPath)); + const migration = await first(secondApp.database.db.select().from(schema.pathMigrations).where(eq(schema.pathMigrations.id, state.migration?.id ?? 0)).limit(1)); + expect(migration).toMatchObject({ status: "failed", errorMessage: expect.stringContaining(injectedError) }); + const item = await first( + secondApp.database.db.select().from(schema.pathMigrationItems).where(eq(schema.pathMigrationItems.mediaLinkId, fixture.linkId)).limit(1) + ); + expect(item).toMatchObject({ validationStatus: "rolled_back", appliedAt: null }); + expect(leaseDbCalls).toBeGreaterThanOrEqual(4); + }); + it("blocks rollback when an applied symlink changes again during recovery", async () => { const firstApp = await openApp(); const cookie = await createAdminSession(firstApp); @@ -442,6 +1706,13 @@ describe("managed path configuration", () => { await fs.symlink(externalTarget, firstNewLinkPath); } return false; + }, + assertLease: async () => undefined, + withLease: (action) => action(), + withLeaseDb: (action) => action(secondApp.database.db), + finishCompleted: async (action) => { + await action(secondApp.database.db); + return true; } }) ).rejects.toThrow("Rollback also failed for 1 symlink(s)"); @@ -483,6 +1754,13 @@ describe("managed path configuration", () => { await reconcileEnvironmentPaths(secondApp.database.db, { symlinkDir: oldSymlinkDir, localDir: oldLocalDir, remoteDir }); } return false; + }, + assertLease: async () => undefined, + withLease: (action) => action(), + withLeaseDb: (action) => action(secondApp.database.db), + finishCompleted: async (action) => { + await action(secondApp.database.db); + return true; } }) ).rejects.toThrow("Detected paths changed again"); @@ -491,6 +1769,11 @@ describe("managed path configuration", () => { const secondNewLinkPath = rebaseFixturePath(secondFixture.linkPath, oldSymlinkDir, newSymlinkDir); expect(path.resolve(await fs.readlink(firstNewLinkPath))).toBe(path.resolve(firstFixture.targetPath)); expect(path.resolve(await fs.readlink(secondNewLinkPath))).toBe(path.resolve(secondFixture.targetPath)); + expect(await first(secondApp.database.db.select().from(schema.pathMigrations).where(eq(schema.pathMigrations.id, migrationId)).limit(1))).toMatchObject({ + status: "rollback_pending" + }); + const recoveryWorker = new JobWorker(secondApp.database.db, { workerId: "restore-rollback-worker", pollIntervalMs: 1, heartbeatIntervalMs: 10, logger: silentLogger }); + await expect(recoveryWorker.runOnce()).resolves.toBe(true); const migration = await first(secondApp.database.db.select().from(schema.pathMigrations).where(eq(schema.pathMigrations.id, migrationId)).limit(1)); expect(migration?.status).toBe("cancelled"); }); diff --git a/tests/pathMigrationDisplay.test.ts b/tests/pathMigrationDisplay.test.ts new file mode 100644 index 0000000..65c3b61 --- /dev/null +++ b/tests/pathMigrationDisplay.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from "vitest"; +import { isActivePathMigrationStatus, pathMigrationProgressTitle, pathMigrationStatusLabel } from "../src/client/appShared"; + +describe("path migration display", () => { + it("keeps rollback recovery active and clearly labelled", () => { + expect(isActivePathMigrationStatus("rollback_pending")).toBe(true); + expect(pathMigrationStatusLabel("rollback_pending")).toBe("Rolling back"); + expect(pathMigrationProgressTitle("rollback_pending", "Restoring symlinks to the active paths")).toBe("Rolling back paths"); + }); + + it("preserves the existing active migration presentation", () => { + expect(isActivePathMigrationStatus("queued")).toBe(true); + expect(isActivePathMigrationStatus("running")).toBe(true); + expect(isActivePathMigrationStatus("planned")).toBe(false); + expect(pathMigrationStatusLabel("planned")).toBe("Ready"); + expect(pathMigrationProgressTitle("running", "Repointing symlinks")).toBe("Repointing symlinks"); + }); +}); diff --git a/tests/workerHeartbeats.test.ts b/tests/workerHeartbeats.test.ts new file mode 100644 index 0000000..744a5e6 --- /dev/null +++ b/tests/workerHeartbeats.test.ts @@ -0,0 +1,222 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { asc, eq } from "drizzle-orm"; +import { describe, expect, it } from "vitest"; +import { createApp, type AppContext } from "../src/server/app"; +import { openDatabase } from "../src/server/db/database"; +import * as schema from "../src/server/db/schema"; +import { pruneWorkerHeartbeatHistory, recordWorkerHeartbeat, workerHeartbeatRetentionMs } from "../src/server/lib/workerHeartbeats"; +import { createTestDatabase } from "./testDb"; + +describe("worker heartbeat history", () => { + it("prunes stopped workers and running workers older than the retention window", async () => { + const testDatabase = await createTestDatabase(); + const database = await openDatabase(testDatabase.databaseUrl); + const nowMs = Date.parse("2026-07-29T18:00:00.000Z"); + const recentHeartbeat = new Date(nowMs - 5 * 60_000).toISOString(); + const oldHeartbeat = new Date(nowMs - workerHeartbeatRetentionMs - 1).toISOString(); + try { + await database.db.insert(schema.workerHeartbeats).values([ + { workerId: "recent-running", startedAt: recentHeartbeat, heartbeatAt: recentHeartbeat, status: "running" }, + { workerId: "old-running", startedAt: oldHeartbeat, heartbeatAt: oldHeartbeat, status: "running" }, + { workerId: "recent-stopped", startedAt: recentHeartbeat, heartbeatAt: recentHeartbeat, status: "stopped" }, + { workerId: "old-stopped", startedAt: oldHeartbeat, heartbeatAt: oldHeartbeat, status: "stopped" } + ]); + + await pruneWorkerHeartbeatHistory(database.db, nowMs); + + expect(await database.db.select().from(schema.workerHeartbeats).orderBy(asc(schema.workerHeartbeats.workerId))).toEqual([ + { workerId: "recent-running", startedAt: recentHeartbeat, heartbeatAt: recentHeartbeat, status: "running", capacity: 1 } + ]); + } finally { + await database.close(); + await testDatabase.cleanup(); + } + }); + + it("upserts one process heartbeat with its advertised capacity", async () => { + const testDatabase = await createTestDatabase(); + const database = await openDatabase(testDatabase.databaseUrl); + const startedAt = "2026-07-29T18:00:00.000Z"; + try { + await recordWorkerHeartbeat(database.db, { + workerId: "worker-process:boot-id", + startedAt, + heartbeatAt: "2026-07-29T18:00:01.000Z", + status: "running", + capacity: 128 + }); + await recordWorkerHeartbeat(database.db, { + workerId: "worker-process:boot-id", + startedAt, + heartbeatAt: "2026-07-29T18:00:02.000Z", + status: "stopped", + capacity: 128 + }); + + expect(await database.db.select().from(schema.workerHeartbeats)).toEqual([ + { + workerId: "worker-process:boot-id", + startedAt, + heartbeatAt: "2026-07-29T18:00:02.000Z", + status: "stopped", + capacity: 128 + } + ]); + } finally { + await database.close(); + await testDatabase.cleanup(); + } + }); + + it("retains a stopped worker heartbeat while a running job still owns its lease", async () => { + const testDatabase = await createTestDatabase(); + const database = await openDatabase(testDatabase.databaseUrl); + const timestamp = "2026-07-29T18:00:00.000Z"; + try { + await database.db.insert(schema.workerHeartbeats).values({ + workerId: "stopped-lease-owner", + startedAt: timestamp, + heartbeatAt: timestamp, + status: "stopped", + capacity: 2 + }); + await database.db.insert(schema.jobs).values({ + type: "copy", + status: "running", + createdAt: timestamp, + startedAt: timestamp, + finishedAt: null, + lockedBy: "stopped-lease-owner", + lockedAt: timestamp, + heartbeatAt: timestamp, + leaseVersion: 1, + exclusive: false, + cancelRequestedAt: null, + progress: "{}" + }); + + await pruneWorkerHeartbeatHistory(database.db, Date.parse(timestamp) + 60_000); + + expect(await database.db.select().from(schema.workerHeartbeats)).toEqual([ + { + workerId: "stopped-lease-owner", + startedAt: timestamp, + heartbeatAt: timestamp, + status: "stopped", + capacity: 2 + } + ]); + } finally { + await database.close(); + await testDatabase.cleanup(); + } + }); + + it("reports not_started when heartbeat history contains only stopped workers", async () => { + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "srtl-worker-health-")); + const testDatabase = await createTestDatabase(); + let app: AppContext | null = null; + const symlinkDir = path.join(tmpDir, "symlinks"); + const localDir = path.join(tmpDir, "local"); + const remoteDir = path.join(tmpDir, "remote"); + try { + await Promise.all([fs.mkdir(symlinkDir), fs.mkdir(localDir), fs.mkdir(remoteDir)]); + app = await createApp({ + rootDir: tmpDir, + dataDir: path.join(tmpDir, "data"), + databaseUrl: testDatabase.databaseUrl, + apiDocsEnabled: false, + autoMigrate: true, + paths: { symlinkDir, localDir, remoteDir }, + jobConcurrency: { + workerCount: 1, + maxRunningJobs: 1, + maxRunningScans: 1, + maxRunningAudits: 1, + maxRunningCopies: 1, + copyFileConcurrency: 1, + maxActiveCopyFiles: 1 + } + }); + const heartbeatAt = new Date().toISOString(); + await app.database.db.insert(schema.workerHeartbeats).values({ workerId: "historical-worker", startedAt: heartbeatAt, heartbeatAt, status: "stopped" }); + + const response = await app.app.inject({ method: "GET", url: "/api/health" }); + + expect(response.statusCode).toBe(200); + expect(response.json()).toMatchObject({ + worker: "not_started", + workerHeartbeatAt: null, + expectedWorkerCount: 1, + readyWorkerCount: 0, + staleWorkerCount: 0 + }); + } finally { + if (app) await app.app.close(); + await testDatabase.cleanup(); + await fs.rm(tmpDir, { recursive: true, force: true }); + } + }); + + it("sums fresh and stale process capacity for worker health", async () => { + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "srtl-worker-capacity-health-")); + const testDatabase = await createTestDatabase(); + let app: AppContext | null = null; + const symlinkDir = path.join(tmpDir, "symlinks"); + const localDir = path.join(tmpDir, "local"); + const remoteDir = path.join(tmpDir, "remote"); + try { + await Promise.all([fs.mkdir(symlinkDir), fs.mkdir(localDir), fs.mkdir(remoteDir)]); + app = await createApp({ + rootDir: tmpDir, + dataDir: path.join(tmpDir, "data"), + databaseUrl: testDatabase.databaseUrl, + apiDocsEnabled: false, + autoMigrate: true, + paths: { symlinkDir, localDir, remoteDir }, + jobConcurrency: { + workerCount: 8, + maxRunningJobs: 5, + maxRunningScans: 5, + maxRunningAudits: 5, + maxRunningCopies: 5, + copyFileConcurrency: 1, + maxActiveCopyFiles: 5 + } + }); + const freshHeartbeatAt = new Date().toISOString(); + const staleHeartbeatAt = new Date(Date.now() - 31_000).toISOString(); + await app.database.db.insert(schema.workerHeartbeats).values([ + { workerId: "fresh-a", startedAt: freshHeartbeatAt, heartbeatAt: freshHeartbeatAt, status: "running", capacity: 2 }, + { workerId: "fresh-b", startedAt: freshHeartbeatAt, heartbeatAt: freshHeartbeatAt, status: "running", capacity: 3 }, + { workerId: "stale", startedAt: staleHeartbeatAt, heartbeatAt: staleHeartbeatAt, status: "running", capacity: 7 }, + { workerId: "stopped", startedAt: freshHeartbeatAt, heartbeatAt: freshHeartbeatAt, status: "stopped", capacity: 99 } + ]); + + const readyResponse = await app.app.inject({ method: "GET", url: "/api/health" }); + expect(readyResponse.statusCode).toBe(200); + expect(readyResponse.json()).toMatchObject({ + worker: "ready", + workerHeartbeatAt: freshHeartbeatAt, + expectedWorkerCount: 5, + readyWorkerCount: 5, + staleWorkerCount: 7 + }); + + await app.database.db.update(schema.workerHeartbeats).set({ status: "stopped" }).where(eq(schema.workerHeartbeats.workerId, "fresh-b")); + const partialResponse = await app.app.inject({ method: "GET", url: "/api/health" }); + expect(partialResponse.json()).toMatchObject({ + worker: "stale", + expectedWorkerCount: 5, + readyWorkerCount: 2, + staleWorkerCount: 7 + }); + } finally { + if (app) await app.app.close(); + await testDatabase.cleanup(); + await fs.rm(tmpDir, { recursive: true, force: true }); + } + }); +}); From 3e47eb374f6d16c3d013b06c75a1048f95616474 Mon Sep 17 00:00:00 2001 From: Aleksey Dmitriev Date: Sat, 1 Aug 2026 18:46:57 -0400 Subject: [PATCH 09/11] Release 0.1.2 beta 4 --- CHANGELOG.md | 10 ++ package-lock.json | 4 +- package.json | 2 +- src/client/api.ts | 16 +- src/client/appShared.ts | 8 +- src/client/jobPresentation.tsx | 12 +- src/client/jobPresentationUtils.ts | 18 +- src/server/db/database.ts | 55 +++++- src/server/jobs/copyReconciliation.ts | 17 ++ src/server/jobs/jobRunner.ts | 71 ++++---- src/server/jobs/resourceMutationGuard.ts | 13 +- src/server/lib/pathConfiguration.ts | 3 +- src/server/lib/scanner.ts | 77 +++------ src/server/lib/storageFilePolicies.ts | 26 +++ src/server/lib/storagePolicies.ts | 12 +- src/server/routes/libraryRoutes.ts | 1 + tests/api.test.ts | 67 ++++++++ tests/app.test.ts | 206 +++++++++++++++++++++-- tests/database.test.ts | 66 +++++++- tests/e2e/app-smoke.spec.ts | 19 ++- tests/jobPresentationUtils.test.ts | 54 +++++- tests/scanner.test.ts | 33 ++-- tests/sectionSummaryDisplay.test.ts | 7 + 23 files changed, 639 insertions(+), 158 deletions(-) create mode 100644 src/server/jobs/copyReconciliation.ts create mode 100644 src/server/lib/storageFilePolicies.ts create mode 100644 tests/api.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d249de..2cb813d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes are documented here. The project follows Semantic Versioning ## [Unreleased] +## [0.1.2-beta.4] - 2026-08-01 + +### Fixed + +- Prevented superseded legacy copy-reconciliation records from blocking newly scanned media while retaining exact media and managed-path safeguards for genuinely unresolved filesystem state. +- Limited newly queued scoped copy jobs to their actionable media so already satisfied title links no longer inflate job totals or selected-title details. +- Batched large selected-link title lookups and restored title tooltips for multi-link jobs without exceeding the API request limit. +- Derived storage-file assignment exclusively from current linked symlinks so unlinked files cannot remain assigned to a storage location. +- Reconciled legacy storage-file policies during migration and after scans or policy updates. + ## [0.1.2-beta.3] - 2026-07-31 ### Changed diff --git a/package-lock.json b/package-lock.json index 65b165f..74578bf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "srtl-manager", - "version": "0.1.2-beta.3", + "version": "0.1.2-beta.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "srtl-manager", - "version": "0.1.2-beta.3", + "version": "0.1.2-beta.4", "license": "MIT", "dependencies": { "@fastify/compress": "^9.1.0", diff --git a/package.json b/package.json index 2573586..b0610b0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "srtl-manager", - "version": "0.1.2-beta.3", + "version": "0.1.2-beta.4", "private": true, "license": "MIT", "homepage": "https://github.com/ramphex/SRTL-Manager#readme", diff --git a/src/client/api.ts b/src/client/api.ts index 6180636..fc97b22 100644 --- a/src/client/api.ts +++ b/src/client/api.ts @@ -58,6 +58,20 @@ async function request(path: string, init: RequestInit = {}): Promise { return (await response.json()) as T; } +const mediaLinkLookupBatchSize = 1000; + +export async function mediaLinksByIds(ids: number[]): Promise { + const uniqueIds = [...new Set(ids)]; + if (uniqueIds.length === 0) return []; + + const rows: MediaLinkRow[] = []; + for (let offset = 0; offset < uniqueIds.length; offset += mediaLinkLookupBatchSize) { + const batch = uniqueIds.slice(offset, offset + mediaLinkLookupBatchSize); + rows.push(...(await request("/api/media-links/by-ids", { method: "POST", body: JSON.stringify({ ids: batch }) }))); + } + return rows; +} + export const api = { me: () => request<{ authenticated: boolean; setupRequired: boolean; user: { id: number; username: string } | null }>("/api/auth/me"), setup: (body: { username: string; password: string; confirmPassword: string }) => @@ -97,7 +111,7 @@ export const api = { startCopy: (body: CopyOptions) => request<{ jobId: number }>("/api/copies", { method: "POST", body: JSON.stringify(body) }), copyConflicts: (body: CopyOptions) => request("/api/copies/conflicts", { method: "POST", body: JSON.stringify(body) }), mediaLinks: (kind?: string) => request(`/api/media-links${kind ? `?kind=${kind}` : ""}`), - mediaLinksByIds: (ids: number[]) => request("/api/media-links/by-ids", { method: "POST", body: JSON.stringify({ ids }) }), + mediaLinksByIds, mediaLinksPage: (params: { kind?: LinkKind; section?: string; storagePolicy?: StoragePolicyKind; relativePathPrefix?: string; search?: string; limit: number; offset: number }) => { const search = new URLSearchParams({ limit: String(params.limit), offset: String(params.offset) }); if (params.kind) search.set("kind", params.kind); diff --git a/src/client/appShared.ts b/src/client/appShared.ts index 4a084c6..5f12c2a 100644 --- a/src/client/appShared.ts +++ b/src/client/appShared.ts @@ -307,12 +307,12 @@ export function sectionPolicyNeededCount(section: Pick): number { - return summary.actionableRemoteLinks + summary.actionableRemoteFiles; +export function inventoryCopyToLocalCount(summary: Pick): number { + return summary.actionableRemoteLinks; } -export function inventoryCopyToRemoteCount(summary: Pick): number { - return summary.actionableLocalLinks + summary.actionableLocalFiles; +export function inventoryCopyToRemoteCount(summary: Pick): number { + return summary.actionableLocalLinks; } export function inventoryAssignedRemoteCount(summary: Pick): number { diff --git a/src/client/jobPresentation.tsx b/src/client/jobPresentation.tsx index 7e920a2..432f99f 100644 --- a/src/client/jobPresentation.tsx +++ b/src/client/jobPresentation.tsx @@ -11,7 +11,7 @@ import { normalizeRecentJobsCompletedWindowMinutes, recentJobsCompletedWindowOpt import { type AuditMode, type AuditResultRecord, type AuditRunRecord, type CopyConflictPreview, type JobEventRecord, type JobRecord, type CopyLocalConflictStrategy, type MediaLinkRow, type TimeFormatPreference } from "../shared/types"; import { JobStatusTerminateAction, LogChipList, Panel, ScanProgressPanel, StatusPill, TerminateJobDialog } from "./App"; import { AuditPrompt, AuditStatusPrompt, canTerminateJob, copyElapsedLabel, CopyPrompt, finiteNumberFromUnknown, formatBytes, formatDate, formatNumber, formatTime, invalidateCopyJobData, recordFromUnknown, scanAgeLabel, ScanStatusPrompt, sectionDisplayTitle, storageLocationName, useJobEventTimeline, useStartCopyJob, useStorageLocations, useTerminateJobMutation, useUserPreferences } from "./appShared"; -import { auditProgressFromJob, auditProgressPercent, auditStageLabel, auditStatusDetail, basenameFromPath, copyCompletedCount, copyCompletedItemSummaries, copyCurrentItem, copyEventChips, copyFailedItemSummaries, copyOverallProgressPercent, copyProgressFromJob, copyRemainingLabel, copyStageLabel, copyStagePercent, copySymlinkedCount, copyThroughputLabel, copyTransferSpeedLabel, copyTransferSpeedSecondaryLabel, formatAuditScope, formatCopyScope, formatScopedFolderParts, formatTitleScanJobDetail, jobDurationLabel, scanFolderScopeParts, scanScopeLabels, selectedLinkIdsFromJobs, selectedLinkTitleSummaries, singleSelectedLinkTitle } from "./jobPresentationUtils"; +import { auditProgressFromJob, auditProgressPercent, auditStageLabel, auditStatusDetail, basenameFromPath, copyCompletedCount, copyCompletedItemSummaries, copyCurrentItem, copyEventChips, copyFailedItemSummaries, copyOverallProgressPercent, copyProgressFromJob, copyRemainingLabel, copyStageLabel, copyStagePercent, copySymlinkedCount, copyThroughputLabel, copyTransferSpeedLabel, copyTransferSpeedSecondaryLabel, copyWorkTotalFromJob, formatAuditScope, formatCopyScope, formatScopedFolderParts, formatTitleScanJobDetail, jobDurationLabel, scanFolderScopeParts, scanScopeLabels, selectedLinkIdsFromJobs, selectedLinkTitleSummaries, singleSelectedLinkTitle } from "./jobPresentationUtils"; function JobEventsHeader({ label, jobId, @@ -626,9 +626,10 @@ export function CopyProgressPanel({ const stagePercent = copyStagePercent(job, progress); const overallPercent = copyOverallProgressPercent(job, progress); const currentIndex = progress.current > 0 ? progress.current : completed; + const workTotal = job ? copyWorkTotalFromJob(job, progress.total) : progress.total; const currentItem = copyCurrentItem(progress, job); const statusLabel = isStarting ? "Starting" : copyStageLabel(progress.stage, progress.direction); - const countLabel = progress.total > 0 ? `${formatNumber(Math.min(Math.max(currentIndex, completed), progress.total))} / ${formatNumber(progress.total)}` : "No matching files"; + const countLabel = workTotal > 0 ? `${formatNumber(Math.min(Math.max(currentIndex, completed), workTotal))} / ${formatNumber(workTotal)}` : "No matching files"; const transferSpeedSecondary = copyTransferSpeedSecondaryLabel(progress); const throughputLabel = copyThroughputLabel(progress); return ( @@ -1035,9 +1036,12 @@ export function JobScope({ if (job.type === "copy") { const options = copyOptionsFromJob(job); - const sectionText = formatCopyScope(options, sections); const directionText = `Copy to ${storageLocationName(storageLocations, options?.direction === "to_remote" ? "remote" : "local")}`; const selectedLinkIds = options?.linkIds ?? []; + const workTotal = copyWorkTotalFromJob(job, selectedLinkIds.length); + const sectionText = selectedLinkIds.length > 0 + ? workTotal === 1 ? "1 selected link" : `${formatNumber(workTotal)} selected links` + : formatCopyScope(options, sections); return ( 0 ? undefined : sectionText}> {directionText} @@ -1074,7 +1078,7 @@ function JobScopeDetail({ linkRowsError?: string | null; }) { if (selectedLinkIds.length === 0) return {text}; - const singleTitle = !linkRowsLoading && !linkRowsError ? singleSelectedLinkTitle(selectedLinkIds, linkRowsById) : null; + const singleTitle = selectedLinkIds.length === 1 && !linkRowsLoading && !linkRowsError ? singleSelectedLinkTitle(selectedLinkIds, linkRowsById) : null; if (singleTitle) return {singleTitle}; const canShowTitleLookup = Boolean(linkRowsById || linkRowsLoading || linkRowsError); return ( diff --git a/src/client/jobPresentationUtils.ts b/src/client/jobPresentationUtils.ts index d02ecbd..f3f8a0d 100644 --- a/src/client/jobPresentationUtils.ts +++ b/src/client/jobPresentationUtils.ts @@ -8,6 +8,7 @@ export type CopyProgressView = { copied: number; repointed: number; skipped: number; + alreadyCompleted: number; conflicts: number; failed: number; stage: string; @@ -131,6 +132,7 @@ export function copyProgressFromJob(job: JobRecord | null): CopyProgressView { copied: finiteNumberFromUnknown(progress?.copied), repointed: finiteNumberFromUnknown(progress?.repointed), skipped: finiteNumberFromUnknown(progress?.skipped), + alreadyCompleted: finiteNumberFromUnknown(progress?.alreadyCompleted), conflicts: finiteNumberFromUnknown(progress?.conflicts), failed: finiteNumberFromUnknown(progress?.failed), stage: typeof progress?.stage === "string" ? progress.stage : job?.status === "queued" ? "queued" : "waiting", @@ -149,6 +151,19 @@ export function copyProgressFromJob(job: JobRecord | null): CopyProgressView { }; } +export function copyWorkTotalFromJob(job: JobRecord, fallbackTotal = 0): number { + const progress = recordFromUnknown(job.progress); + const options = copyOptionsFromJob(job); + const total = finiteNumberFromUnknown(progress?.total) || fallbackTotal; + const skipped = finiteNumberFromUnknown(progress?.skipped); + const alreadyCompleted = finiteNumberFromUnknown(progress?.alreadyCompleted); + const legacyScopedSelection = Boolean(options?.linkIds?.length && (options.section || options.itemName || options.relativePathPrefix)); + if (legacyScopedSelection && alreadyCompleted > 0 && skipped === alreadyCompleted) { + return Math.max(0, total - alreadyCompleted); + } + return total; +} + export function copyCompletedCount(progress: Pick): number { return progress.copied + progress.repointed + progress.skipped + progress.conflicts + progress.failed; } @@ -513,9 +528,10 @@ export function selectedLinkTitleSummaries(linkIds: number[], linkRowsById: Map< counts.set(title, (counts.get(title) ?? 0) + 1); } + const includeLinkCounts = counts.size > 1; const summaries = [...counts.entries()] .sort(([firstTitle], [secondTitle]) => firstTitle.localeCompare(secondTitle, undefined, { numeric: true, sensitivity: "base" })) - .map(([title, count]) => (count > 1 ? `${title} (${formatNumber(count)} links)` : title)); + .map(([title, count]) => (includeLinkCounts && count > 1 ? `${title} (${formatNumber(count)} links)` : title)); if (missingCount > 0) summaries.push(`${formatNumber(missingCount)} link${missingCount === 1 ? "" : "s"} not found in current inventory`); return summaries; } diff --git a/src/server/db/database.ts b/src/server/db/database.ts index 5ea09e0..db484cf 100644 --- a/src/server/db/database.ts +++ b/src/server/db/database.ts @@ -21,7 +21,7 @@ export interface DatabaseOpenOptions { pool?: Pool; } -export const currentSchemaVersion = 7; +export const currentSchemaVersion = 8; const ddl = [ `CREATE TABLE IF NOT EXISTS app_settings (key TEXT PRIMARY KEY, value TEXT NOT NULL, updated_at TEXT NOT NULL)`, @@ -228,6 +228,59 @@ async function initializeDatabase(pool: Pool): Promise { throw error; } } + + if (!applied.has(8)) { + await client.query("BEGIN"); + try { + await client.query(` + DO $$ + BEGIN + IF to_regclass('public.storage_files') IS NOT NULL + AND to_regclass('public.media_links') IS NOT NULL + AND EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'storage_files' AND column_name = 'updated_at' + ) + AND EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'media_links' AND column_name = 'resolved_storage_file_id' + ) + AND EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'media_links' AND column_name = 'missing_since' + ) + THEN + EXECUTE $cleanup$ + WITH desired_policies AS ( + SELECT + storage_files.id, + CASE + WHEN count(DISTINCT media_links.storage_policy) = 1 THEN min(media_links.storage_policy) + ELSE 'unassigned' + END AS storage_policy + FROM storage_files + LEFT JOIN media_links + ON media_links.resolved_storage_file_id = storage_files.id + AND media_links.missing_since IS NULL + GROUP BY storage_files.id + ) + UPDATE storage_files + SET storage_policy = desired_policies.storage_policy, + updated_at = now()::text + FROM desired_policies + WHERE storage_files.id = desired_policies.id + AND storage_files.storage_policy IS DISTINCT FROM desired_policies.storage_policy + $cleanup$; + END IF; + END $$ + `); + await client.query(`INSERT INTO schema_migrations (version, name, applied_at) VALUES (8, 'linked_storage_file_policies', $1)`, [nowIso()]); + await client.query("COMMIT"); + } catch (error) { + await client.query("ROLLBACK"); + throw error; + } + } } finally { await client.query("select pg_advisory_unlock($1)", [bootstrapLockKey]).catch(() => undefined); client.release(); diff --git a/src/server/jobs/copyReconciliation.ts b/src/server/jobs/copyReconciliation.ts new file mode 100644 index 0000000..3a184d6 --- /dev/null +++ b/src/server/jobs/copyReconciliation.ts @@ -0,0 +1,17 @@ +import { sql, type SQL } from "drizzle-orm"; +import * as schema from "../db/schema"; + +export function unresolvedCopyReconciliation(): SQL { + return sql` + ${schema.copyOperations.stage} = 'reconciliation_required' + AND NOT EXISTS ( + SELECT 1 + FROM copy_operations AS superseding_operation + WHERE superseding_operation.id > ${schema.copyOperations.id} + AND superseding_operation.media_link_id = ${schema.copyOperations.mediaLinkId} + AND superseding_operation.link_path = ${schema.copyOperations.linkPath} + AND superseding_operation.stage = 'committed' + AND superseding_operation.result_status IN ('copied', 'repointed') + ) + `; +} diff --git a/src/server/jobs/jobRunner.ts b/src/server/jobs/jobRunner.ts index a3a55ff..4026661 100644 --- a/src/server/jobs/jobRunner.ts +++ b/src/server/jobs/jobRunner.ts @@ -28,6 +28,7 @@ import { normalizeAdvancedSettings } from "../../shared/advancedSettings"; import { evaluateSourceTitleRisk } from "../../shared/sourceTitleRisk"; import { CopyTransferLimiter } from "./copyLimiter"; import { runKeyedPool } from "./copyPool"; +import { unresolvedCopyReconciliation } from "./copyReconciliation"; import { schedulerLockKey } from "./scheduling"; import type { AuditMode, @@ -463,7 +464,7 @@ function filterCopySelectedLinks(links: MediaLinkRow[], options: CopyOptions): M } function orderedCopySelection(links: MediaLinkRow[], options: CopyOptions): MediaLinkRow[] { - const selected = filterCopySelectedLinks(links, options); + const selected = filterCopyLinks(links, options); const requestedOrder = options.linkIds?.length ? new Map(options.linkIds.map((id, index) => [id, index])) : null; return requestedOrder ? [...selected].sort((firstLink, secondLink) => (requestedOrder.get(firstLink.id) ?? 0) - (requestedOrder.get(secondLink.id) ?? 0)) @@ -685,25 +686,23 @@ async function copyPathBindingsForLink( ]; } -async function copyResourceClaims(links: MediaLinkRow[], paths: PathsSettings, direction: CopyOptions["direction"]): Promise { - const claims = await batchedMediaLinkResourceClaims(links, paths, "exclusive"); - const eligibleLinkIds = new Set( - filterCopyLinks(links, { direction, linkIds: links.map((link) => link.id) }).map((link) => link.id) - ); - for (let offset = 0; offset < links.length; offset += 16) { - const batch = links.slice(offset, offset + 16); +async function copyResourceClaims(workLinks: MediaLinkRow[], claimedLinks: MediaLinkRow[], paths: PathsSettings, options: CopyOptions): Promise { + const eligibleLinks = filterCopyLinks(workLinks, options); + const claims = await batchedMediaLinkResourceClaims(claimedLinks, paths, "exclusive"); + for (let offset = 0; offset < eligibleLinks.length; offset += 16) { + const batch = eligibleLinks.slice(offset, offset + 16); const [destinationClaims, pathBindings] = await Promise.all([ Promise.all( batch.map((link) => managedPathResourceClaims( - storageRootForDirection(paths, direction), - copyDestinationPathForLink(link, paths, direction), + storageRootForDirection(paths, options.direction), + copyDestinationPathForLink(link, paths, options.direction), "Copy destination claim", "exclusive" ) ) ), - Promise.all(batch.filter((link) => eligibleLinkIds.has(link.id)).map((link) => copyPathBindingsForLink(link, paths, direction))) + Promise.all(batch.map((link) => copyPathBindingsForLink(link, paths, options.direction))) ]); claims.push(...destinationClaims.flat()); claims.push( @@ -1903,27 +1902,31 @@ export class JobRunner { JOIN jobs ON jobs.id = active.job_id WHERE jobs.status IN ('queued', 'running') UNION - SELECT active.job_id, active.resource_type, active.resource_key, active.access, 'reconciliation_required'::text AS status - FROM job_resource_claims AS active - JOIN copy_operations AS operation ON operation.job_id = active.job_id - WHERE operation.stage = 'reconciliation_required' - AND active.resource_type <> 'title' - UNION - SELECT operation.job_id, 'media'::text, operation.media_link_id::text, 'exclusive'::text, 'reconciliation_required'::text AS status - FROM copy_operations AS operation - WHERE operation.stage = 'reconciliation_required' + SELECT copy_operations.job_id, 'media'::text, copy_operations.media_link_id::text, 'exclusive'::text, 'reconciliation_required'::text AS status + FROM copy_operations + WHERE ${unresolvedCopyReconciliation()} UNION - SELECT operation.job_id, 'path'::text, paths.resource_key, 'exclusive'::text, 'reconciliation_required'::text AS status - FROM copy_operations AS operation + SELECT copy_operations.job_id, 'path'::text, paths.resource_key, 'exclusive'::text, 'reconciliation_required'::text AS status + FROM copy_operations CROSS JOIN LATERAL unnest(ARRAY[ - operation.link_path, - operation.source_path, - operation.destination_path, - operation.temp_path, - operation.displaced_path + copy_operations.link_path, + copy_operations.source_path, + copy_operations.destination_path, + copy_operations.temp_path, + copy_operations.displaced_path ]) AS paths(resource_key) - WHERE operation.stage = 'reconciliation_required' + WHERE ${unresolvedCopyReconciliation()} AND paths.resource_key IS NOT NULL + UNION + SELECT copy_operations.job_id, 'path'::text, binding_paths.resource_key, 'exclusive'::text, 'reconciliation_required'::text AS status + FROM copy_operations + JOIN job_resource_claims AS binding + ON binding.job_id = copy_operations.job_id + AND binding.resource_type = 'copy_path_binding' + AND binding.resource_key::jsonb ->> 0 = copy_operations.media_link_id::text + CROSS JOIN LATERAL (VALUES (binding.resource_key::jsonb ->> 2), (binding.resource_key::jsonb ->> 3)) AS binding_paths(resource_key) + WHERE ${unresolvedCopyReconciliation()} + AND binding_paths.resource_key IS NOT NULL ) SELECT active.job_id AS "jobId", active.status, @@ -2148,19 +2151,21 @@ export class JobRunner { const normalizedOptions = await normalizeCopyOptions(this.db, input); const links = await listMediaLinks(this.db, undefined, "current"); const orderedSelectedLinks = orderedCopySelection(links, normalizedOptions); + const claimedLinks = normalizedOptions.linkIds === undefined ? orderedSelectedLinks : filterCopySelectedLinks(links, normalizedOptions); const optionsWithResolvedLinks = { ...normalizedOptions, linkIds: orderedSelectedLinks.map((link) => link.id) }; const paths = await getJsonSetting(this.db, "paths", { symlinkDir: "", localDir: "", remoteDir: "" }); if (!paths.localDir || !paths.remoteDir) throw new Error("Path settings are incomplete"); const replacementClaims = await copyReplacementResourceClaims(this.db, orderedSelectedLinks, paths, optionsWithResolvedLinks); const expectedSelection = orderedSelectedLinks.map(copyAdmissionFingerprint); + const expectedClaimedSelection = claimedLinks.map(copyAdmissionFingerprint); return this.enqueuePreparedJob("copy", async (transaction) => { - const currentSelection = orderedCopySelection( - await listMediaLinks(transaction, undefined, "current"), - normalizedOptions - ); + const currentLinks = await listMediaLinks(transaction, undefined, "current"); + const currentSelection = orderedCopySelection(currentLinks, normalizedOptions); + const currentClaimedSelection = normalizedOptions.linkIds === undefined ? currentSelection : filterCopySelectedLinks(currentLinks, normalizedOptions); const currentPaths = await getJsonSetting(transaction, "paths", { symlinkDir: "", localDir: "", remoteDir: "" }); if ( JSON.stringify(currentSelection.map(copyAdmissionFingerprint)) !== JSON.stringify(expectedSelection) || + JSON.stringify(currentClaimedSelection.map(copyAdmissionFingerprint)) !== JSON.stringify(expectedClaimedSelection) || currentPaths.symlinkDir !== paths.symlinkDir || currentPaths.localDir !== paths.localDir || currentPaths.remoteDir !== paths.remoteDir @@ -2170,7 +2175,7 @@ export class JobRunner { return { progress: { options: optionsWithResolvedLinks }, exclusive: false, - claims: [...(await copyResourceClaims(orderedSelectedLinks, paths, normalizedOptions.direction)), ...replacementClaims] + claims: [...(await copyResourceClaims(orderedSelectedLinks, claimedLinks, paths, normalizedOptions)), ...replacementClaims] }; }); } diff --git a/src/server/jobs/resourceMutationGuard.ts b/src/server/jobs/resourceMutationGuard.ts index 92b5c2b..90233ae 100644 --- a/src/server/jobs/resourceMutationGuard.ts +++ b/src/server/jobs/resourceMutationGuard.ts @@ -2,6 +2,7 @@ import { sql } from "drizzle-orm"; import type { Db } from "../db/database"; import * as schema from "../db/schema"; import { canonicalTitleKey } from "../lib/storagePolicies"; +import { unresolvedCopyReconciliation } from "./copyReconciliation"; import { schedulerLockKey } from "./scheduling"; export interface MutationResource { @@ -96,15 +97,9 @@ export async function withResourceMutationGuard( join jobs on jobs.id = active.job_id where jobs.status in ('queued', 'running') union - select active.job_id, active.resource_type, active.resource_key, 'copy'::text as type, 'reconciliation_required'::text as status - from job_resource_claims as active - join copy_operations as operation on operation.job_id = active.job_id - where operation.stage = 'reconciliation_required' - and active.resource_type <> 'title' - union - select operation.job_id, 'media'::text, operation.media_link_id::text, 'copy'::text as type, 'reconciliation_required'::text as status - from copy_operations as operation - where operation.stage = 'reconciliation_required' + select copy_operations.job_id, 'media'::text, copy_operations.media_link_id::text, 'copy'::text as type, 'reconciliation_required'::text as status + from copy_operations + where ${unresolvedCopyReconciliation()} ) select active.job_id as "jobId", active.type, active.status from requested_resources as requested diff --git a/src/server/lib/pathConfiguration.ts b/src/server/lib/pathConfiguration.ts index 5de99ca..05d0efe 100644 --- a/src/server/lib/pathConfiguration.ts +++ b/src/server/lib/pathConfiguration.ts @@ -3,6 +3,7 @@ import path from "node:path"; import { and, asc, count, desc, eq, inArray, isNull, ne, notInArray, sql } from "drizzle-orm"; import { first, nowIso, type Db, type DbExecutor } from "../db/database"; import * as schema from "../db/schema"; +import { unresolvedCopyReconciliation } from "../jobs/copyReconciliation"; import { schedulerLockKey } from "../jobs/scheduling"; import { assertPathParentInside } from "./filesystemSafety"; import { inspectMountIdentity, persistentRootIdentityMatch } from "./mountIdentity"; @@ -1009,7 +1010,7 @@ async function assertCopyOperationsReconciledForPathMigration(db: DbExecutor): P db .select({ id: schema.copyOperations.id }) .from(schema.copyOperations) - .where(eq(schema.copyOperations.stage, "reconciliation_required")) + .where(unresolvedCopyReconciliation()) .limit(1) ); if (reconciliationOperation) { diff --git a/src/server/lib/scanner.ts b/src/server/lib/scanner.ts index 991c59d..8225f1a 100644 --- a/src/server/lib/scanner.ts +++ b/src/server/lib/scanner.ts @@ -10,6 +10,7 @@ import { canonicalTitleKey } from "./storagePolicies"; import { applyPendingOnboardingPolicy } from "./onboarding"; import { isMediaFile, isPathInside, safeRelativePath } from "./media"; import { assertReadableRegularFile, withFilesystemTimeout } from "./filesystemSafety"; +import { reconcileStorageFilePolicies } from "./storageFilePolicies"; import type { InventorySummary, InventoryScanTimestamps, @@ -504,7 +505,7 @@ export async function scanLibrary( storageScanIssues.push(...remoteScan.issues); } - const classifiedStorageFiles = applyStorageFilePolicies(storageFiles, settings, storagePolicies); + const classifiedStorageFiles = applyStorageFileMetadata(storageFiles, settings); const reconciledStorageFiles = titleScopesBySection ? uniqueStorageFiles(links.map((link) => storageFileFromTargetedLink(link, paths)).filter((file): file is ClassifiedStorageFile => file !== null)) : []; @@ -595,14 +596,14 @@ function storageFileMetadata(relativePath: string, rootType: StorageRootType, se return { section: "", itemName: parts.length > 1 ? parts[0] : filenameTitle(parts[0]) }; } -function applyStorageFilePolicies(files: ClassifiedStorageFile[], settings: SectionSettings, storagePolicies: StoragePolicyLookup): ClassifiedStorageFile[] { +function applyStorageFileMetadata(files: ClassifiedStorageFile[], settings: SectionSettings): ClassifiedStorageFile[] { const sectionNames = new Set(settings.sections); return files.map((file) => { const metadata = storageFileMetadata(file.relativePath, file.rootType, sectionNames); return { ...file, ...metadata, - storagePolicy: storagePolicyForTitle(storagePolicies, metadata.itemName) + storagePolicy: "unassigned" }; }); } @@ -645,11 +646,11 @@ export function summarizeInventory(links: ClassifiedLink[], storageFiles: Classi unassignedLocalLinks: links.filter((link) => link.kind === "local" && link.storagePolicy === "unassigned").length, localFiles: localFiles.length, remoteFiles: remoteFiles.length, - actionableRemoteFiles: unlinkedRemoteFiles.filter((file) => file.storagePolicy === "location_1").length, - actionableLocalFiles: unlinkedLocalFiles.filter((file) => file.storagePolicy === "location_2").length, - assignedRemoteFiles: unlinkedRemoteFiles.filter((file) => file.storagePolicy === "location_2").length, - unassignedRemoteFiles: unlinkedRemoteFiles.filter((file) => file.storagePolicy === "unassigned").length, - unassignedLocalFiles: unlinkedLocalFiles.filter((file) => file.storagePolicy === "unassigned").length, + actionableRemoteFiles: 0, + actionableLocalFiles: 0, + assignedRemoteFiles: 0, + unassignedRemoteFiles: unlinkedRemoteFiles.length, + unassignedLocalFiles: unlinkedLocalFiles.length, localOrphanFiles: unlinkedLocalFiles.length, remoteOrphanFiles: unlinkedRemoteFiles.length, missingLinks: 0, @@ -725,14 +726,15 @@ export async function persistScanResult(db: Db, result: ScanResult, jobId: numbe for (const file of filesToReconcile) { await throwIfPersistenceCancelled(isCancelled); const existing = await first(db.select().from(schema.storageFiles).where(eq(schema.storageFiles.filePath, file.filePath)).limit(1)); + const persistedFile: ClassifiedStorageFile = { ...file, storagePolicy: normalizeStoragePolicy(existing?.storagePolicy) }; const firstSeenAt = existing?.firstSeenAt ?? timestamp; - const lastChangedAt = storageFileChanged(existing, file) ? timestamp : existing?.lastChangedAt ?? timestamp; + const lastChangedAt = storageFileChanged(existing, persistedFile) ? timestamp : existing?.lastChangedAt ?? timestamp; await db .insert(schema.storageFiles) - .values({ ...file, firstSeenAt, lastSeenAt: timestamp, lastChangedAt, missingSince: null, lastSeenJobId: jobId, updatedAt: timestamp }) + .values({ ...persistedFile, firstSeenAt, lastSeenAt: timestamp, lastChangedAt, missingSince: null, lastSeenJobId: jobId, updatedAt: timestamp }) .onConflictDoUpdate({ target: schema.storageFiles.filePath, - set: { ...file, lastSeenAt: timestamp, lastChangedAt, missingSince: null, lastSeenJobId: jobId, updatedAt: timestamp } + set: { ...persistedFile, lastSeenAt: timestamp, lastChangedAt, missingSince: null, lastSeenJobId: jobId, updatedAt: timestamp } }); } @@ -818,6 +820,8 @@ export async function persistScanResult(db: Db, result: ScanResult, jobId: numbe await throwIfPersistenceCancelled(isCancelled); await applyPendingOnboardingPolicy(db, jobId); await throwIfPersistenceCancelled(isCancelled); + await reconcileStorageFilePolicies(db, timestamp); + await throwIfPersistenceCancelled(isCancelled); const scannedLinks = (await db.select().from(schema.mediaLinks)).filter((link) => seenLinkPaths.has(link.linkPath) && !link.missingSince); const persistedLinkInventory = summarizeInventory( @@ -866,11 +870,11 @@ export async function persistScanResult(db: Db, result: ScanResult, jobId: numbe unassignedLocalLinks: persistedLinkInventory.unassignedLocalLinks, localFiles: localFiles.length, remoteFiles: remoteFiles.length, - actionableRemoteFiles: unlinkedRemoteFiles.filter((file) => file.storagePolicy === "location_1").length, - actionableLocalFiles: unlinkedLocalFiles.filter((file) => file.storagePolicy === "location_2").length, - assignedRemoteFiles: unlinkedRemoteFiles.filter((file) => file.storagePolicy === "location_2").length, - unassignedRemoteFiles: unlinkedRemoteFiles.filter((file) => file.storagePolicy === "unassigned").length, - unassignedLocalFiles: unlinkedLocalFiles.filter((file) => file.storagePolicy === "unassigned").length, + actionableRemoteFiles: 0, + actionableLocalFiles: 0, + assignedRemoteFiles: 0, + unassignedRemoteFiles: unlinkedRemoteFiles.length, + unassignedLocalFiles: unlinkedLocalFiles.length, localOrphanFiles: unlinkedLocalFiles.length, remoteOrphanFiles: unlinkedRemoteFiles.length, missingLinks, @@ -1475,49 +1479,15 @@ export async function getInventorySummary(db: Db): Promise { unassignedLocalLinks: await rawCount(db, sql`select count(*) as value from media_links where missing_since is null and kind = 'local' and storage_policy = 'unassigned'`), localFiles: await rawCount(db, sql`select count(*) as value from storage_files where missing_since is null and root_type = 'local'`), remoteFiles: await rawCount(db, sql`select count(*) as value from storage_files where missing_since is null and root_type = 'remote'`), - actionableRemoteFiles: await rawCount( - db, - sql`select count(*) as value - from storage_files sf - where sf.missing_since is null - and sf.root_type = 'remote' - and sf.storage_policy = 'location_1' - and not exists ( - select 1 from media_links ml - where ml.missing_since is null and ml.resolved_storage_file_id = sf.id - )` - ), - actionableLocalFiles: await rawCount( - db, - sql`select count(*) as value - from storage_files sf - where sf.missing_since is null - and sf.root_type = 'local' - and sf.storage_policy = 'location_2' - and not exists ( - select 1 from media_links ml - where ml.missing_since is null and ml.resolved_storage_file_id = sf.id - )` - ), - assignedRemoteFiles: await rawCount( - db, - sql`select count(*) as value - from storage_files sf - where sf.missing_since is null - and sf.root_type = 'remote' - and sf.storage_policy = 'location_2' - and not exists ( - select 1 from media_links ml - where ml.missing_since is null and ml.resolved_storage_file_id = sf.id - )` - ), + actionableRemoteFiles: 0, + actionableLocalFiles: 0, + assignedRemoteFiles: 0, unassignedRemoteFiles: await rawCount( db, sql`select count(*) as value from storage_files sf where sf.missing_since is null and sf.root_type = 'remote' - and sf.storage_policy = 'unassigned' and not exists ( select 1 from media_links ml where ml.missing_since is null and ml.resolved_storage_file_id = sf.id @@ -1529,7 +1499,6 @@ export async function getInventorySummary(db: Db): Promise { from storage_files sf where sf.missing_since is null and sf.root_type = 'local' - and sf.storage_policy = 'unassigned' and not exists ( select 1 from media_links ml where ml.missing_since is null and ml.resolved_storage_file_id = sf.id diff --git a/src/server/lib/storageFilePolicies.ts b/src/server/lib/storageFilePolicies.ts new file mode 100644 index 0000000..e4108fe --- /dev/null +++ b/src/server/lib/storageFilePolicies.ts @@ -0,0 +1,26 @@ +import { sql } from "drizzle-orm"; +import { nowIso, type DbExecutor } from "../db/database"; + +export async function reconcileStorageFilePolicies(db: DbExecutor, timestamp = nowIso()): Promise { + await db.execute(sql` + with desired_policies as ( + select + storage_files.id, + case + when count(distinct media_links.storage_policy) = 1 then min(media_links.storage_policy) + else 'unassigned' + end as storage_policy + from storage_files + left join media_links + on media_links.resolved_storage_file_id = storage_files.id + and media_links.missing_since is null + group by storage_files.id + ) + update storage_files + set storage_policy = desired_policies.storage_policy, + updated_at = ${timestamp} + from desired_policies + where storage_files.id = desired_policies.id + and storage_files.storage_policy is distinct from desired_policies.storage_policy + `); +} diff --git a/src/server/lib/storagePolicies.ts b/src/server/lib/storagePolicies.ts index 37e3ea0..97ec825 100644 --- a/src/server/lib/storagePolicies.ts +++ b/src/server/lib/storagePolicies.ts @@ -10,6 +10,7 @@ import type { StoragePolicyTitle } from "../../shared/types"; import { inferSectionContentType, normalizeSectionContentType } from "../../shared/sections"; +import { reconcileStorageFilePolicies } from "./storageFilePolicies"; export function normalizeTitle(value: string): string { return value.trim().toLowerCase(); @@ -347,11 +348,7 @@ async function syncStoragePolicyMediaLinksForTitleKeys( await db.update(schema.mediaLinks).set({ storagePolicy: policy, updatedAt: timestamp }).where(eq(schema.mediaLinks.id, link.id)); } } - for (const file of await db.select().from(schema.storageFiles)) { - const policy = titlePolicies.get(canonicalTitleKey(file.itemName)); - if (!policy || file.storagePolicy === policy) continue; - await db.update(schema.storageFiles).set({ storagePolicy: policy, updatedAt: timestamp }).where(eq(schema.storageFiles.id, file.id)); - } + await reconcileStorageFilePolicies(db, timestamp); } export async function bootstrapLocalStoragePolicies( @@ -475,10 +472,7 @@ export async function syncStoragePolicyMediaLinks(db: Db, normalizedTitle: strin if (canonicalTitleKey(link.itemName) !== titleKey) continue; await db.update(schema.mediaLinks).set({ storagePolicy, updatedAt: timestamp }).where(eq(schema.mediaLinks.id, link.id)); } - for (const file of await db.select().from(schema.storageFiles)) { - if (canonicalTitleKey(file.itemName) !== titleKey) continue; - await db.update(schema.storageFiles).set({ storagePolicy, updatedAt: timestamp }).where(eq(schema.storageFiles.id, file.id)); - } + await reconcileStorageFilePolicies(db, timestamp); } export async function setStoragePolicyTitles( diff --git a/src/server/routes/libraryRoutes.ts b/src/server/routes/libraryRoutes.ts index 9587c42..be5cb7d 100644 --- a/src/server/routes/libraryRoutes.ts +++ b/src/server/routes/libraryRoutes.ts @@ -279,6 +279,7 @@ export function registerLibraryRoutes(app: FastifyInstance, db: Db, jobs: JobRun try { return { jobId: await jobs.startCopy(body) }; } catch (error: unknown) { + request.log.warn({ err: error }, "Copy admission rejected"); return reply.code(400).send({ error: error instanceof Error ? error.message : String(error) }); } }); diff --git a/tests/api.test.ts b/tests/api.test.ts new file mode 100644 index 0000000..3450167 --- /dev/null +++ b/tests/api.test.ts @@ -0,0 +1,67 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { mediaLinksByIds } from "../src/client/api"; +import type { MediaLinkRow } from "../src/shared/types"; + +function mediaLink(id: number): MediaLinkRow { + const timestamp = "2026-08-01T08:00:00.000Z"; + return { + id, + section: "shows", + itemName: `Selected Title ${id}`, + relativePath: `Selected Title ${id}/episode.mkv`, + linkPath: `/links/Selected Title ${id}/episode.mkv`, + targetPath: `/remote/Selected Title ${id}/episode.mkv`, + kind: "remote", + targetExists: true, + isMedia: true, + storagePolicy: "location_1", + resolvedStorageFileId: null, + sizeBytes: null, + firstSeenAt: timestamp, + lastSeenAt: timestamp, + lastChangedAt: timestamp, + missingSince: null, + updatedAt: timestamp + }; +} + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("media link lookup", () => { + it("splits more than 1,000 selected IDs into bounded requests and preserves order", async () => { + const requestIds: number[][] = []; + vi.stubGlobal( + "fetch", + vi.fn(async (_path: string, init?: RequestInit) => { + const ids = (JSON.parse(String(init?.body)) as { ids: number[] }).ids; + requestIds.push(ids); + return new Response(JSON.stringify(ids.map(mediaLink)), { status: 200, headers: { "Content-Type": "application/json" } }); + }) + ); + + const ids = Array.from({ length: 1176 }, (_, index) => index + 1); + const rows = await mediaLinksByIds(ids); + + expect(requestIds.map((batch) => batch.length)).toEqual([1000, 176]); + expect(rows.map((row) => row.id)).toEqual(ids); + }); + + it("deduplicates IDs before batching", async () => { + const requestIds: number[][] = []; + vi.stubGlobal( + "fetch", + vi.fn(async (_path: string, init?: RequestInit) => { + const ids = (JSON.parse(String(init?.body)) as { ids: number[] }).ids; + requestIds.push(ids); + return new Response(JSON.stringify(ids.map(mediaLink)), { status: 200, headers: { "Content-Type": "application/json" } }); + }) + ); + + const rows = await mediaLinksByIds([2, 1, 2, 3, 1]); + + expect(requestIds).toEqual([[2, 1, 3]]); + expect(rows.map((row) => row.id)).toEqual([2, 1, 3]); + }); +}); diff --git a/tests/app.test.ts b/tests/app.test.ts index 927f396..cc79353 100644 --- a/tests/app.test.ts +++ b/tests/app.test.ts @@ -2,7 +2,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { eq, inArray, sql } from "drizzle-orm"; +import { and, eq, inArray, sql } from "drizzle-orm"; import { createApp, type AppContext } from "../src/server/app"; import { first, getJsonSetting, setSetting } from "../src/server/db/database"; import * as schema from "../src/server/db/schema"; @@ -12,6 +12,7 @@ import type { AuditCommandRunner } from "../src/server/lib/auditor"; import { readCopyFileIdentity, serializeCopyFileIdentity, type CopyCommandRunner, type CopyFileProgressReporter } from "../src/server/lib/copier"; import { reconcileEnvironmentPaths } from "../src/server/lib/pathConfiguration"; import { markOnboardingCompleteForExistingInstall } from "../src/server/lib/onboarding"; +import { getInventorySummary } from "../src/server/lib/scanner"; import { bootstrapLocalStoragePolicies, normalizeTitle } from "../src/server/lib/storagePolicies"; import type { AuditMode, SectionContentType, StoragePolicyKind } from "../src/shared/types"; import { createTestDatabase, type TestDatabaseHandle } from "./testDb"; @@ -313,6 +314,54 @@ async function markCopyFixtureInstalled(jobId: number, fixture: Awaited>; + stage: "committed" | "reconciliation_required"; + resultStatus?: "copied" | "repointed" | null; + errorMessage?: string | null; +}): Promise { + const mediaLink = await first(ctx.database.db.select().from(schema.mediaLinks).where(eq(schema.mediaLinks.id, fixture.id)).limit(1)); + if (!mediaLink) throw new Error("Copy operation fixture media link was not found"); + const timestamp = new Date().toISOString(); + const operation = await first( + ctx.database.db + .insert(schema.copyOperations) + .values({ + jobId, + mediaLinkId: fixture.id, + linkPath: fixture.linkPath, + sourcePath: fixture.sourcePath, + destinationPath: fixture.destinationPath, + originalTargetPath: fixture.sourcePath, + originalLinkState: JSON.stringify(mediaLink), + previousCopySource: null, + tempPath: null, + displacedPath: null, + tempIdentity: null, + destinationIdentity: null, + displacedIdentity: null, + stage, + resultStatus, + localConflictStrategy: null, + sizeBytes: mediaLink.sizeBytes, + errorMessage, + createdAt: timestamp, + updatedAt: timestamp, + completedAt: stage === "committed" ? timestamp : null + }) + .returning({ id: schema.copyOperations.id }) + ); + if (!operation) throw new Error("Copy operation fixture was not inserted"); + return operation.id; +} + describe("api app", () => { beforeEach(async () => { copyFfmpegModes = []; @@ -818,7 +867,7 @@ describe("api app", () => { expect(version.statusCode).toBe(200); expect(version.json()).toMatchObject({ - currentVersion: "0.1.2-beta.3", + currentVersion: "0.1.2-beta.4", currentChannel: "beta", currentChannelLabel: "Beta", latestVersion: null, @@ -885,7 +934,7 @@ describe("api app", () => { expect(version.statusCode).toBe(200); expect(version.json()).toMatchObject({ - currentVersion: "0.1.2-beta.3", + currentVersion: "0.1.2-beta.4", currentChannel: "beta", currentChannelLabel: "Beta", latestVersion: "0.2.0-beta.1", @@ -1867,6 +1916,99 @@ describe("api app", () => { }); }); + it("limits reconciliation blockers to the exact uncertain media from a legacy multi-item job", async () => { + const blockedFixture = await insertCopySymlink({ itemName: "Uncertain Legacy Movie", kind: "remote", storagePolicy: "location_1", content: "uncertain source" }); + const relatedFixture = await insertCopySymlink({ itemName: "Independent Legacy Movie", kind: "remote", storagePolicy: "location_1", content: "independent source" }); + const legacyJobId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: [blockedFixture.id, relatedFixture.id] }); + const timestamp = new Date().toISOString(); + await ctx.database.db.update(schema.jobs).set({ status: "failed", finishedAt: timestamp }).where(eq(schema.jobs.id, legacyJobId)); + await insertCopyOperationFixture({ + jobId: legacyJobId, + fixture: blockedFixture, + stage: "reconciliation_required", + errorMessage: "The first media item has unresolved filesystem state" + }); + + const relatedJobId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: [relatedFixture.id] }); + await expect(ctx.jobs.getJob(relatedJobId)).resolves.toMatchObject({ status: "queued" }); + await expect(ctx.jobs.startCopy({ direction: "to_local", linkIds: [blockedFixture.id] })).rejects.toThrow( + `Copy data from job #${legacyJobId} requires manual reconciliation` + ); + await expect(ctx.jobs.terminate(relatedJobId)).resolves.toBe(true); + }); + + it("does not let a stale local title item block new actionable copy work", async () => { + const itemName = "Superseded Title Copy"; + const localFixture = await insertCopySymlink({ + itemName, + kind: "local", + storagePolicy: "location_1", + section: "shows", + relativePath: path.join(itemName, "Season 01", "episode-09.mkv"), + content: "existing local episode" + }); + const remoteFixture = await insertCopySymlink({ + itemName, + kind: "remote", + storagePolicy: "location_1", + section: "shows", + relativePath: path.join(itemName, "Season 01", "episode-10.mkv"), + content: "new remote episode" + }); + const legacyJobId = await ctx.jobs.createJob("copy"); + const timestamp = new Date().toISOString(); + await ctx.database.db.update(schema.jobs).set({ status: "failed", finishedAt: timestamp }).where(eq(schema.jobs.id, legacyJobId)); + await insertCopyOperationFixture({ + jobId: legacyJobId, + fixture: localFixture, + stage: "reconciliation_required", + errorMessage: "Legacy local episode state is uncertain" + }); + + const copyJobId = await ctx.jobs.startCopy({ direction: "to_local", section: "shows", itemName }); + const mediaClaims = await ctx.database.db + .select({ resourceKey: schema.jobResourceClaims.resourceKey }) + .from(schema.jobResourceClaims) + .where(and(eq(schema.jobResourceClaims.jobId, copyJobId), eq(schema.jobResourceClaims.resourceType, "media"))); + expect(mediaClaims).toEqual([{ resourceKey: String(remoteFixture.id) }]); + await expect(runQueuedJob(copyJobId, { copyRunner: testCopyRunner })).resolves.toMatchObject({ + status: "completed", + progress: expect.objectContaining({ total: 1, copied: 1, skipped: 0, failed: 0 }) + }); + await expect(fs.readlink(localFixture.linkPath)).resolves.toBe(localFixture.sourcePath); + await expect(fs.readlink(remoteFixture.linkPath)).resolves.toBe(remoteFixture.destinationPath); + }); + + it("ignores legacy reconciliation state superseded by a later committed copy", async () => { + const cookie = await createAdminSession(); + const fixture = await insertCopySymlink({ itemName: "Later Commit Wins", kind: "remote", storagePolicy: "location_1", content: "later committed copy" }); + const legacyJobId = await ctx.jobs.createJob("copy"); + const legacyTimestamp = new Date().toISOString(); + await ctx.database.db.update(schema.jobs).set({ status: "failed", finishedAt: legacyTimestamp }).where(eq(schema.jobs.id, legacyJobId)); + const legacyOperationId = await insertCopyOperationFixture({ + jobId: legacyJobId, + fixture, + stage: "reconciliation_required", + errorMessage: "Legacy copy ownership cannot be proven" + }); + + const committedJobId = await ctx.jobs.createJob("copy"); + const committedTimestamp = new Date().toISOString(); + await markCopyFixtureInstalled(committedJobId, fixture, committedTimestamp); + const committedOperationId = await insertCopyOperationFixture({ jobId: committedJobId, fixture, stage: "committed", resultStatus: "copied" }); + await ctx.database.db.update(schema.jobs).set({ status: "completed", finishedAt: committedTimestamp }).where(eq(schema.jobs.id, committedJobId)); + expect(committedOperationId).toBeGreaterThan(legacyOperationId); + + const policyMutation = await ctx.app.inject({ + method: "POST", + url: "/api/storage-policies", + headers: { cookie }, + payload: { title: "Later Commit Wins", policy: "location_2" } + }); + expect(policyMutation.statusCode).toBe(200); + expect(policyMutation.json()).toMatchObject({ title: "Later Commit Wins", policy: "location_2" }); + }); + it("rejects duplicate copy jobs while matching media is already queued", async () => { const cookie = await createAdminSession(); const fixture = await insertCopySymlink({ itemName: "Duplicate Queue Movie", kind: "remote", storagePolicy: "location_1", content: "copy once" }); @@ -2257,12 +2399,18 @@ describe("api app", () => { await expect(runQueuedJob(jobId, { copyRunner: testCopyRunner })).resolves.toMatchObject({ status: "completed", progress: expect.objectContaining({ - options: expect.objectContaining({ direction: "to_local", section: "shows", itemName: "Scoped Copy Show", relativePathPrefix: "Scoped Copy Show/Season 01" }), - current: 2, - total: 2, + options: expect.objectContaining({ + direction: "to_local", + linkIds: [needsCopy.id], + section: "shows", + itemName: "Scoped Copy Show", + relativePathPrefix: "Scoped Copy Show/Season 01" + }), + current: 1, + total: 1, copied: 1, - skipped: 1, - alreadyCompleted: 1, + skipped: 0, + alreadyCompleted: 0, conflicts: 0, failed: 0 }) @@ -3864,7 +4012,7 @@ describe("api app", () => { }); }); - it("applies storage policies to canonical-equivalent media and storage titles", async () => { + it("applies canonical-equivalent media policies without assigning unlinked storage files", async () => { const cookie = await createAdminSession(); const ampersandLinkId = await insertMediaLink("Rock & Roll Movie"); const wordLinkId = await insertMediaLink("Rock and Roll Movie"); @@ -3891,7 +4039,41 @@ describe("api app", () => { .from(schema.storageFiles) .where(inArray(schema.storageFiles.itemName, ["Rock & Roll Movie", "Rock and Roll Movie"])); expect(files).toHaveLength(2); - expect(files.every((file) => file.storagePolicy === "location_2")).toBe(true); + expect(files.every((file) => file.storagePolicy === "unassigned")).toBe(true); + }); + + it("does not treat a completed copy source as storage policy work after its symlink moves local", async () => { + const cookie = await createAdminSession(); + const itemName = "Completed Source Movie"; + const remoteStorageFileId = await insertStorageFile("remote", path.join("movies", itemName, "remote-source.mkv")); + const localStorageFileId = await insertStorageFile("local", path.join("movies", itemName, "local-copy.mkv")); + const linkId = await insertMediaLink(itemName, "remote", "movies", undefined, "location_1", remoteStorageFileId); + const localStorageFile = await first(ctx.database.db.select().from(schema.storageFiles).where(eq(schema.storageFiles.id, localStorageFileId)).limit(1)); + if (!localStorageFile) throw new Error("Local storage fixture was not found"); + await ctx.database.db + .update(schema.mediaLinks) + .set({ kind: "local", targetPath: localStorageFile.filePath, resolvedStorageFileId: localStorageFileId, updatedAt: new Date().toISOString() }) + .where(eq(schema.mediaLinks.id, linkId)); + await ctx.database.db.update(schema.storageFiles).set({ storagePolicy: "location_1" }).where(eq(schema.storageFiles.id, remoteStorageFileId)); + + const response = await ctx.app.inject({ + method: "POST", + url: "/api/storage-policies", + headers: { cookie }, + payload: { title: itemName, policy: "location_1" } + }); + + expect(response.statusCode).toBe(200); + expect(await first(ctx.database.db.select().from(schema.storageFiles).where(eq(schema.storageFiles.id, remoteStorageFileId)).limit(1))).toMatchObject({ + storagePolicy: "unassigned" + }); + expect(await first(ctx.database.db.select().from(schema.storageFiles).where(eq(schema.storageFiles.id, localStorageFileId)).limit(1))).toMatchObject({ + storagePolicy: "location_1" + }); + await expect(getInventorySummary(ctx.database.db)).resolves.toMatchObject({ + actionableRemoteFiles: 0, + unassignedRemoteFiles: 1 + }); }); it("manages storage policies and bulk assignments", async () => { @@ -4032,8 +4214,8 @@ describe("api app", () => { const filesAfterBulkAssign = await ctx.app.inject({ method: "GET", url: "/api/storage-files?rootType=remote", headers: { cookie } }); expect(filesAfterBulkAssign.json>()).toEqual( expect.arrayContaining([ - expect.objectContaining({ itemName: "Manual Movie", storagePolicy: "location_2" }), - expect.objectContaining({ itemName: "Manual Show", storagePolicy: "location_2" }) + expect.objectContaining({ itemName: "Manual Movie", storagePolicy: "unassigned" }), + expect.objectContaining({ itemName: "Manual Show", storagePolicy: "unassigned" }) ]) ); diff --git a/tests/database.test.ts b/tests/database.test.ts index f8f7c0b..8941560 100644 --- a/tests/database.test.ts +++ b/tests/database.test.ts @@ -35,7 +35,7 @@ describe("database bootstrap", () => { expect(indexes.rows.map((index) => index.indexname)).toEqual(expect.arrayContaining(["jobs_status_idx", "jobs_heartbeat_idx"])); const migrations = await database.pool.query<{ version: number }>("select version from schema_migrations order by version"); - expect(migrations.rows).toEqual([{ version: 1 }, { version: 2 }, { version: 3 }, { version: 4 }, { version: 5 }, { version: 6 }, { version: 7 }]); + expect(migrations.rows).toEqual([{ version: 1 }, { version: 2 }, { version: 3 }, { version: 4 }, { version: 5 }, { version: 6 }, { version: 7 }, { version: 8 }]); const workerColumns = await database.pool.query<{ column_name: string }>(` select column_name @@ -163,7 +163,66 @@ describe("database bootstrap", () => { { version: 4 }, { version: 5 }, { version: 6 }, - { version: 7 } + { version: 7 }, + { version: 8 } + ]); + } finally { + await database.close(); + await testDatabase.cleanup(); + } + }); + + it("clears storage policies that are not backed by one current symlink policy", async () => { + const testDatabase = await createTestDatabase(); + const legacyPool = new Pool({ connectionString: testDatabase.databaseUrl }); + try { + await legacyPool.query(`CREATE TABLE schema_migrations (version INTEGER PRIMARY KEY, name TEXT NOT NULL, applied_at TEXT NOT NULL)`); + await legacyPool.query(` + INSERT INTO schema_migrations (version, name, applied_at) + VALUES + (1, 'initial_postgres_schema', now()::text), + (2, 'beta_security_and_recovery', now()::text), + (3, 'beta_runtime_health_and_cleanup', now()::text), + (4, 'location_identity_storage_policies', now()::text), + (5, 'multi_worker_job_claims', now()::text), + (6, 'copy_operation_file_identities', now()::text), + (7, 'path_migration_target_identities', now()::text) + `); + await legacyPool.query(`CREATE TABLE storage_files (id SERIAL PRIMARY KEY, storage_policy TEXT NOT NULL, updated_at TEXT NOT NULL)`); + await legacyPool.query(` + CREATE TABLE media_links ( + id SERIAL PRIMARY KEY, + resolved_storage_file_id INTEGER, + storage_policy TEXT NOT NULL, + missing_since TEXT + ) + `); + await legacyPool.query(` + INSERT INTO storage_files (storage_policy, updated_at) + VALUES ('location_1', now()::text), ('location_1', now()::text), ('location_1', now()::text), ('location_2', now()::text) + `); + await legacyPool.query(` + INSERT INTO media_links (resolved_storage_file_id, storage_policy, missing_since) + VALUES + (1, 'location_1', NULL), + (3, 'location_1', NULL), + (3, 'location_2', NULL), + (4, 'location_2', now()::text) + `); + } finally { + await legacyPool.end(); + } + + const database = await openDatabase(testDatabase.databaseUrl); + try { + expect((await database.pool.query<{ id: number; storage_policy: string }>(`SELECT id, storage_policy FROM storage_files ORDER BY id`)).rows).toEqual([ + { id: 1, storage_policy: "location_1" }, + { id: 2, storage_policy: "unassigned" }, + { id: 3, storage_policy: "unassigned" }, + { id: 4, storage_policy: "unassigned" } + ]); + expect((await database.pool.query<{ version: number; name: string }>(`SELECT version, name FROM schema_migrations WHERE version = 8`)).rows).toEqual([ + { version: 8, name: "linked_storage_file_policies" } ]); } finally { await database.close(); @@ -287,7 +346,8 @@ describe("database bootstrap", () => { ).toEqual([ { version: 5, name: "multi_worker_job_claims" }, { version: 6, name: "copy_operation_file_identities" }, - { version: 7, name: "path_migration_target_identities" } + { version: 7, name: "path_migration_target_identities" }, + { version: 8, name: "linked_storage_file_policies" } ]); } finally { await database.close(); diff --git a/tests/e2e/app-smoke.spec.ts b/tests/e2e/app-smoke.spec.ts index 3eeb649..a6a1a6e 100644 --- a/tests/e2e/app-smoke.spec.ts +++ b/tests/e2e/app-smoke.spec.ts @@ -89,7 +89,7 @@ test("dashboard task notifications overlay without shifting content", async ({ p else if (url.pathname === "/api/settings/storage-locations") body = { locations: [{ key: "location_1", rootType: "local", displayName: "Local", path: "/mnt/local" }, { key: "location_2", rootType: "remote", displayName: "Remote", path: "/mnt/remote" }] }; else if (url.pathname === "/api/system/version") { const unavailableRelease = { latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, releaseNotes: null, message: "Unavailable" }; - body = { currentVersion: "0.1.2-beta.3", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; + body = { currentVersion: "0.1.2-beta.4", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; } else if (url.pathname === "/api/settings/sections") body = { sections: ["shows"], sectionTitles: { shows: "Shows" }, sectionTypes: { shows: "shows" } }; else if (url.pathname === "/api/settings/paths") body = { symlinkDir: "/mnt/links", localDir: "/mnt/local", remoteDir: "/mnt/remote" }; else if (url.pathname === "/api/settings/scan") body = { scanSymlinks: true, scanLocal: false, scanRemote: false, symlinkSections: ["shows"], localSections: [] }; @@ -220,7 +220,7 @@ test("refreshes an open work list when an inventory job finishes", async ({ page else if (url.pathname === "/api/settings/storage-locations") body = { locations: [{ key: "location_1", rootType: "local", displayName: "Local", path: "/mnt/local" }, { key: "location_2", rootType: "remote", displayName: "Remote", path: "/mnt/remote" }] }; else if (url.pathname === "/api/system/version") { const unavailableRelease = { latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, releaseNotes: null, message: "Unavailable" }; - body = { currentVersion: "0.1.2-beta.3", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; + body = { currentVersion: "0.1.2-beta.4", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; } else if (url.pathname === "/api/settings/sections") body = { sections: ["shows"], sectionTitles: { shows: "Shows" }, sectionTypes: { shows: "shows" } }; else if (url.pathname === "/api/settings/paths") body = { symlinkDir: "/mnt/links", localDir: "/mnt/local", remoteDir: "/mnt/remote" }; @@ -313,7 +313,7 @@ test("loads every work-list page and scopes show copies beyond the first page", else if (url.pathname === "/api/settings/storage-locations") body = { locations: [{ key: "location_1", rootType: "local", displayName: "Local", path: "/mnt/local" }, { key: "location_2", rootType: "remote", displayName: "Remote", path: "/mnt/remote" }] }; else if (url.pathname === "/api/system/version") { const unavailableRelease = { latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, releaseNotes: null, message: "Unavailable" }; - body = { currentVersion: "0.1.2-beta.3", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; + body = { currentVersion: "0.1.2-beta.4", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; } else if (url.pathname === "/api/settings/sections") body = { sections: ["shows"], sectionTitles: { shows: "Shows" }, sectionTypes: { shows: "shows" } }; else if (url.pathname === "/api/settings/paths") body = { symlinkDir: "/mnt/links", localDir: "/mnt/local", remoteDir: "/mnt/remote" }; @@ -896,7 +896,7 @@ test("failed copy admission does not display a waiting job", async ({ page }) => else if (url.pathname === "/api/settings/storage-locations") body = { locations: [{ key: "location_1", rootType: "local", displayName: "Local", path: "/mnt/local" }, { key: "location_2", rootType: "remote", displayName: "Remote", path: "/mnt/remote" }] }; else if (url.pathname === "/api/system/version") { const unavailableRelease = { latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, releaseNotes: null, message: "Unavailable" }; - body = { currentVersion: "0.1.2-beta.3", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; + body = { currentVersion: "0.1.2-beta.4", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; } else if (url.pathname === "/api/settings/sections") body = { sections: ["shows"], sectionTitles: { shows: "Shows" }, sectionTypes: { shows: "shows" } }; else if (url.pathname === "/api/settings/paths") body = { symlinkDir: "/mnt/links", localDir: "/mnt/local", remoteDir: "/mnt/remote" }; @@ -974,7 +974,7 @@ test("recent jobs identifies a targeted scan by title instead of only its parent await expect(row.locator(".job-scope-cell > small")).toContainText("Movies 4K"); }); -test("recent copy jobs show a single title directly and retain the title list for multi-title jobs", async ({ page }) => { +test("recent copy jobs show a single link title directly and retain title inspection for multi-link jobs", async ({ page }) => { test.skip(!sessionToken, "Set SRTL_E2E_SESSION_TOKEN to exercise authenticated pages."); const timestamp = new Date().toISOString(); const singleMovieTitle = "Single Copy Title (2026)"; @@ -1056,8 +1056,11 @@ test("recent copy jobs show a single title directly and retain the title list fo await expect(singleMovieRow.locator(".job-scope-cell > small")).toHaveText(singleMovieTitle); await expect(singleMovieRow.getByLabel("View selected titles")).toHaveCount(0); - await expect(singleSeriesRow.locator(".job-scope-cell > small")).toHaveText(singleSeriesTitle); - await expect(singleSeriesRow.getByLabel("View selected titles")).toHaveCount(0); + await expect(singleSeriesRow.locator(".job-scope-detail-line > span:first-child")).toHaveText("2 selected links"); + const singleSeriesTrigger = singleSeriesRow.getByLabel("View selected titles"); + await expect(singleSeriesTrigger).toHaveCount(1); + await singleSeriesTrigger.hover(); + await expect(singleSeriesTrigger.locator("li")).toHaveText([singleSeriesTitle]); await expect(multiTitleRow.locator(".job-scope-detail-line > span:first-child")).toHaveText("2 selected links"); const multiTitleTrigger = multiTitleRow.getByLabel("View selected titles"); @@ -1353,7 +1356,7 @@ test("copy progress opens a persistent, scrollable completed item summary", asyn else if (url.pathname === "/api/settings/storage-locations") body = { locations: [{ key: "location_1", rootType: "local", displayName: "Local", path: "/mnt/local" }, { key: "location_2", rootType: "remote", displayName: "Remote", path: "/mnt/remote" }] }; else if (url.pathname === "/api/system/version") { const unavailableRelease = { latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, releaseNotes: null, message: "Unavailable" }; - body = { currentVersion: "0.1.2-beta.3", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; + body = { currentVersion: "0.1.2-beta.4", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; } else if (url.pathname === `/api/jobs/${jobId}/events/page`) body = { events, total: events.length, hasOlder: false }; else if (url.pathname === `/api/jobs/${jobId}`) body = job; else if (url.pathname === "/api/jobs") body = [job]; diff --git a/tests/jobPresentationUtils.test.ts b/tests/jobPresentationUtils.test.ts index 1d85eb5..cca47d7 100644 --- a/tests/jobPresentationUtils.test.ts +++ b/tests/jobPresentationUtils.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; -import { copyCompletedItemSummaries, copyFailedItemSummaries, singleSelectedLinkTitle } from "../src/client/jobPresentationUtils"; -import type { JobEventRecord, MediaLinkRow } from "../src/shared/types"; +import { copyCompletedItemSummaries, copyFailedItemSummaries, copyWorkTotalFromJob, selectedLinkTitleSummaries, singleSelectedLinkTitle } from "../src/client/jobPresentationUtils"; +import type { JobEventRecord, JobRecord, MediaLinkRow } from "../src/shared/types"; function event(id: number, message: string, data: unknown, level: JobEventRecord["level"] = "error"): JobEventRecord { return { id, jobId: 42, timestamp: "2026-07-17T12:00:00.000Z", level, message, data }; @@ -52,6 +52,56 @@ describe("selected link title display", () => { expect(singleSelectedLinkTitle([1, 4], rows)).toBeNull(); expect(singleSelectedLinkTitle([], rows)).toBeNull(); }); + + it("keeps a single title concise when several selected links share it", () => { + const rows = new Map([ + [1, mediaLink(1, "shows", "Single Series (2026)")], + [2, mediaLink(2, "shows", "Single Series (2026)")] + ]); + + expect(selectedLinkTitleSummaries([1, 2], rows)).toEqual(["Single Series (2026)"]); + }); +}); + +describe("copy work total display", () => { + function copyJob(progress: Record): JobRecord { + const timestamp = "2026-08-01T08:00:00.000Z"; + return { + id: 386, + type: "copy", + status: "completed", + createdAt: timestamp, + startedAt: timestamp, + finishedAt: timestamp, + progress + }; + } + + it("removes legacy pre-existing title links from the displayed work total", () => { + expect(copyWorkTotalFromJob(copyJob({ + options: { direction: "to_local", section: "shows", itemName: "Example Show", linkIds: Array.from({ length: 540 }, (_, index) => index + 1) }, + total: 540, + copied: 3, + repointed: 0, + skipped: 537, + alreadyCompleted: 537, + conflicts: 0, + failed: 0 + }), 540)).toBe(3); + }); + + it("preserves the original total for an explicitly selected resumed job", () => { + expect(copyWorkTotalFromJob(copyJob({ + options: { direction: "to_local", linkIds: [1, 2, 3] }, + total: 3, + copied: 3, + repointed: 0, + skipped: 0, + alreadyCompleted: 1, + conflicts: 0, + failed: 0 + }), 3)).toBe(3); + }); }); describe("copy failure summaries", () => { diff --git a/tests/scanner.test.ts b/tests/scanner.test.ts index 295875b..bea5c2d 100644 --- a/tests/scanner.test.ts +++ b/tests/scanner.test.ts @@ -543,7 +543,7 @@ describe("scanner", () => { expect(result.storageFiles.map((file) => file.relativePath)).toEqual([path.join("shows", "Show One", "show.mkv")]); }); - it("tracks bidirectional policy and copy work for symlinks and storage-only files", async () => { + it("limits policy and copy work to current symlinks while retaining storage-only files as unassigned orphans", async () => { const symlinkDir = path.join(tmpDir, "plex"); const localDir = path.join(tmpDir, "local"); const remoteDir = path.join(tmpDir, "remote"); @@ -610,11 +610,11 @@ describe("scanner", () => { actionableLocalLinks: 1, unassignedRemoteLinks: 1, unassignedLocalLinks: 1, - actionableRemoteFiles: 1, - actionableLocalFiles: 1, - assignedRemoteFiles: 1, - unassignedRemoteFiles: 1, - unassignedLocalFiles: 1 + actionableRemoteFiles: 0, + actionableLocalFiles: 0, + assignedRemoteFiles: 0, + unassignedRemoteFiles: 3, + unassignedLocalFiles: 2 }); await persistScanResult(database.db, result, 1); @@ -622,11 +622,18 @@ describe("scanner", () => { actionableRemoteLinks: 1, actionableLocalLinks: 1, unassignedRemoteLinks: 1, - actionableRemoteFiles: 1, - actionableLocalFiles: 1, - assignedRemoteFiles: 1, - unassignedRemoteFiles: 1, - unassignedLocalFiles: 1 + actionableRemoteFiles: 0, + actionableLocalFiles: 0, + assignedRemoteFiles: 0, + unassignedRemoteFiles: 3, + unassignedLocalFiles: 2 + }); + + expect((await listStorageFiles(database.db, "remote", true)).find((file) => file.itemName === "Remote File Copy Local")).toMatchObject({ + storagePolicy: "unassigned" + }); + expect((await listStorageFiles(database.db, "remote")).find((file) => file.filePath === remoteCopyLinkTarget)).toMatchObject({ + storagePolicy: "location_1" }); expect(await listStoragePolicyCandidates(database.db, "Local File Needs", 10)).toEqual([]); @@ -641,8 +648,8 @@ describe("scanner", () => { expect(await getInventorySummary(database.db)).toMatchObject({ actionableRemoteLinks: 2, unassignedRemoteLinks: 0, - actionableRemoteFiles: 1, - unassignedRemoteFiles: 1 + actionableRemoteFiles: 0, + unassignedRemoteFiles: 3 }); } finally { await database.close(); diff --git a/tests/sectionSummaryDisplay.test.ts b/tests/sectionSummaryDisplay.test.ts index b315c3e..356c5fa 100644 --- a/tests/sectionSummaryDisplay.test.ts +++ b/tests/sectionSummaryDisplay.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "vitest"; +import { inventoryCopyToLocalCount, inventoryCopyToRemoteCount } from "../src/client/appShared"; import { inventoryPolicyNeededCount, mediaLinkTreeStatusCounts, orderSectionSummaries, sectionActionUnit, sectionCompositionParts } from "../src/client/sectionSummaryDisplay"; import type { SectionSummary } from "../src/shared/types"; @@ -25,6 +26,12 @@ function sectionSummary(section: string): SectionSummary { } describe("section summary display helpers", () => { + it("does not present unlinked storage files as copyable library work", () => { + const summary = { actionableRemoteLinks: 195, actionableRemoteFiles: 54, actionableLocalLinks: 7, actionableLocalFiles: 3 }; + expect(inventoryCopyToLocalCount(summary)).toBe(195); + expect(inventoryCopyToRemoteCount(summary)).toBe(7); + }); + it("uses episode units for show section action counts", () => { expect(sectionActionUnit({ section: "shows", type: "shows" }, 92)).toBe("episodes"); expect(sectionActionUnit({ section: "anime", type: "shows" }, 1)).toBe("episode"); From 453e001647e75cd20a9c10b00630926e4e5cd7c4 Mon Sep 17 00:00:00 2001 From: Aleksey Dmitriev Date: Tue, 4 Aug 2026 04:18:48 -0400 Subject: [PATCH 10/11] Release SRTL Manager 0.1.2 --- .env.example | 6 +- .github/workflows/ci.yml | 6 +- .github/workflows/release.yml | 2 +- CHANGELOG.md | 52 ++-- README.md | 6 +- docker-compose.yml | 4 +- package-lock.json | 36 +-- package.json | 2 +- scripts/build.mjs | 2 +- scripts/dev-sync.sh | 86 ++++-- src/client/App.tsx | 8 +- src/client/api.ts | 13 +- src/client/appShared.ts | 20 +- src/client/jobPresentation.tsx | 118 ++++++-- src/client/jobPresentationUtils.ts | 5 +- src/client/jobScopeLocks.ts | 13 +- src/client/libraryRoutes.tsx | 8 +- src/client/operationsRoutes.tsx | 111 ++++---- src/client/styles.css | 133 +++------ src/server/app.ts | 15 +- src/server/auth.ts | 16 +- src/server/config.ts | 7 + src/server/db/database.ts | 215 ++++++++++++++- src/server/db/schema.ts | 103 ++++--- src/server/jobs/copyReconciliation.ts | 151 ++++++++++- src/server/jobs/jobRunner.ts | 309 +++++++++++++++++---- src/server/jobs/resourceMutationGuard.ts | 29 +- src/server/jobs/scheduling.ts | 1 + src/server/lib/env.ts | 2 + src/server/lib/historyRetention.ts | 55 ++++ src/server/lib/pathConfiguration.ts | 2 +- src/server/lib/scanner.ts | 159 +++++------ src/server/routes/auditRoutes.ts | 45 ++- src/server/routes/jobRoutes.ts | 35 +-- src/server/routes/libraryRoutes.ts | 23 +- src/server/routes/settingsRoutes.ts | 8 +- src/server/worker.ts | 44 ++- src/shared/types.ts | 52 ++++ tests/app.test.ts | 331 +++++++++++++++++++++-- tests/config.test.ts | 17 +- tests/database.test.ts | 85 +++++- tests/e2e/app-smoke.spec.ts | 90 +++++- tests/env.test.ts | 4 +- tests/historyRetention.test.ts | 188 +++++++++++++ tests/jobPresentationUtils.test.ts | 40 +++ tests/jobScheduler.test.ts | 39 ++- tests/workerHeartbeats.test.ts | 9 + 47 files changed, 2128 insertions(+), 577 deletions(-) create mode 100644 src/server/lib/historyRetention.ts create mode 100644 tests/historyRetention.test.ts diff --git a/.env.example b/.env.example index 26d2096..b7ecffb 100644 --- a/.env.example +++ b/.env.example @@ -27,7 +27,7 @@ SRTL_TRUST_PROXY=false # Parallel job slots hosted by the single worker service. One preserves the # existing serial behavior. The total limit defaults to SRTL_WORKER_COUNT and # per-type limits default to that total. Copy file concurrency defaults to one; -# the active-file limit defaults to the worker count. +# the independent process-wide active-file limit defaults to the worker count. SRTL_WORKER_COUNT=1 # SRTL_MAX_RUNNING_JOBS=1 # SRTL_MAX_RUNNING_SCANS=1 @@ -35,3 +35,7 @@ SRTL_WORKER_COUNT=1 # SRTL_MAX_RUNNING_COPIES=1 # SRTL_COPY_FILE_CONCURRENCY=1 # SRTL_MAX_ACTIVE_COPY_FILES=1 + +# Terminal job, event, audit, and scan history retention. Zero preserves all +# history. Jobs with unresolved copy recovery state are never removed. +SRTL_JOB_HISTORY_RETENTION_DAYS=90 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 251322e..1dd64c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: timeout-minutes: 30 services: postgres: - image: postgres:17-alpine + image: postgres:17-alpine@sha256:742f40ea20b9ff2ff31db5458d127452988a2164df9e17441e191f3b72252193 env: POSTGRES_DB: postgres POSTGRES_USER: srtl_test @@ -81,7 +81,7 @@ jobs: npm run start:api > /tmp/srtl-api.log 2>&1 & echo $! > /tmp/srtl-api.pid for _ in {1..30}; do - curl -fsS http://127.0.0.1:3010/api/health && exit 0 + curl -fsS http://127.0.0.1:3010/api/health/live && exit 0 sleep 1 done cat /tmp/srtl-api.log @@ -177,7 +177,7 @@ jobs: run: docker build --check . - uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4 - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4 - - uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4 + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1313ca4..9426c6a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -58,7 +58,7 @@ jobs: run: docker build --check . - uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4 - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4 - - uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4 + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cb813d..d6a46ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,26 +4,44 @@ All notable changes are documented here. The project follows Semantic Versioning ## [Unreleased] -## [0.1.2-beta.4] - 2026-08-01 +## [0.1.2] - 2026-08-04 -### Fixed - -- Prevented superseded legacy copy-reconciliation records from blocking newly scanned media while retaining exact media and managed-path safeguards for genuinely unresolved filesystem state. -- Limited newly queued scoped copy jobs to their actionable media so already satisfied title links no longer inflate job totals or selected-title details. -- Batched large selected-link title lookups and restored title tooltips for multi-link jobs without exceeding the API request limit. -- Derived storage-file assignment exclusively from current linked symlinks so unlinked files cannot remain assigned to a storage location. -- Reconciled legacy storage-file policies during migration and after scans or policy updates. +### Added -## [0.1.2-beta.3] - 2026-07-31 +- Added immutable database-backed job selections so large jobs retain exact, bounded title details without embedding unbounded link arrays in progress payloads. +- Added operator-assisted copy-reconciliation status and safe automatic recovery for journal entries whose current filesystem identity proves their final state. +- Added separate liveness and readiness health checks, periodic configurable terminal-history and expired-session retention, and indexes for worker, audit, and cleanup workloads. ### Changed +- Paginated audit findings, pushed library filters and inventory counts into Postgres, and removed per-result scanner lookups that made large libraries progressively slower. +- Frozen copy behavior when jobs are admitted, including explicit per-job overrides for source-title mismatches, so later settings changes cannot alter queued work. +- Hardened development synchronization with destination ownership checks, argv-safe transfer options, and a non-mutating dry-run mode. +- Decoupled per-file copy concurrency from job-slot count while retaining an explicit process-wide active-file ceiling, validated legacy integrity constraints, and pinned the PostgreSQL runtime image. - Added configurable in-process worker slots and independent global, per-job-type, and copy-transfer concurrency limits without an arbitrary worker-count ceiling. - Kept example deployments at one worker slot by default while honoring any positive `SRTL_WORKER_COUNT` value from `.env`. - Allowed non-overlapping copy, audit, and targeted title-rescan work to run concurrently while broad scans and path migrations remain exclusive. +- Made targeted title rescans validate readable symlink targets, reconcile their exact storage files, and report persistent read failures. +- Retried transient source and transfer I/O failures before failing a copy. +- Matched administrator usernames case-insensitively for login and account conflicts while preserving display capitalization. +- Replaced layout-shifting dashboard action messages with responsive overlay notifications. +- Displayed a copy job's title directly when all selected links belong to one title, while retaining the title list for multi-title jobs. ### Fixed +- Isolated concurrent copy progress so one file transfer cannot leak totals or current-file details into another job or transfer. +- Prevented replaced destination files from being automatically reconciled unless their recorded identity still matches the durable copy journal. +- Closed legacy destination-only reconciliation journals when the original symlink is intact and no temporary or displaced artifacts remain, preserving the unlinked destination for normal conflict handling instead of indefinitely blocking retries. +- Persisted automatic copy-reconciliation resolutions so later service restarts and path checks cannot reactivate already-settled legacy journals. +- Cleared failed copy-submission state whenever the copy dialog selection changes so an earlier title's admission error cannot appear on a later title. +- Allowed superseded recovery records to age out with their terminal jobs while continuing to preserve genuinely unresolved copy state. +- Prevented queued jobs from being reinterpreted after section settings change, rejected malformed password hashes safely, expired stale sessions promptly, and surfaced corrupt stored settings instead of silently substituting defaults. +- Accepted selections beyond the former 1,000-link request limit and loaded large audit result sets incrementally in the interface. +- Prevented superseded legacy copy-reconciliation records from blocking newly scanned media while retaining exact media and managed-path safeguards for genuinely unresolved filesystem state. +- Limited newly queued scoped copy jobs to their actionable media so already satisfied title links no longer inflate job totals or selected-title details. +- Batched large selected-link title lookups and restored title tooltips for multi-link jobs without exceeding the API request limit. +- Derived storage-file assignment exclusively from current linked symlinks so unlinked files cannot remain assigned to a storage location. +- Reconciled legacy storage-file policies during migration and after scans or policy updates. - Made queue admission, worker claims, stale-job recovery, and job updates lease-aware so overlapping or superseded workers cannot mutate the same job. - Preserved immutable job resource scopes so later inventory changes cannot remove an active job's overlap protection. - Scoped legacy failed-copy reconciliation locks to their exact media records and managed paths so newly scanned items from the same title can still be queued. @@ -31,20 +49,10 @@ All notable changes are documented here. The project follows Semantic Versioning - Loaded every page of dashboard work lists and made show and season copy actions server-scoped so large sections are never truncated to the first 250 links. - Accepted routine FUSE and NFS remounts without a false path migration when the canonical path and stable mount signature are unchanged, while retaining exact identity checks during active mutations. -## [0.1.2-beta.2] - 2026-07-29 - -### Changed - -- Made targeted title rescans validate readable symlink targets, reconcile their exact storage files, and report persistent read failures. -- Retried transient source and transfer I/O failures before failing a copy. -- Matched administrator usernames case-insensitively for login and account conflicts while preserving display capitalization. - -## [0.1.2-beta.1] - 2026-07-27 - -### Changed +### Security -- Replaced layout-shifting dashboard action messages with responsive overlay notifications. -- Displayed a copy job's title directly when all selected links belong to one title, while retaining the title list for multi-title jobs. +- Patched current high- and moderate-severity transitive dependency advisories in `fast-uri`, `brace-expansion`, `undici`, and `postcss`. +- Updated the pinned Docker registry login action to its hardened 4.6.0 release. ## [0.1.1] - 2026-07-25 diff --git a/README.md b/README.md index ed8a6cd..15fb607 100644 --- a/README.md +++ b/README.md @@ -80,10 +80,12 @@ The single Compose worker service can host any positive number of independent jo The optional `SRTL_MAX_RUNNING_JOBS` setting limits total simultaneous jobs and must not exceed `SRTL_WORKER_COUNT`. `SRTL_MAX_RUNNING_SCANS`, `SRTL_MAX_RUNNING_AUDITS`, and `SRTL_MAX_RUNNING_COPIES` apply per-type limits, may be zero to pause that job type, and must not exceed the total-job limit. When omitted, the total limit follows the configured worker count and the per-type limits follow that total limit. -`SRTL_COPY_FILE_CONCURRENCY` controls how many files one copy job may transfer at once. `SRTL_MAX_ACTIVE_COPY_FILES` is the worker process-wide copy-file ceiling and must be at least the per-job value. Their defaults keep one file active per copy job while allowing separate copy jobs to use separate slots. Start conservatively and raise copy limits only when the storage endpoints and network can sustain the additional I/O. +`SRTL_COPY_FILE_CONCURRENCY` controls how many files one copy job may transfer at once. `SRTL_MAX_ACTIVE_COPY_FILES` is the independent worker process-wide copy-file ceiling and must be at least the per-job value. Their defaults keep one file active per copy job while allowing separate copy jobs to use separate slots. Setting both values above the worker count is supported, including parallel file transfers from one copy job in a one-slot worker; start conservatively and raise copy limits only when the storage endpoints and network can sustain the additional I/O. Scale with `SRTL_WORKER_COUNT`; do not simultaneously run `docker compose up --scale worker=...`. The supported deployment model keeps job slots and the active-copy-file safeguard inside one worker process. Compose gives that process two minutes to stop active jobs and perform safe rollback during shutdown. +`SRTL_JOB_HISTORY_RETENTION_DAYS` controls automatic cleanup of terminal job, event, audit, and scan history. The default is 90 days; set it to `0` to preserve all history. Jobs with unresolved copy-recovery state are always retained. + The API checks the public GitHub Releases endpoint for stable and beta version information at startup and when version status is refreshed. This request does not include credentials, paths, or inventory data. When a configured root changes, restart the stack. The UI enters maintenance mode until it validates and applies a path migration or the prior value is restored. This rebases managed paths; it does not move stored content. Routine Linux remounts are accepted automatically when the canonical path, mount point, filesystem type, and mount source are unchanged; exact device and inode checks still fence active filesystem mutations. @@ -132,6 +134,8 @@ npm run test:e2e Stop the disposable database with `docker stop srtl-manager-dev-postgres`. To use an existing Postgres instance instead, provide the `SRTL_POSTGRES_*` settings or `SRTL_DATABASE_URL` in a local `.env`. +The optional `npm run sync:server` helper synchronizes source to an existing development server selected by `SRTL_REMOTE` (or `SRTL_REMOTE_HOST`). It preserves remote `.env`, data, dependencies, and logs, refuses an unrelated non-empty destination, and supports a non-mutating preview with `SRTL_SYNC_DRY_RUN=1`. A normal first sync initializes the destination marker that later syncs require. + ## Releases And Contributions - Pull requests target `beta`. diff --git a/docker-compose.yml b/docker-compose.yml index 51706a6..d45f20a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ services: postgres: - image: postgres:17-alpine + image: postgres:17-alpine@sha256:742f40ea20b9ff2ff31db5458d127452988a2164df9e17441e191f3b72252193 restart: unless-stopped environment: POSTGRES_DB: ${SRTL_POSTGRES_DB:?Set SRTL_POSTGRES_DB in .env} @@ -65,7 +65,7 @@ services: tmpfs: - /tmp:size=256m,mode=1777 healthcheck: - test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:3010/api/health').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"] + test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:3010/api/health/live').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"] interval: 10s timeout: 5s retries: 12 diff --git a/package-lock.json b/package-lock.json index 74578bf..195e71f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "srtl-manager", - "version": "0.1.2-beta.4", + "version": "0.1.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "srtl-manager", - "version": "0.1.2-beta.4", + "version": "0.1.2", "license": "MIT", "dependencies": { "@fastify/compress": "^9.1.0", @@ -2697,9 +2697,9 @@ } }, "node_modules/brace-expansion": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.8.tgz", - "integrity": "sha512-JZyDyq3D4AUifKTPOB7DELf6XsB3WdPuNxCtob1vFXPsSXhdAiHBWJ/tJ8HAc9aH84BK+5JFZLNkJKx3G9kzQg==", + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" @@ -3544,9 +3544,9 @@ } }, "node_modules/fast-uri": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.4.tgz", - "integrity": "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.5.tgz", + "integrity": "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==", "funding": [ { "type": "github", @@ -3633,9 +3633,9 @@ } }, "node_modules/fastify/node_modules/fast-uri": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-4.1.1.tgz", - "integrity": "sha512-YPOs1zD5TG2+EZt+r88LwF6mclA7TPkpwMP7ZN3TO2HiHS8TXvq7QA/17iJsV9dubcLo/f8eEYqMBruyQV21hQ==", + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-4.1.2.tgz", + "integrity": "sha512-TyGmBcbDTZXcb2cj5MV89DrF42DKvb3y5DDUNh95iO+IMeAzMkVSxK1PZRrRIpc9yg8U2GhGdbofNa0LS/a4Bw==", "funding": [ { "type": "github", @@ -4957,9 +4957,9 @@ } }, "node_modules/postcss": { - "version": "8.5.19", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.19.tgz", - "integrity": "sha512-Mz8SaolMd8nB+G13WkORcxQKHZ/NE4xXevtkJHVuG+guo9/wYKlIMTKAqGdEmYOXR2ijPjTYNHssizdaVSUNdQ==", + "version": "8.5.25", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.25.tgz", + "integrity": "sha512-DTPx3RWSSnWyzLxQnlH0rJP+EW5ekl16ZU4/psbIhA0e53kJfdgaN5vKM+xP7yJtXVu+nfdVFmlgFDEKAe4Pyw==", "dev": true, "funding": [ { @@ -4977,7 +4977,7 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.12", + "nanoid": "^3.3.16", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" }, @@ -5712,9 +5712,9 @@ } }, "node_modules/undici": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.28.0.tgz", - "integrity": "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==", + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz", + "integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==", "dev": true, "license": "MIT", "engines": { diff --git a/package.json b/package.json index b0610b0..6d6774f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "srtl-manager", - "version": "0.1.2-beta.4", + "version": "0.1.2", "private": true, "license": "MIT", "homepage": "https://github.com/ramphex/SRTL-Manager#readme", diff --git a/scripts/build.mjs b/scripts/build.mjs index 48a8c13..c762fca 100644 --- a/scripts/build.mjs +++ b/scripts/build.mjs @@ -10,7 +10,7 @@ await esbuild({ packages: "external", platform: "node", format: "esm", - target: "node22", + target: "node24", outdir: "dist/server", sourcemap: true, logLevel: "info" diff --git a/scripts/dev-sync.sh b/scripts/dev-sync.sh index 265cbc0..bbf26b8 100644 --- a/scripts/dev-sync.sh +++ b/scripts/dev-sync.sh @@ -16,31 +16,77 @@ if [[ "$REMOTE" == -* || "$REMOTE" == *$'\n'* ]]; then printf 'Unsafe remote target: %s\n' "$REMOTE" >&2 exit 2 fi -if [[ ! "$REMOTE_DIR" =~ ^[A-Za-z0-9._/-]+$ || "$REMOTE_DIR" == /* || "/$REMOTE_DIR/" == */../* ]]; then +if [[ ! "$REMOTE_DIR" =~ ^[A-Za-z0-9._/-]+$ || "$REMOTE_DIR" == -* || "$REMOTE_DIR" == /* || "$REMOTE_DIR" == "." || "/$REMOTE_DIR/" == */../* || "/$REMOTE_DIR/" == */./* ]]; then printf 'SRTL_REMOTE_DIR must be a safe relative path: %s\n' "$REMOTE_DIR" >&2 exit 2 fi +SYNC_DRY_RUN="${SRTL_SYNC_DRY_RUN:-0}" +if [[ "$SYNC_DRY_RUN" != "0" && "$SYNC_DRY_RUN" != "1" ]]; then + printf 'SRTL_SYNC_DRY_RUN must be 0 or 1.\n' >&2 + exit 2 +fi printf 'Syncing %s -> %s:%s/\n' "$ROOT_DIR" "$REMOTE" "$REMOTE_DIR" -# The path is expanded locally only after the strict relative-path validation above. -# shellcheck disable=SC2029 -ssh "$REMOTE" "mkdir -p -- '$REMOTE_DIR'" +ssh "$REMOTE" sh -s -- "$REMOTE_DIR" "$SYNC_DRY_RUN" <<'REMOTE_SCRIPT' +set -eu +remote_dir=$1 +dry_run=$2 +sentinel="$remote_dir/.srtl-dev-workspace" +if [ -e "$remote_dir" ] && [ ! -d "$remote_dir" ]; then + printf 'Remote development target is not a directory: %s\n' "$remote_dir" >&2 + exit 2 +fi +if [ ! -e "$remote_dir" ]; then + if [ "$dry_run" = 1 ]; then + printf 'Remote development target does not exist; run a normal sync once to initialize it: %s\n' "$remote_dir" >&2 + exit 2 + fi + mkdir -p -- "$remote_dir" +fi +if [ ! -f "$sentinel" ]; then + if [ -f "$remote_dir/package.json" ]; then + if ! grep -Eq '"name"[[:space:]]*:[[:space:]]*"srtl-manager"' "$remote_dir/package.json"; then + printf 'Refusing to adopt a remote directory that is not SRTL Manager: %s\n' "$remote_dir" >&2 + exit 2 + fi + else + existing_entry=$(find "$remote_dir" -mindepth 1 -maxdepth 1 -print -quit) + fi + if [ ! -f "$remote_dir/package.json" ] && [ -n "$existing_entry" ]; then + printf 'Refusing to delete into a non-empty remote directory without an SRTL Manager package: %s\n' "$remote_dir" >&2 + exit 2 + fi + if [ "$dry_run" = 0 ]; then + : > "$sentinel" + fi +fi +REMOTE_SCRIPT + +rsync_args=( + -az + --delete + --exclude .git + --exclude node_modules + --exclude dist + --exclude data + --exclude .env + --exclude coverage + --exclude .cache + --exclude test-results + --exclude playwright-report + --exclude verification_logs + --exclude '*.log' + --exclude .srtl-dev-workspace +) +if [[ "$SYNC_DRY_RUN" == "1" ]]; then + rsync_args+=(--dry-run --itemize-changes) +fi -rsync -az --delete \ - --exclude .git \ - --exclude node_modules \ - --exclude dist \ - --exclude data \ - --exclude .env \ - --exclude coverage \ - --exclude .cache \ - --exclude test-results \ - --exclude playwright-report \ - --exclude verification_logs \ - --exclude '*.log' \ - -- \ - "$ROOT_DIR/" \ - "$REMOTE:$REMOTE_DIR/" +rsync "${rsync_args[@]}" -- "$ROOT_DIR/" "$REMOTE:$REMOTE_DIR/" -printf 'Sync complete.\n' +if [[ "$SYNC_DRY_RUN" == "1" ]]; then + printf 'Dry run complete; no files were changed.\n' +else + printf 'Sync complete.\n' +fi diff --git a/src/client/App.tsx b/src/client/App.tsx index 1511ef7..3fbd8bd 100644 --- a/src/client/App.tsx +++ b/src/client/App.tsx @@ -1,7 +1,7 @@ import { useEffect, useRef, useState, type FormEvent, type ReactNode } from "react"; import { createRootRoute, createRoute, createRouter, lazyRouteComponent, Link, Outlet, RouterProvider, useLocation } from "@tanstack/react-router"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import { Activity, ArrowDown, ArrowLeft, ArrowRight, ArrowUp, Blocks, CheckCircle2, ChevronRight, Database, FileText, Gauge, HardDrive, HardDriveDownload, Info, Library, Link2, ListChecks, LogIn, LogOut, OctagonX, Plus, RefreshCw, Search, Settings, Shield, Trash2, TriangleAlert, X, UserCog, UserPlus } from "lucide-react"; +import { Activity, ArrowDown, ArrowLeft, ArrowRight, ArrowUp, CheckCircle2, ChevronRight, Database, FileText, Gauge, HardDrive, HardDriveDownload, Info, Library, Link2, ListChecks, LogIn, LogOut, OctagonX, Plus, RefreshCw, Search, Settings, Shield, Trash2, TriangleAlert, X, UserCog, UserPlus } from "lucide-react"; import { api } from "./api"; import { formatJobType, jobProgressChips } from "./logDisplay"; import { formatCurrentVersionDisplay } from "./versionDisplay"; @@ -264,7 +264,6 @@ function RootLayout() { const nav = [ { to: "/", label: "Dashboard", icon: Gauge }, { to: "/library", label: "Library", icon: Library }, - { to: "/integrations", label: "Integrations", icon: Blocks }, { to: "/logs", label: "Logs", icon: FileText } ]; const toggleGroup = (group: SidebarGroup) => { @@ -1203,17 +1202,14 @@ const loadOperationsRoutes = () => import("./operationsRoutes"); const DashboardPage = lazyRouteComponent(loadLibraryRoutes, "DashboardPage"); const LibraryPage = lazyRouteComponent(loadLibraryRoutes, "LibraryPage"); const RunsPage = lazyRouteComponent(loadOperationsRoutes, "RunsPage"); -const IntegrationsPage = lazyRouteComponent(loadOperationsRoutes, "IntegrationsPage"); const SettingsPage = lazyRouteComponent(loadOperationsRoutes, "SettingsPage"); const LogsPage = lazyRouteComponent(loadOperationsRoutes, "LogsPage"); const indexRoute = createRoute({ getParentRoute: () => rootRoute, path: "/", component: DashboardPage }); const libraryRoute = createRoute({ getParentRoute: () => rootRoute, path: "/library", component: LibraryPage }); const scansRoute = createRoute({ getParentRoute: () => rootRoute, path: "/scans", component: () => }); const auditsRoute = createRoute({ getParentRoute: () => rootRoute, path: "/audits", component: () => }); -const integrationsRoute = createRoute({ getParentRoute: () => rootRoute, path: "/integrations", component: IntegrationsPage }); const settingsRoute = createRoute({ getParentRoute: () => rootRoute, path: "/settings", component: () => }); const settingsSectionsRoute = createRoute({ getParentRoute: () => rootRoute, path: "/settings/sections", component: () => }); -const settingsIntegrationsRoute = createRoute({ getParentRoute: () => rootRoute, path: "/settings/integrations", component: () => }); const settingsAdvancedRoute = createRoute({ getParentRoute: () => rootRoute, path: "/settings/advanced", component: () => }); const settingsUserRoute = createRoute({ getParentRoute: () => rootRoute, path: "/settings/user", component: () => }); const logsRoute = createRoute({ getParentRoute: () => rootRoute, path: "/logs", validateSearch: parseLogsRouteSearch, component: LogsPage }); @@ -1222,10 +1218,8 @@ const routeTree = rootRoute.addChildren([ libraryRoute, scansRoute, auditsRoute, - integrationsRoute, settingsRoute, settingsSectionsRoute, - settingsIntegrationsRoute, settingsAdvancedRoute, settingsUserRoute, logsRoute diff --git a/src/client/api.ts b/src/client/api.ts index fc97b22..0b1af9f 100644 --- a/src/client/api.ts +++ b/src/client/api.ts @@ -1,11 +1,12 @@ import type { AuditOptions, - AuditResultRecord, + AuditResultPage, AuditRunRecord, AuditSettings, AdvancedSettings, AppVersionInfo, CopyConflictPreview, + CopyReconciliationState, InventorySummary, InventoryScanTimestamps, JobEventPage, @@ -110,6 +111,8 @@ export const api = { inventoryScanTimestamps: () => request("/api/inventory/scan-timestamps"), startCopy: (body: CopyOptions) => request<{ jobId: number }>("/api/copies", { method: "POST", body: JSON.stringify(body) }), copyConflicts: (body: CopyOptions) => request("/api/copies/conflicts", { method: "POST", body: JSON.stringify(body) }), + copyReconciliation: () => request("/api/job-reconciliation"), + recheckCopyReconciliation: () => request("/api/job-reconciliation/recheck", { method: "POST" }), mediaLinks: (kind?: string) => request(`/api/media-links${kind ? `?kind=${kind}` : ""}`), mediaLinksByIds, mediaLinksPage: (params: { kind?: LinkKind; section?: string; storagePolicy?: StoragePolicyKind; relativePathPrefix?: string; search?: string; limit: number; offset: number }) => { @@ -141,7 +144,13 @@ export const api = { startAudit: (body: AuditOptions) => request<{ jobId: number }>("/api/audits", { method: "POST", body: JSON.stringify(body) }), audits: () => request("/api/audits"), auditByJob: (jobId: number) => request(`/api/audits/job/${jobId}`), - auditResults: (id: number) => request(`/api/audits/${id}/results`), + auditResultPage: (id: number, options: { offset?: number; limit?: number; attentionOnly?: boolean } = {}) => { + const search = new URLSearchParams(); + if (options.offset) search.set("offset", String(options.offset)); + if (options.limit) search.set("limit", String(options.limit)); + if (options.attentionOnly) search.set("attentionOnly", "true"); + return request(`/api/audits/${id}/results/page${search.size > 0 ? `?${search.toString()}` : ""}`); + }, jobs: (options: { activeOnly?: boolean; completedWithinMinutes?: number; limit?: number } = {}) => { const search = new URLSearchParams(); if (options.activeOnly) search.set("activeOnly", "true"); diff --git a/src/client/appShared.ts b/src/client/appShared.ts index 5f12c2a..395468e 100644 --- a/src/client/appShared.ts +++ b/src/client/appShared.ts @@ -1,6 +1,6 @@ import { createContext, useContext, useEffect, useMemo, useState } from "react"; import { useInfiniteQuery, useMutation, useQueryClient, type QueryClient } from "@tanstack/react-query"; -import { Blocks, FolderCog, Gauge, ListChecks, Monitor, Moon, Search, Sun, UserCog } from "lucide-react"; +import { FolderCog, Gauge, ListChecks, Monitor, Moon, Search, Sun, UserCog } from "lucide-react"; import { api } from "./api"; import { scanOptionsFromJob } from "./jobScopeLocks"; import { mergeJobEventPages } from "./jobEvents"; @@ -12,7 +12,7 @@ export type ThemePreference = "light" | "dark" | "system"; export type SymlinkKindFilter = Exclude | "all"; -export type SettingsView = "library" | "integrations" | "advanced" | "user"; +export type SettingsView = "library" | "advanced" | "user"; export type SidebarGroup = "history" | "settings"; @@ -112,26 +112,10 @@ export const historySections = [ export const settingsSections = [ { view: "library", to: "/settings", label: "Library", icon: FolderCog }, - { view: "integrations", to: "/settings/integrations", label: "Integrations", icon: Blocks }, { view: "advanced", to: "/settings/advanced", label: "Advanced", icon: Gauge }, { view: "user", to: "/settings/user", label: "User settings", icon: UserCog } ] as const; -export const integrationPlaceholders = [ - { - name: "Metadata integration", - description: "Future metadata lookup, health checks, and candidate mapping.", - urlPlaceholder: "Connection URL", - keyPlaceholder: "Encrypted API key" - }, - { - name: "Automation integration", - description: "Future refresh hooks and event-driven inventory updates.", - urlPlaceholder: "Connection URL", - keyPlaceholder: "Encrypted API key" - } -] as const; - export const copyProfileOptions: Array<{ value: CopyVerificationProfile; label: string; detail: string }> = [ { value: "off", label: "Off", detail: "Skip post-transfer byte compare and media validation" }, { value: "fast", label: "Fast", detail: "Byte compare only" }, diff --git a/src/client/jobPresentation.tsx b/src/client/jobPresentation.tsx index 432f99f..225c8b3 100644 --- a/src/client/jobPresentation.tsx +++ b/src/client/jobPresentation.tsx @@ -1,6 +1,6 @@ import { useEffect, useId, useMemo, useRef, useState, type ReactNode } from "react"; import { Link } from "@tanstack/react-router"; -import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { useInfiniteQuery, useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { Activity, CheckCircle2, Copy, File, FileText, Folder, Info, ListChecks, OctagonX, Play, Search, Trash2, TriangleAlert, X } from "lucide-react"; import { api } from "./api"; import { defaultAdvancedSettings, normalizeAdvancedSettings } from "../shared/advancedSettings"; @@ -8,7 +8,7 @@ import { eventDataChips, formatEventLevel, formatJobType, formatLogData, hasLogD import { auditOptionsFromJob, copyOptionsFromJob, scanOptionsFromJob } from "./jobScopeLocks"; import { jobEventCountLabel } from "./jobEvents"; import { normalizeRecentJobsCompletedWindowMinutes, recentJobsCompletedWindowOptions, visibleDashboardJobs } from "./recentJobs"; -import { type AuditMode, type AuditResultRecord, type AuditRunRecord, type CopyConflictPreview, type JobEventRecord, type JobRecord, type CopyLocalConflictStrategy, type MediaLinkRow, type TimeFormatPreference } from "../shared/types"; +import { type AuditMode, type AuditResultRecord, type AuditRunRecord, type CopyConflictPreview, type JobEventRecord, type JobRecord, type JobSelectionSummary, type CopyLocalConflictStrategy, type MediaLinkRow, type TimeFormatPreference } from "../shared/types"; import { JobStatusTerminateAction, LogChipList, Panel, ScanProgressPanel, StatusPill, TerminateJobDialog } from "./App"; import { AuditPrompt, AuditStatusPrompt, canTerminateJob, copyElapsedLabel, CopyPrompt, finiteNumberFromUnknown, formatBytes, formatDate, formatNumber, formatTime, invalidateCopyJobData, recordFromUnknown, scanAgeLabel, ScanStatusPrompt, sectionDisplayTitle, storageLocationName, useJobEventTimeline, useStartCopyJob, useStorageLocations, useTerminateJobMutation, useUserPreferences } from "./appShared"; import { auditProgressFromJob, auditProgressPercent, auditStageLabel, auditStatusDetail, basenameFromPath, copyCompletedCount, copyCompletedItemSummaries, copyCurrentItem, copyEventChips, copyFailedItemSummaries, copyOverallProgressPercent, copyProgressFromJob, copyRemainingLabel, copyStageLabel, copyStagePercent, copySymlinkedCount, copyThroughputLabel, copyTransferSpeedLabel, copyTransferSpeedSecondaryLabel, copyWorkTotalFromJob, formatAuditScope, formatCopyScope, formatScopedFolderParts, formatTitleScanJobDetail, jobDurationLabel, scanFolderScopeParts, scanScopeLabels, selectedLinkIdsFromJobs, selectedLinkTitleSummaries, singleSelectedLinkTitle } from "./jobPresentationUtils"; @@ -185,6 +185,7 @@ export function CopyDialog({ const [jobId, setJobId] = useState(prompt?.jobId ?? null); const [startedPromptKey, setStartedPromptKey] = useState(null); const [localConflictStrategy, setLocalConflictStrategy] = useState(null); + const [allowSourceTitleMismatch, setAllowSourceTitleMismatch] = useState(false); const [terminatePrompt, setTerminatePrompt] = useState(null); const terminateJob = useTerminateJobMutation(() => setTerminatePrompt(null)); const jobQuery = useQuery({ @@ -199,23 +200,31 @@ export function CopyDialog({ const jobActive = currentJob ? currentJob.status === "queued" || currentJob.status === "running" : Boolean(jobId); const events = useJobEventTimeline({ jobId, enabled: Boolean(prompt && jobId), refetchInterval: jobActive ? 500 : 2500, loadAll: true }); const startCopy = useStartCopyJob((result) => setJobId(result.jobId)); + const resetStartCopy = startCopy.reset; useEffect(() => { + resetStartCopy(); setJobId(prompt?.jobId ?? null); setStartedPromptKey(null); setLocalConflictStrategy(null); - }, [prompt?.key, prompt?.jobId]); + setAllowSourceTitleMismatch(false); + }, [prompt?.key, prompt?.jobId, resetStartCopy]); useEffect(() => { if (!prompt?.autoStart || !prompt.options || jobId || startedPromptKey === prompt.key || startCopy.isPending) return; const requiresLocalResolution = Boolean(prompt.conflicts?.totalConflicts && !prompt.options.localConflictStrategy && !localConflictStrategy); - if (requiresLocalResolution) return; - const options = localConflictStrategy ? { ...prompt.options, localConflictStrategy } : prompt.options; - const resolvedPromptKey = localConflictStrategy ? `${prompt.key}:${localConflictStrategy}` : prompt.key; + const requiresSourceResolution = Boolean((prompt.conflicts?.totalSourceTitleBlocks ?? 0) > 0 && !prompt.options.allowSourceTitleMismatch && !allowSourceTitleMismatch); + if (requiresLocalResolution || requiresSourceResolution) return; + const options = { + ...prompt.options, + ...(localConflictStrategy ? { localConflictStrategy } : {}), + ...(allowSourceTitleMismatch ? { allowSourceTitleMismatch: true } : {}) + }; + const resolvedPromptKey = [prompt.key, localConflictStrategy, allowSourceTitleMismatch ? "source-title-override" : null].filter(Boolean).join(":"); if (startedPromptKey === resolvedPromptKey) return; setStartedPromptKey(resolvedPromptKey); startCopy.mutate({ ...prompt, key: resolvedPromptKey, options }); - }, [jobId, localConflictStrategy, prompt, startCopy, startedPromptKey]); + }, [allowSourceTitleMismatch, jobId, localConflictStrategy, prompt, startCopy, startedPromptKey]); useEffect(() => { if (!currentJobId || currentJobStatus === "queued" || currentJobStatus === "running") return; @@ -226,6 +235,7 @@ export function CopyDialog({ const displayedEvents = [...events.events].reverse(); const needsLocalConflictResolution = Boolean(prompt.conflicts?.totalConflicts && !prompt.options?.localConflictStrategy && !localConflictStrategy && !jobId); + const needsSourceTitleResolution = Boolean((prompt.conflicts?.totalSourceTitleBlocks ?? 0) > 0 && !prompt.options?.allowSourceTitleMismatch && !allowSourceTitleMismatch && !jobId); return (
event.target === event.currentTarget && onClose()}> @@ -243,6 +253,8 @@ export function CopyDialog({ {startCopy.error && !jobId ? (

The copy job was not queued. No files were changed.

+ ) : needsSourceTitleResolution && prompt.conflicts ? ( + setAllowSourceTitleMismatch(true)} /> ) : needsLocalConflictResolution && prompt.conflicts ? ( setLocalConflictStrategy("keep_both")} onReplace={() => setLocalConflictStrategy("replace")} /> ) : ( @@ -261,7 +273,8 @@ export function CopyDialog({ {jobId && events.isLoading ?

Loading copy events...

: null} {events.error ?

{events.error.message}

: null} {jobId && !events.isLoading && !events.error && displayedEvents.length === 0 ?

No events yet.

: null} - {!jobId && !startCopy.error && !needsLocalConflictResolution ?

Starting the copy job. Closing this window after start leaves the job running in the background.

: null} + {!jobId && !startCopy.error && !needsLocalConflictResolution && !needsSourceTitleResolution ?

Starting the copy job. Closing this window after start leaves the job running in the background.

: null} + {needsSourceTitleResolution ?

Review and explicitly accept the source-title mismatch before starting this copy.

: null} {needsLocalConflictResolution ?

Choose how to handle the existing local file before starting this copy.

: null} {displayedEvents.length > 0 ? (
@@ -283,6 +296,36 @@ export function CopyDialog({ ); } +function CopySourceTitleResolution({ conflicts, onContinue }: { conflicts: CopyConflictPreview; onContinue: () => void }) { + const risks = conflicts.sourceTitleRisks ?? []; + return ( +
+
+ + + Source title mismatch + {formatNumber(risks.length)} source file{risks.length === 1 ? " does" : "s do"} not look like the selected library title. + +
+
+ {risks.map((risk) => ( +
+ {risk.itemName} + {risk.relativePath} + {risk.reason}: {risk.sourcePath} +
+ ))} +
+
+ +
+
+ ); +} + function CopyConflictResolution({ conflicts, onKeepBoth, @@ -432,9 +475,11 @@ export function AuditStatusDialog({ }); const auditRun = auditRunQuery.data ?? null; const auditRunId = prompt?.auditRunId ?? auditRun?.id ?? null; - const auditResults = useQuery({ + const auditResults = useInfiniteQuery({ queryKey: ["audit-results", auditRunId], - queryFn: () => api.auditResults(auditRunId!), + queryFn: ({ pageParam }) => api.auditResultPage(auditRunId!, { offset: pageParam, limit: 100, attentionOnly: true }), + initialPageParam: 0, + getNextPageParam: (page) => (page.hasMore ? page.offset + page.results.length : undefined), enabled: Boolean(prompt && auditRunId), refetchInterval: jobActive ? 1500 : false }); @@ -443,6 +488,7 @@ export function AuditStatusDialog({ if (!prompt) return null; const displayedEvents = [...events.events].reverse(); + const auditResultRecords = auditResults.data?.pages.flatMap((page) => page.results) ?? []; return (
event.target === event.currentTarget && onClose()}> @@ -462,7 +508,13 @@ export function AuditStatusDialog({ - {auditRunId && !auditResults.isLoading && !auditResults.error ? : null} + {auditRunId && !auditResults.isLoading && !auditResults.error ? : null} + {auditResults.hasNextPage && jobActive ?

Additional findings can be loaded after the audit finishes so newly inserted rows cannot shift the current page.

: null} + {auditResults.hasNextPage && !jobActive ? ( + + ) : null}
@@ -1023,13 +1075,14 @@ export function JobScope({ if (job.type === "audit") { const options = auditOptionsFromJob(job); - const sectionText = formatAuditScope(options, sections); + const selectedCount = job.selection?.total ?? options.linkIds?.length ?? 0; + const sectionText = selectedCount > 0 ? (selectedCount === 1 ? "1 selected link" : `${formatNumber(selectedCount)} selected links`) : formatAuditScope(options, sections); const modeText = options.mode === "fast" ? "Fast audit" : options.mode === "deep" ? "Deep audit" : "Audit"; const selectedLinkIds = options.linkIds ?? []; return ( - 0 ? undefined : sectionText}> + 0 ? undefined : sectionText}> {modeText} - + ); } @@ -1038,14 +1091,15 @@ export function JobScope({ const options = copyOptionsFromJob(job); const directionText = `Copy to ${storageLocationName(storageLocations, options?.direction === "to_remote" ? "remote" : "local")}`; const selectedLinkIds = options?.linkIds ?? []; - const workTotal = copyWorkTotalFromJob(job, selectedLinkIds.length); - const sectionText = selectedLinkIds.length > 0 + const selectedCount = job.selection?.total ?? selectedLinkIds.length; + const workTotal = copyWorkTotalFromJob(job, selectedCount); + const sectionText = selectedCount > 0 ? workTotal === 1 ? "1 selected link" : `${formatNumber(workTotal)} selected links` : formatCopyScope(options, sections); return ( - 0 ? undefined : sectionText}> + 0 ? undefined : sectionText}> {directionText} - + ); } @@ -1066,17 +1120,35 @@ export function JobScope({ function JobScopeDetail({ text, + selection, selectedLinkIds, linkRowsById, linkRowsLoading, linkRowsError }: { text: string; + selection?: JobSelectionSummary; selectedLinkIds: number[]; linkRowsById?: Map; linkRowsLoading?: boolean; linkRowsError?: string | null; }) { + if (selection && selection.total > 0) { + const snapshotSummaries = selection.titles.map((title) => + selection.titles.length > 1 && title.count > 1 ? `${title.itemName} (${formatNumber(title.count)} links)` : title.itemName + ); + if (selection.omittedTitles) snapshotSummaries.push(`${formatNumber(selection.omittedTitles)} additional title${selection.omittedTitles === 1 ? "" : "s"}`); + if (selection.unavailable > 0) snapshotSummaries.push(`${formatNumber(selection.unavailable)} unavailable historical link${selection.unavailable === 1 ? "" : "s"}`); + if (selection.total === 1 && selection.titles.length === 1 && selection.unavailable === 0) { + return {selection.titles[0]?.itemName}; + } + return ( + + {text} + + + ); + } if (selectedLinkIds.length === 0) return {text}; const singleTitle = selectedLinkIds.length === 1 && !linkRowsLoading && !linkRowsError ? singleSelectedLinkTitle(selectedLinkIds, linkRowsById) : null; if (singleTitle) return {singleTitle}; @@ -1090,17 +1162,19 @@ function JobScopeDetail({ } function SelectedLinkTitlesTooltip({ + summaries: suppliedSummaries, linkIds, linkRowsById, isLoading, error }: { - linkIds: number[]; + summaries?: string[]; + linkIds?: number[]; linkRowsById?: Map; isLoading?: boolean; error?: string | null; }) { - const summaries = selectedLinkTitleSummaries(linkIds, linkRowsById); + const summaries = suppliedSummaries ?? selectedLinkTitleSummaries(linkIds ?? [], linkRowsById); return ( @@ -1111,8 +1185,8 @@ function SelectedLinkTitlesTooltip({ {!isLoading && !error && summaries.length === 0 ? No matching titles found in the current inventory. : null} {!isLoading && !error && summaries.length > 0 ? (
    - {summaries.map((title) => ( -
  • {title}
    • + {summaries.map((title, index) => ( +
  • {title}
    • ))}
) : null} diff --git a/src/client/jobPresentationUtils.ts b/src/client/jobPresentationUtils.ts index f3f8a0d..4af1207 100644 --- a/src/client/jobPresentationUtils.ts +++ b/src/client/jobPresentationUtils.ts @@ -152,6 +152,7 @@ export function copyProgressFromJob(job: JobRecord | null): CopyProgressView { } export function copyWorkTotalFromJob(job: JobRecord, fallbackTotal = 0): number { + if (job.selection) return Math.max(0, job.selection.total - job.selection.unavailable); const progress = recordFromUnknown(job.progress); const options = copyOptionsFromJob(job); const total = finiteNumberFromUnknown(progress?.total) || fallbackTotal; @@ -508,6 +509,7 @@ export function selectedLinkIdsFromJob(job: JobRecord): number[] { export function selectedLinkIdsFromJobs(jobs: JobRecord[]): number[] { const ids = new Set(); for (const job of jobs) { + if (job.selection) continue; for (const id of selectedLinkIdsFromJob(job)) ids.add(id); } return [...ids].sort((first, second) => first - second); @@ -615,6 +617,7 @@ export function copyPromptKey(options: CopyOptions): string { options.itemName ?? "", options.relativePathPrefix ?? "", options.linkIds?.join(",") ?? "", - options.localConflictStrategy ?? "" + options.localConflictStrategy ?? "", + options.allowSourceTitleMismatch ? "allow-source-title-mismatch" : "" ].join(":"); } diff --git a/src/client/jobScopeLocks.ts b/src/client/jobScopeLocks.ts index dfb5b0d..3311b29 100644 --- a/src/client/jobScopeLocks.ts +++ b/src/client/jobScopeLocks.ts @@ -23,12 +23,12 @@ export function normalizeAuditTargets(targets: unknown): StorageRootType[] { export function auditOptionsFromJob(job: JobRecord): AuditJobOptions { const progress = recordFromUnknown(job.progress); - const options = recordFromUnknown(progress?.options) ?? progress; + const options = recordFromUnknown(job.options) ?? recordFromUnknown(progress?.options) ?? progress; return { mode: options?.mode === "fast" || options?.mode === "deep" ? options.mode : null, sections: Array.isArray(options?.sections) ? options.sections.filter((section): section is string => typeof section === "string") : undefined, targets: Array.isArray(options?.targets) ? normalizeAuditTargets(options.targets) : undefined, - linkIds: Array.isArray(options?.linkIds) ? options.linkIds.filter((id): id is number => Number.isInteger(id)) : undefined, + linkIds: job.selection?.linkIds ?? (Array.isArray(options?.linkIds) ? options.linkIds.filter((id): id is number => Number.isInteger(id)) : undefined), section: typeof options?.section === "string" ? options.section : undefined, itemName: typeof options?.itemName === "string" ? options.itemName : undefined, relativePathPrefix: typeof options?.relativePathPrefix === "string" ? options.relativePathPrefix : undefined, @@ -38,16 +38,17 @@ export function auditOptionsFromJob(job: JobRecord): AuditJobOptions { export function copyOptionsFromJob(job: JobRecord): CopyOptions | null { const progress = recordFromUnknown(job.progress); - const options = recordFromUnknown(progress?.options) ?? progress; + const options = recordFromUnknown(job.options) ?? recordFromUnknown(progress?.options) ?? progress; const direction = options?.direction === "to_local" || options?.direction === "to_remote" ? options.direction : null; if (!direction) return null; return { direction, - linkIds: Array.isArray(options?.linkIds) ? options.linkIds.filter((id): id is number => Number.isInteger(id)) : undefined, + linkIds: job.selection?.linkIds ?? (Array.isArray(options?.linkIds) ? options.linkIds.filter((id): id is number => Number.isInteger(id)) : undefined), section: typeof options?.section === "string" ? options.section : undefined, itemName: typeof options?.itemName === "string" ? options.itemName : undefined, relativePathPrefix: typeof options?.relativePathPrefix === "string" ? options.relativePathPrefix : undefined, - localConflictStrategy: options?.localConflictStrategy === "keep_both" || options?.localConflictStrategy === "replace" ? options.localConflictStrategy : undefined + localConflictStrategy: options?.localConflictStrategy === "keep_both" || options?.localConflictStrategy === "replace" ? options.localConflictStrategy : undefined, + allowSourceTitleMismatch: options?.allowSourceTitleMismatch === true ? true : undefined }; } @@ -62,7 +63,7 @@ function scanTitleScopesFromUnknown(value: unknown): ScanTitleScope[] | undefine export function scanOptionsFromJob(job: JobRecord): Partial | null { const progress = recordFromUnknown(job.progress); - const options = recordFromUnknown(progress?.options) ?? progress; + const options = recordFromUnknown(job.options) ?? recordFromUnknown(progress?.options) ?? progress; if (!options) return null; return { scanSymlinks: options.scanSymlinks === true, diff --git a/src/client/libraryRoutes.tsx b/src/client/libraryRoutes.tsx index 74f1433..e9d02f3 100644 --- a/src/client/libraryRoutes.tsx +++ b/src/client/libraryRoutes.tsx @@ -103,7 +103,7 @@ export function DashboardPage() { return api.copyConflicts(prompt.options); }, onSuccess: (conflicts, prompt) => { - if (conflicts.totalConflicts > 0) { + if (conflicts.totalConflicts > 0 || (conflicts.totalSourceTitleBlocks ?? 0) > 0) { setCopyPrompt({ ...prompt, conflicts }); return; } @@ -312,7 +312,7 @@ export function DashboardPage() { function handleCopyRequest(prompt: CopyPrompt) { startScan.reset(); - if (prompt.options?.direction === "to_local" && !prompt.options.localConflictStrategy) { + if (prompt.options && (!prompt.options.allowSourceTitleMismatch || (prompt.options.direction === "to_local" && !prompt.options.localConflictStrategy))) { copyConflictCheck.mutate(prompt); return; } @@ -1723,7 +1723,7 @@ export function LibraryPage() { return api.copyConflicts(prompt.options); }, onSuccess: (conflicts, prompt) => { - if (conflicts.totalConflicts > 0) { + if (conflicts.totalConflicts > 0 || (conflicts.totalSourceTitleBlocks ?? 0) > 0) { setCopyPrompt({ ...prompt, conflicts }); return; } @@ -1736,7 +1736,7 @@ export function LibraryPage() { }); function handleCopyRequest(prompt: CopyPrompt) { - if (prompt.options?.direction === "to_local" && !prompt.options.localConflictStrategy) { + if (prompt.options && (!prompt.options.allowSourceTitleMismatch || (prompt.options.direction === "to_local" && !prompt.options.localConflictStrategy))) { copyConflictCheck.mutate(prompt); return; } diff --git a/src/client/operationsRoutes.tsx b/src/client/operationsRoutes.tsx index ca1c2e8..efd0e50 100644 --- a/src/client/operationsRoutes.tsx +++ b/src/client/operationsRoutes.tsx @@ -1,7 +1,7 @@ import { useEffect, useMemo, useState, type FormEvent } from "react"; -import { getRouteApi, Link } from "@tanstack/react-router"; +import { getRouteApi } from "@tanstack/react-router"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import { Activity, ArrowUp, Blocks, CheckCircle2, FileText, FolderCog, ListChecks, Plus, Radar, Search, ServerCog, Settings, TriangleAlert, UserCog } from "lucide-react"; +import { Activity, ArrowUp, CheckCircle2, FileText, FolderCog, ListChecks, Plus, Search, ServerCog, Settings, TriangleAlert, UserCog } from "lucide-react"; import { api } from "./api"; import { copyBehaviorForProfile, defaultAdvancedSettings, normalizeAdvancedSettings } from "../shared/advancedSettings"; import { formatJobType, jobProgressChips, matchesEventFilters, matchesJobFilters, type EventLevelFilter, type JobStatusFilter, type JobTypeFilter } from "./logDisplay"; @@ -10,7 +10,7 @@ import { inventoryPolicyNeededCount } from "./sectionSummaryDisplay"; import { jobEventCountLabel } from "./jobEvents"; import { type AuditMode, type AuditRunRecord, type AdvancedSettings, type JobRecord, type CopyMediaValidationMode, type CopyVerificationProfile, type PathsSettings, type ScanOptions, type ScanRunRecord, type StorageLocationKey, type StorageLocationsSettings, type TimeFormatPreference, type UserPreferences } from "../shared/types"; import { LogChipList, Page, Panel, ScanProgressPanel, SectionDraftList, StatusPill } from "./App"; -import { auditModeOptions, AuditStatusPrompt, copyPipelineLabels, copyProfileOptions, createEmptySectionDraft, defaultStorageLocations, defaultUserPreferences, formatDate, formatDuration, formatNumber, integrationPlaceholders, inventoryAssignedRemoteCount, inventoryCopyToLocalCount, inventoryCopyToRemoteCount, mediaValidationOptions, ScanStatusPrompt, SectionDraft, sectionDraftsToSettings, sectionSettingsToDrafts, SettingsView, storageLocationName, timeFormatOptions, useJobEventTimeline, useStorageLocations, useUserPreferences } from "./appShared"; +import { auditModeOptions, AuditStatusPrompt, copyPipelineLabels, copyProfileOptions, createEmptySectionDraft, defaultStorageLocations, defaultUserPreferences, formatDate, formatDuration, formatNumber, inventoryAssignedRemoteCount, inventoryCopyToLocalCount, inventoryCopyToRemoteCount, mediaValidationOptions, ScanStatusPrompt, SectionDraft, sectionDraftsToSettings, sectionSettingsToDrafts, SettingsView, storageLocationName, timeFormatOptions, useJobEventTimeline, useStorageLocations, useUserPreferences } from "./appShared"; import { AuditProgressPanel, AuditStatusDialog, CopyProgressPanel, JobScope, LogEventRow, ScanStatusDialog } from "./jobPresentation"; import { AuditScopeDisplayOptions, auditStatusPromptFromRun, countEventsByLevel, countJobsByStatus, countJobsByType, formatFolderScope, formatTitleScanScope, jobDurationLabel, scanStatusPromptFromRun } from "./jobPresentationUtils"; @@ -426,60 +426,6 @@ function MetricGroup({ title, description, metrics }: { title: string; descripti ); } -export function IntegrationsPage() { - return ( - -
-
- -

Coming soon

-

Integration-driven workflows are not active yet. Connection settings live under Settings > Integrations while this page is being built out.

-
- - - Open integration settings - -
- - ); -} - -function IntegrationSettingsPanel() { - return ( - <> -

Integration settings are placeholders until the adapters are built. These fields are intentionally disabled and do not connect, save, sync, or run health checks yet.

-
- {integrationPlaceholders.map((integration) => ( - }> -
-

{integration.description}

- Coming soon -
-
- - - -
- - ))} -
- - - ); -} - export function SettingsPage({ activeView }: { activeView: SettingsView }) { const queryClient = useQueryClient(); const paths = useQuery({ queryKey: ["paths"], queryFn: api.getPaths, enabled: activeView === "library" }); @@ -505,12 +451,10 @@ export function SettingsPage({ activeView }: { activeView: SettingsView }) { const subtitle = activeView === "library" ? "Name storage locations, review mounted paths, and manage library sections." - : activeView === "integrations" - ? "Preview planned external connection settings." : activeView === "advanced" ? "Tune job behavior and verification defaults." : "Manage display preferences and account access."; - const activeTitle = activeView === "library" ? "Library" : activeView === "integrations" ? "Integrations" : activeView === "advanced" ? "Advanced" : "User settings"; + const activeTitle = activeView === "library" ? "Library" : activeView === "advanced" ? "Advanced" : "User settings"; return ( ${activeTitle}`} subtitle={subtitle}> {activeView === "library" ? ( @@ -534,7 +478,6 @@ export function SettingsPage({ activeView }: { activeView: SettingsView }) { ) : null} - {activeView === "integrations" ? : null} {activeView === "advanced" ? : null} {activeView === "user" ? : null} @@ -642,6 +585,7 @@ function StorageLocationsPanel({ paths, isLoadingPaths, pathsError }: { paths: P function AdvancedSettingsPanel() { const queryClient = useQueryClient(); const settings = useQuery({ queryKey: ["advanced-settings"], queryFn: api.getAdvancedSettings }); + const reconciliation = useQuery({ queryKey: ["copy-reconciliation"], queryFn: api.copyReconciliation, refetchInterval: 30_000 }); const [draft, setDraft] = useState(defaultAdvancedSettings); const [message, setMessage] = useState(null); const savedAdvancedSettings = normalizeAdvancedSettings(settings.data ?? defaultAdvancedSettings); @@ -659,6 +603,13 @@ function AdvancedSettingsPanel() { setMessage("Advanced settings saved."); } }); + const recheckReconciliation = useMutation({ + mutationFn: api.recheckCopyReconciliation, + onSuccess: (state) => { + queryClient.setQueryData(["copy-reconciliation"], state); + queryClient.invalidateQueries({ queryKey: ["jobs"] }); + } + }); useEffect(() => { if (settings.data) setDraft(normalizeAdvancedSettings(settings.data)); }, [settings.data]); @@ -762,6 +713,11 @@ function AdvancedSettingsPanel() { ))}
+ {copyVerificationDisabled ? ( +

+ Verification is disabled. Copies still use a temporary file and guarded promotion, but source bytes and media integrity are not checked before the symlink is repointed. +

+ ) : null} {validationError ?

{validationError}

: null} @@ -801,6 +757,39 @@ function AdvancedSettingsPanel() {
+
+
+
+

Copy recovery

+

Rechecks interrupted copy journals and automatically closes only states proven safe from the current symlink and journal files.

+
+ + {reconciliation.isLoading ? "Checking" : `${formatNumber(reconciliation.data?.unresolvedCount ?? 0)} unresolved`} + +
+ {reconciliation.error ?

{reconciliation.error.message}

: null} + {recheckReconciliation.data?.resolvedNow ? ( +

Resolved {formatNumber(recheckReconciliation.data.resolvedNow)} stale copy journal {recheckReconciliation.data.resolvedNow === 1 ? "entry" : "entries"}.

+ ) : null} + {(reconciliation.data?.unresolved ?? []).length > 0 ? ( +
    + {(reconciliation.data?.unresolved ?? []).slice(0, 20).map((operation) => ( +
  • + Operation #{operation.id} / job #{operation.jobId} + {operation.errorMessage ?? "Filesystem state is still uncertain."} +
    • + ))} +
+ ) : ( +

No copy journals currently require operator attention.

+ )} + + {recheckReconciliation.error ? {recheckReconciliation.error.message} : null} +
+
{message ? {message} : null} {save.error ? {save.error.message} : null} diff --git a/src/client/styles.css b/src/client/styles.css index 8cd9a35..07b25c0 100644 --- a/src/client/styles.css +++ b/src/client/styles.css @@ -3215,6 +3215,36 @@ tbody tr:has(.job-link-title-tooltip:focus-within) > td:has(.job-link-title-tool margin-right: auto; } +.settings-reconciliation-list { + display: grid; + gap: 8px; + list-style: none; + margin: 0; + max-height: 280px; + overflow: auto; + padding: 0; +} + +.settings-reconciliation-list li { + background: color-mix(in srgb, var(--surface), var(--bg-subtle) 24%); + border: 1px solid var(--border-soft); + border-radius: 8px; + display: grid; + gap: 3px; + padding: 9px 10px; +} + +.settings-reconciliation-list strong { + color: var(--text-strong); + font-size: 12px; +} + +.settings-reconciliation-list span { + color: var(--muted); + font-size: 12px; + line-height: 1.4; +} + .inline-form { align-items: end; display: grid; @@ -3437,6 +3467,15 @@ tbody tr:has(.job-link-title-tooltip:focus-within) > td:has(.job-link-title-tool font-size: 13px; } +.verification-disabled-warning { + background: color-mix(in srgb, var(--warn), var(--surface) 91%); + border: 1px solid color-mix(in srgb, var(--warn), var(--border) 68%); + border-radius: 8px; + color: var(--warn); + line-height: 1.45; + padding: 10px 12px; +} + .panel-message.action-progress { align-items: center; color: var(--muted-strong); @@ -3486,91 +3525,6 @@ button.danger-button:hover:not(:disabled) { text-align: center; } -.integration-grid { - display: grid; - gap: 14px; - grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); -} - -.integration-placeholder-copy { - align-items: flex-start; - display: flex; - gap: 12px; - justify-content: space-between; - margin-bottom: 14px; -} - -.integration-placeholder-copy p { - color: var(--muted); - line-height: 1.45; - margin: 0; -} - -.integration-placeholder-fields { - border: 0; - margin: 0; - opacity: 0.72; - padding: 0; -} - -.integration-placeholder-fields input { - cursor: not-allowed; -} - -.coming-soon-panel { - align-items: center; - background: var(--surface); - border: 1px solid var(--border-soft); - border-radius: 8px; - box-shadow: var(--shadow); - display: flex; - gap: 16px; - justify-content: space-between; - padding: 18px; -} - -.coming-soon-panel > div { - display: grid; - gap: 8px; - min-width: 0; -} - -.coming-soon-panel svg { - color: var(--muted); -} - -.coming-soon-panel h2 { - color: var(--text-strong); - font-size: 18px; - margin: 0; -} - -.coming-soon-panel p { - color: var(--muted); - font-size: 13px; - line-height: 1.45; - margin: 0; - max-width: 620px; -} - -.button-link { - align-items: center; - background: var(--primary); - border: 1px solid var(--primary); - border-radius: 7px; - color: var(--primary-text); - display: inline-flex; - flex: 0 0 auto; - gap: 8px; - min-height: 36px; - padding: 0 12px; - text-decoration: none; -} - -.button-link:hover { - background: var(--primary-hover); -} - .split { display: grid; gap: 16px; @@ -5802,15 +5756,6 @@ button.danger-button:hover:not(:disabled) { grid-template-columns: 1fr; } - .coming-soon-panel { - align-items: stretch; - flex-direction: column; - } - - .coming-soon-panel .button-link { - justify-content: center; - } - .split { grid-template-columns: 1fr; } diff --git a/src/server/app.ts b/src/server/app.ts index bef5fe9..1400842 100644 --- a/src/server/app.ts +++ b/src/server/app.ts @@ -119,7 +119,7 @@ export async function createApp(overrides: Partial = {}): Promise { + const workerHealth = async () => { await database.pool.query("select 1"); const workers = await database.db .select() @@ -137,16 +137,23 @@ export async function createApp(overrides: Partial = {}): Promise 30_000 ? capacity + worker.capacity : capacity; }, 0); const expectedWorkerCount = config.jobConcurrency.maxRunningJobs; + const ready = readyWorkerCount >= expectedWorkerCount; return { - ok: true, + ok: ready, database: "ready", - worker: readyWorkerCount >= expectedWorkerCount ? "ready" : workers.length > 0 ? "stale" : "not_started", + worker: ready ? "ready" : workers.length > 0 ? "stale" : "not_started", workerHeartbeatAt: latestWorker?.heartbeatAt ?? null, expectedWorkerCount, readyWorkerCount, staleWorkerCount }; + }; + app.get("/api/health/live", async () => ({ ok: true, service: "running" })); + app.get("/api/health/ready", async (_request, reply) => { + const health = await workerHealth(); + return health.ok ? health : reply.code(503).send(health); }); + app.get("/api/health", async () => ({ ...(await workerHealth()), ok: true })); registerAuthRoutes(app, database.db, { cookieName: config.sessionCookieName, cookieSecure: config.sessionCookieSecure @@ -155,7 +162,7 @@ export async function createApp(overrides: Partial = {}): Promise { const documentationRequest = config.apiDocsEnabled && request.url.startsWith("/documentation"); if (!request.url.startsWith("/api/") && !documentationRequest) return; - if (request.url.startsWith("/api/auth/") || request.url === "/api/health") return; + if (request.url.startsWith("/api/auth/") || request.url === "/api/health" || request.url.startsWith("/api/health/")) return; await requireAuth(database.db, config.sessionCookieName)(request, reply); if (reply.sent) return; if (documentationRequest) return; diff --git a/src/server/auth.ts b/src/server/auth.ts index 279bc53..e51b934 100644 --- a/src/server/auth.ts +++ b/src/server/auth.ts @@ -1,6 +1,6 @@ import crypto from "node:crypto"; import { promisify } from "node:util"; -import { eq, sql } from "drizzle-orm"; +import { eq, lt, sql } from "drizzle-orm"; import type { FastifyReply, FastifyRequest } from "fastify"; import type { Db } from "./db/database"; import { first, nowIso } from "./db/database"; @@ -18,8 +18,10 @@ export async function hashPassword(password: string): Promise { export async function verifyPassword(password: string, hash: string): Promise { const [scheme, salt, digest] = hash.split("$"); if (scheme !== "scrypt" || !salt || !digest) return false; + const storedDigest = Buffer.from(digest, "base64"); + if (storedDigest.length !== 64) return false; const derived = (await scrypt(password, salt, 64)) as Buffer; - return crypto.timingSafeEqual(Buffer.from(digest, "base64"), derived); + return crypto.timingSafeEqual(storedDigest, derived); } export async function hasAdmin(db: Db): Promise { @@ -52,6 +54,7 @@ export async function login(db: Db, username: string, password: string): Promise if (!(await verifyPassword(password, user.passwordHash))) return null; const token = crypto.randomBytes(32).toString("base64url"); const tokenHash = hashToken(token); + await db.delete(schema.sessions).where(lt(schema.sessions.expiresAt, nowIso())); const expiresAt = new Date(Date.now() + 1000 * 60 * 60 * 24 * 14).toISOString(); await db.insert(schema.sessions).values({ tokenHash, userId: user.id, expiresAt, createdAt: nowIso() }); return token; @@ -63,8 +66,13 @@ export function hashToken(token: string): string { export async function getSessionUser(db: Db, token: string | undefined): Promise<{ id: number; username: string } | null> { if (!token) return null; - const session = await first(db.select().from(schema.sessions).where(eq(schema.sessions.tokenHash, hashToken(token))).limit(1)); - if (!session || Date.parse(session.expiresAt) < Date.now()) return null; + const tokenHash = hashToken(token); + const session = await first(db.select().from(schema.sessions).where(eq(schema.sessions.tokenHash, tokenHash)).limit(1)); + if (!session) return null; + if (Date.parse(session.expiresAt) < Date.now()) { + await db.delete(schema.sessions).where(eq(schema.sessions.tokenHash, tokenHash)); + return null; + } const user = await first(db.select().from(schema.adminUsers).where(eq(schema.adminUsers.id, session.userId)).limit(1)); return user ? { id: user.id, username: user.username } : null; } diff --git a/src/server/config.ts b/src/server/config.ts index d004814..f4d832b 100644 --- a/src/server/config.ts +++ b/src/server/config.ts @@ -16,6 +16,7 @@ export interface AppConfig { autoMigrate: boolean; trustProxy: boolean; webRoot: string; + jobHistoryRetentionDays: number; jobConcurrency: JobConcurrencySettings; paths: PathsSettings; } @@ -197,6 +198,12 @@ export function loadConfig(overrides: Partial = {}): AppConfig { overrides.autoMigrate ?? booleanSetting(process.env.SRTL_AUTO_MIGRATE ?? envFile.SRTL_AUTO_MIGRATE, process.env.NODE_ENV !== "production"), trustProxy: overrides.trustProxy ?? booleanSetting(process.env.SRTL_TRUST_PROXY ?? envFile.SRTL_TRUST_PROXY, false), webRoot: overrides.webRoot ?? process.env.SRTL_WEB_ROOT ?? envFile.SRTL_WEB_ROOT ?? path.join(rootDir, "dist", "client"), + jobHistoryRetentionDays: integerSetting( + overrides.jobHistoryRetentionDays ?? process.env.SRTL_JOB_HISTORY_RETENTION_DAYS ?? envFile.SRTL_JOB_HISTORY_RETENTION_DAYS, + 90, + "SRTL_JOB_HISTORY_RETENTION_DAYS", + 0 + ), jobConcurrency: loadJobConcurrency(envFile, overrides.jobConcurrency), paths: overrides.paths ?? { symlinkDir: environment.SYMLINK_DIR ?? "", diff --git a/src/server/db/database.ts b/src/server/db/database.ts index db484cf..b4e13ad 100644 --- a/src/server/db/database.ts +++ b/src/server/db/database.ts @@ -21,7 +21,7 @@ export interface DatabaseOpenOptions { pool?: Pool; } -export const currentSchemaVersion = 8; +export const currentSchemaVersion = 12; const ddl = [ `CREATE TABLE IF NOT EXISTS app_settings (key TEXT PRIMARY KEY, value TEXT NOT NULL, updated_at TEXT NOT NULL)`, @@ -62,7 +62,7 @@ const ddl = [ ]; const hardeningDdl = [ - `CREATE TABLE IF NOT EXISTS copy_operations (id SERIAL PRIMARY KEY, job_id INTEGER NOT NULL, media_link_id INTEGER NOT NULL, link_path TEXT NOT NULL, source_path TEXT NOT NULL, destination_path TEXT NOT NULL, original_target_path TEXT NOT NULL, original_link_state TEXT NOT NULL, previous_copy_source TEXT, temp_path TEXT, displaced_path TEXT, temp_identity TEXT, destination_identity TEXT, displaced_identity TEXT, stage TEXT NOT NULL, result_status TEXT, local_conflict_strategy TEXT, size_bytes BIGINT, error_message TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, completed_at TEXT, UNIQUE(job_id, media_link_id))`, + `CREATE TABLE IF NOT EXISTS copy_operations (id SERIAL PRIMARY KEY, job_id INTEGER NOT NULL, media_link_id INTEGER NOT NULL, link_path TEXT NOT NULL, source_path TEXT NOT NULL, destination_path TEXT NOT NULL, original_target_path TEXT NOT NULL, original_link_state TEXT NOT NULL, previous_copy_source TEXT, temp_path TEXT, displaced_path TEXT, temp_identity TEXT, destination_identity TEXT, displaced_identity TEXT, stage TEXT NOT NULL, result_status TEXT, local_conflict_strategy TEXT, size_bytes BIGINT, error_message TEXT, reconciliation_resolved_at TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, completed_at TEXT, UNIQUE(job_id, media_link_id))`, `CREATE INDEX IF NOT EXISTS copy_operations_job_stage_idx ON copy_operations(job_id, stage, id)`, `CREATE UNIQUE INDEX IF NOT EXISTS admin_users_singleton_idx ON admin_users ((true))`, `DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'sessions_user_fk') THEN ALTER TABLE sessions ADD CONSTRAINT sessions_user_fk FOREIGN KEY (user_id) REFERENCES admin_users(id) ON DELETE CASCADE NOT VALID; END IF; END $$`, @@ -281,6 +281,205 @@ async function initializeDatabase(pool: Pool): Promise { throw error; } } + + if (!applied.has(9)) { + await client.query("BEGIN"); + try { + const tables = await client.query<{ jobs: boolean; mediaLinks: boolean; mediaLinkSnapshots: boolean; jobProgress: boolean }>(` + SELECT to_regclass('public.jobs') IS NOT NULL AS jobs, + to_regclass('public.media_links') IS NOT NULL AS "mediaLinks", + ( + SELECT count(*) = 4 FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'media_links' + AND column_name IN ('section', 'item_name', 'relative_path', 'link_path') + ) AS "mediaLinkSnapshots", + EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'jobs' AND column_name = 'progress' + ) AS "jobProgress" + `); + const available = tables.rows[0]; + if (available?.jobs) { + await client.query(`ALTER TABLE jobs ADD COLUMN IF NOT EXISTS options TEXT NOT NULL DEFAULT '{}'`); + await client.query(`ALTER TABLE jobs ADD COLUMN IF NOT EXISTS selection_frozen BOOLEAN NOT NULL DEFAULT FALSE`); + if (available.jobProgress) { + await client.query(` + UPDATE jobs + SET options = CASE + WHEN jsonb_typeof(progress::jsonb -> 'options') = 'object' THEN (progress::jsonb -> 'options')::text + ELSE '{}' + END + WHERE options = '{}' + `); + } + await client.query(` + UPDATE jobs + SET selection_frozen = TRUE + WHERE jsonb_typeof(options::jsonb -> 'linkIds') = 'array' + `); + await client.query(` + CREATE TABLE IF NOT EXISTS job_selection_items ( + id SERIAL PRIMARY KEY, + job_id INTEGER NOT NULL REFERENCES jobs(id) ON DELETE CASCADE, + media_link_id INTEGER NOT NULL, + selection_order INTEGER NOT NULL, + section TEXT NOT NULL, + item_name TEXT NOT NULL, + relative_path TEXT NOT NULL, + link_path TEXT NOT NULL, + created_at TEXT NOT NULL, + CONSTRAINT job_selection_items_job_media_idx UNIQUE (job_id, media_link_id), + CONSTRAINT job_selection_items_job_order_idx UNIQUE (job_id, selection_order) + ) + `); + await client.query(`CREATE INDEX IF NOT EXISTS job_selection_items_job_title_idx ON job_selection_items(job_id, section, item_name)`); + if (available.mediaLinks && available.mediaLinkSnapshots) { + await client.query(` + INSERT INTO job_selection_items ( + job_id, media_link_id, selection_order, section, item_name, relative_path, link_path, created_at + ) + SELECT jobs.id, + selected.media_link_id, + selected.selection_order, + media_links.section, + media_links.item_name, + media_links.relative_path, + media_links.link_path, + jobs.created_at + FROM jobs + CROSS JOIN LATERAL ( + SELECT value::integer AS media_link_id, ordinality::integer - 1 AS selection_order + FROM jsonb_array_elements_text(jobs.options::jsonb -> 'linkIds') WITH ORDINALITY + ) AS selected + JOIN media_links ON media_links.id = selected.media_link_id + WHERE jobs.selection_frozen = TRUE + ON CONFLICT (job_id, media_link_id) DO NOTHING + `); + } + } + await client.query(`INSERT INTO schema_migrations (version, name, applied_at) VALUES (9, 'immutable_job_inputs_and_selections', $1)`, [nowIso()]); + await client.query("COMMIT"); + } catch (error) { + await client.query("ROLLBACK"); + throw error; + } + } + + if (!applied.has(10)) { + await client.query("BEGIN"); + try { + await client.query(` + DO $$ + BEGIN + IF to_regclass('public.sessions') IS NOT NULL THEN + EXECUTE 'CREATE INDEX IF NOT EXISTS sessions_expires_idx ON sessions(expires_at)'; + END IF; + IF to_regclass('public.jobs') IS NOT NULL + AND EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema = 'public' AND table_name = 'jobs' AND column_name = 'finished_at') + THEN + EXECUTE 'CREATE INDEX IF NOT EXISTS jobs_terminal_retention_idx ON jobs(status, finished_at, id)'; + END IF; + IF to_regclass('public.audit_results') IS NOT NULL THEN + EXECUTE 'CREATE INDEX IF NOT EXISTS audit_results_run_id_idx ON audit_results(audit_run_id, id)'; + END IF; + IF to_regclass('public.audit_runs') IS NOT NULL THEN + EXECUTE 'CREATE INDEX IF NOT EXISTS audit_runs_job_id_idx ON audit_runs(job_id, id)'; + END IF; + IF to_regclass('public.scan_runs') IS NOT NULL THEN + EXECUTE 'CREATE INDEX IF NOT EXISTS scan_runs_job_id_idx ON scan_runs(job_id, id)'; + END IF; + IF to_regclass('public.copy_operations') IS NOT NULL + AND EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema = 'public' AND table_name = 'copy_operations' AND column_name = 'media_link_id') + THEN + EXECUTE 'CREATE INDEX IF NOT EXISTS copy_operations_media_recovery_idx ON copy_operations(media_link_id, link_path, stage, id)'; + END IF; + END $$ + `); + await client.query(`INSERT INTO schema_migrations (version, name, applied_at) VALUES (10, 'retention_and_history_indexes', $1)`, [nowIso()]); + await client.query("COMMIT"); + } catch (error) { + await client.query("ROLLBACK"); + throw error; + } + } + + if (!applied.has(11)) { + await client.query("BEGIN"); + try { + await client.query(` + DO $$ + DECLARE + constraint_record RECORD; + BEGIN + FOR constraint_record IN + SELECT constraint_data.conrelid::regclass AS table_name, constraint_data.conname + FROM pg_constraint AS constraint_data + JOIN pg_namespace AS constraint_namespace ON constraint_namespace.oid = constraint_data.connamespace + WHERE constraint_namespace.nspname = 'public' + AND constraint_data.convalidated = FALSE + AND constraint_data.conname = ANY (ARRAY[ + 'sessions_user_fk', + 'job_events_job_fk', + 'scan_runs_job_fk', + 'audit_runs_job_fk', + 'audit_results_run_fk', + 'copy_operations_job_fk', + 'copy_operations_media_link_fk', + 'path_migration_items_migration_fk', + 'jobs_status_check', + 'jobs_type_check', + 'media_links_kind_check', + 'media_links_policy_check', + 'storage_files_policy_check', + 'storage_files_root_type_check', + 'storage_policies_policy_check', + 'copy_operations_stage_check' + ]) + LOOP + EXECUTE format('ALTER TABLE %s VALIDATE CONSTRAINT %I', constraint_record.table_name, constraint_record.conname); + END LOOP; + END $$ + `); + await client.query(`INSERT INTO schema_migrations (version, name, applied_at) VALUES (11, 'validate_integrity_constraints', $1)`, [nowIso()]); + await client.query("COMMIT"); + } catch (error) { + await client.query("ROLLBACK"); + throw error; + } + } + + if (!applied.has(12)) { + await client.query("BEGIN"); + try { + await client.query(` + DO $$ + BEGIN + IF to_regclass('public.copy_operations') IS NOT NULL THEN + ALTER TABLE copy_operations ADD COLUMN IF NOT EXISTS reconciliation_resolved_at TEXT; + IF ( + SELECT count(*) = 3 + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = 'copy_operations' + AND column_name IN ('error_message', 'updated_at', 'completed_at') + ) THEN + EXECUTE $backfill$ + UPDATE copy_operations + SET reconciliation_resolved_at = coalesce(completed_at, updated_at, now()::text) + WHERE reconciliation_resolved_at IS NULL + AND error_message LIKE 'Automatically closed after recheck:%' + $backfill$; + END IF; + END IF; + END $$ + `); + await client.query(`INSERT INTO schema_migrations (version, name, applied_at) VALUES (12, 'durable_copy_reconciliation_resolution', $1)`, [nowIso()]); + await client.query("COMMIT"); + } catch (error) { + await client.query("ROLLBACK"); + throw error; + } + } } finally { await client.query("select pg_advisory_unlock($1)", [bootstrapLockKey]).catch(() => undefined); client.release(); @@ -296,11 +495,13 @@ export async function openDatabase(options: DatabaseOpenOptions | string): Promi if (shouldMigrate) { await initializeDatabase(pool); } else { - const migration = await pool.query<{ version: number | null }>("select max(version) as version from schema_migrations").catch((error: unknown) => { + const migration = await pool.query<{ version: number }>("select version from schema_migrations order by version").catch((error: unknown) => { throw new Error("Database schema is not initialized. Run the migration service before starting SRTL Manager.", { cause: error }); }); - if (Number(migration.rows[0]?.version ?? 0) < currentSchemaVersion) { - throw new Error(`Database schema is out of date. Expected migration ${currentSchemaVersion}; run the migration service.`); + const appliedVersions = new Set(migration.rows.map((row) => Number(row.version))); + const missingVersions = Array.from({ length: currentSchemaVersion }, (_unused, index) => index + 1).filter((version) => !appliedVersions.has(version)); + if (missingVersions.length > 0) { + throw new Error(`Database schema is out of date. Missing migration${missingVersions.length === 1 ? "" : "s"} ${missingVersions.join(", ")}; run the migration service.`); } } } catch (error) { @@ -359,8 +560,8 @@ export async function getJsonSetting(db: Db, key: string, fallback: T): Promi if (!raw) return fallback; try { return JSON.parse(raw) as T; - } catch { - return fallback; + } catch (error) { + throw new Error(`Stored setting "${key}" contains invalid JSON; repair or remove that setting before continuing.`, { cause: error }); } } diff --git a/src/server/db/schema.ts b/src/server/db/schema.ts index beb2980..497b9bd 100644 --- a/src/server/db/schema.ts +++ b/src/server/db/schema.ts @@ -59,12 +59,16 @@ export const adminUsers = pgTable("admin_users", { createdAt: text("created_at").notNull() }); -export const sessions = pgTable("sessions", { - tokenHash: text("token_hash").primaryKey(), - userId: integer("user_id").notNull(), - expiresAt: text("expires_at").notNull(), - createdAt: text("created_at").notNull() -}); +export const sessions = pgTable( + "sessions", + { + tokenHash: text("token_hash").primaryKey(), + userId: integer("user_id").notNull(), + expiresAt: text("expires_at").notNull(), + createdAt: text("created_at").notNull() + }, + (table) => [index("sessions_expires_idx").on(table.expiresAt)] +); export const sections = pgTable("sections", { id: serial("id").primaryKey(), @@ -148,21 +152,49 @@ export const copySources = pgTable( (table) => [uniqueIndex("copy_sources_destination_idx").on(table.destinationPath)] ); -export const jobs = pgTable("jobs", { - id: serial("id").primaryKey(), - type: text("type").notNull(), - status: text("status").notNull(), - createdAt: text("created_at").notNull(), - startedAt: text("started_at"), - finishedAt: text("finished_at"), - lockedBy: text("locked_by"), - lockedAt: text("locked_at"), - heartbeatAt: text("heartbeat_at"), - leaseVersion: integer("lease_version").notNull().default(0), - exclusive: boolean("exclusive").notNull().default(true), - cancelRequestedAt: text("cancel_requested_at"), - progress: text("progress").notNull() -}); +export const jobs = pgTable( + "jobs", + { + id: serial("id").primaryKey(), + type: text("type").notNull(), + status: text("status").notNull(), + createdAt: text("created_at").notNull(), + startedAt: text("started_at"), + finishedAt: text("finished_at"), + lockedBy: text("locked_by"), + lockedAt: text("locked_at"), + heartbeatAt: text("heartbeat_at"), + leaseVersion: integer("lease_version").notNull().default(0), + exclusive: boolean("exclusive").notNull().default(true), + options: text("options").notNull().default("{}"), + selectionFrozen: boolean("selection_frozen").notNull().default(false), + cancelRequestedAt: text("cancel_requested_at"), + progress: text("progress").notNull() + }, + (table) => [index("jobs_terminal_retention_idx").on(table.status, table.finishedAt, table.id)] +); + +export const jobSelectionItems = pgTable( + "job_selection_items", + { + id: serial("id").primaryKey(), + jobId: integer("job_id") + .notNull() + .references(() => jobs.id, { onDelete: "cascade" }), + mediaLinkId: integer("media_link_id").notNull(), + selectionOrder: integer("selection_order").notNull(), + section: text("section").notNull(), + itemName: text("item_name").notNull(), + relativePath: text("relative_path").notNull(), + linkPath: text("link_path").notNull(), + createdAt: text("created_at").notNull() + }, + (table) => [ + uniqueIndex("job_selection_items_job_media_idx").on(table.jobId, table.mediaLinkId), + uniqueIndex("job_selection_items_job_order_idx").on(table.jobId, table.selectionOrder), + index("job_selection_items_job_title_idx").on(table.jobId, table.section, table.itemName) + ] +); export const jobResourceClaims = pgTable( "job_resource_claims", @@ -217,6 +249,7 @@ export const copyOperations = pgTable( localConflictStrategy: text("local_conflict_strategy"), sizeBytes: bigint("size_bytes", { mode: "number" }), errorMessage: text("error_message"), + reconciliationResolvedAt: text("reconciliation_resolved_at"), createdAt: text("created_at").notNull(), updatedAt: text("updated_at").notNull(), completedAt: text("completed_at") @@ -283,15 +316,19 @@ export const auditRuns = pgTable("audit_runs", { errorMessage: text("error_message") }); -export const auditResults = pgTable("audit_results", { - id: serial("id").primaryKey(), - auditRunId: integer("audit_run_id").notNull(), - linkPath: text("link_path").notNull(), - targetPath: text("target_path").notNull(), - sourcePath: text("source_path"), - status: text("status").notNull(), - ffmpegStatus: text("ffmpeg_status").notNull(), - cmpStatus: text("cmp_status").notNull(), - message: text("message").notNull(), - createdAt: text("created_at").notNull() -}); +export const auditResults = pgTable( + "audit_results", + { + id: serial("id").primaryKey(), + auditRunId: integer("audit_run_id").notNull(), + linkPath: text("link_path").notNull(), + targetPath: text("target_path").notNull(), + sourcePath: text("source_path"), + status: text("status").notNull(), + ffmpegStatus: text("ffmpeg_status").notNull(), + cmpStatus: text("cmp_status").notNull(), + message: text("message").notNull(), + createdAt: text("created_at").notNull() + }, + (table) => [index("audit_results_run_id_idx").on(table.auditRunId, table.id)] +); diff --git a/src/server/jobs/copyReconciliation.ts b/src/server/jobs/copyReconciliation.ts index 3a184d6..62423b9 100644 --- a/src/server/jobs/copyReconciliation.ts +++ b/src/server/jobs/copyReconciliation.ts @@ -1,5 +1,12 @@ -import { sql, type SQL } from "drizzle-orm"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { and, eq, inArray, sql, type SQL } from "drizzle-orm"; +import { nowIso, type Db } from "../db/database"; import * as schema from "../db/schema"; +import { withFilesystemTimeout } from "../lib/filesystemSafety"; +import { copyFileIdentitiesMatch, parseCopyFileIdentity, readCopyFileIdentity } from "../lib/copier"; +import type { CopyReconciliationRecord, CopyReconciliationState } from "../../shared/types"; +import { schedulerLockKey } from "./scheduling"; export function unresolvedCopyReconciliation(): SQL { return sql` @@ -15,3 +22,145 @@ export function unresolvedCopyReconciliation(): SQL { ) `; } + +type CopyOperation = typeof schema.copyOperations.$inferSelect; +type PathPresence = "exists" | "missing" | "unknown"; +type LinkState = { kind: "symlink"; target: string } | { kind: "missing" | "other" | "unknown" }; + +async function pathPresence(filePath: string | null, label: string): Promise { + if (!filePath) return "missing"; + try { + await withFilesystemTimeout(fs.lstat(filePath), label); + return "exists"; + } catch (error) { + if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") return "missing"; + return "unknown"; + } +} + +async function linkState(linkPath: string): Promise { + try { + const stat = await withFilesystemTimeout(fs.lstat(linkPath), `Copy reconciliation link inspection for ${linkPath}`); + if (!stat.isSymbolicLink()) return { kind: "other" }; + const target = await withFilesystemTimeout(fs.readlink(linkPath), `Copy reconciliation target inspection for ${linkPath}`); + return { kind: "symlink", target: path.resolve(path.dirname(linkPath), target) }; + } catch (error) { + if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") return { kind: "missing" }; + return { kind: "unknown" }; + } +} + +async function destinationMatchesJournal(operation: CopyOperation): Promise { + if (!operation.destinationIdentity) return false; + try { + const expected = parseCopyFileIdentity(operation.destinationIdentity); + const actual = await withFilesystemTimeout( + readCopyFileIdentity(operation.destinationPath), + `Copy reconciliation destination identity inspection for ${operation.destinationPath}` + ); + return copyFileIdentitiesMatch(actual, expected); + } catch { + return false; + } +} + +type ProvenResolution = { stage: "committed" | "rolled_back" | "failed"; resultStatus?: "copied" | "repointed"; message?: string }; + +async function provableResolution(operation: CopyOperation): Promise { + const [link, destination, temporary, displaced] = await Promise.all([ + linkState(operation.linkPath), + pathPresence(operation.destinationPath, `Copy reconciliation destination inspection for ${operation.destinationPath}`), + pathPresence(operation.tempPath, `Copy reconciliation temporary-file inspection for operation #${operation.id}`), + pathPresence(operation.displacedPath, `Copy reconciliation displaced-file inspection for operation #${operation.id}`) + ]); + if (link.kind === "unknown" || destination === "unknown" || temporary === "unknown" || displaced === "unknown") return null; + if (temporary === "exists" || displaced === "exists" || link.kind === "other") return null; + + if (link.kind === "symlink") { + if (link.target === path.resolve(operation.originalTargetPath)) { + if (destination === "missing") return { stage: "rolled_back" }; + return { + stage: "failed", + message: "Automatically closed after recheck: the original symlink is intact and no temporary or displaced journal artifacts remain; the existing destination was left untouched for normal conflict handling." + }; + } + if (link.target === path.resolve(operation.destinationPath) && destination === "exists") { + if (!(await destinationMatchesJournal(operation))) return null; + return { + stage: "committed", + resultStatus: operation.resultStatus === "repointed" || operation.sourcePath === operation.destinationPath ? "repointed" : "copied" + }; + } + if (destination === "missing") { + return { stage: "failed", message: "Automatically closed after recheck: the symlink moved elsewhere and no journaled copy artifacts remain." }; + } + return null; + } + + if (link.kind === "missing" && destination === "missing") return { stage: "rolled_back" }; + return null; +} + +function serializeOperation(operation: CopyOperation): CopyReconciliationRecord { + return { + id: operation.id, + jobId: operation.jobId, + mediaLinkId: operation.mediaLinkId, + linkPath: operation.linkPath, + errorMessage: operation.errorMessage, + updatedAt: operation.updatedAt + }; +} + +export async function listCopyReconciliation(db: Db): Promise { + const rows = await db.select().from(schema.copyOperations).where(unresolvedCopyReconciliation()).orderBy(schema.copyOperations.id); + return rows.map(serializeOperation); +} + +export async function reconcileProvablySettledCopyOperations(db: Db): Promise { + const operations = await db.select().from(schema.copyOperations).where(unresolvedCopyReconciliation()).orderBy(schema.copyOperations.id); + if (operations.length === 0) return { unresolved: [], unresolvedCount: 0, resolvedNow: 0 }; + const jobIds = [...new Set(operations.map((operation) => operation.jobId))]; + const jobs = await db.select({ id: schema.jobs.id, status: schema.jobs.status }).from(schema.jobs).where(inArray(schema.jobs.id, jobIds)); + const terminalJobIds = new Set(jobs.filter((job) => !["queued", "running"].includes(job.status)).map((job) => job.id)); + const candidates = operations.filter((operation) => terminalJobIds.has(operation.jobId)); + const resolutions = new Map(); + let nextIndex = 0; + await Promise.all( + Array.from({ length: Math.min(8, candidates.length) }, async () => { + while (nextIndex < candidates.length) { + const operation = candidates[nextIndex]; + nextIndex += 1; + if (!operation) continue; + const resolution = await provableResolution(operation); + if (resolution) resolutions.set(operation.id, resolution); + } + }) + ); + + if (resolutions.size > 0) { + await db.transaction(async (transaction) => { + await transaction.execute(sql`select pg_advisory_xact_lock(${schedulerLockKey})`); + const timestamp = nowIso(); + for (const [operationId, resolution] of resolutions) { + await transaction + .update(schema.copyOperations) + .set({ + stage: resolution.stage, + ...(resolution.resultStatus ? { resultStatus: resolution.resultStatus } : {}), + tempPath: null, + displacedPath: null, + tempIdentity: null, + displacedIdentity: null, + errorMessage: resolution.message ?? null, + reconciliationResolvedAt: timestamp, + updatedAt: timestamp, + completedAt: timestamp + }) + .where(and(eq(schema.copyOperations.id, operationId), eq(schema.copyOperations.stage, "reconciliation_required"))); + } + }); + } + const unresolved = await listCopyReconciliation(db); + return { unresolved, unresolvedCount: unresolved.length, resolvedNow: operations.length - unresolved.length }; +} diff --git a/src/server/jobs/jobRunner.ts b/src/server/jobs/jobRunner.ts index 4026661..224b309 100644 --- a/src/server/jobs/jobRunner.ts +++ b/src/server/jobs/jobRunner.ts @@ -3,7 +3,7 @@ import type { Dirent } from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; import type { JobConcurrencySettings } from "../config"; -import { dbGet, first, getJsonSetting, getSectionSettings, nowIso, type Db, type DbExecutor } from "../db/database"; +import { dbAll, dbGet, first, getJsonSetting, getSectionSettings, nowIso, type Db, type DbExecutor } from "../db/database"; import * as schema from "../db/schema"; import { auditMediaLink, defaultAuditRunner, type AuditCommandRunner } from "../lib/auditor"; import { @@ -28,7 +28,7 @@ import { normalizeAdvancedSettings } from "../../shared/advancedSettings"; import { evaluateSourceTitleRisk } from "../../shared/sourceTitleRisk"; import { CopyTransferLimiter } from "./copyLimiter"; import { runKeyedPool } from "./copyPool"; -import { unresolvedCopyReconciliation } from "./copyReconciliation"; +import { listCopyReconciliation, reconcileProvablySettledCopyOperations, unresolvedCopyReconciliation } from "./copyReconciliation"; import { schedulerLockKey } from "./scheduling"; import type { AuditMode, @@ -38,11 +38,14 @@ import type { CopyLocalConflict, CopyLocalConflictCandidate, CopyLocalConflictStrategy, + CopyJobBehaviorSettings, CopyOptions, InventorySummary, JobEventPage, JobEventRecord, JobRecord, + CopyReconciliationState, + JobSelectionSummary, JobStatus, MediaLinkRow, PathsSettings, @@ -53,7 +56,10 @@ import type { } from "../../shared/types"; type JobRow = typeof schema.jobs.$inferSelect; +type StoredCopyOptions = CopyOptions & { behavior?: CopyJobBehaviorSettings }; type CopyProgressStage = CopyProgressUpdate["stage"] | "queued" | "done" | "skipped" | "conflict" | "partially_failed" | "failed" | "completed" | "cancelled"; +const maxSelectionTitlesPerJob = 100; +const maxAdvisorySelectionLinkIds = 1_000; class WorkerShutdownError extends Error { constructor() { @@ -96,6 +102,9 @@ interface ResourceClaim { interface PreparedJob { progress: unknown; + options?: unknown; + selection?: MediaLinkRow[]; + selectionFrozen?: boolean; exclusive: boolean; claims: ResourceClaim[]; } @@ -241,12 +250,34 @@ function isRecord(value: unknown): value is Record { return value !== null && typeof value === "object" && !Array.isArray(value); } +function compactJobProgress(progress: unknown): unknown { + if (!isRecord(progress) || !("options" in progress)) return progress; + const { options: _options, ...compact } = progress; + return compact; +} + +function progressOptions(progress: unknown): unknown { + return isRecord(progress) && "options" in progress ? progress.options : {}; +} + +function compactFrozenOptions(options: unknown): unknown { + if (!isRecord(options) || !("linkIds" in options)) return options; + const { linkIds: _linkIds, ...compact } = options; + return compact; +} + +function progressWithOptions(progress: unknown, options: unknown): unknown { + if (!isRecord(progress) || !isRecord(options) || Object.keys(options).length === 0) return progress; + return { ...progress, options }; +} + function finiteNumberFromRecord(record: Record, key: string): number { const value = record[key]; return typeof value === "number" && Number.isFinite(value) ? value : 0; } function jobProgressOptions(job: JobRecord): T | null { + if (isRecord(job.options) && Object.keys(job.options).length > 0) return job.options as T; if (!isRecord(job.progress)) return null; return "options" in job.progress ? (job.progress.options as T) : null; } @@ -305,7 +336,7 @@ async function normalizeAuditOptions(db: Db, input: AuditMode | AuditOptions): P }; } -function readAuditOptions(job: JobRecord): AuditOptions { +function readAuditOptions(job: JobRecord, frozenLinkIds?: number[]): AuditOptions { const options = jobProgressOptions(job); if (!options || (options.mode !== "fast" && options.mode !== "deep")) { throw new Error("Audit job is missing valid options"); @@ -314,7 +345,11 @@ function readAuditOptions(job: JobRecord): AuditOptions { mode: options.mode, ...(Array.isArray(options.sections) ? { sections: options.sections.filter((section) => typeof section === "string" && section.trim()).map((section) => section.trim()) } : {}), ...(Array.isArray(options.targets) ? { targets: normalizeAuditTargets(options.targets) } : {}), - ...(Array.isArray(options.linkIds) ? { linkIds: options.linkIds.filter((id) => Number.isInteger(id) && id > 0) } : {}), + ...(frozenLinkIds !== undefined + ? { linkIds: frozenLinkIds } + : Array.isArray(options.linkIds) + ? { linkIds: options.linkIds.filter((id) => Number.isInteger(id) && id > 0) } + : {}), ...(typeof options.section === "string" && options.section.trim() ? { section: options.section.trim() } : {}), ...(typeof options.itemName === "string" && options.itemName.trim() ? { itemName: options.itemName.trim() } : {}), ...(typeof options.relativePathPrefix === "string" && options.relativePathPrefix.trim() ? { relativePathPrefix: normalizeRelativePrefix(options.relativePathPrefix) } : {}), @@ -349,27 +384,31 @@ async function normalizeCopyOptions(db: Db, options: CopyOptions): Promise(job); +function readCopyOptions(job: JobRecord, frozenLinkIds?: number[]): StoredCopyOptions { + const options = jobProgressOptions(job); if (!options) throw new Error("Copy job is missing options"); - return normalizeCopyOptionsFromProgress(options); + return normalizeCopyOptionsFromProgress({ ...options, ...(frozenLinkIds !== undefined ? { linkIds: frozenLinkIds } : {}) }); } -function normalizeCopyOptionsFromProgress(options: CopyOptions): CopyOptions { +function normalizeCopyOptionsFromProgress(options: StoredCopyOptions): StoredCopyOptions { if (options.direction !== "to_local" && options.direction !== "to_remote") throw new Error("Copy job has invalid direction"); + const behavior = options.behavior ? normalizeAdvancedSettings({ copy: options.behavior }).copy : undefined; return { direction: options.direction, ...(Array.isArray(options.linkIds) ? { linkIds: options.linkIds.filter((id) => Number.isInteger(id) && id > 0) } : {}), ...(typeof options.section === "string" && options.section.trim() ? { section: options.section.trim() } : {}), ...(typeof options.itemName === "string" && options.itemName.trim() ? { itemName: options.itemName.trim() } : {}), ...(typeof options.relativePathPrefix === "string" && options.relativePathPrefix.trim() ? { relativePathPrefix: normalizeRelativePrefix(options.relativePathPrefix) } : {}), - ...(options.localConflictStrategy === "keep_both" || options.localConflictStrategy === "replace" ? { localConflictStrategy: options.localConflictStrategy } : {}) + ...(options.localConflictStrategy === "keep_both" || options.localConflictStrategy === "replace" ? { localConflictStrategy: options.localConflictStrategy } : {}), + ...(options.allowSourceTitleMismatch === true ? { allowSourceTitleMismatch: true } : {}), + ...(behavior ? { behavior } : {}) }; } @@ -489,6 +528,10 @@ function copyAdmissionFingerprint(link: MediaLinkRow): string { ]); } +export function copyAdmissionSelectionFingerprint(links: readonly MediaLinkRow[]): string { + return JSON.stringify(links.map(copyAdmissionFingerprint).sort()); +} + function resourceClaimKey(claim: Pick): string { return `${claim.resourceType}\0${claim.resourceKey}`; } @@ -1006,19 +1049,28 @@ async function copyLocalConflictForLink( } async function previewCopyConflicts(db: Db, paths: PathsSettings, options: CopyOptions): Promise { - if (options.direction !== "to_local") return { conflicts: [], totalConflicts: 0, totalCandidates: 0 }; const selectedLinks = orderedCopySelection(await listMediaLinks(db), options); const links = filterCopyLinks(selectedLinks, { ...options, linkIds: selectedLinks.map((link) => link.id) }); + const sourceTitleRisks = links.flatMap((link) => { + const risk = evaluateSourceTitleRisk({ expectedTitle: link.itemName, sourcePath: link.targetPath }); + return risk.severity === "block" + ? [{ linkId: link.id, itemName: link.itemName, relativePath: link.relativePath, sourcePath: link.targetPath, reason: risk.reason }] + : []; + }); const selectedDestinations = await copySelectedDestinationsForLinks(selectedLinks, paths, options.direction); const conflicts: CopyLocalConflict[] = []; - for (const link of links) { - const conflict = await copyLocalConflictForLink(db, link, paths, selectedDestinations); - if (conflict) conflicts.push(conflict); + if (options.direction === "to_local") { + for (const link of links) { + const conflict = await copyLocalConflictForLink(db, link, paths, selectedDestinations); + if (conflict) conflicts.push(conflict); + } } return { conflicts, totalConflicts: conflicts.length, - totalCandidates: conflicts.reduce((total, conflict) => total + conflict.candidates.length, 0) + totalCandidates: conflicts.reduce((total, conflict) => total + conflict.candidates.length, 0), + sourceTitleRisks, + totalSourceTitleBlocks: sourceTitleRisks.length }; } @@ -1823,18 +1875,108 @@ async function readCopyResumeState(db: Db, jobId: number, selectedLinks: MediaLi function toJobRecord(row: JobRow): LeasedJob { const progress = parseJson(row.progress); - const normalizedProgress = normalizeJobProgress(row.type, row.status, progress); + const options = parseJson(row.options); + const normalizedProgress = normalizeJobProgress(row.type, row.status, progressWithOptions(progress, options)); const status = normalizeJobStatus(row.type, row.status, progress); return { ...row, type: row.type as JobRecord["type"], status, + options, + selectionFrozen: row.selectionFrozen, progress: normalizedProgress, leaseVersion: row.leaseVersion, exclusive: row.exclusive }; } +function legacySelectionCount(job: JobRecord): number { + const options = jobProgressOptions<{ linkIds?: unknown }>(job); + return Array.isArray(options?.linkIds) ? options.linkIds.length : 0; +} + +async function attachJobSelections(db: DbExecutor, jobs: JobRecord[]): Promise { + const selectedJobs = jobs.filter((job) => job.selectionFrozen); + if (selectedJobs.length === 0) return jobs; + const jobIds = selectedJobs.map((job) => job.id); + const titleRows = await dbAll<{ jobId: number; section: string; itemName: string; count: number; titleCount: number; selectionCount: number }>(db as Db, sql` + WITH selected_job_ids AS ( + SELECT value::integer AS job_id + FROM jsonb_array_elements_text(${JSON.stringify(jobIds)}::jsonb) + ), title_counts AS ( + SELECT items.job_id, + items.section, + items.item_name, + count(*)::integer AS count + FROM job_selection_items AS items + JOIN selected_job_ids ON selected_job_ids.job_id = items.job_id + GROUP BY items.job_id, items.section, items.item_name + ), ranked_titles AS ( + SELECT title_counts.*, + count(*) OVER (PARTITION BY title_counts.job_id)::integer AS title_count, + sum(title_counts.count) OVER (PARTITION BY title_counts.job_id)::integer AS selection_count, + row_number() OVER (PARTITION BY title_counts.job_id ORDER BY title_counts.item_name, title_counts.section) AS title_order + FROM title_counts + ) + SELECT ranked_titles.job_id AS "jobId", + ranked_titles.section, + ranked_titles.item_name AS "itemName", + ranked_titles.count, + ranked_titles.title_count AS "titleCount", + ranked_titles.selection_count AS "selectionCount" + FROM ranked_titles + WHERE ranked_titles.title_order <= ${maxSelectionTitlesPerJob} + ORDER BY ranked_titles.job_id, ranked_titles.title_order + `); + const activeJobIds = selectedJobs.filter((job) => job.status === "queued" || job.status === "running").map((job) => job.id); + const activeLinkRows = activeJobIds.length === 0 + ? [] + : await dbAll<{ jobId: number; mediaLinkId: number; selectionOrder: number }>(db as Db, sql` + WITH active_job_ids AS ( + SELECT value::integer AS job_id + FROM jsonb_array_elements_text(${JSON.stringify(activeJobIds)}::jsonb) + ), eligible_jobs AS ( + SELECT items.job_id + FROM job_selection_items AS items + JOIN active_job_ids ON active_job_ids.job_id = items.job_id + GROUP BY items.job_id + HAVING count(*) <= ${maxAdvisorySelectionLinkIds} + ) + SELECT items.job_id AS "jobId", + items.media_link_id AS "mediaLinkId", + items.selection_order AS "selectionOrder" + FROM job_selection_items AS items + JOIN eligible_jobs ON eligible_jobs.job_id = items.job_id + ORDER BY items.job_id, items.selection_order + `); + const summaryByJobId = new Map(); + const titleCountByJobId = new Map(); + const selectionCountByJobId = new Map(); + for (const job of selectedJobs) { + summaryByJobId.set(job.id, { total: legacySelectionCount(job), titles: [], unavailable: 0 }); + } + for (const row of titleRows) { + const summary = summaryByJobId.get(row.jobId); + if (!summary) continue; + summary.titles.push({ section: row.section, itemName: row.itemName, count: Number(row.count) }); + titleCountByJobId.set(row.jobId, Number(row.titleCount)); + selectionCountByJobId.set(row.jobId, Number(row.selectionCount)); + } + for (const [jobId, summary] of summaryByJobId) { + const snapshotCount = selectionCountByJobId.get(jobId) ?? 0; + summary.total = Math.max(summary.total, snapshotCount); + summary.unavailable = Math.max(0, summary.total - snapshotCount); + const omittedTitles = Math.max(0, (titleCountByJobId.get(jobId) ?? summary.titles.length) - summary.titles.length); + if (omittedTitles > 0) summary.omittedTitles = omittedTitles; + } + for (const row of activeLinkRows) { + const summary = summaryByJobId.get(row.jobId); + if (!summary) continue; + (summary.linkIds ??= []).push(row.mediaLinkId); + } + return jobs.map((job) => ({ ...job, ...(summaryByJobId.has(job.id) ? { selection: summaryByJobId.get(job.id) } : {}) })); +} + function normalizeJobStatus(type: string, status: string, progress: unknown): JobStatus { if (!isPartiallyFailedCopyProgress(type, status, progress)) return status as JobStatus; return "partially_failed"; @@ -1875,10 +2017,11 @@ export class JobRunner { } private async enqueueJob(type: JobRecord["type"], progress: unknown, exclusive: boolean, requestedClaims: ResourceClaim[]): Promise { - return this.enqueuePreparedJob(type, async () => ({ progress, exclusive, claims: requestedClaims })); + return this.enqueuePreparedJob(type, async () => ({ progress, options: progressOptions(progress), exclusive, claims: requestedClaims })); } private async enqueuePreparedJob(type: JobRecord["type"], prepare: (db: DbExecutor) => Promise): Promise { + await reconcileProvablySettledCopyOperations(this.db); if (type !== "path_migration" && (await isPathConfigurationBlocked(this.db))) { throw new Error("Managed storage paths changed. Resolve the required path migration before starting another job."); } @@ -1953,6 +2096,9 @@ export class JobRunner { } const timestamp = nowIso(); + const selection = prepared.selection ?? []; + const selectionFrozen = prepared.selectionFrozen === true; + const immutableOptions = selectionFrozen ? compactFrozenOptions(prepared.options ?? progressOptions(prepared.progress)) : (prepared.options ?? progressOptions(prepared.progress)); const row = await first( transaction .insert(schema.jobs) @@ -1967,12 +2113,28 @@ export class JobRunner { heartbeatAt: null, leaseVersion: 0, exclusive: prepared.exclusive, + options: JSON.stringify(immutableOptions), + selectionFrozen, cancelRequestedAt: null, - progress: JSON.stringify(prepared.progress) + progress: JSON.stringify(compactJobProgress(prepared.progress)) }) .returning({ id: schema.jobs.id }) ); if (!row) throw new Error("Job was not queued"); + for (let offset = 0; offset < selection.length; offset += 500) { + await transaction.insert(schema.jobSelectionItems).values( + selection.slice(offset, offset + 500).map((link, index) => ({ + jobId: row.id, + mediaLinkId: link.id, + selectionOrder: offset + index, + section: link.section, + itemName: link.itemName, + relativePath: link.relativePath, + linkPath: link.linkPath, + createdAt: timestamp + })) + ); + } for (let offset = 0; offset < claims.length; offset += 500) { await transaction.insert(schema.jobResourceClaims).values( claims.slice(offset, offset + 500).map((claim) => ({ @@ -1994,7 +2156,7 @@ export class JobRunner { const activeStatuses: JobStatus[] = ["queued", "running"]; const terminalStatuses: JobStatus[] = ["completed", "partially_failed", "failed", "cancelled"]; const activeRows = await this.db.select().from(schema.jobs).where(inArray(schema.jobs.status, activeStatuses)).orderBy(desc(schema.jobs.id)); - if (options.activeOnly) return activeRows.map(toJobRecord); + if (options.activeOnly) return attachJobSelections(this.db, activeRows.map(toJobRecord)); const terminalRows = options.completedSince ? await this.db @@ -2009,12 +2171,22 @@ export class JobRunner { .where(inArray(schema.jobs.status, terminalStatuses)) .orderBy(desc(schema.jobs.id)) .limit(limit); - return [...activeRows, ...terminalRows].sort((a, b) => b.id - a.id).map(toJobRecord); + return attachJobSelections(this.db, [...activeRows, ...terminalRows].sort((a, b) => b.id - a.id).map(toJobRecord)); } async getJob(jobId: number): Promise { const row = await first(this.db.select().from(schema.jobs).where(eq(schema.jobs.id, jobId)).limit(1)); - return row ? toJobRecord(row) : null; + if (!row) return null; + return (await attachJobSelections(this.db, [toJobRecord(row)]))[0] ?? null; + } + + async copyReconciliationState(): Promise { + const unresolved = await listCopyReconciliation(this.db); + return { unresolved, unresolvedCount: unresolved.length, resolvedNow: 0 }; + } + + async recheckCopyReconciliation(): Promise { + return reconcileProvablySettledCopyOperations(this.db); } async listEvents(jobId: number, afterId = 0, limit = 100): Promise { @@ -2118,6 +2290,7 @@ export class JobRunner { } return { progress: { options: normalizedOptions }, + options: normalizedOptions, exclusive: false, claims: await titleScanResourceClaims(normalizedOptions, scanLinks, paths) }; @@ -2141,6 +2314,9 @@ export class JobRunner { const frozenOptions = { ...optionsWithDefaults, linkIds: auditLinks.map((link) => link.id) }; return { progress: { options: frozenOptions }, + options: frozenOptions, + selection: auditLinks, + selectionFrozen: true, exclusive: false, claims: await auditResourceClaims(auditLinks, paths) }; @@ -2153,19 +2329,21 @@ export class JobRunner { const orderedSelectedLinks = orderedCopySelection(links, normalizedOptions); const claimedLinks = normalizedOptions.linkIds === undefined ? orderedSelectedLinks : filterCopySelectedLinks(links, normalizedOptions); const optionsWithResolvedLinks = { ...normalizedOptions, linkIds: orderedSelectedLinks.map((link) => link.id) }; + const advancedSettings = normalizeAdvancedSettings(await getJsonSetting(this.db, "advancedSettings", {})); + const storedOptions: StoredCopyOptions = { ...optionsWithResolvedLinks, behavior: advancedSettings.copy }; const paths = await getJsonSetting(this.db, "paths", { symlinkDir: "", localDir: "", remoteDir: "" }); if (!paths.localDir || !paths.remoteDir) throw new Error("Path settings are incomplete"); const replacementClaims = await copyReplacementResourceClaims(this.db, orderedSelectedLinks, paths, optionsWithResolvedLinks); - const expectedSelection = orderedSelectedLinks.map(copyAdmissionFingerprint); - const expectedClaimedSelection = claimedLinks.map(copyAdmissionFingerprint); + const expectedSelection = copyAdmissionSelectionFingerprint(orderedSelectedLinks); + const expectedClaimedSelection = copyAdmissionSelectionFingerprint(claimedLinks); return this.enqueuePreparedJob("copy", async (transaction) => { const currentLinks = await listMediaLinks(transaction, undefined, "current"); const currentSelection = orderedCopySelection(currentLinks, normalizedOptions); const currentClaimedSelection = normalizedOptions.linkIds === undefined ? currentSelection : filterCopySelectedLinks(currentLinks, normalizedOptions); const currentPaths = await getJsonSetting(transaction, "paths", { symlinkDir: "", localDir: "", remoteDir: "" }); if ( - JSON.stringify(currentSelection.map(copyAdmissionFingerprint)) !== JSON.stringify(expectedSelection) || - JSON.stringify(currentClaimedSelection.map(copyAdmissionFingerprint)) !== JSON.stringify(expectedClaimedSelection) || + copyAdmissionSelectionFingerprint(currentSelection) !== expectedSelection || + copyAdmissionSelectionFingerprint(currentClaimedSelection) !== expectedClaimedSelection || currentPaths.symlinkDir !== paths.symlinkDir || currentPaths.localDir !== paths.localDir || currentPaths.remoteDir !== paths.remoteDir @@ -2173,7 +2351,10 @@ export class JobRunner { throw new Error("Copy selection changed while the job was being prepared. Review the current inventory and queue it again."); } return { - progress: { options: optionsWithResolvedLinks }, + progress: { options: storedOptions }, + options: storedOptions, + selection: orderedSelectedLinks, + selectionFrozen: true, exclusive: false, claims: [...(await copyResourceClaims(orderedSelectedLinks, claimedLinks, paths, normalizedOptions)), ...replacementClaims] }; @@ -2786,17 +2967,32 @@ export class JobWorker { } private async runHandler(job: JobRecord, ctx: JobContext): Promise { + const frozenLinkIds = job.selectionFrozen + ? (() => { + const legacyIds = jobProgressOptions<{ linkIds?: unknown }>(job)?.linkIds; + return Array.isArray(legacyIds) && legacyIds.every((id) => Number.isInteger(id) && Number(id) > 0) + ? legacyIds.map(Number) + : null; + })() ?? + ( + await this.db + .select({ mediaLinkId: schema.jobSelectionItems.mediaLinkId }) + .from(schema.jobSelectionItems) + .where(eq(schema.jobSelectionItems.jobId, job.id)) + .orderBy(asc(schema.jobSelectionItems.selectionOrder)) + ).map((item) => item.mediaLinkId) + : undefined; if (job.type === "scan") { const options = jobProgressOptions(job) ?? defaultScanOptions; await this.runScanJob(job.id, await normalizeScanOptions(this.db, { ...defaultScanOptions, ...options }), ctx); return; } if (job.type === "audit") { - await this.runAuditJob(job.id, readAuditOptions(job), ctx); + await this.runAuditJob(job.id, readAuditOptions(job, frozenLinkIds), ctx); return; } if (job.type === "copy") { - await this.runCopyJob(readCopyOptions(job), ctx); + await this.runCopyJob(readCopyOptions(job, frozenLinkIds), ctx); return; } if (job.type === "path_migration") { @@ -2871,7 +3067,7 @@ export class JobWorker { const row = await first( this.db .update(schema.jobs) - .set({ progress: JSON.stringify(progress) }) + .set({ progress: JSON.stringify(compactJobProgress(progress)) }) .where( and( eq(schema.jobs.id, job.id), @@ -3125,7 +3321,7 @@ export class JobWorker { await completeOnboardingScan(leaseDb, jobId); await leaseDb .update(schema.jobs) - .set({ progress: JSON.stringify(scanProgressPayload(normalizedOptions, "completed", completionMessage, inventory)) }) + .set({ progress: JSON.stringify(compactJobProgress(scanProgressPayload(normalizedOptions, "completed", completionMessage, inventory))) }) .where(eq(schema.jobs.id, jobId)); await leaseDb.insert(schema.jobEvents).values({ jobId, @@ -3225,7 +3421,7 @@ export class JobWorker { sections: normalizedOptions.sections, section: normalizedOptions.section, itemName: normalizedOptions.itemName, - linkIds: normalizedOptions.linkIds, + selectedLinkCount: normalizedOptions.linkIds?.length, relativePathPrefix: normalizedOptions.relativePathPrefix }); const sourceLookup = await this.db.select().from(schema.copySources); @@ -3301,7 +3497,7 @@ export class JobWorker { .update(schema.auditRuns) .set({ status: "completed", finishedAt: nowIso(), checked, passed, failed, sourceUnknown, sourceMissing, sourceCompareErrors, byteMismatches, targetValidationFailures, errorMessage: null }) .where(eq(schema.auditRuns.id, auditRun.id)); - await leaseDb.update(schema.jobs).set({ progress: JSON.stringify(completedProgress) }).where(eq(schema.jobs.id, jobId)); + await leaseDb.update(schema.jobs).set({ progress: JSON.stringify(compactJobProgress(completedProgress)) }).where(eq(schema.jobs.id, jobId)); await leaseDb.insert(schema.jobEvents).values({ jobId, timestamp: nowIso(), @@ -3338,7 +3534,7 @@ export class JobWorker { } } - private async runCopyJob(normalizedOptions: CopyOptions, ctx: JobContext): Promise { + private async runCopyJob(normalizedOptions: StoredCopyOptions, ctx: JobContext): Promise { const paths = await getJsonSetting(this.db, "paths", { symlinkDir: "", localDir: "", remoteDir: "" }); if (!paths.symlinkDir || !paths.localDir || !paths.remoteDir) throw new Error("Path settings are incomplete"); const durableClaims = await this.db.select().from(schema.jobResourceClaims).where(eq(schema.jobResourceClaims.jobId, ctx.jobId)); @@ -3376,7 +3572,7 @@ export class JobWorker { return expected?.lexicalPath === binding.lexicalPath && expected.canonicalPath === binding.canonicalPath; }); }; - const advancedSettings = normalizeAdvancedSettings(await getJsonSetting(this.db, "advancedSettings", {})); + const copyBehavior = normalizedOptions.behavior ?? normalizeAdvancedSettings(await getJsonSetting(this.db, "advancedSettings", {})).copy; await ctx.assertLease(); // Path-blocked copy jobs are admitted only to reconcile and roll back their // durable journal. Mark the context cancelled before replaying that journal. @@ -3418,8 +3614,6 @@ export class JobWorker { let failed = unavailable; const resumedCopied = copied; const resumedRepointed = repointed; - let activeLink: MediaLinkRow | undefined; - let activeUpdate: Partial | undefined; const replacementCandidateEntries = new Map< number, { linkId: number; destinationPath: string; candidates: CopyLocalConflictCandidate[]; expectedIdentities: Map } @@ -3488,7 +3682,7 @@ export class JobWorker { skipped, unavailable, alreadyCompleted, - copyBehavior: advancedSettings.copy + copyBehavior }); if (links.length === 0) { if (failed > 0) { @@ -3513,9 +3707,7 @@ export class JobWorker { let cancellationReported = false; const processLink = async (link: MediaLinkRow): Promise => { if (await ctx.isCancelled()) return; - activeLink = link; let linkUpdate: Partial | undefined; - activeUpdate = linkUpdate; let activeOperationId: number | null = null; let filesystemMutationCompleted = false; current += 1; @@ -3528,14 +3720,13 @@ export class JobWorker { return; } const sourceTitleRisk = evaluateSourceTitleRisk({ expectedTitle: link.itemName, sourcePath: link.targetPath }); - if (sourceTitleRisk.severity === "block") { + if (sourceTitleRisk.severity === "block" && !normalizedOptions.allowSourceTitleMismatch) { conflicts += 1; - activeUpdate = { + linkUpdate = { sourcePath: link.targetPath, linkPath: link.linkPath, sizeBytes: link.sizeBytes ?? undefined }; - linkUpdate = activeUpdate; await setCopyProgress("conflict", "Source title mismatch blocked copy", link, linkUpdate); await ctx.event("warn", "Source title mismatch blocked copy", { direction: normalizedOptions.direction, @@ -3546,6 +3737,15 @@ export class JobWorker { }); return; } + if (sourceTitleRisk.severity === "block" && normalizedOptions.allowSourceTitleMismatch) { + await ctx.event("warn", "Source title mismatch override accepted", { + direction: normalizedOptions.direction, + itemName: link.itemName, + linkPath: link.linkPath, + sourcePath: link.targetPath, + risk: sourceTitleRisk + }); + } if (sourceTitleRisk.severity === "warn") { await ctx.event("warn", "Source title risk warning", { direction: normalizedOptions.direction, @@ -3579,13 +3779,12 @@ export class JobWorker { } if (unclaimedCandidates.length > 0) { conflicts += 1; - activeUpdate = { + linkUpdate = { sourcePath: link.targetPath, destinationPath: localConflict.destinationPath, linkPath: link.linkPath, sizeBytes: link.sizeBytes ?? undefined }; - linkUpdate = activeUpdate; await setCopyProgress("conflict", "Local replacement candidates changed after copy admission; queue the copy again", link, linkUpdate); await ctx.event("warn", "Local replacement candidates changed after copy admission", { ...localConflict, @@ -3596,13 +3795,12 @@ export class JobWorker { } if (localConflict && !normalizedOptions.localConflictStrategy) { conflicts += 1; - activeUpdate = { + linkUpdate = { sourcePath: link.targetPath, destinationPath: localConflict.destinationPath, linkPath: link.linkPath, sizeBytes: link.sizeBytes ?? undefined }; - linkUpdate = activeUpdate; await setCopyProgress("conflict", "Existing local file requires copy resolution", link, linkUpdate); await ctx.event("warn", "Existing local file requires copy resolution", localConflict); return; @@ -3630,7 +3828,6 @@ export class JobWorker { this.copyRunner, async (update) => { linkUpdate = update; - activeUpdate = linkUpdate; await setCopyProgress(update.stage, update.message, link, linkUpdate); if (update.stage === "copying" || (update.stage === "preparing" && !/retry/i.test(update.message))) return; const progressEventKey = `${update.stage}:${update.message}`; @@ -3657,7 +3854,7 @@ export class JobWorker { }) ); }, - advancedSettings.copy, + copyBehavior, ctx.signal, normalizedOptions.localConflictStrategy, (update) => ctx.withLeaseDb((leaseDb) => updateCopyOperation(leaseDb, operation.id, update)), @@ -3671,9 +3868,8 @@ export class JobWorker { await ctx.withLeaseDb((leaseDb) => commitCopyOperation(leaseDb, operation.id, link, result)); copied += 1; linkUpdate = { ...(linkUpdate ?? {}), ...result }; - activeUpdate = linkUpdate; await setCopyProgress("done", result.message, link, linkUpdate); - await ctx.event("info", advancedSettings.copy.profile === "off" ? "Copy installed without verification" : "Verified copy installed", { ...result, itemName: link.itemName }); + await ctx.event("info", copyBehavior.profile === "off" ? "Copy installed without verification" : "Verified copy installed", { ...result, itemName: link.itemName }); if (normalizedOptions.localConflictStrategy === "replace" && localConflict) { replacementCandidateEntries.set(link.id, { linkId: link.id, @@ -3686,7 +3882,6 @@ export class JobWorker { await ctx.withLeaseDb((leaseDb) => commitCopyOperation(leaseDb, operation.id, link, result)); repointed += 1; linkUpdate = { ...(linkUpdate ?? {}), ...result }; - activeUpdate = linkUpdate; await setCopyProgress("done", result.message, link, linkUpdate); await ctx.event("info", "Symlink repointed to existing verified file", { ...result, itemName: link.itemName }); if (normalizedOptions.localConflictStrategy === "replace" && localConflict) { @@ -3701,14 +3896,12 @@ export class JobWorker { conflicts += 1; await ctx.withLeaseDb((leaseDb) => completeCopyOperationWithoutMutation(leaseDb, operation.id, result)); linkUpdate = { ...(linkUpdate ?? {}), ...result }; - activeUpdate = linkUpdate; await setCopyProgress("conflict", result.message, link, linkUpdate); await ctx.event("warn", "Destination conflict; file was not overwritten", result); } else { skipped += 1; await ctx.withLeaseDb((leaseDb) => completeCopyOperationWithoutMutation(leaseDb, operation.id, result)); linkUpdate = { ...(linkUpdate ?? {}), ...result }; - activeUpdate = linkUpdate; await setCopyProgress("skipped", result.message, link, linkUpdate); await ctx.event("info", "Copy skipped", result); } @@ -3764,7 +3957,7 @@ export class JobWorker { throw (ctx.signal.reason instanceof Error ? ctx.signal.reason : new WorkerShutdownError()); } const rollbackCancelledCopy = async () => { - await setCopyProgress("cancelled", "Rolling back completed copy changes", activeLink, activeUpdate); + await setCopyProgress("cancelled", "Rolling back completed copy changes"); const { rolledBack, warnings } = await rollbackDurableCopyOperations(this.db, ctx.jobId, paths, ctx); copied = resumedCopied; repointed = resumedRepointed; @@ -3779,7 +3972,7 @@ export class JobWorker { const completed = copied + repointed + skipped + conflicts; const partialFailure = completed > 0; const failureMessage = partialFailure ? `Copy job partially failed: ${failed} of ${total} ${itemLabel} failed` : `Copy job failed: ${failed} of ${total} ${itemLabel} failed`; - await setCopyProgress(partialFailure ? "partially_failed" : "failed", failureMessage, activeLink, activeUpdate); + await setCopyProgress(partialFailure ? "partially_failed" : "failed", failureMessage); await ctx.event(partialFailure ? "warn" : "error", partialFailure ? "Copy job partially failed processing media" : "Copy job failed processing media", { total, copied, @@ -3794,9 +3987,9 @@ export class JobWorker { } if (!cancelled) { if (replacementCandidateEntries.size > 0) { - await setCopyProgress("symlinking", "Finalizing previous local-file replacements", activeLink, activeUpdate); + await setCopyProgress("symlinking", "Finalizing previous local-file replacements"); } - await setCopyProgress("completed", links.length === 0 && alreadyCompleted === 0 ? "No matching media found" : "Copy job finished", activeLink, activeUpdate); + await setCopyProgress("completed", links.length === 0 && alreadyCompleted === 0 ? "No matching media found" : "Copy job finished"); await ctx.event("info", "Copy job finished processing media", { total, copied, repointed, skipped, conflicts, failed, unavailable }); const finalized = await ctx.finishCompletedIsolated(async (leaseDb) => { const finalizationWarnings: string[] = []; @@ -3857,7 +4050,7 @@ export class JobWorker { if (finalized) return; await rollbackCancelledCopy(); } - await setCopyProgress("cancelled", "Copy job terminated", activeLink, activeUpdate); + await setCopyProgress("cancelled", "Copy job terminated"); await ctx.event("warn", "Copy job terminated", { total, copied, repointed, skipped, conflicts, failed, unavailable }); } } diff --git a/src/server/jobs/resourceMutationGuard.ts b/src/server/jobs/resourceMutationGuard.ts index 90233ae..16ae420 100644 --- a/src/server/jobs/resourceMutationGuard.ts +++ b/src/server/jobs/resourceMutationGuard.ts @@ -2,7 +2,7 @@ import { sql } from "drizzle-orm"; import type { Db } from "../db/database"; import * as schema from "../db/schema"; import { canonicalTitleKey } from "../lib/storagePolicies"; -import { unresolvedCopyReconciliation } from "./copyReconciliation"; +import { reconcileProvablySettledCopyOperations, unresolvedCopyReconciliation } from "./copyReconciliation"; import { schedulerLockKey } from "./scheduling"; export interface MutationResource { @@ -38,6 +38,15 @@ export class ActiveJobResourceConflictError extends Error { } } +export class ActiveJobConfigurationConflictError extends Error { + readonly statusCode = 409; + + constructor(job: ActiveJobConflict) { + super(`Library folders cannot be changed while ${job.type} job #${job.jobId} is ${job.status}. Wait for it to finish or terminate it first.`); + this.name = "ActiveJobConfigurationConflictError"; + } +} + function normalizeResources(resources: MutationResource[]): MutationResource[] { const unique = new Map(); for (const resource of resources) unique.set(`${resource.resourceType}\0${resource.resourceKey}`, resource); @@ -66,6 +75,7 @@ export async function withResourceMutationGuard( db: Db, prepare: (transaction: Db) => Promise> ): Promise { + await reconcileProvablySettledCopyOperations(db); return db.transaction(async (transaction) => { await transaction.execute(sql`select pg_advisory_xact_lock(${schedulerLockKey})`); const prepared = await prepare(transaction); @@ -116,3 +126,20 @@ export async function withResourceMutationGuard( return prepared.mutate(); }); } + +export async function withQueueConfigurationGuard(db: Db, mutate: (transaction: Db) => Promise): Promise { + return db.transaction(async (transaction) => { + await transaction.execute(sql`select pg_advisory_xact_lock(${schedulerLockKey})`); + const activeJob = ( + await transaction.execute(sql` + select id as "jobId", type, status + from jobs + where status in ('queued', 'running') + order by id + limit 1 + `) + ).rows[0]; + if (activeJob) throw new ActiveJobConfigurationConflictError(activeJob); + return mutate(transaction); + }); +} diff --git a/src/server/jobs/scheduling.ts b/src/server/jobs/scheduling.ts index 0860d1e..0f633bd 100644 --- a/src/server/jobs/scheduling.ts +++ b/src/server/jobs/scheduling.ts @@ -1,3 +1,4 @@ // Serializes queue admission, worker claims, and path-configuration barriers. // Keep this value stable so every process coordinates on the same advisory lock. export const schedulerLockKey = 1_672_148_903; +export const workerProcessLockKey = 1_672_148_904; diff --git a/src/server/lib/env.ts b/src/server/lib/env.ts index 81fa474..efb34b8 100644 --- a/src/server/lib/env.ts +++ b/src/server/lib/env.ts @@ -21,6 +21,7 @@ export interface EnvSettings { SRTL_MAX_RUNNING_COPIES?: string; SRTL_COPY_FILE_CONCURRENCY?: string; SRTL_MAX_ACTIVE_COPY_FILES?: string; + SRTL_JOB_HISTORY_RETENTION_DAYS?: string; SRTL_ALLOWED_ORIGINS?: string; SRTL_COOKIE_SECURE?: string; SRTL_API_DOCS?: string; @@ -50,6 +51,7 @@ const supportedKeys = [ "SRTL_MAX_RUNNING_COPIES", "SRTL_COPY_FILE_CONCURRENCY", "SRTL_MAX_ACTIVE_COPY_FILES", + "SRTL_JOB_HISTORY_RETENTION_DAYS", "SRTL_ALLOWED_ORIGINS", "SRTL_COOKIE_SECURE", "SRTL_API_DOCS", diff --git a/src/server/lib/historyRetention.ts b/src/server/lib/historyRetention.ts new file mode 100644 index 0000000..45f393d --- /dev/null +++ b/src/server/lib/historyRetention.ts @@ -0,0 +1,55 @@ +import { sql } from "drizzle-orm"; +import { dbAll, type Db } from "../db/database"; + +const retentionBatchSize = 500; + +export async function pruneTerminalJobHistory(db: Db, retentionDays: number, nowMs = Date.now()): Promise { + if (retentionDays === 0) return 0; + const cutoff = new Date(nowMs - retentionDays * 24 * 60 * 60 * 1_000).toISOString(); + let removed = 0; + + while (true) { + const rows = await dbAll<{ id: number }>(db, sql` + WITH expired_jobs AS ( + SELECT jobs.id + FROM jobs + WHERE jobs.status IN ('completed', 'partially_failed', 'failed', 'cancelled') + AND jobs.type <> 'path_migration' + AND jobs.finished_at IS NOT NULL + AND jobs.finished_at < ${cutoff} + AND NOT EXISTS ( + SELECT 1 + FROM copy_operations AS recovery_operation + WHERE recovery_operation.job_id = jobs.id + AND recovery_operation.stage = 'reconciliation_required' + AND NOT EXISTS ( + SELECT 1 + FROM copy_operations AS superseding_operation + WHERE superseding_operation.id > recovery_operation.id + AND superseding_operation.media_link_id = recovery_operation.media_link_id + AND superseding_operation.link_path = recovery_operation.link_path + AND superseding_operation.stage = 'committed' + AND superseding_operation.result_status IN ('copied', 'repointed') + ) + ) + ORDER BY jobs.id + LIMIT ${retentionBatchSize} + ) + DELETE FROM jobs + USING expired_jobs + WHERE jobs.id = expired_jobs.id + RETURNING jobs.id + `); + removed += rows.length; + if (rows.length < retentionBatchSize) return removed; + } +} + +export async function pruneExpiredSessions(db: Db, now = new Date().toISOString()): Promise { + const rows = await dbAll<{ tokenHash: string }>(db, sql` + DELETE FROM sessions + WHERE expires_at < ${now} + RETURNING token_hash AS "tokenHash" + `); + return rows.length; +} diff --git a/src/server/lib/pathConfiguration.ts b/src/server/lib/pathConfiguration.ts index 05d0efe..1a8fc79 100644 --- a/src/server/lib/pathConfiguration.ts +++ b/src/server/lib/pathConfiguration.ts @@ -504,7 +504,7 @@ async function markUncertainLegacyFailedCopyOperations(db: DbExecutor): Promise< await db .select() .from(schema.copyOperations) - .where(eq(schema.copyOperations.stage, "failed")) + .where(and(eq(schema.copyOperations.stage, "failed"), isNull(schema.copyOperations.reconciliationResolvedAt))) ).filter( (operation) => operation.tempIdentity == null && diff --git a/src/server/lib/scanner.ts b/src/server/lib/scanner.ts index 8225f1a..e37ae4c 100644 --- a/src/server/lib/scanner.ts +++ b/src/server/lib/scanner.ts @@ -722,10 +722,12 @@ export async function persistScanResult(db: Db, result: ScanResult, jobId: numbe let missingRemoteFiles = 0; if (result.options.scanLocal) scannedStorageRootTypes.add("local"); if (result.options.scanRemote) scannedStorageRootTypes.add("remote"); + const existingStorageFiles = await db.select().from(schema.storageFiles); + const existingStorageFileByPath = new Map(existingStorageFiles.map((file) => [file.filePath, file])); for (const file of filesToReconcile) { await throwIfPersistenceCancelled(isCancelled); - const existing = await first(db.select().from(schema.storageFiles).where(eq(schema.storageFiles.filePath, file.filePath)).limit(1)); + const existing = existingStorageFileByPath.get(file.filePath); const persistedFile: ClassifiedStorageFile = { ...file, storagePolicy: normalizeStoragePolicy(existing?.storagePolicy) }; const firstSeenAt = existing?.firstSeenAt ?? timestamp; const lastChangedAt = storageFileChanged(existing, persistedFile) ? timestamp : existing?.lastChangedAt ?? timestamp; @@ -738,7 +740,7 @@ export async function persistScanResult(db: Db, result: ScanResult, jobId: numbe }); } - for (const file of await db.select().from(schema.storageFiles)) { + for (const file of existingStorageFiles) { await throwIfPersistenceCancelled(isCancelled); const fileSection = firstRelativePathPart(file.relativePath); const scannedFileSection = file.rootType === "remote" || !scannedLocalSections || scannedLocalSections.has(fileSection); @@ -758,11 +760,13 @@ export async function persistScanResult(db: Db, result: ScanResult, jobId: numbe if (result.options.scanSymlinks) { const currentStorageFiles = (await db.select().from(schema.storageFiles)).filter((file) => !file.missingSince); const storageFileIdByPath = new Map(currentStorageFiles.map((file) => [file.filePath, file.id])); + const existingMediaLinks = await db.select().from(schema.mediaLinks); + const existingMediaLinkByPath = new Map(existingMediaLinks.map((link) => [link.linkPath, link])); for (const link of result.links) { await throwIfPersistenceCancelled(isCancelled); const resolvedStorageFileId = storageFileIdByPath.get(link.targetPath) ?? null; - const existing = await first(db.select().from(schema.mediaLinks).where(eq(schema.mediaLinks.linkPath, link.linkPath)).limit(1)); + const existing = existingMediaLinkByPath.get(link.linkPath); const firstSeenAt = existing?.firstSeenAt ?? existing?.updatedAt ?? timestamp; const lastChangedAt = linkChanged(existing, link, resolvedStorageFileId) ? timestamp : existing?.lastChangedAt ?? existing?.updatedAt ?? timestamp; const values = { @@ -793,7 +797,7 @@ export async function persistScanResult(db: Db, result: ScanResult, jobId: numbe }); } - for (const link of await db.select().from(schema.mediaLinks)) { + for (const link of existingMediaLinks) { await throwIfPersistenceCancelled(isCancelled); const scannedLinkScope = scannedSymlinkTitles ? scannedSymlinkTitles.has(scanTitleScopeKey(link.section, link.itemName)) @@ -976,12 +980,9 @@ function mediaLinkFilters(options: MediaLinkListFilters) { } export async function listMediaLinks(db: Db, kind?: LinkKind, status: MediaLinkStatusFilter = "current", filters: Pick = {}): Promise { - return (await db.select().from(schema.mediaLinks)) - .filter((row) => !kind || row.kind === kind) - .filter((row) => !filters.section || row.section === filters.section) - .filter((row) => !filters.storagePolicy || normalizeStoragePolicy(row.storagePolicy) === filters.storagePolicy) - .filter((row) => status === "all" || (status === "current" ? !row.missingSince : Boolean(row.missingSince))) - .map(serializeMediaLink); + const where = mediaLinkFilters({ kind, status, ...filters }); + const rows = where ? await db.select().from(schema.mediaLinks).where(where) : await db.select().from(schema.mediaLinks); + return rows.map(serializeMediaLink); } export async function listMediaLinksByIds(db: Db, ids: number[]): Promise { @@ -1459,76 +1460,75 @@ async function listStorageFileTreeRows( }; } -async function rawCount(db: Db, query: SQL): Promise { - const row = await dbGet<{ value: number }>(db, query); - return Number(row?.value ?? 0); -} - export async function getInventorySummary(db: Db): Promise { + const [links, files] = await Promise.all([ + dbGet>(db, sql` + select + count(*) filter (where missing_since is null) as "totalLinks", + count(*) filter (where missing_since is null and kind = 'remote') as "remoteLinks", + count(*) filter (where missing_since is null and kind = 'local') as "localLinks", + count(*) filter (where missing_since is null and kind = 'broken') as "brokenLinks", + count(*) filter (where missing_since is null and kind = 'other') as "otherLinks", + count(*) filter (where missing_since is null and kind = 'non_media') as "nonMediaLinks", + count(*) filter (where missing_since is null and kind = 'remote' and storage_policy = 'location_1') as "actionableRemoteLinks", + count(*) filter (where missing_since is null and kind = 'local' and storage_policy = 'location_2') as "actionableLocalLinks", + count(*) filter (where missing_since is null and kind = 'remote' and storage_policy = 'location_2') as "assignedRemoteLinks", + count(*) filter (where missing_since is null and kind = 'remote' and storage_policy = 'unassigned') as "unassignedRemoteLinks", + count(*) filter (where missing_since is null and kind = 'local' and storage_policy = 'unassigned') as "unassignedLocalLinks", + count(*) filter (where missing_since is not null) as "missingLinks" + from media_links + `), + dbGet>(db, sql` + with file_state as ( + select + sf.root_type, + sf.missing_since, + not exists ( + select 1 + from media_links ml + where ml.missing_since is null + and ml.resolved_storage_file_id = sf.id + ) as orphaned + from storage_files sf + ) + select + count(*) filter (where missing_since is null and root_type = 'local') as "localFiles", + count(*) filter (where missing_since is null and root_type = 'remote') as "remoteFiles", + count(*) filter (where missing_since is null and root_type = 'remote' and orphaned) as "unassignedRemoteFiles", + count(*) filter (where missing_since is null and root_type = 'local' and orphaned) as "unassignedLocalFiles", + count(*) filter (where missing_since is null and root_type = 'local' and orphaned) as "localOrphanFiles", + count(*) filter (where missing_since is null and root_type = 'remote' and orphaned) as "remoteOrphanFiles", + count(*) filter (where missing_since is not null and root_type = 'local') as "missingLocalFiles", + count(*) filter (where missing_since is not null and root_type = 'remote') as "missingRemoteFiles" + from file_state + `) + ]); + const linkCount = (key: string) => Number(links?.[key] ?? 0); + const fileCount = (key: string) => Number(files?.[key] ?? 0); return { - totalLinks: await rawCount(db, sql`select count(*) as value from media_links where missing_since is null`), - remoteLinks: await rawCount(db, sql`select count(*) as value from media_links where missing_since is null and kind = 'remote'`), - localLinks: await rawCount(db, sql`select count(*) as value from media_links where missing_since is null and kind = 'local'`), - brokenLinks: await rawCount(db, sql`select count(*) as value from media_links where missing_since is null and kind = 'broken'`), - otherLinks: await rawCount(db, sql`select count(*) as value from media_links where missing_since is null and kind = 'other'`), - nonMediaLinks: await rawCount(db, sql`select count(*) as value from media_links where missing_since is null and kind = 'non_media'`), - actionableRemoteLinks: await rawCount(db, sql`select count(*) as value from media_links where missing_since is null and kind = 'remote' and storage_policy = 'location_1'`), - actionableLocalLinks: await rawCount(db, sql`select count(*) as value from media_links where missing_since is null and kind = 'local' and storage_policy = 'location_2'`), - assignedRemoteLinks: await rawCount(db, sql`select count(*) as value from media_links where missing_since is null and kind = 'remote' and storage_policy = 'location_2'`), - unassignedRemoteLinks: await rawCount(db, sql`select count(*) as value from media_links where missing_since is null and kind = 'remote' and storage_policy = 'unassigned'`), - unassignedLocalLinks: await rawCount(db, sql`select count(*) as value from media_links where missing_since is null and kind = 'local' and storage_policy = 'unassigned'`), - localFiles: await rawCount(db, sql`select count(*) as value from storage_files where missing_since is null and root_type = 'local'`), - remoteFiles: await rawCount(db, sql`select count(*) as value from storage_files where missing_since is null and root_type = 'remote'`), + totalLinks: linkCount("totalLinks"), + remoteLinks: linkCount("remoteLinks"), + localLinks: linkCount("localLinks"), + brokenLinks: linkCount("brokenLinks"), + otherLinks: linkCount("otherLinks"), + nonMediaLinks: linkCount("nonMediaLinks"), + actionableRemoteLinks: linkCount("actionableRemoteLinks"), + actionableLocalLinks: linkCount("actionableLocalLinks"), + assignedRemoteLinks: linkCount("assignedRemoteLinks"), + unassignedRemoteLinks: linkCount("unassignedRemoteLinks"), + unassignedLocalLinks: linkCount("unassignedLocalLinks"), + localFiles: fileCount("localFiles"), + remoteFiles: fileCount("remoteFiles"), actionableRemoteFiles: 0, actionableLocalFiles: 0, assignedRemoteFiles: 0, - unassignedRemoteFiles: await rawCount( - db, - sql`select count(*) as value - from storage_files sf - where sf.missing_since is null - and sf.root_type = 'remote' - and not exists ( - select 1 from media_links ml - where ml.missing_since is null and ml.resolved_storage_file_id = sf.id - )` - ), - unassignedLocalFiles: await rawCount( - db, - sql`select count(*) as value - from storage_files sf - where sf.missing_since is null - and sf.root_type = 'local' - and not exists ( - select 1 from media_links ml - where ml.missing_since is null and ml.resolved_storage_file_id = sf.id - )` - ), - localOrphanFiles: await rawCount( - db, - sql`select count(*) as value - from storage_files sf - where sf.missing_since is null - and sf.root_type = 'local' - and not exists ( - select 1 from media_links ml - where ml.missing_since is null and ml.resolved_storage_file_id = sf.id - )` - ), - remoteOrphanFiles: await rawCount( - db, - sql`select count(*) as value - from storage_files sf - where sf.missing_since is null - and sf.root_type = 'remote' - and not exists ( - select 1 from media_links ml - where ml.missing_since is null and ml.resolved_storage_file_id = sf.id - )` - ), - missingLinks: await rawCount(db, sql`select count(*) as value from media_links where missing_since is not null`), - missingLocalFiles: await rawCount(db, sql`select count(*) as value from storage_files where missing_since is not null and root_type = 'local'`), - missingRemoteFiles: await rawCount(db, sql`select count(*) as value from storage_files where missing_since is not null and root_type = 'remote'`) + unassignedRemoteFiles: fileCount("unassignedRemoteFiles"), + unassignedLocalFiles: fileCount("unassignedLocalFiles"), + localOrphanFiles: fileCount("localOrphanFiles"), + remoteOrphanFiles: fileCount("remoteOrphanFiles"), + missingLinks: linkCount("missingLinks"), + missingLocalFiles: fileCount("missingLocalFiles"), + missingRemoteFiles: fileCount("missingRemoteFiles") }; } @@ -1541,6 +1541,7 @@ type CompletedScanTimestampRow = { startedAt: string; finishedAt: string | null; progress: string; + options: string; }; function recordFromUnknown(value: unknown): Record | null { @@ -1560,10 +1561,10 @@ function scanTitleScopesFromUnknown(value: unknown): ScanTitleScope[] | undefine .map((scope) => ({ section: String(scope.section), itemName: String(scope.itemName) })); } -function scanOptionsFromProgress(progress: string): Partial | null { +function scanOptionsFromProgress(progress: string, optionsJson: string): Partial | null { try { const parsed = recordFromUnknown(JSON.parse(progress)); - const options = recordFromUnknown(parsed?.options); + const options = recordFromUnknown(JSON.parse(optionsJson)) ?? recordFromUnknown(parsed?.options); if (!options) return null; return { scanSymlinks: options.scanSymlinks === true, @@ -1598,7 +1599,7 @@ export async function getInventoryScanTimestamps(db: Db): Promise(db, sql` - select sr.started_at as "startedAt", sr.finished_at as "finishedAt", j.progress as progress + select sr.started_at as "startedAt", sr.finished_at as "finishedAt", j.progress as progress, j.options as options from scan_runs sr join jobs j on j.id = sr.job_id where sr.status = 'completed' @@ -1606,7 +1607,7 @@ export async function getInventoryScanTimestamps(db: Db): Promise { } function serializeAuditRun(run: typeof schema.auditRuns.$inferSelect, jobById: Map): AuditRunRecord { + const job = jobById.get(run.jobId); return { ...run, mode: run.mode as AuditRunRecord["mode"], status: run.status as JobStatus, - options: auditOptionsFromProgress(jobById.get(run.jobId)?.progress ?? "") + options: auditOptionsFromJob(job?.progress ?? "", job?.options ?? "") }; } @@ -63,7 +68,7 @@ async function findAuditRunByJobId(db: Db, jobId: number): Promise { + app.post("/api/audits", { bodyLimit: largeSelectionBodyLimitBytes }, async (request, reply) => { const body = auditOptionsSchema.parse(request.body); const configuredSections = (await getSectionSettings(db)).sections; let sections: string[] | undefined; @@ -103,8 +108,28 @@ export function registerAuditRoutes(app: FastifyInstance, db: Db, jobs: JobRunne return row; }); - app.get("/api/audits/:id/results", async (request) => { - const params = z.object({ id: z.coerce.number() }).parse(request.params); - return db.select().from(schema.auditResults).where(eq(schema.auditResults.auditRunId, params.id)).orderBy(desc(schema.auditResults.id)); + app.get("/api/audits/:id/results/page", async (request): Promise => { + const params = z.object({ id: z.coerce.number().int().positive() }).parse(request.params); + const query = z + .object({ + limit: z.coerce.number().int().min(1).max(500).default(100), + offset: z.coerce.number().int().min(0).default(0), + attentionOnly: z.string().transform((value) => value === "true").default(false) + }) + .parse(request.query); + const where = query.attentionOnly + ? and(eq(schema.auditResults.auditRunId, params.id), ne(schema.auditResults.status, "pass")) + : eq(schema.auditResults.auditRunId, params.id); + const [results, totalRow] = await Promise.all([ + db.select().from(schema.auditResults).where(where).orderBy(desc(schema.auditResults.id)).limit(query.limit).offset(query.offset), + first(db.select({ value: count() }).from(schema.auditResults).where(where).limit(1)) + ]); + const total = Number(totalRow?.value ?? 0); + return { + results: results as AuditResultPage["results"], + total, + offset: query.offset, + hasMore: query.offset + results.length < total + }; }); } diff --git a/src/server/routes/jobRoutes.ts b/src/server/routes/jobRoutes.ts index fae52db..cbd977a 100644 --- a/src/server/routes/jobRoutes.ts +++ b/src/server/routes/jobRoutes.ts @@ -22,6 +22,10 @@ export function registerJobRoutes(app: FastifyInstance, jobs: JobRunner): void { return row; }); + app.get("/api/job-reconciliation", async () => jobs.copyReconciliationState()); + + app.post("/api/job-reconciliation/recheck", async () => jobs.recheckCopyReconciliation()); + app.post("/api/jobs/:id/terminate", async (request, reply) => { const params = z.object({ id: z.coerce.number() }).parse(request.params); if (!(await jobs.terminate(params.id))) return reply.code(409).send({ error: "Job cannot be terminated" }); @@ -34,35 +38,10 @@ export function registerJobRoutes(app: FastifyInstance, jobs: JobRunner): void { return { ok: true, jobId: params.id }; }); - app.get("/api/jobs/:id/events", async (request, reply) => { + app.get("/api/jobs/:id/events", async (request) => { const params = z.object({ id: z.coerce.number() }).parse(request.params); - const query = z.object({ afterId: z.coerce.number().int().min(0).default(0), limit: z.coerce.number().int().min(1).max(500).default(100), stream: z.coerce.boolean().default(false) }).parse(request.query); - if (!query.stream) return jobs.listEvents(params.id, query.afterId, query.limit); - - reply.raw.writeHead(200, { - "Content-Type": "text/event-stream", - "Cache-Control": "no-cache", - Connection: "keep-alive" - }); - - let lastId = query.afterId; - const sendNewEvents = async () => { - for (const event of await jobs.listEvents(params.id, lastId, query.limit)) { - lastId = Math.max(lastId, event.id); - reply.raw.write(`id: ${event.id}\n`); - reply.raw.write(`event: ${event.level}\n`); - reply.raw.write(`data: ${JSON.stringify(event)}\n\n`); - } - }; - - await sendNewEvents(); - const interval = setInterval(() => { - void sendNewEvents().catch((error: unknown) => { - reply.raw.write(`event: error\n`); - reply.raw.write(`data: ${JSON.stringify({ error: error instanceof Error ? error.message : String(error) })}\n\n`); - }); - }, 1000); - request.raw.on("close", () => clearInterval(interval)); + const query = z.object({ afterId: z.coerce.number().int().min(0).default(0), limit: z.coerce.number().int().min(1).max(500).default(100) }).parse(request.query); + return jobs.listEvents(params.id, query.afterId, query.limit); }); app.get("/api/jobs/:id/events/page", async (request) => { diff --git a/src/server/routes/libraryRoutes.ts b/src/server/routes/libraryRoutes.ts index be5cb7d..8d2bee7 100644 --- a/src/server/routes/libraryRoutes.ts +++ b/src/server/routes/libraryRoutes.ts @@ -42,6 +42,8 @@ const scanOptionsSchema = z }); const storagePolicySchema = z.enum(["unassigned", "location_1", "location_2"]); +const maxBulkSelectionItems = 100_000; +const largeSelectionBodyLimitBytes = 4 * 1024 * 1024; const storagePolicyInputSchema = z.object({ title: z.string().trim().min(1, "Title is required").max(500, "Title is too long"), @@ -77,11 +79,12 @@ function mediaLinkIdsFromMutationResources(resources: Array<{ resourceType: stri const copyInputSchema = z .object({ direction: z.enum(["to_local", "to_remote"]), - linkIds: z.array(z.coerce.number().int().positive()).max(1000).optional(), + linkIds: z.array(z.coerce.number().int().positive()).max(maxBulkSelectionItems).optional(), section: z.string().trim().min(1).max(200).optional(), itemName: z.string().trim().min(1).max(500).optional(), relativePathPrefix: z.string().trim().min(1).max(2000).optional(), - localConflictStrategy: z.enum(["keep_both", "replace"]).optional() + localConflictStrategy: z.enum(["keep_both", "replace"]).optional(), + allowSourceTitleMismatch: z.boolean().optional() }) .refine((value) => Boolean(value.linkIds?.length || value.section || value.itemName), { message: "Copy requires link IDs, a folder scope, or a title" }); @@ -122,10 +125,11 @@ async function latestErrorMessages(db: Db, jobIds: Set): Promise { const orphanedFailedJobs = scanJobs.filter((job) => terminalFailureStatuses.has(job.status as JobStatus) && !recordedJobIds.has(job.id)); const errorMessages = await latestErrorMessages(db, new Set(orphanedFailedJobs.map((job) => job.id))); const zeroTotals = emptyScanTotals(); - const recordedRuns: ScanRunRecord[] = scanRuns.map((run) => ({ ...run, status: run.status as JobStatus, options: scanOptionsFromProgress(jobById.get(run.jobId)?.progress ?? "") })); + const recordedRuns: ScanRunRecord[] = scanRuns.map((run) => { + const job = jobById.get(run.jobId); + return { ...run, status: run.status as JobStatus, options: scanOptionsFromJob(job?.progress ?? "", job?.options ?? "") }; + }); const legacyRuns: ScanRunRecord[] = orphanedFailedJobs.map((job) => ({ id: null, jobId: job.id, @@ -150,7 +157,7 @@ async function listScanHistory(db: Db): Promise { startedAt: job.startedAt ?? job.createdAt, finishedAt: job.finishedAt, errorMessage: errorMessages.get(job.id) ?? (job.status === "failed" ? "Scan failed before a history row was created." : null), - options: scanOptionsFromProgress(job.progress), + options: scanOptionsFromJob(job.progress, job.options), ...zeroTotals })); return [...recordedRuns, ...legacyRuns].sort((a, b) => b.jobId - a.jobId).slice(0, 25); @@ -274,7 +281,7 @@ export function registerLibraryRoutes(app: FastifyInstance, db: Db, jobs: JobRun app.get("/api/inventory/scan-timestamps", async () => getInventoryScanTimestamps(db)); - app.post("/api/copies", async (request, reply) => { + app.post("/api/copies", { bodyLimit: largeSelectionBodyLimitBytes }, async (request, reply) => { const body = copyInputSchema.parse(request.body); try { return { jobId: await jobs.startCopy(body) }; @@ -284,7 +291,7 @@ export function registerLibraryRoutes(app: FastifyInstance, db: Db, jobs: JobRun } }); - app.post("/api/copies/conflicts", async (request) => { + app.post("/api/copies/conflicts", { bodyLimit: largeSelectionBodyLimitBytes }, async (request) => { const body = copyInputSchema.parse(request.body); return jobs.previewCopyConflicts(body); }); diff --git a/src/server/routes/settingsRoutes.ts b/src/server/routes/settingsRoutes.ts index 60ab7bc..79b01c2 100644 --- a/src/server/routes/settingsRoutes.ts +++ b/src/server/routes/settingsRoutes.ts @@ -2,6 +2,7 @@ import { z } from "zod"; import type { FastifyInstance } from "fastify"; import { getJsonSetting, getSectionSettings, setSetting, type Db } from "../db/database"; import { parseSectionSettings, persistSectionSettings } from "../lib/sectionSettings"; +import { withQueueConfigurationGuard } from "../jobs/resourceMutationGuard"; import { getStorageLocationsSettings, persistStorageLocationNames, @@ -106,7 +107,7 @@ export function registerSettingsRoutes(app: FastifyInstance, db: Db): void { app.put("/api/settings/sections", async (request) => { const body: SectionSettings = parseSectionSettings(request.body); - await persistSectionSettings(db, body); + await withQueueConfigurationGuard(db, async (transaction) => persistSectionSettings(transaction, body)); return body; }); @@ -150,9 +151,4 @@ export function registerSettingsRoutes(app: FastifyInstance, db: Db): void { return body; }); - app.get("/api/settings/integrations", async () => []); - - app.put("/api/settings/integrations", async (_request, reply) => - reply.code(501).send({ error: "External integrations are not available in this release" }) - ); } diff --git a/src/server/worker.ts b/src/server/worker.ts index 5d2a7a3..701f864 100644 --- a/src/server/worker.ts +++ b/src/server/worker.ts @@ -4,7 +4,9 @@ import { loadConfig } from "./config"; import { nowIso, openDatabase } from "./db/database"; import { CopyTransferLimiter } from "./jobs/copyLimiter"; import { JobWorker } from "./jobs/jobRunner"; +import { workerProcessLockKey } from "./jobs/scheduling"; import { reconcileEnvironmentPaths } from "./lib/pathConfiguration"; +import { pruneExpiredSessions, pruneTerminalJobHistory } from "./lib/historyRetention"; import { pruneWorkerHeartbeatHistory, recordWorkerHeartbeat } from "./lib/workerHeartbeats"; // Media outputs must remain readable by services running under a different account. @@ -12,8 +14,29 @@ process.umask(0o022); const config = loadConfig(); const database = await openDatabase({ databaseUrl: config.databaseUrl, migrate: config.autoMigrate }); -await pruneWorkerHeartbeatHistory(database.db); -await reconcileEnvironmentPaths(database.db, config.paths); +const workerProcessLock = await database.pool.connect(); +let workerProcessLockAcquired = false; +async function runDataMaintenance(): Promise { + await pruneWorkerHeartbeatHistory(database.db); + await pruneExpiredSessions(database.db); + await pruneTerminalJobHistory(database.db, config.jobHistoryRetentionDays); +} +try { + const workerLockResult = await workerProcessLock.query<{ acquired: boolean }>("select pg_try_advisory_lock($1) as acquired", [workerProcessLockKey]); + workerProcessLockAcquired = workerLockResult.rows[0]?.acquired === true; + if (!workerProcessLockAcquired) { + throw new Error("Another SRTL worker process is already active. Scale safe job slots with SRTL_WORKER_COUNT inside one worker container; do not scale the worker service."); + } + await runDataMaintenance(); + await reconcileEnvironmentPaths(database.db, config.paths); +} catch (error) { + if (workerProcessLockAcquired) { + await workerProcessLock.query("select pg_advisory_unlock($1)", [workerProcessLockKey]).catch(() => undefined); + } + workerProcessLock.release(); + await database.close(); + throw error; +} const workerBaseId = process.env.SRTL_WORKER_ID?.trim() || `${os.hostname()}-${process.pid}`; const bootId = randomUUID(); @@ -49,6 +72,8 @@ process.once("SIGTERM", shutdown); let heartbeatTimer: NodeJS.Timeout | null = null; let heartbeatRun: Promise | null = null; +let maintenanceTimer: NodeJS.Timeout | null = null; +let maintenanceRun: Promise | null = null; let workerRun: Promise | null = null; try { await recordHeartbeat("running"); @@ -66,16 +91,31 @@ try { }); }, 5_000); heartbeatTimer.unref(); + maintenanceTimer = setInterval(() => { + if (shuttingDown || maintenanceRun) return; + maintenanceRun = runDataMaintenance() + .catch((error: unknown) => { + console.error("Worker data maintenance failed", error); + }) + .finally(() => { + maintenanceRun = null; + }); + }, 6 * 60 * 60 * 1_000); + maintenanceTimer.unref(); workerRun = worker.start(); await workerRun; } } finally { shutdown(); if (heartbeatTimer) clearInterval(heartbeatTimer); + if (maintenanceTimer) clearInterval(maintenanceTimer); if (workerRun) await Promise.allSettled([workerRun]); await heartbeatRun; + await maintenanceRun; await recordHeartbeat("stopped").catch((error: unknown) => { console.error("Unable to record stopped worker status", error); }); + await workerProcessLock.query("select pg_advisory_unlock($1)", [workerProcessLockKey]).catch(() => undefined); + workerProcessLock.release(); await database.close(); } diff --git a/src/shared/types.ts b/src/shared/types.ts index 5ed4a07..99d5b67 100644 --- a/src/shared/types.ts +++ b/src/shared/types.ts @@ -256,6 +256,14 @@ export interface CopyConflictPreview { conflicts: CopyLocalConflict[]; totalConflicts: number; totalCandidates: number; + sourceTitleRisks?: Array<{ + linkId: number; + itemName: string; + relativePath: string; + sourcePath: string; + reason: string; + }>; + totalSourceTitleBlocks?: number; } export interface CopyOptions { @@ -265,6 +273,7 @@ export interface CopyOptions { itemName?: string; relativePathPrefix?: string; localConflictStrategy?: CopyLocalConflictStrategy; + allowSourceTitleMismatch?: boolean; } export interface SectionSummary { @@ -497,9 +506,45 @@ export interface JobRecord { lockedAt?: string | null; heartbeatAt?: string | null; cancelRequestedAt?: string | null; + /** Immutable admission options. Legacy records may expose these through progress.options instead. */ + options?: unknown; + /** True when the job owns an immutable selected-media snapshot, including an intentionally empty selection. */ + selectionFrozen?: boolean; + selection?: JobSelectionSummary; progress: unknown; } +export interface JobSelectionTitle { + section: string; + itemName: string; + count: number; +} + +export interface JobSelectionSummary { + total: number; + titles: JobSelectionTitle[]; + unavailable: number; + /** Additional title groups omitted from the bounded job-list payload. */ + omittedTitles?: number; + /** Present only for active jobs so the client can show advisory per-item locks. */ + linkIds?: number[]; +} + +export interface CopyReconciliationRecord { + id: number; + jobId: number; + mediaLinkId: number; + linkPath: string; + errorMessage: string | null; + updatedAt: string; +} + +export interface CopyReconciliationState { + unresolved: CopyReconciliationRecord[]; + unresolvedCount: number; + resolvedNow: number; +} + export interface ScanRunRecord { id: number | null; jobId: number; @@ -565,6 +610,13 @@ export interface AuditResultRecord { createdAt: string; } +export interface AuditResultPage { + results: AuditResultRecord[]; + total: number; + offset: number; + hasMore: boolean; +} + export interface JobEventRecord { id: number; jobId: number; diff --git a/tests/app.test.ts b/tests/app.test.ts index cc79353..8ef4dd5 100644 --- a/tests/app.test.ts +++ b/tests/app.test.ts @@ -460,11 +460,6 @@ describe("api app", () => { const setupPaths = await ctx.app.inject({ method: "GET", url: "/api/settings/paths", headers: { cookie: String(setupCookie) } }); expect(setupPaths.statusCode).toBe(200); - const integrationPlaceholders = await ctx.app.inject({ method: "GET", url: "/api/settings/integrations", headers: { cookie: String(setupCookie) } }); - expect(integrationPlaceholders.json()).toEqual([]); - const disabledIntegrationSave = await ctx.app.inject({ method: "PUT", url: "/api/settings/integrations", headers: { cookie: String(setupCookie) }, payload: [] }); - expect(disabledIntegrationSave.statusCode).toBe(501); - const defaultScanSettings = await ctx.app.inject({ method: "GET", url: "/api/settings/scan", headers: { cookie: String(setupCookie) } }); expect(defaultScanSettings.statusCode).toBe(200); expect(defaultScanSettings.json()).toEqual({ scanSymlinks: true, scanLocal: false, scanRemote: false, symlinkSections: ["movies", "shows"], localSections: ["movies", "shows"] }); @@ -712,6 +707,18 @@ describe("api app", () => { expect(paths.json()).toMatchObject({ localDir: path.join(tmpDir, "local"), remoteDir: path.join(tmpDir, "remote") }); }); + it("fails loudly when a stored JSON setting is corrupt", async () => { + await ctx.database.db.insert(schema.appSettings).values({ + key: "corruptSetting", + value: "{not-json", + updatedAt: new Date().toISOString() + }); + + await expect(getJsonSetting(ctx.database.db, "corruptSetting", { fallback: true })).rejects.toThrow( + 'Stored setting "corruptSetting" contains invalid JSON' + ); + }); + it("bounds authentication input sizes before password hashing", async () => { const oversizedUsername = await ctx.app.inject({ method: "POST", @@ -731,6 +738,31 @@ describe("api app", () => { expect(response.json()).toMatchObject({ error: "Password must be 256 characters or fewer" }); }); + it("rejects malformed stored password digests without throwing", async () => { + await createAdminSession(); + await ctx.database.db.update(schema.adminUsers).set({ passwordHash: "scrypt$valid-salt$invalid" }); + + const response = await ctx.app.inject({ + method: "POST", + url: "/api/auth/login", + payload: { username: "admin", password: "password123" } + }); + + expect(response.statusCode).toBe(401); + expect(response.json()).toMatchObject({ error: "Invalid username or password" }); + }); + + it("deletes expired sessions when they are presented", async () => { + const cookie = await createAdminSession(); + await ctx.database.db.update(schema.sessions).set({ expiresAt: new Date(Date.now() - 1_000).toISOString() }); + + const response = await ctx.app.inject({ method: "GET", url: "/api/auth/me", headers: { cookie } }); + + expect(response.statusCode).toBe(200); + expect(response.json()).toMatchObject({ authenticated: false, user: null }); + await expect(ctx.database.db.select().from(schema.sessions)).resolves.toHaveLength(0); + }); + it("serializes concurrent first-admin setup attempts", async () => { const payloads = [ { username: "first-admin", password: "password123", confirmPassword: "password123" }, @@ -753,11 +785,20 @@ describe("api app", () => { expect(health.headers["x-content-type-options"]).toBe("nosniff"); expect(health.headers["content-security-policy"]).toContain("default-src 'self'"); expect(health.headers["content-security-policy"]).not.toContain("upgrade-insecure-requests"); + const liveHealth = await ctx.app.inject({ method: "GET", url: "/api/health/live" }); + expect(liveHealth.statusCode).toBe(200); + expect(liveHealth.json()).toEqual({ ok: true, service: "running" }); + const unavailableReadiness = await ctx.app.inject({ method: "GET", url: "/api/health/ready" }); + expect(unavailableReadiness.statusCode).toBe(503); + expect(unavailableReadiness.json()).toMatchObject({ ok: false, worker: "not_started", readyWorkerCount: 0 }); const heartbeatAt = new Date().toISOString(); await ctx.database.db.insert(schema.workerHeartbeats).values({ workerId: "test-worker", startedAt: heartbeatAt, heartbeatAt, status: "running" }); const workerReadyHealth = await ctx.app.inject({ method: "GET", url: "/api/health" }); expect(workerReadyHealth.json()).toMatchObject({ worker: "ready", workerHeartbeatAt: heartbeatAt }); + const availableReadiness = await ctx.app.inject({ method: "GET", url: "/api/health/ready" }); + expect(availableReadiness.statusCode).toBe(200); + expect(availableReadiness.json()).toMatchObject({ ok: true, worker: "ready", workerHeartbeatAt: heartbeatAt }); const cookie = await createAdminSession(); const crossSite = await ctx.app.inject({ @@ -867,9 +908,9 @@ describe("api app", () => { expect(version.statusCode).toBe(200); expect(version.json()).toMatchObject({ - currentVersion: "0.1.2-beta.4", - currentChannel: "beta", - currentChannelLabel: "Beta", + currentVersion: "0.1.2", + currentChannel: "stable", + currentChannelLabel: "Stable", latestVersion: null, updateAvailable: false, status: "unavailable", @@ -934,13 +975,13 @@ describe("api app", () => { expect(version.statusCode).toBe(200); expect(version.json()).toMatchObject({ - currentVersion: "0.1.2-beta.4", - currentChannel: "beta", - currentChannelLabel: "Beta", - latestVersion: "0.2.0-beta.1", + currentVersion: "0.1.2", + currentChannel: "stable", + currentChannelLabel: "Stable", + latestVersion: "0.1.1", updateAvailable: true, status: "update_available", - releaseUrl: "https://github.com/ramphex/srtl-manager/releases/tag/v0.2.0-beta.1", + releaseUrl: "https://github.com/ramphex/srtl-manager/releases/tag/v0.1.1", message: "Beta v0.2.0-beta.1 available", checkedAt: expect.any(String), stable: { @@ -1325,6 +1366,22 @@ describe("api app", () => { path.join(tmpDir, "remote", "Remote Show", "remote-show.mkv") ].sort()); + const firstResultPage = await ctx.app.inject({ + method: "GET", + url: `/api/audits/${auditRun?.id}/results/page?attentionOnly=true&limit=2&offset=0`, + headers: { cookie } + }); + expect(firstResultPage.statusCode).toBe(200); + expect(firstResultPage.json()).toMatchObject({ total: 4, offset: 0, hasMore: true }); + expect(firstResultPage.json().results).toHaveLength(2); + const secondResultPage = await ctx.app.inject({ + method: "GET", + url: `/api/audits/${auditRun?.id}/results/page?attentionOnly=true&limit=2&offset=2`, + headers: { cookie } + }); + expect(secondResultPage.json()).toMatchObject({ total: 4, offset: 2, hasMore: false }); + expect(secondResultPage.json().results).toHaveLength(2); + const unknownFolder = await ctx.app.inject({ method: "POST", url: "/api/audits", @@ -1434,17 +1491,18 @@ describe("api app", () => { const { jobId } = audit.json<{ jobId: number }>(); await expect(runQueuedJob(jobId, { auditRunner })).resolves.toMatchObject({ status: "completed", - progress: expect.objectContaining({ options: { mode: "fast", linkIds: [fixture.id], byteCompare: false }, checked: 1, passed: 1 }) + selection: { total: 1, titles: [{ section: "movies", itemName: "No Compare Movie", count: 1 }], unavailable: 0 }, + progress: expect.objectContaining({ options: { mode: "fast", byteCompare: false }, checked: 1, passed: 1 }) }); const history = await ctx.app.inject({ method: "GET", url: "/api/audits", headers: { cookie } }); expect(history.statusCode).toBe(200); expect(history.json>()).toEqual( - expect.arrayContaining([expect.objectContaining({ jobId, options: { mode: "fast", linkIds: [fixture.id], byteCompare: false } })]) + expect.arrayContaining([expect.objectContaining({ jobId, options: { mode: "fast", byteCompare: false } })]) ); const directRun = await ctx.app.inject({ method: "GET", url: `/api/audits/job/${jobId}`, headers: { cookie } }); expect(directRun.statusCode).toBe(200); - expect(directRun.json()).toMatchObject({ jobId, options: { mode: "fast", linkIds: [fixture.id], byteCompare: false } }); + expect(directRun.json()).toMatchObject({ jobId, options: { mode: "fast", byteCompare: false } }); expect(await ctx.database.db.select().from(schema.auditResults)).toMatchObject([expect.objectContaining({ cmpStatus: "skipped", status: "pass" })]); }); @@ -1510,10 +1568,19 @@ describe("api app", () => { }); expect(audit.statusCode).toBe(200); const { jobId } = audit.json<{ jobId: number }>(); + const queuedAuditRow = await first(ctx.database.db.select().from(schema.jobs).where(eq(schema.jobs.id, jobId)).limit(1)); + expect(JSON.parse(queuedAuditRow?.options ?? "{}")).not.toHaveProperty("linkIds"); + expect(JSON.parse(queuedAuditRow?.progress ?? "{}")).not.toHaveProperty("options.linkIds"); + await expect(ctx.database.db.select().from(schema.jobSelectionItems).where(eq(schema.jobSelectionItems.jobId, jobId))).resolves.toMatchObject([ + expect.objectContaining({ mediaLinkId: remoteLink?.id, section: "movies", itemName: "Remote Scoped Movie", selectionOrder: 0 }) + ]); + await expect(ctx.jobs.getJob(jobId)).resolves.toMatchObject({ + selection: { total: 1, unavailable: 0, titles: [{ section: "movies", itemName: "Remote Scoped Movie", count: 1 }], linkIds: [remoteLink?.id] } + }); await expect(runQueuedJob(jobId)).resolves.toMatchObject({ status: "completed", progress: { - options: { mode: "fast", linkIds: [remoteLink?.id] }, + options: { mode: "fast" }, checked: 1, total: 1, passed: 0, @@ -1589,7 +1656,8 @@ describe("api app", () => { await expect(runQueuedJob(jobId, { auditRunner })).resolves.toMatchObject({ status: "completed", - progress: expect.objectContaining({ options: expect.objectContaining({ linkIds: [firstFixture.id] }), checked: 1, total: 1 }) + selection: expect.objectContaining({ total: 1 }), + progress: expect.objectContaining({ options: expect.not.objectContaining({ linkIds: expect.anything() }), checked: 1, total: 1 }) }); expect(auditedPaths).toEqual([firstFixture.sourcePath]); expect(auditedPaths).not.toContain(secondFixture.sourcePath); @@ -1615,7 +1683,8 @@ describe("api app", () => { await expect(runQueuedJob(jobId, { auditRunner })).resolves.toMatchObject({ status: "completed", - progress: expect.objectContaining({ options: expect.objectContaining({ linkIds: [] }), checked: 0, total: 0 }) + selection: { total: 0, titles: [], unavailable: 0 }, + progress: expect.objectContaining({ checked: 0, total: 0 }) }); expect(auditCalls).toBe(0); }); @@ -1644,9 +1713,12 @@ describe("api app", () => { conflicts: 0, failed: 0, stage: "completed", - currentTitle: "Copy Local Movie", - currentFile: fixture.relativePath, - destinationPath: fixture.destinationPath, + currentTitle: null, + currentFile: null, + sourcePath: null, + destinationPath: null, + linkPath: null, + sizeBytes: null, bytesCopied: null, bytesProcessed: null, totalBytes: null, @@ -1916,7 +1988,7 @@ describe("api app", () => { }); }); - it("limits reconciliation blockers to the exact uncertain media from a legacy multi-item job", async () => { + it("auto-closes provably empty legacy reconciliation state without blocking related or retried media", async () => { const blockedFixture = await insertCopySymlink({ itemName: "Uncertain Legacy Movie", kind: "remote", storagePolicy: "location_1", content: "uncertain source" }); const relatedFixture = await insertCopySymlink({ itemName: "Independent Legacy Movie", kind: "remote", storagePolicy: "location_1", content: "independent source" }); const legacyJobId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: [blockedFixture.id, relatedFixture.id] }); @@ -1931,10 +2003,57 @@ describe("api app", () => { const relatedJobId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: [relatedFixture.id] }); await expect(ctx.jobs.getJob(relatedJobId)).resolves.toMatchObject({ status: "queued" }); - await expect(ctx.jobs.startCopy({ direction: "to_local", linkIds: [blockedFixture.id] })).rejects.toThrow( - `Copy data from job #${legacyJobId} requires manual reconciliation` - ); + const retriedJobId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: [blockedFixture.id] }); + await expect(ctx.jobs.getJob(retriedJobId)).resolves.toMatchObject({ status: "queued" }); + await expect(first(ctx.database.db.select().from(schema.copyOperations).where(eq(schema.copyOperations.jobId, legacyJobId)).limit(1))).resolves.toMatchObject({ + stage: "rolled_back", + errorMessage: null + }); await expect(ctx.jobs.terminate(relatedJobId)).resolves.toBe(true); + await expect(ctx.jobs.terminate(retriedJobId)).resolves.toBe(true); + }); + + it("preserves an unlinked legacy destination and returns it to normal copy conflict handling", async () => { + const fixture = await insertCopySymlink({ itemName: "Legacy Destination Conflict", kind: "remote", storagePolicy: "location_1", content: "current remote source" }); + const legacyJobId = await ctx.jobs.createJob("copy"); + const timestamp = new Date().toISOString(); + await ctx.database.db.update(schema.jobs).set({ status: "failed", finishedAt: timestamp }).where(eq(schema.jobs.id, legacyJobId)); + await fs.mkdir(path.dirname(fixture.destinationPath), { recursive: true }); + await fs.writeFile(fixture.destinationPath, "preserved unowned destination"); + await insertCopyOperationFixture({ + jobId: legacyJobId, + fixture, + stage: "reconciliation_required", + errorMessage: "Legacy destination ownership cannot be proven" + }); + + const retriedJobId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: [fixture.id] }); + + await expect(fs.readlink(fixture.linkPath)).resolves.toBe(fixture.sourcePath); + await expect(fs.readFile(fixture.destinationPath, "utf8")).resolves.toBe("preserved unowned destination"); + const resolvedOperation = await first(ctx.database.db.select().from(schema.copyOperations).where(eq(schema.copyOperations.jobId, legacyJobId)).limit(1)); + expect(resolvedOperation).toMatchObject({ + stage: "failed", + completedAt: expect.any(String), + reconciliationResolvedAt: expect.any(String), + errorMessage: expect.stringContaining("existing destination was left untouched for normal conflict handling") + }); + await expect(ctx.jobs.getJob(retriedJobId)).resolves.toMatchObject({ status: "queued" }); + await expect(ctx.jobs.terminate(retriedJobId)).resolves.toBe(true); + + await reconcileEnvironmentPaths(ctx.database.db, { + symlinkDir: path.join(tmpDir, "plex"), + localDir: path.join(tmpDir, "local"), + remoteDir: path.join(tmpDir, "remote") + }); + + await expect(first(ctx.database.db.select().from(schema.copyOperations).where(eq(schema.copyOperations.jobId, legacyJobId)).limit(1))).resolves.toMatchObject({ + stage: "failed", + reconciliationResolvedAt: resolvedOperation?.reconciliationResolvedAt + }); + const secondRetryJobId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: [fixture.id] }); + await expect(ctx.jobs.getJob(secondRetryJobId)).resolves.toMatchObject({ status: "queued" }); + await expect(ctx.jobs.terminate(secondRetryJobId)).resolves.toBe(true); }); it("does not let a stale local title item block new actionable copy work", async () => { @@ -2186,6 +2305,31 @@ describe("api app", () => { expect(copyFfmpegModes).toEqual(["deep"]); }); + it("freezes copy verification settings when a job is queued", async () => { + const cookie = await createAdminSession(); + const fixture = await insertCopySymlink({ itemName: "Frozen Verify Movie", kind: "remote", storagePolicy: "location_1", content: "copy with queued settings" }); + const jobId = await ctx.jobs.startCopy({ direction: "to_local", linkIds: [fixture.id] }); + + const settings = await ctx.app.inject({ + method: "PUT", + url: "/api/settings/advanced", + headers: { cookie }, + payload: { + copy: { profile: "off", byteCompare: false, mediaValidation: "off" }, + audit: { defaultMode: "fast", byteCompareWhenSourceKnown: true } + } + }); + expect(settings.statusCode).toBe(200); + + await expect(runQueuedJob(jobId, { copyRunner: testCopyRunner })).resolves.toMatchObject({ + status: "completed", + options: expect.objectContaining({ behavior: { profile: "balanced", byteCompare: true, mediaValidation: "fast" } }), + progress: expect.objectContaining({ copied: 1, failed: 0 }) + }); + expect(copyCmpCalls).toBe(1); + expect(copyFfmpegModes).toEqual(["fast"]); + }); + it("copies with verification disabled while retaining guarded transfer and promotion", async () => { const cookie = await createAdminSession(); const fixture = await insertCopySymlink({ itemName: "No Verify Movie", kind: "remote", storagePolicy: "location_1", content: "copy without content verification" }); @@ -2239,6 +2383,18 @@ describe("api app", () => { content: "wrong but valid media" }); + const preview = await ctx.app.inject({ + method: "POST", + url: "/api/copies/conflicts", + headers: { cookie }, + payload: { direction: "to_local", linkIds: [fixture.id] } + }); + expect(preview.statusCode).toBe(200); + expect(preview.json()).toMatchObject({ + totalSourceTitleBlocks: 1, + sourceTitleRisks: [expect.objectContaining({ linkId: fixture.id, itemName: "Mother Jugs and Speed (1976)" })] + }); + const copy = await ctx.app.inject({ method: "POST", url: "/api/copies", @@ -2275,6 +2431,23 @@ describe("api app", () => { await expect(fs.stat(fixture.destinationPath)).rejects.toMatchObject({ code: "ENOENT" }); await expect(fs.readlink(fixture.linkPath)).resolves.toBe(fixture.sourcePath); expect(copyFfmpegModes).toEqual([]); + + const override = await ctx.app.inject({ + method: "POST", + url: "/api/copies", + headers: { cookie }, + payload: { direction: "to_local", linkIds: [fixture.id], allowSourceTitleMismatch: true } + }); + expect(override.statusCode).toBe(200); + const overrideJobId = override.json<{ jobId: number }>().jobId; + await expect(runQueuedJob(overrideJobId, { copyRunner: testCopyRunner })).resolves.toMatchObject({ + status: "completed", + progress: expect.objectContaining({ copied: 1, conflicts: 0, failed: 0 }) + }); + await expect(ctx.jobs.listEvents(overrideJobId)).resolves.toEqual( + expect.arrayContaining([expect.objectContaining({ level: "warn", message: "Source title mismatch override accepted" })]) + ); + await expect(fs.readFile(fixture.destinationPath, "utf8")).resolves.toBe("wrong but valid media"); }); it("copies assign-remote local symlinks to remote storage with verification", async () => { @@ -2401,7 +2574,6 @@ describe("api app", () => { progress: expect.objectContaining({ options: expect.objectContaining({ direction: "to_local", - linkIds: [needsCopy.id], section: "shows", itemName: "Scoped Copy Show", relativePathPrefix: "Scoped Copy Show/Season 01" @@ -2447,8 +2619,8 @@ describe("api app", () => { await expect(runQueuedJob(jobId, { copyRunner: testCopyRunner })).resolves.toMatchObject({ status: "completed", + selection: { total: 0, titles: [], unavailable: 0 }, progress: expect.objectContaining({ - options: expect.objectContaining({ linkIds: [] }), current: 0, total: 0, copied: 0, @@ -2460,6 +2632,46 @@ describe("api app", () => { await expect(fs.readlink(laterFixture.linkPath)).resolves.toBe(laterFixture.sourcePath); }); + it("bounds large immutable-selection metadata without changing the selected total", async () => { + const timestamp = new Date().toISOString(); + const job = await first( + ctx.database.db + .insert(schema.jobs) + .values({ + type: "copy", + status: "queued", + createdAt: timestamp, + options: JSON.stringify({ direction: "to_local" }), + selectionFrozen: true, + progress: "{}" + }) + .returning({ id: schema.jobs.id }) + ); + if (!job) throw new Error("Large selection job was not inserted"); + await ctx.database.pool.query( + ` + INSERT INTO job_selection_items ( + job_id, media_link_id, selection_order, section, item_name, relative_path, link_path, created_at + ) + SELECT $1, + value, + value - 1, + 'shows', + 'Large title ' || lpad(value::text, 4, '0'), + 'Large title ' || value || '/episode.mkv', + '/links/Large title ' || value || '/episode.mkv', + $2 + FROM generate_series(1, 1001) AS value + `, + [job.id, timestamp] + ); + + const record = await ctx.jobs.getJob(job.id); + expect(record?.selection).toMatchObject({ total: 1001, unavailable: 0, omittedTitles: 901 }); + expect(record?.selection?.titles).toHaveLength(100); + expect(record?.selection?.linkIds).toBeUndefined(); + }); + it("resumes stale copy jobs without shrinking the original selected total", async () => { const fixtures = await Promise.all([ insertCopySymlink({ itemName: "Resume Copy One", kind: "remote", storagePolicy: "location_1", content: "resume one" }), @@ -2469,7 +2681,8 @@ describe("api app", () => { const linkIds = fixtures.map((fixture) => fixture.id); const jobId = await ctx.jobs.startCopy({ direction: "to_local", linkIds }); await expect(ctx.jobs.getJob(jobId)).resolves.toMatchObject({ - progress: expect.objectContaining({ options: expect.objectContaining({ direction: "to_local", linkIds }) }) + selection: expect.objectContaining({ total: 3, linkIds }), + progress: expect.objectContaining({ options: expect.objectContaining({ direction: "to_local" }) }) }); const staleStartedAt = new Date(Date.now() - 30 * 60_000).toISOString(); @@ -3443,7 +3656,21 @@ describe("api app", () => { expect(await worker.runOnce()).toBe(true); expect(maximumActiveTransfers).toBe(2); - expect(await ctx.jobs.getJob(jobId)).toMatchObject({ status: "completed", progress: expect.objectContaining({ copied: 2, failed: 0 }) }); + expect(await ctx.jobs.getJob(jobId)).toMatchObject({ + status: "completed", + progress: expect.objectContaining({ + current: 2, + total: 2, + copied: 2, + failed: 0, + currentTitle: null, + currentFile: null, + sourcePath: null, + destinationPath: null, + linkPath: null, + sizeBytes: null + }) + }); }); it("copies links for the same title concurrently within one copy job", async () => { @@ -3937,6 +4164,22 @@ describe("api app", () => { ); }); + it("rejects section configuration changes while queued work could depend on them", async () => { + const cookie = await createAdminSession(); + const jobId = await ctx.jobs.startScan({ scanSymlinks: true, scanLocal: false, scanRemote: false }); + + const response = await ctx.app.inject({ + method: "PUT", + url: "/api/settings/sections", + headers: { cookie }, + payload: { sections: ["movies"], sectionTitles: {}, sectionTypes: { movies: "movies" } } + }); + + expect(response.statusCode).toBe(409); + expect(response.json()).toMatchObject({ error: expect.stringContaining(`scan job #${jobId} is queued`) }); + await expect(ctx.jobs.terminate(jobId)).resolves.toBe(true); + }); + it("rejects storage policy changes overlapping a queued copy while allowing a disjoint title", async () => { const cookie = await createAdminSession(); const copyFixture = await insertCopySymlink({ @@ -4320,6 +4563,34 @@ describe("api app", () => { ]); }); + it("accepts explicit copy and audit selections larger than the old 1000-item preview limit", async () => { + const cookie = await createAdminSession(); + const linkIds = Array.from({ length: 1_001 }, (_unused, index) => index + 1); + + const copy = await ctx.app.inject({ + method: "POST", + url: "/api/copies", + headers: { cookie }, + payload: { direction: "to_local", linkIds } + }); + expect(copy.statusCode).toBe(200); + const copyJobId = copy.json<{ jobId: number }>().jobId; + await expect(ctx.jobs.getJob(copyJobId)).resolves.toMatchObject({ status: "queued", selection: { total: 0, titles: [] } }); + + const audit = await ctx.app.inject({ + method: "POST", + url: "/api/audits", + headers: { cookie }, + payload: { mode: "fast", linkIds } + }); + expect(audit.statusCode).toBe(200); + const auditJobId = audit.json<{ jobId: number }>().jobId; + await expect(ctx.jobs.getJob(auditJobId)).resolves.toMatchObject({ status: "queued", selection: { total: 0, titles: [] } }); + + await expect(ctx.jobs.terminate(copyJobId)).resolves.toBe(true); + await expect(ctx.jobs.terminate(auditJobId)).resolves.toBe(true); + }); + it("filters media link pages by section and storage policy", async () => { const cookie = await createAdminSession(); await insertMediaLink("Remote Movie", "remote", "movies"); diff --git a/tests/config.test.ts b/tests/config.test.ts index 843c684..e281a89 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -26,6 +26,7 @@ describe("config", () => { expect(config.host).toBe("0.0.0.0"); expect(config.port).toBe(3009); expect(config.sessionCookieName).toBe("srtl_session_5178"); + expect(config.jobHistoryRetentionDays).toBe(90); expect(config.paths).toEqual({ symlinkDir: "/mnt/links", localDir: "/mnt/local", remoteDir: "/mnt/remote" }); }); @@ -96,6 +97,19 @@ describe("config", () => { expect(loadConfig({ rootDir: tmpDir }).jobConcurrency.maxRunningScans).toBe(0); }); + it("allows one job slot to transfer several files under an explicit process-wide ceiling", async () => { + await fs.writeFile( + path.join(tmpDir, ".env"), + ["SRTL_WORKER_COUNT=1", "SRTL_COPY_FILE_CONCURRENCY=4", "SRTL_MAX_ACTIVE_COPY_FILES=4"].join("\n") + ); + + expect(loadConfig({ rootDir: tmpDir }).jobConcurrency).toMatchObject({ + workerCount: 1, + copyFileConcurrency: 4, + maxActiveCopyFiles: 4 + }); + }); + it("rejects malformed and contradictory worker concurrency settings", async () => { const invalidSettings = [ ["SRTL_WORKER_COUNT=0", "SRTL_WORKER_COUNT must be a positive safe integer"], @@ -103,7 +117,8 @@ describe("config", () => { ["SRTL_WORKER_COUNT=9007199254740992", "SRTL_WORKER_COUNT must be a positive safe integer"], [["SRTL_WORKER_COUNT=2", "SRTL_MAX_RUNNING_JOBS=3"].join("\n"), "SRTL_MAX_RUNNING_JOBS must not exceed SRTL_WORKER_COUNT"], [["SRTL_WORKER_COUNT=4", "SRTL_MAX_RUNNING_JOBS=2", "SRTL_MAX_RUNNING_COPIES=3"].join("\n"), "SRTL_MAX_RUNNING_COPIES must not exceed SRTL_MAX_RUNNING_JOBS"], - [["SRTL_COPY_FILE_CONCURRENCY=3", "SRTL_MAX_ACTIVE_COPY_FILES=2"].join("\n"), "SRTL_COPY_FILE_CONCURRENCY must not exceed SRTL_MAX_ACTIVE_COPY_FILES"] + [["SRTL_COPY_FILE_CONCURRENCY=3", "SRTL_MAX_ACTIVE_COPY_FILES=2"].join("\n"), "SRTL_COPY_FILE_CONCURRENCY must not exceed SRTL_MAX_ACTIVE_COPY_FILES"], + ["SRTL_JOB_HISTORY_RETENTION_DAYS=-1", "SRTL_JOB_HISTORY_RETENTION_DAYS must be a non-negative safe integer"] ] as const; for (const [contents, message] of invalidSettings) { diff --git a/tests/database.test.ts b/tests/database.test.ts index 8941560..58f690c 100644 --- a/tests/database.test.ts +++ b/tests/database.test.ts @@ -13,6 +13,21 @@ describe("database bootstrap", () => { } }); + it("fails closed when a migration gap exists below the current version", async () => { + const testDatabase = await createTestDatabase(); + const database = await openDatabase(testDatabase.databaseUrl); + try { + await database.pool.query(`DELETE FROM schema_migrations WHERE version = 4`); + } finally { + await database.close(); + } + try { + await expect(openDatabase({ databaseUrl: testDatabase.databaseUrl, migrate: false })).rejects.toThrow("Missing migration 4"); + } finally { + await testDatabase.cleanup(); + } + }); + it("creates the current schema and applies versioned migrations idempotently", async () => { const testDatabase = await createTestDatabase(); const database = await openDatabase(testDatabase.databaseUrl); @@ -35,7 +50,15 @@ describe("database bootstrap", () => { expect(indexes.rows.map((index) => index.indexname)).toEqual(expect.arrayContaining(["jobs_status_idx", "jobs_heartbeat_idx"])); const migrations = await database.pool.query<{ version: number }>("select version from schema_migrations order by version"); - expect(migrations.rows).toEqual([{ version: 1 }, { version: 2 }, { version: 3 }, { version: 4 }, { version: 5 }, { version: 6 }, { version: 7 }, { version: 8 }]); + expect(migrations.rows).toEqual([{ version: 1 }, { version: 2 }, { version: 3 }, { version: 4 }, { version: 5 }, { version: 6 }, { version: 7 }, { version: 8 }, { version: 9 }, { version: 10 }, { version: 11 }, { version: 12 }]); + + const unvalidatedConstraints = await database.pool.query<{ conname: string }>(` + SELECT conname + FROM pg_constraint + WHERE connamespace = 'public'::regnamespace + AND convalidated = FALSE + `); + expect(unvalidatedConstraints.rows).toEqual([]); const workerColumns = await database.pool.query<{ column_name: string }>(` select column_name @@ -52,7 +75,7 @@ describe("database bootstrap", () => { const copyOperationColumns = await database.pool.query<{ column_name: string; is_nullable: string }>(` SELECT column_name, is_nullable FROM information_schema.columns WHERE table_name = 'copy_operations' `); - for (const columnName of ["temp_identity", "destination_identity", "displaced_identity"]) { + for (const columnName of ["temp_identity", "destination_identity", "displaced_identity", "reconciliation_resolved_at"]) { expect(copyOperationColumns.rows).toContainEqual({ column_name: columnName, is_nullable: "YES" }); } const pathMigrationColumns = await database.pool.query<{ column_name: string; is_nullable: string }>(` @@ -105,6 +128,52 @@ describe("database bootstrap", () => { } }); + it("backfills durable resolution markers for copy journals already closed by reconciliation", async () => { + const testDatabase = await createTestDatabase(); + const legacyPool = new Pool({ connectionString: testDatabase.databaseUrl }); + try { + await legacyPool.query(`CREATE TABLE schema_migrations (version INTEGER PRIMARY KEY, name TEXT NOT NULL, applied_at TEXT NOT NULL)`); + await legacyPool.query(`INSERT INTO schema_migrations (version, name, applied_at) SELECT version, 'legacy', now()::text FROM generate_series(1, 11) AS version`); + await legacyPool.query(` + CREATE TABLE copy_operations ( + id SERIAL PRIMARY KEY, + error_message TEXT, + updated_at TEXT, + completed_at TEXT + ) + `); + await legacyPool.query( + ` + INSERT INTO copy_operations (error_message, updated_at, completed_at) + VALUES + ('Automatically closed after recheck: settled journal', $1, $2), + ('Unrelated copy failure', $1, $2) + `, + ["2026-08-02T21:00:00.000Z", "2026-08-02T21:01:00.000Z"] + ); + } finally { + await legacyPool.end(); + } + + const database = await openDatabase(testDatabase.databaseUrl); + try { + expect( + (await database.pool.query<{ id: number; reconciliation_resolved_at: string | null }>( + `SELECT id, reconciliation_resolved_at FROM copy_operations ORDER BY id` + )).rows + ).toEqual([ + { id: 1, reconciliation_resolved_at: "2026-08-02T21:01:00.000Z" }, + { id: 2, reconciliation_resolved_at: null } + ]); + expect((await database.pool.query<{ version: number; name: string }>(`SELECT version, name FROM schema_migrations WHERE version = 12`)).rows).toEqual([ + { version: 12, name: "durable_copy_reconciliation_resolution" } + ]); + } finally { + await database.close(); + await testDatabase.cleanup(); + } + }); + it("migrates legacy assignment policies and copy snapshots to location identities", async () => { const testDatabase = await createTestDatabase(); const legacyPool = new Pool({ connectionString: testDatabase.databaseUrl }); @@ -164,7 +233,11 @@ describe("database bootstrap", () => { { version: 5 }, { version: 6 }, { version: 7 }, - { version: 8 } + { version: 8 }, + { version: 9 }, + { version: 10 }, + { version: 11 }, + { version: 12 } ]); } finally { await database.close(); @@ -347,7 +420,11 @@ describe("database bootstrap", () => { { version: 5, name: "multi_worker_job_claims" }, { version: 6, name: "copy_operation_file_identities" }, { version: 7, name: "path_migration_target_identities" }, - { version: 8, name: "linked_storage_file_policies" } + { version: 8, name: "linked_storage_file_policies" }, + { version: 9, name: "immutable_job_inputs_and_selections" }, + { version: 10, name: "retention_and_history_indexes" }, + { version: 11, name: "validate_integrity_constraints" }, + { version: 12, name: "durable_copy_reconciliation_resolution" } ]); } finally { await database.close(); diff --git a/tests/e2e/app-smoke.spec.ts b/tests/e2e/app-smoke.spec.ts index a6a1a6e..bbc898c 100644 --- a/tests/e2e/app-smoke.spec.ts +++ b/tests/e2e/app-smoke.spec.ts @@ -5,6 +5,10 @@ const baseUrl = process.env.SRTL_E2E_BASE_URL; const sessionToken = process.env.SRTL_E2E_SESSION_TOKEN; const sessionCookieName = process.env.SRTL_E2E_SESSION_COOKIE_NAME ?? (baseUrl && new URL(baseUrl).port ? `srtl_session_${new URL(baseUrl).port}` : "srtl_session"); +if (process.env.CI && baseUrl && !sessionToken) { + throw new Error("CI browser checks require SRTL_E2E_SESSION_TOKEN so authenticated coverage cannot be skipped."); +} + test.skip(!baseUrl, "Set SRTL_E2E_BASE_URL to run browser smoke checks."); test.beforeEach(async ({ context }) => { @@ -89,7 +93,7 @@ test("dashboard task notifications overlay without shifting content", async ({ p else if (url.pathname === "/api/settings/storage-locations") body = { locations: [{ key: "location_1", rootType: "local", displayName: "Local", path: "/mnt/local" }, { key: "location_2", rootType: "remote", displayName: "Remote", path: "/mnt/remote" }] }; else if (url.pathname === "/api/system/version") { const unavailableRelease = { latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, releaseNotes: null, message: "Unavailable" }; - body = { currentVersion: "0.1.2-beta.4", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; + body = { currentVersion: "0.1.2", currentChannel: "stable", currentChannelLabel: "Stable", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; } else if (url.pathname === "/api/settings/sections") body = { sections: ["shows"], sectionTitles: { shows: "Shows" }, sectionTypes: { shows: "shows" } }; else if (url.pathname === "/api/settings/paths") body = { symlinkDir: "/mnt/links", localDir: "/mnt/local", remoteDir: "/mnt/remote" }; else if (url.pathname === "/api/settings/scan") body = { scanSymlinks: true, scanLocal: false, scanRemote: false, symlinkSections: ["shows"], localSections: [] }; @@ -220,7 +224,7 @@ test("refreshes an open work list when an inventory job finishes", async ({ page else if (url.pathname === "/api/settings/storage-locations") body = { locations: [{ key: "location_1", rootType: "local", displayName: "Local", path: "/mnt/local" }, { key: "location_2", rootType: "remote", displayName: "Remote", path: "/mnt/remote" }] }; else if (url.pathname === "/api/system/version") { const unavailableRelease = { latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, releaseNotes: null, message: "Unavailable" }; - body = { currentVersion: "0.1.2-beta.4", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; + body = { currentVersion: "0.1.2", currentChannel: "stable", currentChannelLabel: "Stable", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; } else if (url.pathname === "/api/settings/sections") body = { sections: ["shows"], sectionTitles: { shows: "Shows" }, sectionTypes: { shows: "shows" } }; else if (url.pathname === "/api/settings/paths") body = { symlinkDir: "/mnt/links", localDir: "/mnt/local", remoteDir: "/mnt/remote" }; @@ -313,7 +317,7 @@ test("loads every work-list page and scopes show copies beyond the first page", else if (url.pathname === "/api/settings/storage-locations") body = { locations: [{ key: "location_1", rootType: "local", displayName: "Local", path: "/mnt/local" }, { key: "location_2", rootType: "remote", displayName: "Remote", path: "/mnt/remote" }] }; else if (url.pathname === "/api/system/version") { const unavailableRelease = { latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, releaseNotes: null, message: "Unavailable" }; - body = { currentVersion: "0.1.2-beta.4", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; + body = { currentVersion: "0.1.2", currentChannel: "stable", currentChannelLabel: "Stable", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; } else if (url.pathname === "/api/settings/sections") body = { sections: ["shows"], sectionTitles: { shows: "Shows" }, sectionTypes: { shows: "shows" } }; else if (url.pathname === "/api/settings/paths") body = { symlinkDir: "/mnt/links", localDir: "/mnt/local", remoteDir: "/mnt/remote" }; @@ -367,10 +371,8 @@ test("renders every authenticated route without page errors or global overflow", { path: "/library", expected: "Library" }, { path: "/scans", expected: "History > Scans" }, { path: "/audits", expected: "History > Audits" }, - { path: "/integrations", expected: "Coming soon" }, { path: "/logs", expected: "Logs" }, { path: "/settings", expected: "Settings > Library" }, - { path: "/settings/integrations", expected: "Settings > Integrations" }, { path: "/settings/advanced", expected: "Settings > Advanced" }, { path: "/settings/user", expected: "Settings > User settings" } ]; @@ -774,6 +776,7 @@ test("advanced settings can disable copy verification and identify the recommend await expect(byteCompare).not.toBeChecked(); await expect(byteCompare).toBeDisabled(); await expect(copySection.locator(".advanced-readonly-value")).toContainText("Off"); + await expect(copySection.getByRole("alert")).toContainText("source bytes and media integrity are not checked"); await page.getByRole("button", { name: "Save advanced settings", exact: true }).click(); await expect(copySection.getByText("Current: Off", { exact: true })).toBeVisible(); expect(savedSettings).toMatchObject({ copy: { profile: "off", byteCompare: false, mediaValidation: "off" } }); @@ -865,6 +868,7 @@ test("title rescan controls explain their scope and lock sibling actions while q test("failed copy admission does not display a waiting job", async ({ page }) => { const timestamp = "2026-07-29T20:42:08.000Z"; const title = "Newly Scanned Title (2026)"; + const retryTitle = "Independent Copy Candidate (2026)"; const item = { id: 42, title, @@ -882,7 +886,14 @@ test("failed copy admission does not display a waiting job", async ({ page }) => source: "scan", updatedAt: timestamp }; + const retryItem = { + ...item, + id: 43, + title: retryTitle, + normalizedTitle: "independent copy candidate (2026)" + }; const admissionError = "Copy data from job #199 requires manual reconciliation before another action can touch the same media item or managed path."; + let copyAttempts = 0; await page.route("**/api/**", async (route) => { const url = new URL(route.request().url()); @@ -896,7 +907,7 @@ test("failed copy admission does not display a waiting job", async ({ page }) => else if (url.pathname === "/api/settings/storage-locations") body = { locations: [{ key: "location_1", rootType: "local", displayName: "Local", path: "/mnt/local" }, { key: "location_2", rootType: "remote", displayName: "Remote", path: "/mnt/remote" }] }; else if (url.pathname === "/api/system/version") { const unavailableRelease = { latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, releaseNotes: null, message: "Unavailable" }; - body = { currentVersion: "0.1.2-beta.4", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; + body = { currentVersion: "0.1.2", currentChannel: "stable", currentChannelLabel: "Stable", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; } else if (url.pathname === "/api/settings/sections") body = { sections: ["shows"], sectionTitles: { shows: "Shows" }, sectionTypes: { shows: "shows" } }; else if (url.pathname === "/api/settings/paths") body = { symlinkDir: "/mnt/links", localDir: "/mnt/local", remoteDir: "/mnt/remote" }; @@ -904,11 +915,32 @@ test("failed copy admission does not display a waiting job", async ({ page }) => else if (url.pathname === "/api/inventory/summary") body = {}; else if (url.pathname === "/api/inventory/scan-timestamps") body = { symlinkSections: { shows: timestamp }, localSections: { shows: null }, remoteRoot: null }; else if (url.pathname === "/api/jobs") body = []; - else if (url.pathname === "/api/storage-policies") body = url.searchParams.get("policy") === "location_1" ? [item] : []; + else if (url.pathname === "/api/jobs/200/events/page") body = { events: [], total: 0, hasOlder: false }; + else if (url.pathname === "/api/jobs/200") { + body = { + id: 200, + type: "copy", + status: "queued", + createdAt: timestamp, + startedAt: null, + finishedAt: null, + lockedBy: null, + lockedAt: null, + heartbeatAt: null, + cancelRequestedAt: null, + progress: { options: { direction: "to_local" }, stage: "queued", current: 0, total: 1 } + }; + } + else if (url.pathname === "/api/storage-policies") body = url.searchParams.get("policy") === "location_1" ? [item, retryItem] : []; else if (url.pathname === "/api/copies/conflicts") body = { conflicts: [], totalConflicts: 0, totalCandidates: 0 }; else if (url.pathname === "/api/copies") { - status = 409; - body = { error: admissionError }; + copyAttempts += 1; + if (copyAttempts === 1) { + status = 409; + body = { error: admissionError }; + } else { + body = { jobId: 200 }; + } } await route.fulfill({ status, contentType: "application/json", body: JSON.stringify(body) }); @@ -927,6 +959,17 @@ test("failed copy admission does not display a waiting job", async ({ page }) => await expect(dialog.getByText("The copy job was not queued. No files were changed.", { exact: true })).toBeVisible(); await expect(dialog.locator(".copy-progress-panel")).toHaveCount(0); await expect(dialog.locator(".audit-dialog-events")).toHaveCount(0); + + await dialog.getByRole("button", { name: "Close copy window", exact: true }).click(); + await page.getByPlaceholder("Filter scanned titles", { exact: true }).fill(retryTitle); + const retryRow = page.locator("tbody tr").filter({ hasText: retryTitle }); + await expect(retryRow).toHaveCount(1); + await retryRow.getByRole("button", { name: "Copy to Local", exact: true }).click(); + + await expect(dialog.getByRole("heading", { name: `Copy ${retryTitle} to Local`, exact: true })).toBeVisible(); + await expect(dialog.locator(".action-error")).toHaveCount(0); + await expect(dialog.getByText(admissionError, { exact: true })).toHaveCount(0); + await expect(dialog.getByText(`Job #200 - ${retryTitle}`, { exact: true })).toBeVisible(); }); test("recent jobs identifies a targeted scan by title instead of only its parent folder", async ({ page }) => { @@ -987,7 +1030,8 @@ test("recent copy jobs show a single link title directly and retain title inspec createdAt: timestamp, startedAt: timestamp, finishedAt: timestamp, - progress: { options: { direction: "to_local", linkIds: [9101] }, stage: "completed", total: 1, current: 1, copied: 1 } + selection: { total: 1, unavailable: 0, titles: [{ section: "movies", itemName: singleMovieTitle, count: 1 }] }, + progress: { options: { direction: "to_local" }, stage: "completed", total: 1, current: 1, copied: 1 } }, { id: 999992, @@ -996,7 +1040,8 @@ test("recent copy jobs show a single link title directly and retain title inspec createdAt: timestamp, startedAt: timestamp, finishedAt: timestamp, - progress: { options: { direction: "to_local", linkIds: [9201, 9202] }, stage: "completed", total: 2, current: 2, copied: 2 } + selection: { total: 2, unavailable: 0, titles: [{ section: "shows", itemName: singleSeriesTitle, count: 2 }] }, + progress: { options: { direction: "to_local" }, stage: "completed", total: 2, current: 2, copied: 2 } }, { id: 999993, @@ -1005,7 +1050,15 @@ test("recent copy jobs show a single link title directly and retain title inspec createdAt: timestamp, startedAt: timestamp, finishedAt: timestamp, - progress: { options: { direction: "to_local", linkIds: [9301, 9302] }, stage: "completed", total: 2, current: 2, copied: 2 } + selection: { + total: 2, + unavailable: 0, + titles: [ + { section: "movies", itemName: "Alpha Multi Title (2026)", count: 1 }, + { section: "movies", itemName: "Zulu Multi Title (2026)", count: 1 } + ] + }, + progress: { options: { direction: "to_local" }, stage: "completed", total: 2, current: 2, copied: 2 } } ]; const link = (id: number, section: string, itemName: string) => ({ @@ -1060,13 +1113,24 @@ test("recent copy jobs show a single link title directly and retain title inspec const singleSeriesTrigger = singleSeriesRow.getByLabel("View selected titles"); await expect(singleSeriesTrigger).toHaveCount(1); await singleSeriesTrigger.hover(); + await expect(singleSeriesTrigger.getByRole("tooltip")).toBeVisible(); await expect(singleSeriesTrigger.locator("li")).toHaveText([singleSeriesTitle]); await expect(multiTitleRow.locator(".job-scope-detail-line > span:first-child")).toHaveText("2 selected links"); const multiTitleTrigger = multiTitleRow.getByLabel("View selected titles"); await expect(multiTitleTrigger).toHaveCount(1); await multiTitleTrigger.hover(); + const multiTitleTooltip = multiTitleTrigger.getByRole("tooltip"); + await expect(multiTitleTooltip).toBeVisible(); await expect(multiTitleTrigger.locator("li")).toHaveText(["Alpha Multi Title (2026)", "Zulu Multi Title (2026)"]); + const tooltipBox = await multiTitleTooltip.boundingBox(); + const viewport = page.viewportSize(); + expect(tooltipBox).not.toBeNull(); + expect(viewport).not.toBeNull(); + expect(tooltipBox!.x).toBeGreaterThanOrEqual(0); + expect(tooltipBox!.y).toBeGreaterThanOrEqual(0); + expect(tooltipBox!.x + tooltipBox!.width).toBeLessThanOrEqual(viewport!.width); + expect(tooltipBox!.y + tooltipBox!.height).toBeLessThanOrEqual(viewport!.height); }); test("job progress shows the complete event timeline and opens the selected full log", async ({ page }) => { @@ -1356,7 +1420,7 @@ test("copy progress opens a persistent, scrollable completed item summary", asyn else if (url.pathname === "/api/settings/storage-locations") body = { locations: [{ key: "location_1", rootType: "local", displayName: "Local", path: "/mnt/local" }, { key: "location_2", rootType: "remote", displayName: "Remote", path: "/mnt/remote" }] }; else if (url.pathname === "/api/system/version") { const unavailableRelease = { latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, releaseNotes: null, message: "Unavailable" }; - body = { currentVersion: "0.1.2-beta.4", currentChannel: "beta", currentChannelLabel: "Beta", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; + body = { currentVersion: "0.1.2", currentChannel: "stable", currentChannelLabel: "Stable", stable: { channel: "stable", ...unavailableRelease }, beta: { channel: "beta", ...unavailableRelease }, latestVersion: null, updateAvailable: false, status: "unavailable", releaseUrl: null, checkedAt: timestamp, message: "Unavailable" }; } else if (url.pathname === `/api/jobs/${jobId}/events/page`) body = { events, total: events.length, hasOlder: false }; else if (url.pathname === `/api/jobs/${jobId}`) body = job; else if (url.pathname === "/api/jobs") body = [job]; diff --git a/tests/env.test.ts b/tests/env.test.ts index a8fb9f3..d713990 100644 --- a/tests/env.test.ts +++ b/tests/env.test.ts @@ -14,6 +14,7 @@ SRTL_MAX_RUNNING_AUDITS=3 SRTL_MAX_RUNNING_COPIES=2 SRTL_COPY_FILE_CONCURRENCY=2 SRTL_MAX_ACTIVE_COPY_FILES=4 +SRTL_JOB_HISTORY_RETENTION_DAYS=120 IGNORED=value `); expect(env).toEqual({ @@ -26,7 +27,8 @@ IGNORED=value SRTL_MAX_RUNNING_AUDITS: "3", SRTL_MAX_RUNNING_COPIES: "2", SRTL_COPY_FILE_CONCURRENCY: "2", - SRTL_MAX_ACTIVE_COPY_FILES: "4" + SRTL_MAX_ACTIVE_COPY_FILES: "4", + SRTL_JOB_HISTORY_RETENTION_DAYS: "120" }); }); diff --git a/tests/historyRetention.test.ts b/tests/historyRetention.test.ts new file mode 100644 index 0000000..3c76b5f --- /dev/null +++ b/tests/historyRetention.test.ts @@ -0,0 +1,188 @@ +import { eq } from "drizzle-orm"; +import { describe, expect, it } from "vitest"; +import { first, openDatabase } from "../src/server/db/database"; +import * as schema from "../src/server/db/schema"; +import { pruneExpiredSessions, pruneTerminalJobHistory } from "../src/server/lib/historyRetention"; +import { createTestDatabase } from "./testDb"; + +describe("job history retention", () => { + it("removes expired sessions without deleting current sessions", async () => { + const testDatabase = await createTestDatabase(); + const database = await openDatabase(testDatabase.databaseUrl); + try { + const user = await first( + database.db + .insert(schema.adminUsers) + .values({ username: "retention-admin", passwordHash: "unused", createdAt: "2026-01-01T00:00:00.000Z" }) + .returning({ id: schema.adminUsers.id }) + ); + if (!user) throw new Error("Session retention user was not created"); + await database.db.insert(schema.sessions).values([ + { tokenHash: "expired", userId: user.id, expiresAt: "2026-07-31T00:00:00.000Z", createdAt: "2026-07-01T00:00:00.000Z" }, + { tokenHash: "current", userId: user.id, expiresAt: "2026-08-02T00:00:00.000Z", createdAt: "2026-07-31T00:00:00.000Z" } + ]); + + await expect(pruneExpiredSessions(database.db, "2026-08-01T00:00:00.000Z")).resolves.toBe(1); + await expect(database.db.select({ tokenHash: schema.sessions.tokenHash }).from(schema.sessions)).resolves.toEqual([{ tokenHash: "current" }]); + } finally { + await database.close(); + await testDatabase.cleanup(); + } + }); + + it("removes old terminal history while preserving current, migration, and unresolved recovery jobs", async () => { + const testDatabase = await createTestDatabase(); + const database = await openDatabase(testDatabase.databaseUrl); + const nowMs = Date.parse("2026-08-01T12:00:00.000Z"); + const oldTimestamp = "2026-04-01T12:00:00.000Z"; + const recentTimestamp = "2026-07-31T12:00:00.000Z"; + try { + const [oldAudit, recentJob, pathMigrationJob, recoveryJob, supersededRecoveryJob, laterCommittedJob] = await database.db + .insert(schema.jobs) + .values([ + { type: "audit", status: "completed", createdAt: oldTimestamp, finishedAt: oldTimestamp, progress: "{}" }, + { type: "scan", status: "completed", createdAt: recentTimestamp, finishedAt: recentTimestamp, progress: "{}" }, + { type: "path_migration", status: "failed", createdAt: oldTimestamp, finishedAt: oldTimestamp, progress: "{}" }, + { type: "copy", status: "failed", createdAt: oldTimestamp, finishedAt: oldTimestamp, progress: "{}" }, + { type: "copy", status: "failed", createdAt: oldTimestamp, finishedAt: oldTimestamp, progress: "{}" }, + { type: "copy", status: "completed", createdAt: recentTimestamp, finishedAt: recentTimestamp, progress: "{}" } + ]) + .returning({ id: schema.jobs.id }); + if (!oldAudit || !recentJob || !pathMigrationJob || !recoveryJob || !supersededRecoveryJob || !laterCommittedJob) { + throw new Error("Retention fixtures were not created"); + } + + await database.db.insert(schema.jobEvents).values({ + jobId: oldAudit.id, + timestamp: oldTimestamp, + level: "info", + message: "old event", + data: "{}" + }); + const auditRun = await first( + database.db + .insert(schema.auditRuns) + .values({ + jobId: oldAudit.id, + mode: "fast", + status: "completed", + startedAt: oldTimestamp, + finishedAt: oldTimestamp, + checked: 1, + passed: 1, + failed: 0, + sourceUnknown: 0, + sourceMissing: 0, + sourceCompareErrors: 0, + byteMismatches: 0, + targetValidationFailures: 0, + errorMessage: null + }) + .returning({ id: schema.auditRuns.id }) + ); + if (!auditRun) throw new Error("Audit retention fixture was not created"); + await database.db.insert(schema.auditResults).values({ + auditRunId: auditRun.id, + linkPath: "/links/old.mkv", + targetPath: "/remote/old.mkv", + sourcePath: null, + status: "pass", + ffmpegStatus: "pass", + cmpStatus: "skipped", + message: "ok", + createdAt: oldTimestamp + }); + + const mediaLink = await first( + database.db + .insert(schema.mediaLinks) + .values({ + section: "movies", + itemName: "Recovery", + relativePath: "Recovery/file.mkv", + linkPath: "/links/Recovery/file.mkv", + targetPath: "/remote/Recovery/file.mkv", + kind: "remote", + targetExists: true, + isMedia: true, + storagePolicy: "location_1", + updatedAt: oldTimestamp + }) + .returning({ id: schema.mediaLinks.id }) + ); + if (!mediaLink) throw new Error("Media-link retention fixture was not created"); + await database.db.insert(schema.copyOperations).values({ + jobId: recoveryJob.id, + mediaLinkId: mediaLink.id, + linkPath: "/links/Recovery/file.mkv", + sourcePath: "/remote/Recovery/file.mkv", + destinationPath: "/local/Recovery/file.mkv", + originalTargetPath: "/remote/Recovery/file.mkv", + originalLinkState: "{}", + stage: "reconciliation_required", + createdAt: oldTimestamp, + updatedAt: oldTimestamp + }); + + const supersededLink = await first( + database.db + .insert(schema.mediaLinks) + .values({ + section: "movies", + itemName: "Superseded recovery", + relativePath: "Superseded/file.mkv", + linkPath: "/links/Superseded/file.mkv", + targetPath: "/local/Superseded/file.mkv", + kind: "local", + targetExists: true, + isMedia: true, + storagePolicy: "location_1", + updatedAt: recentTimestamp + }) + .returning({ id: schema.mediaLinks.id }) + ); + if (!supersededLink) throw new Error("Superseded media-link fixture was not created"); + await database.db.insert(schema.copyOperations).values([ + { + jobId: supersededRecoveryJob.id, + mediaLinkId: supersededLink.id, + linkPath: "/links/Superseded/file.mkv", + sourcePath: "/remote/Superseded/file.mkv", + destinationPath: "/local/Superseded/file.mkv", + originalTargetPath: "/remote/Superseded/file.mkv", + originalLinkState: "{}", + stage: "reconciliation_required", + createdAt: oldTimestamp, + updatedAt: oldTimestamp + }, + { + jobId: laterCommittedJob.id, + mediaLinkId: supersededLink.id, + linkPath: "/links/Superseded/file.mkv", + sourcePath: "/remote/Superseded/file.mkv", + destinationPath: "/local/Superseded/file.mkv", + originalTargetPath: "/remote/Superseded/file.mkv", + originalLinkState: "{}", + stage: "committed", + resultStatus: "copied", + createdAt: recentTimestamp, + updatedAt: recentTimestamp, + completedAt: recentTimestamp + } + ]); + + await expect(pruneTerminalJobHistory(database.db, 90, nowMs)).resolves.toBe(2); + const remainingJobs = await database.db.select({ id: schema.jobs.id }).from(schema.jobs); + expect(remainingJobs.map((job) => job.id).sort((left, right) => left - right)).toEqual( + [recentJob.id, pathMigrationJob.id, recoveryJob.id, laterCommittedJob.id].sort((left, right) => left - right) + ); + await expect(database.db.select().from(schema.jobEvents).where(eq(schema.jobEvents.jobId, oldAudit.id))).resolves.toHaveLength(0); + await expect(database.db.select().from(schema.auditRuns).where(eq(schema.auditRuns.id, auditRun.id))).resolves.toHaveLength(0); + await expect(database.db.select().from(schema.auditResults).where(eq(schema.auditResults.auditRunId, auditRun.id))).resolves.toHaveLength(0); + await expect(pruneTerminalJobHistory(database.db, 0, nowMs)).resolves.toBe(0); + } finally { + await database.close(); + await testDatabase.cleanup(); + } + }); +}); diff --git a/tests/jobPresentationUtils.test.ts b/tests/jobPresentationUtils.test.ts index cca47d7..c279bc1 100644 --- a/tests/jobPresentationUtils.test.ts +++ b/tests/jobPresentationUtils.test.ts @@ -102,6 +102,46 @@ describe("copy work total display", () => { failed: 0 }), 3)).toBe(3); }); + + it("uses the immutable selection total instead of unrelated legacy progress totals", () => { + const job = copyJob({ + options: { direction: "to_local", section: "shows", itemName: "Example Show" }, + total: 540, + copied: 3, + repointed: 0, + skipped: 0, + alreadyCompleted: 0, + conflicts: 0, + failed: 0 + }); + job.selection = { + total: 3, + titles: [{ section: "shows", itemName: "Example Show", count: 3 }], + unavailable: 0 + }; + + expect(copyWorkTotalFromJob(job, 540)).toBe(3); + }); + + it("excludes unavailable migrated links from the immutable copy work total", () => { + const job = copyJob({ + options: { direction: "to_local", section: "shows", itemName: "Example Show", linkIds: Array.from({ length: 540 }, (_, index) => index + 1) }, + total: 540, + copied: 3, + repointed: 0, + skipped: 537, + alreadyCompleted: 537, + conflicts: 0, + failed: 0 + }); + job.selection = { + total: 540, + titles: [{ section: "shows", itemName: "Example Show", count: 3 }], + unavailable: 537 + }; + + expect(copyWorkTotalFromJob(job, 540)).toBe(3); + }); }); describe("copy failure summaries", () => { diff --git a/tests/jobScheduler.test.ts b/tests/jobScheduler.test.ts index f04df2e..1e78757 100644 --- a/tests/jobScheduler.test.ts +++ b/tests/jobScheduler.test.ts @@ -7,10 +7,11 @@ import { createApp, type AppContext } from "../src/server/app"; import type { JobConcurrencySettings } from "../src/server/config"; import { first, setSetting } from "../src/server/db/database"; import * as schema from "../src/server/db/schema"; -import { JobWorker } from "../src/server/jobs/jobRunner"; +import { copyAdmissionSelectionFingerprint, JobWorker } from "../src/server/jobs/jobRunner"; import { schedulerLockKey } from "../src/server/jobs/scheduling"; import type { AuditCommandRunner } from "../src/server/lib/auditor"; import type { CopyCommandRunner } from "../src/server/lib/copier"; +import type { MediaLinkRow } from "../src/shared/types"; import { createTestDatabase, type TestDatabaseHandle } from "./testDb"; const silentLogger = { @@ -29,6 +30,42 @@ const twoJobConcurrency: JobConcurrencySettings = { maxActiveCopyFiles: 2 }; +function admissionLink(id: number, overrides: Partial = {}): MediaLinkRow { + const timestamp = "2026-08-02T00:00:00.000Z"; + return { + id, + section: "shows", + itemName: `Admission Show ${id}`, + relativePath: `Admission Show ${id}/Season 01/episode-${id}.mkv`, + linkPath: `/symlinks/shows/Admission Show ${id}/Season 01/episode-${id}.mkv`, + targetPath: `/remote/shows/Admission Show ${id}/Season 01/episode-${id}.mkv`, + kind: "remote", + targetExists: true, + isMedia: true, + storagePolicy: "location_1", + resolvedStorageFileId: id, + sizeBytes: id * 1_000, + firstSeenAt: timestamp, + lastSeenAt: timestamp, + lastChangedAt: timestamp, + missingSince: null, + updatedAt: timestamp, + ...overrides + }; +} + +describe("copy admission selection fingerprint", () => { + it("ignores database row order while preserving metadata change detection", () => { + const first = admissionLink(11); + const second = admissionLink(12); + + expect(copyAdmissionSelectionFingerprint([first, second])).toBe(copyAdmissionSelectionFingerprint([second, first])); + expect(copyAdmissionSelectionFingerprint([first, second])).not.toBe( + copyAdmissionSelectionFingerprint([first, { ...second, targetPath: `${second.targetPath}.changed` }]) + ); + }); +}); + let tmpDir: string; let ctx: AppContext; let testDatabase: TestDatabaseHandle; diff --git a/tests/workerHeartbeats.test.ts b/tests/workerHeartbeats.test.ts index 744a5e6..4370c4b 100644 --- a/tests/workerHeartbeats.test.ts +++ b/tests/workerHeartbeats.test.ts @@ -144,6 +144,7 @@ describe("worker heartbeat history", () => { await app.database.db.insert(schema.workerHeartbeats).values({ workerId: "historical-worker", startedAt: heartbeatAt, heartbeatAt, status: "stopped" }); const response = await app.app.inject({ method: "GET", url: "/api/health" }); + const readiness = await app.app.inject({ method: "GET", url: "/api/health/ready" }); expect(response.statusCode).toBe(200); expect(response.json()).toMatchObject({ @@ -153,6 +154,8 @@ describe("worker heartbeat history", () => { readyWorkerCount: 0, staleWorkerCount: 0 }); + expect(readiness.statusCode).toBe(503); + expect(readiness.json()).toMatchObject({ ok: false, worker: "not_started" }); } finally { if (app) await app.app.close(); await testDatabase.cleanup(); @@ -196,6 +199,7 @@ describe("worker heartbeat history", () => { ]); const readyResponse = await app.app.inject({ method: "GET", url: "/api/health" }); + const readyReadiness = await app.app.inject({ method: "GET", url: "/api/health/ready" }); expect(readyResponse.statusCode).toBe(200); expect(readyResponse.json()).toMatchObject({ worker: "ready", @@ -204,15 +208,20 @@ describe("worker heartbeat history", () => { readyWorkerCount: 5, staleWorkerCount: 7 }); + expect(readyReadiness.statusCode).toBe(200); + expect(readyReadiness.json()).toMatchObject({ ok: true, worker: "ready" }); await app.database.db.update(schema.workerHeartbeats).set({ status: "stopped" }).where(eq(schema.workerHeartbeats.workerId, "fresh-b")); const partialResponse = await app.app.inject({ method: "GET", url: "/api/health" }); + const partialReadiness = await app.app.inject({ method: "GET", url: "/api/health/ready" }); expect(partialResponse.json()).toMatchObject({ worker: "stale", expectedWorkerCount: 5, readyWorkerCount: 2, staleWorkerCount: 7 }); + expect(partialReadiness.statusCode).toBe(503); + expect(partialReadiness.json()).toMatchObject({ ok: false, worker: "stale" }); } finally { if (app) await app.app.close(); await testDatabase.cleanup(); From 03c6c52fa65629a9205eb4a7419a9f26161023ed Mon Sep 17 00:00:00 2001 From: Aleksey Dmitriev Date: Tue, 4 Aug 2026 04:23:10 -0400 Subject: [PATCH 11/11] Rate-limit database health checks --- src/server/app.ts | 4 ++-- tests/app.test.ts | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/server/app.ts b/src/server/app.ts index 1400842..c5d3a07 100644 --- a/src/server/app.ts +++ b/src/server/app.ts @@ -149,11 +149,11 @@ export async function createApp(overrides: Partial = {}): Promise ({ ok: true, service: "running" })); - app.get("/api/health/ready", async (_request, reply) => { + app.get("/api/health/ready", { config: { rateLimit: { max: 120, timeWindow: "1 minute" } } }, async (_request, reply) => { const health = await workerHealth(); return health.ok ? health : reply.code(503).send(health); }); - app.get("/api/health", async () => ({ ...(await workerHealth()), ok: true })); + app.get("/api/health", { config: { rateLimit: { max: 120, timeWindow: "1 minute" } } }, async () => ({ ...(await workerHealth()), ok: true })); registerAuthRoutes(app, database.db, { cookieName: config.sessionCookieName, cookieSecure: config.sessionCookieSecure diff --git a/tests/app.test.ts b/tests/app.test.ts index 8ef4dd5..54b85c4 100644 --- a/tests/app.test.ts +++ b/tests/app.test.ts @@ -782,6 +782,7 @@ describe("api app", () => { const health = await ctx.app.inject({ method: "GET", url: "/api/health" }); expect(health.statusCode).toBe(200); expect(health.json()).toMatchObject({ ok: true, database: "ready", worker: "not_started", workerHeartbeatAt: null }); + expect(health.headers["x-ratelimit-limit"]).toBe("120"); expect(health.headers["x-content-type-options"]).toBe("nosniff"); expect(health.headers["content-security-policy"]).toContain("default-src 'self'"); expect(health.headers["content-security-policy"]).not.toContain("upgrade-insecure-requests"); @@ -791,6 +792,7 @@ describe("api app", () => { const unavailableReadiness = await ctx.app.inject({ method: "GET", url: "/api/health/ready" }); expect(unavailableReadiness.statusCode).toBe(503); expect(unavailableReadiness.json()).toMatchObject({ ok: false, worker: "not_started", readyWorkerCount: 0 }); + expect(unavailableReadiness.headers["x-ratelimit-limit"]).toBe("120"); const heartbeatAt = new Date().toISOString(); await ctx.database.db.insert(schema.workerHeartbeats).values({ workerId: "test-worker", startedAt: heartbeatAt, heartbeatAt, status: "running" });