feat: keep yt-dlp current, harden downloads, fix notification & photo handling

docs/deployment/configuration.md

@@ -80,6 +80,7 @@ Push notifications won't work without these. The app will still function, but us
 | `SMS_DEV_MODE` | `false` | Set to `true` to bypass Twilio and auto-approve any verification code (dev/testing only) |
 | `GIPHY_API_KEY` | — | Giphy API key for GIF search in comments. Get one at [developers.giphy.com](https://developers.giphy.com/) |
 | `DATA_DIR` | `./data` | Directory for database and media files |
+| `YTDLP_COOKIES_FILE` | — | Path to a Netscape-format cookies file passed to yt-dlp as `--cookies`. Supplies a logged-in session so login-gated / audience-restricted Instagram (and similar) posts download. Use a **secondary** account — bulk downloads can get an account rate-limited or flagged. Treat the file as a secret (mount read-only, keep out of git). |
 | `LOG_LEVEL` | `info` | Logging level (`trace`, `debug`, `info`, `warn`, `error`, `fatal`) |
 | `BACKUP_RETENTION_COUNT` | `7` | Number of daily database backups to keep |
 | `RATE_LIMITING` | `false` | Set to `true` to enable API rate limiting. Recommended for public-facing deployments. |

src/lib/server/__tests__/download-notify.test.ts

@@ -0,0 +1,185 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { v4 as uuid } from 'uuid';
+import { eq } from 'drizzle-orm';
+import * as schema from '../db/schema';
+
+vi.mock('$lib/server/db', async () => {
+	const { createTestDb } = await import('../../../../tests/helpers/db');
+	return createTestDb();
+});
+
+// Spy on the group-notify call so we can assert it fires ONLY on a genuine 'ready'.
+vi.mock('$lib/server/push', () => ({
+	notifyNewClip: vi.fn(async () => {}),
+	sendNotification: vi.fn(async () => {})
+}));
+
+// Provider is injected per-test.
+const getActiveProvider = vi.fn();
+vi.mock('$lib/server/providers/registry', () => ({
+	getActiveProvider: (...args: unknown[]) => getActiveProvider(...args)
+}));
+
+// Pass-through the dedup wrapper so the inner download runs synchronously in-test.
+vi.mock('$lib/server/download-lock', () => ({
+	deduplicatedDownload: (
+		clipId: string,
+		url: string,
+		fn: (clipId: string, url: string) => Promise<void>
+	) => fn(clipId, url)
+}));
+
+const { db } = await import('$lib/server/db');
+const push = await import('$lib/server/push');
+const { downloadVideo, isNotAVideoError, NOT_A_VIDEO_TITLE } = await import('../video/download');
+
+function setup() {
+	const groupId = uuid();
+	const userId = uuid();
+	const now = new Date();
+	(db as any)
+		.insert(schema.groups)
+		.values({
+			id: groupId,
+			name: 'G',
+			inviteCode: `i-${groupId.slice(0, 6)}`,
+			accentColor: 'coral',
+			reciprocitySeed: 5,
+			reciprocityCap: 5,
+			reciprocityRatio: 1,
+			queueLimit: 10,
+			downloadProvider: 'ytdlp',
+			createdAt: now
+		})
+		.run();
+	(db as any)
+		.insert(schema.users)
+		.values({
+			id: userId,
+			username: `u-${userId.slice(0, 6)}`,
+			phone: `+1${Math.floor(2_000_000_000 + Math.random() * 7_999_999_999)}`,
+			groupId,
+			postCredits: 5,
+			createdAt: now
+		})
+		.run();
+	return { groupId, userId };
+}
+
+function insertDownloadingClip(groupId: string, userId: string) {
+	const id = uuid();
+	(db as any)
+		.insert(schema.clips)
+		.values({
+			id,
+			groupId,
+			addedBy: userId,
+			originalUrl: `https://x/${id}`,
+			platform: 'instagram',
+			status: 'downloading',
+			contentType: 'video',
+			createdAt: new Date()
+		})
+		.run();
+	return id;
+}
+
+function statusOf(clipId: string): string {
+	const row = (db as any)
+		.select({ status: schema.clips.status })
+		.from(schema.clips)
+		.where(eq(schema.clips.id, clipId))
+		.get();
+	return row.status;
+}
+
+beforeEach(() => {
+	(push.notifyNewClip as any).mockClear();
+	getActiveProvider.mockReset();
+});
+
+describe('download notifications only fire on a genuine ready post', () => {
+	it('does NOT notify when the download fails', async () => {
+		const { groupId, userId } = setup();
+		const clipId = insertDownloadingClip(groupId, userId);
+		getActiveProvider.mockResolvedValue({
+			downloadVideo: vi.fn(async () => {
+				throw new Error('yt-dlp exited with code 1: empty media response');
+			})
+		});
+
+		await downloadVideo(clipId, 'https://x/fail');
+
+		expect(statusOf(clipId)).toBe('failed');
+		expect(push.notifyNewClip).not.toHaveBeenCalled();
+	});
+
+	it('does NOT notify when no provider is configured', async () => {
+		const { groupId, userId } = setup();
+		const clipId = insertDownloadingClip(groupId, userId);
+		getActiveProvider.mockResolvedValue(null);
+
+		await downloadVideo(clipId, 'https://x/noprovider');
+
+		expect(statusOf(clipId)).toBe('failed');
+		expect(push.notifyNewClip).not.toHaveBeenCalled();
+	});
+
+	it('DOES notify once when the download succeeds and the clip goes ready', async () => {
+		const { groupId, userId } = setup();
+		const clipId = insertDownloadingClip(groupId, userId);
+		getActiveProvider.mockResolvedValue({
+			downloadVideo: vi.fn(async () => ({
+				videoPath: '/nonexistent/video.mp4',
+				thumbnailPath: null,
+				title: 'a caption',
+				duration: 12,
+				creatorName: 'someone',
+				creatorUrl: null,
+				sourceViewCount: 42
+			}))
+		});
+
+		await downloadVideo(clipId, 'https://x/ok');
+
+		expect(statusOf(clipId)).toBe('ready');
+		expect(push.notifyNewClip).toHaveBeenCalledTimes(1);
+		expect(push.notifyNewClip).toHaveBeenCalledWith(clipId);
+	});
+});
+
+describe('confirmed photo shares are rejected as not-a-video', () => {
+	it('isNotAVideoError matches the yt-dlp no-video signatures only', () => {
+		expect(isNotAVideoError('ERROR: [Instagram] X: There is no video in this post')).toBe(true);
+		expect(isNotAVideoError('ERROR: [Instagram] X: No video formats found!')).toBe(true);
+		expect(isNotAVideoError('ERROR: [Instagram] X: Instagram sent an empty media response')).toBe(
+			false
+		);
+		expect(isNotAVideoError('ERROR: unable to download webpage: network is unreachable')).toBe(
+			false
+		);
+	});
+
+	it('fails a photo share with a clear reason and does NOT notify', async () => {
+		const { groupId, userId } = setup();
+		const clipId = insertDownloadingClip(groupId, userId);
+		getActiveProvider.mockResolvedValue({
+			downloadVideo: vi.fn(async () => {
+				throw new Error(
+					'yt-dlp exited with code 1: ERROR: [Instagram] X: There is no video in this post'
+				);
+			})
+		});
+
+		await downloadVideo(clipId, 'https://instagram.com/p/photo');
+
+		const row = (db as any)
+			.select({ status: schema.clips.status, title: schema.clips.title })
+			.from(schema.clips)
+			.where(eq(schema.clips.id, clipId))
+			.get();
+		expect(row.status).toBe('failed');
+		expect(row.title).toBe(NOT_A_VIDEO_TITLE);
+		expect(push.notifyNewClip).not.toHaveBeenCalled();
+	});
+});

src/lib/server/providers/__tests__/ytdlp.test.ts

@@ -0,0 +1,88 @@
+import { describe, it, expect } from 'vitest';
+import {
+	parseYtdlpDate,
+	ytdlpAgeDays,
+	isYtdlpStale,
+	YTDLP_STALE_DAYS,
+	extractCaption
+} from '../ytdlp/version';
+
+describe('yt-dlp version parsing', () => {
+	it('parses a date-stamped version to a UTC date', () => {
+		const d = parseYtdlpDate('2026.07.04');
+		expect(d?.toISOString()).toBe('2026-07-04T00:00:00.000Z');
+	});
+
+	it('parses versions with a build suffix', () => {
+		const d = parseYtdlpDate('2026.07.04.123456');
+		expect(d?.toISOString()).toBe('2026-07-04T00:00:00.000Z');
+	});
+
+	it('returns null for null / unparseable / dev-hash versions', () => {
+		expect(parseYtdlpDate(null)).toBeNull();
+		expect(parseYtdlpDate('')).toBeNull();
+		expect(parseYtdlpDate('deadbeef')).toBeNull();
+		expect(parseYtdlpDate('nightly')).toBeNull();
+	});
+});
+
+describe('ytdlpAgeDays', () => {
+	const now = Date.UTC(2026, 6, 4); // 2026-07-04
+
+	it('computes whole-day age from a version date', () => {
+		expect(ytdlpAgeDays('2026.07.04', now)).toBe(0);
+		expect(ytdlpAgeDays('2026.06.04', now)).toBe(30);
+		expect(ytdlpAgeDays('2026.02.21', now)).toBe(133);
+	});
+
+	it('returns null when unparseable', () => {
+		expect(ytdlpAgeDays(null, now)).toBeNull();
+		expect(ytdlpAgeDays('nightly', now)).toBeNull();
+	});
+});
+
+describe('isYtdlpStale', () => {
+	const now = Date.UTC(2026, 6, 4);
+
+	it('is false for a fresh build and true past the threshold', () => {
+		expect(isYtdlpStale('2026.07.04', now)).toBe(false);
+		expect(isYtdlpStale('2026.06.04', now)).toBe(false); // 30 days, under 90
+		expect(isYtdlpStale('2026.02.21', now)).toBe(true); // 133 days
+	});
+
+	it('is exactly boundary-safe at the threshold', () => {
+		// A build exactly YTDLP_STALE_DAYS old is not yet "older than" the threshold.
+		const boundary = new Date(now - YTDLP_STALE_DAYS * 86_400_000);
+		const v = `${boundary.getUTCFullYear()}.${String(boundary.getUTCMonth() + 1).padStart(2, '0')}.${String(boundary.getUTCDate()).padStart(2, '0')}`;
+		expect(isYtdlpStale(v, now)).toBe(false);
+	});
+
+	it('never flags an unparseable version as stale', () => {
+		expect(isYtdlpStale(null, now)).toBe(false);
+		expect(isYtdlpStale('nightly', now)).toBe(false);
+	});
+});
+
+describe('extractCaption', () => {
+	it('prefers description over title', () => {
+		expect(extractCaption({ description: 'real caption', title: 'Video by @x' })).toBe(
+			'real caption'
+		);
+	});
+
+	it('falls back to title then fulltitle', () => {
+		expect(extractCaption({ title: 'the title' })).toBe('the title');
+		expect(extractCaption({ fulltitle: 'the fulltitle' })).toBe('the fulltitle');
+	});
+
+	it('truncates very long captions with an ellipsis', () => {
+		const long = 'x'.repeat(600);
+		const out = extractCaption({ description: long });
+		expect(out?.length).toBe(501); // 500 chars + ellipsis
+		expect(out?.endsWith('…')).toBe(true);
+	});
+
+	it('returns null when there is nothing usable', () => {
+		expect(extractCaption({})).toBeNull();
+	});
+});

src/lib/server/providers/registry.ts

@@ -3,6 +3,7 @@ import { groups } from '../db/schema';
 import { eq } from 'drizzle-orm';
 import type { KnownProvider, DownloadProvider } from './types';
 import { YtDlpProvider } from './ytdlp';
+import { isYtdlpStale } from './ytdlp/version';
 import { FakeProvider } from './fake';
 
 const TEST_MODE = process.env.TEST_MODE === 'true';
@@ -15,7 +16,10 @@ export const KNOWN_PROVIDERS: KnownProvider[] = [
 		homepage: 'https://github.com/yt-dlp/yt-dlp',
 		license: 'Unlicense',
 		capabilities: ['video', 'music'],
-		binaryUrl: 'https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp',
+		// Nightly channel: extractor fixes for IG/TikTok/YT land here days-to-weeks before
+		// stable. The scheduler self-updates this binary, so tracking nightly keeps the app
+		// resilient to the frequent site changes that break social-media downloads.
+		binaryUrl: 'https://github.com/yt-dlp/yt-dlp-nightly-builds/releases/latest/download/yt-dlp',
 		dependencies: ['ffmpeg', 'python3']
 	},
 	...(TEST_MODE
@@ -72,6 +76,8 @@ export interface ProviderStatus {
 	installed: boolean;
 	active: boolean;
 	version: string | null;
+	/** yt-dlp only: true when the installed build is older than the staleness threshold. */
+	stale: boolean;
 }
 
 export async function listProvidersWithStatus(groupId: string): Promise<ProviderStatus[]> {
@@ -87,7 +93,8 @@ export async function listProvidersWithStatus(groupId: string): Promise<Provider
 				...known,
 				installed,
 				active: known.id === activeId && installed,
-				version
+				version,
+				stale: known.id === 'ytdlp' ? isYtdlpStale(version) : false
 			};
 		})
 	);