From 89ee1c5a21afb8759bf76d69191d9ddb77b20789 Mon Sep 17 00:00:00 2001 From: Brian Bornino Date: Mon, 13 Jul 2026 07:33:16 -0700 Subject: [PATCH 1/5] feat(worker): add cleanup task for unreferenced S3 attachments Sweeps S3 objects under posts/*/attachments/* on a repeatable interval and removes any with no matching post_attachments row, catching attachments orphaned by interrupted sync-post jobs. --- apps/worker/src/index.ts | 21 +++++- .../cleanup-attachments/processor.test.ts | 71 +++++++++++++++++++ .../tasks/cleanup-attachments/processor.ts | 32 +++++++++ apps/worker/test-utils/setup.ts | 1 + packages/bullmq/src/tasks/types.ts | 3 + packages/s3/src/utils.ts | 26 +++++++ 6 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 apps/worker/src/tasks/cleanup-attachments/processor.test.ts create mode 100644 apps/worker/src/tasks/cleanup-attachments/processor.ts diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts index 5114b521..720af9bc 100644 --- a/apps/worker/src/index.ts +++ b/apps/worker/src/index.ts @@ -1,6 +1,6 @@ import { createHealthcheck } from "./createHealthcheck.ts"; import { createWorker } from "./createWorker.ts"; -import { Tasks } from "@playfulprogramming/bullmq"; +import { Tasks, createQueue } from "@playfulprogramming/bullmq"; createWorker(Tasks.POST_IMAGES, "./tasks/post-images/processor.ts"); createWorker(Tasks.URL_METADATA, "./tasks/url-metadata/processor.ts"); @@ -12,4 +12,23 @@ createWorker( Tasks.GRANT_AUTHOR_ACHIEVEMENTS, "./tasks/grant-author-achievements/processor.ts", ); +createWorker( + Tasks.CLEANUP_ATTACHMENTS, + "./tasks/cleanup-attachments/processor.ts", +); createHealthcheck(); + +// Repeatable job: BullMQ dedupes repeatable schedulers by name + repeat +// options, so re-registering this on every worker restart is a no-op rather +// than creating duplicate schedules. +const CLEANUP_ATTACHMENTS_INTERVAL_MS = 24 * 60 * 60 * 1000; + +createQueue(Tasks.CLEANUP_ATTACHMENTS) + .add( + Tasks.CLEANUP_ATTACHMENTS, + {}, + { repeat: { every: CLEANUP_ATTACHMENTS_INTERVAL_MS } }, + ) + .catch((err) => + console.error("Failed to schedule cleanup-attachments job:", err), + ); diff --git a/apps/worker/src/tasks/cleanup-attachments/processor.test.ts b/apps/worker/src/tasks/cleanup-attachments/processor.test.ts new file mode 100644 index 00000000..f69502c5 --- /dev/null +++ b/apps/worker/src/tasks/cleanup-attachments/processor.test.ts @@ -0,0 +1,71 @@ +import processor from "./processor.ts"; +import { db, postAttachments } from "@playfulprogramming/db"; +import { s3 } from "@playfulprogramming/s3"; + +test("Removes an attachment from S3 when no post_attachments row references it", async () => { + vi.mocked(s3.list).mockResolvedValue([ + "posts/example-post/en/content.md", + "posts/example-post/attachments/referenced-sha.pdf", + "posts/example-post/attachments/orphaned-sha.jpeg", + ]); + + vi.mocked(db.select).mockReturnValue({ + from: vi.fn().mockResolvedValue([ + { + attachmentKey: "posts/example-post/attachments/referenced-sha.pdf", + }, + ]), + } as never); + + await processor({} as never); + + expect(s3.remove).toBeCalledWith( + "example-bucket", + "posts/example-post/attachments/orphaned-sha.jpeg", + ); + expect(s3.remove).toBeCalledTimes(1); +}); + +test("Leaves an attachment alone when a post_attachments row still references it", async () => { + vi.mocked(s3.list).mockResolvedValue([ + "posts/example-post/attachments/referenced-sha.pdf", + ]); + + vi.mocked(db.select).mockReturnValue({ + from: vi.fn().mockResolvedValue([ + { + attachmentKey: "posts/example-post/attachments/referenced-sha.pdf", + }, + ]), + } as never); + + await processor({} as never); + + expect(s3.remove).not.toBeCalled(); +}); + +test("Does nothing when there are no attachment objects in S3", async () => { + vi.mocked(s3.list).mockResolvedValue(["posts/example-post/en/content.md"]); + + await processor({} as never); + + expect(db.select).not.toBeCalled(); + expect(s3.remove).not.toBeCalled(); +}); + +test("Queries the full attachment table with no per-post filter", async () => { + vi.mocked(s3.list).mockResolvedValue([ + "posts/example-post/attachments/orphaned-sha.jpeg", + ]); + + const from = vi.fn().mockResolvedValue([]); + vi.mocked(db.select).mockReturnValue({ from } as never); + + await processor({} as never); + + expect(from).toBeCalledWith(postAttachments); + expect(s3.remove).toBeCalledWith( + "example-bucket", + "posts/example-post/attachments/orphaned-sha.jpeg", + ); +}); diff --git a/apps/worker/src/tasks/cleanup-attachments/processor.ts b/apps/worker/src/tasks/cleanup-attachments/processor.ts new file mode 100644 index 00000000..8449e925 --- /dev/null +++ b/apps/worker/src/tasks/cleanup-attachments/processor.ts @@ -0,0 +1,32 @@ +import { env } from "@playfulprogramming/common"; +import { Tasks } from "@playfulprogramming/bullmq"; +import { db, postAttachments } from "@playfulprogramming/db"; +import { s3 } from "@playfulprogramming/s3"; +import { createProcessor } from "../../createProcessor.ts"; + +const ATTACHMENTS_PREFIX = "posts/"; + +export default createProcessor(Tasks.CLEANUP_ATTACHMENTS, async () => { + const bucket = await s3.ensureBucket(env.S3_BUCKET); + + const objectKeys = await s3.list(bucket, ATTACHMENTS_PREFIX); + const attachmentKeys = objectKeys.filter((key) => + key.includes("/attachments/"), + ); + + if (attachmentKeys.length === 0) return; + + const referencedRows = await db + .select({ attachmentKey: postAttachments.attachmentKey }) + .from(postAttachments); + const referencedKeys = new Set( + referencedRows.map((row) => row.attachmentKey), + ); + + for (const key of attachmentKeys) { + if (referencedKeys.has(key)) continue; + + await s3.remove(bucket, key); + console.log(`Removed unreferenced attachment ${key} from S3`); + } +}); diff --git a/apps/worker/test-utils/setup.ts b/apps/worker/test-utils/setup.ts index de6a3618..783a4564 100644 --- a/apps/worker/test-utils/setup.ts +++ b/apps/worker/test-utils/setup.ts @@ -23,6 +23,7 @@ vi.mock("@playfulprogramming/s3", () => { ensureBucket: vi.fn(() => "example-bucket"), upload: vi.fn(), remove: vi.fn(), + list: vi.fn(() => []), }, }; }); diff --git a/packages/bullmq/src/tasks/types.ts b/packages/bullmq/src/tasks/types.ts index 465ff1f3..b594b288 100644 --- a/packages/bullmq/src/tasks/types.ts +++ b/packages/bullmq/src/tasks/types.ts @@ -20,6 +20,7 @@ export const Tasks = { URL_METADATA: "url-metadata", POST_IMAGES: "post-images", GRANT_AUTHOR_ACHIEVEMENTS: "grant-author-achievements", + CLEANUP_ATTACHMENTS: "cleanup-attachments", } as const; export type TasksKeys = keyof typeof Tasks; @@ -33,6 +34,7 @@ export interface TaskInputs { [Tasks.URL_METADATA]: UrlMetadataInput; [Tasks.POST_IMAGES]: PostImageInput; [Tasks.GRANT_AUTHOR_ACHIEVEMENTS]: GrantAuthorAchievementsInput; + [Tasks.CLEANUP_ATTACHMENTS]: object; } export type TaskInputsValues = TaskInputs[TasksValues]; @@ -45,6 +47,7 @@ export interface TaskOutputs { [Tasks.URL_METADATA]: UrlMetadataOutput; [Tasks.POST_IMAGES]: PostImageOutput; [Tasks.GRANT_AUTHOR_ACHIEVEMENTS]: GrantAuthorAchievementsOutput; + [Tasks.CLEANUP_ATTACHMENTS]: object; } export type TaskOutputsValues = TaskOutputs[TasksValues]; diff --git a/packages/s3/src/utils.ts b/packages/s3/src/utils.ts index 7c4eed77..a95fcd64 100644 --- a/packages/s3/src/utils.ts +++ b/packages/s3/src/utils.ts @@ -5,6 +5,7 @@ import { DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, + ListObjectsV2Command, NoSuchKey, PutBucketPolicyCommand, } from "@aws-sdk/client-s3"; @@ -70,6 +71,31 @@ export async function remove(bucket: string, key: string) { await client.send(new DeleteObjectCommand({ Bucket: bucket, Key: key })); } +export async function list(bucket: string, prefix: string): Promise { + const keys: string[] = []; + let continuationToken: string | undefined; + + do { + const response = await client.send( + new ListObjectsV2Command({ + Bucket: bucket, + Prefix: prefix, + ContinuationToken: continuationToken, + }), + ); + + for (const object of response.Contents ?? []) { + if (object.Key) keys.push(object.Key); + } + + continuationToken = response.IsTruncated + ? response.NextContinuationToken + : undefined; + } while (continuationToken); + + return keys; +} + export async function matchesEtag( bucket: string, key: string, From 76e8422f59d49b142e62c4b63b99c9570af584fd Mon Sep 17 00:00:00 2001 From: Brian Bornino Date: Mon, 13 Jul 2026 08:00:50 -0700 Subject: [PATCH 2/5] feat(worker): add grace period to unreferenced attachment cleanup sync-post uploads an attachment to S3 before its post_attachments row commits, so a very recent orphan-looking object may just be mid-flight. S3's list now returns lastModified alongside each key, and the cleanup task skips anything younger than a one-hour grace period. --- .../cleanup-attachments/processor.test.ts | 62 +++++++++++++++++-- .../tasks/cleanup-attachments/processor.ts | 19 ++++-- packages/s3/src/utils.ts | 18 ++++-- 3 files changed, 83 insertions(+), 16 deletions(-) diff --git a/apps/worker/src/tasks/cleanup-attachments/processor.test.ts b/apps/worker/src/tasks/cleanup-attachments/processor.test.ts index f69502c5..9a347c13 100644 --- a/apps/worker/src/tasks/cleanup-attachments/processor.test.ts +++ b/apps/worker/src/tasks/cleanup-attachments/processor.test.ts @@ -2,11 +2,27 @@ import processor from "./processor.ts"; import { db, postAttachments } from "@playfulprogramming/db"; import { s3 } from "@playfulprogramming/s3"; +const NOW = new Date("2025-05-05T12:00:00Z"); +const ONE_HOUR_MS = 60 * 60 * 1000; +const OUTSIDE_GRACE_PERIOD = new Date(NOW.getTime() - ONE_HOUR_MS - 1000); +const INSIDE_GRACE_PERIOD = new Date(NOW.getTime() - 30 * 60 * 1000); + test("Removes an attachment from S3 when no post_attachments row references it", async () => { + vi.setSystemTime(NOW); + vi.mocked(s3.list).mockResolvedValue([ - "posts/example-post/en/content.md", - "posts/example-post/attachments/referenced-sha.pdf", - "posts/example-post/attachments/orphaned-sha.jpeg", + { + key: "posts/example-post/en/content.md", + lastModified: OUTSIDE_GRACE_PERIOD, + }, + { + key: "posts/example-post/attachments/referenced-sha.pdf", + lastModified: OUTSIDE_GRACE_PERIOD, + }, + { + key: "posts/example-post/attachments/orphaned-sha.jpeg", + lastModified: OUTSIDE_GRACE_PERIOD, + }, ]); vi.mocked(db.select).mockReturnValue({ @@ -27,8 +43,13 @@ test("Removes an attachment from S3 when no post_attachments row references it", }); test("Leaves an attachment alone when a post_attachments row still references it", async () => { + vi.setSystemTime(NOW); + vi.mocked(s3.list).mockResolvedValue([ - "posts/example-post/attachments/referenced-sha.pdf", + { + key: "posts/example-post/attachments/referenced-sha.pdf", + lastModified: OUTSIDE_GRACE_PERIOD, + }, ]); vi.mocked(db.select).mockReturnValue({ @@ -45,7 +66,14 @@ test("Leaves an attachment alone when a post_attachments row still references it }); test("Does nothing when there are no attachment objects in S3", async () => { - vi.mocked(s3.list).mockResolvedValue(["posts/example-post/en/content.md"]); + vi.setSystemTime(NOW); + + vi.mocked(s3.list).mockResolvedValue([ + { + key: "posts/example-post/en/content.md", + lastModified: OUTSIDE_GRACE_PERIOD, + }, + ]); await processor({} as never); @@ -54,8 +82,13 @@ test("Does nothing when there are no attachment objects in S3", async () => { }); test("Queries the full attachment table with no per-post filter", async () => { + vi.setSystemTime(NOW); + vi.mocked(s3.list).mockResolvedValue([ - "posts/example-post/attachments/orphaned-sha.jpeg", + { + key: "posts/example-post/attachments/orphaned-sha.jpeg", + lastModified: OUTSIDE_GRACE_PERIOD, + }, ]); const from = vi.fn().mockResolvedValue([]); @@ -69,3 +102,20 @@ test("Queries the full attachment table with no per-post filter", async () => { "posts/example-post/attachments/orphaned-sha.jpeg", ); }); + +test("Leaves an unreferenced attachment alone when it's within the grace period", async () => { + vi.setSystemTime(NOW); + + vi.mocked(s3.list).mockResolvedValue([ + { + key: "posts/example-post/attachments/fresh-sha.jpeg", + lastModified: INSIDE_GRACE_PERIOD, + }, + ]); + + await processor({} as never); + + // The grace period filters it out before the post_attachments query even runs + expect(db.select).not.toBeCalled(); + expect(s3.remove).not.toBeCalled(); +}); diff --git a/apps/worker/src/tasks/cleanup-attachments/processor.ts b/apps/worker/src/tasks/cleanup-attachments/processor.ts index 8449e925..8a17c61c 100644 --- a/apps/worker/src/tasks/cleanup-attachments/processor.ts +++ b/apps/worker/src/tasks/cleanup-attachments/processor.ts @@ -5,16 +5,23 @@ import { s3 } from "@playfulprogramming/s3"; import { createProcessor } from "../../createProcessor.ts"; const ATTACHMENTS_PREFIX = "posts/"; +const GRACE_PERIOD_MS = 60 * 60 * 1000; export default createProcessor(Tasks.CLEANUP_ATTACHMENTS, async () => { const bucket = await s3.ensureBucket(env.S3_BUCKET); - const objectKeys = await s3.list(bucket, ATTACHMENTS_PREFIX); - const attachmentKeys = objectKeys.filter((key) => - key.includes("/attachments/"), - ); + const objects = await s3.list(bucket, ATTACHMENTS_PREFIX); + const now = Date.now(); + + // sync-post uploads an attachment to S3 before its post_attachments row is + // committed, so a very recent object may just be mid-flight rather than + // truly orphaned - skip anything younger than the grace period. + const candidateKeys = objects + .filter((object) => object.key.includes("/attachments/")) + .filter((object) => now - object.lastModified.getTime() > GRACE_PERIOD_MS) + .map((object) => object.key); - if (attachmentKeys.length === 0) return; + if (candidateKeys.length === 0) return; const referencedRows = await db .select({ attachmentKey: postAttachments.attachmentKey }) @@ -23,7 +30,7 @@ export default createProcessor(Tasks.CLEANUP_ATTACHMENTS, async () => { referencedRows.map((row) => row.attachmentKey), ); - for (const key of attachmentKeys) { + for (const key of candidateKeys) { if (referencedKeys.has(key)) continue; await s3.remove(bucket, key); diff --git a/packages/s3/src/utils.ts b/packages/s3/src/utils.ts index a95fcd64..d19d7f36 100644 --- a/packages/s3/src/utils.ts +++ b/packages/s3/src/utils.ts @@ -71,8 +71,16 @@ export async function remove(bucket: string, key: string) { await client.send(new DeleteObjectCommand({ Bucket: bucket, Key: key })); } -export async function list(bucket: string, prefix: string): Promise { - const keys: string[] = []; +export interface S3Object { + key: string; + lastModified: Date; +} + +export async function list( + bucket: string, + prefix: string, +): Promise { + const objects: S3Object[] = []; let continuationToken: string | undefined; do { @@ -85,7 +93,9 @@ export async function list(bucket: string, prefix: string): Promise { ); for (const object of response.Contents ?? []) { - if (object.Key) keys.push(object.Key); + if (object.Key && object.LastModified) { + objects.push({ key: object.Key, lastModified: object.LastModified }); + } } continuationToken = response.IsTruncated @@ -93,7 +103,7 @@ export async function list(bucket: string, prefix: string): Promise { : undefined; } while (continuationToken); - return keys; + return objects; } export async function matchesEtag( From ca598c657a169779af5cb537747adfb2c400c5b7 Mon Sep 17 00:00:00 2001 From: Brian Bornino Date: Mon, 13 Jul 2026 08:21:41 -0700 Subject: [PATCH 3/5] fix(bullmq): correct CLEANUP_ATTACHMENTS output type to void TaskOutputs[Tasks.CLEANUP_ATTACHMENTS] was typed as object but the processor returns Promise, breaking the build. Also adds a test locking in that an S3 removal failure rejects the job instead of being swallowed. --- .../cleanup-attachments/processor.test.ts | 20 +++++++++++++++++++ packages/bullmq/src/tasks/types.ts | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/apps/worker/src/tasks/cleanup-attachments/processor.test.ts b/apps/worker/src/tasks/cleanup-attachments/processor.test.ts index 9a347c13..9ab6f2de 100644 --- a/apps/worker/src/tasks/cleanup-attachments/processor.test.ts +++ b/apps/worker/src/tasks/cleanup-attachments/processor.test.ts @@ -103,6 +103,26 @@ test("Queries the full attachment table with no per-post filter", async () => { ); }); +test("Fails the job when an S3 removal rejects, rather than continuing past the error", async () => { + vi.setSystemTime(NOW); + + vi.mocked(s3.list).mockResolvedValue([ + { + key: "posts/example-post/attachments/orphaned-sha.jpeg", + lastModified: OUTSIDE_GRACE_PERIOD, + }, + ]); + + vi.mocked(db.select).mockReturnValue({ + from: vi.fn().mockResolvedValue([]), + } as never); + + const s3Error = new Error("S3 removal failed"); + vi.mocked(s3.remove).mockRejectedValue(s3Error); + + await expect(processor({} as never)).rejects.toThrow(s3Error); +}); + test("Leaves an unreferenced attachment alone when it's within the grace period", async () => { vi.setSystemTime(NOW); diff --git a/packages/bullmq/src/tasks/types.ts b/packages/bullmq/src/tasks/types.ts index b594b288..b171ae2b 100644 --- a/packages/bullmq/src/tasks/types.ts +++ b/packages/bullmq/src/tasks/types.ts @@ -47,7 +47,7 @@ export interface TaskOutputs { [Tasks.URL_METADATA]: UrlMetadataOutput; [Tasks.POST_IMAGES]: PostImageOutput; [Tasks.GRANT_AUTHOR_ACHIEVEMENTS]: GrantAuthorAchievementsOutput; - [Tasks.CLEANUP_ATTACHMENTS]: object; + [Tasks.CLEANUP_ATTACHMENTS]: void; } export type TaskOutputsValues = TaskOutputs[TasksValues]; From c17d85edc2a41f0eeb383f472a410469f5bcf6f0 Mon Sep 17 00:00:00 2001 From: Brian Bornino Date: Fri, 7 Aug 2026 07:12:17 -0700 Subject: [PATCH 4/5] feat(worker): query attachments table directly for cleanup, per review feedback Replaces the S3-list-based orphan sweep with a DB-native DELETE ... WHERE NOT EXISTS ... RETURNING loop against the attachments table, per James's review feedback on #195. Adds a lastModified column to attachments, used as the grace-period condition in place of S3 object timestamps, and switches the sync-post insert from onConflictDoNothing to onConflictDoUpdate so a conflicting insert (content reused across posts/ branches) still refreshes the timestamp instead of silently no-opping. --- .../cleanup-attachments/processor.test.ts | 166 +- .../tasks/cleanup-attachments/processor.ts | 83 +- .../src/tasks/sync-post/processor.test.ts | 3 + apps/worker/src/tasks/sync-post/processor.ts | 11 +- .../migration.sql | 1 + .../snapshot.json | 1906 +++++++++++++++++ packages/db/src/schema/attachments.ts | 5 +- packages/test-fixtures/src/db-mock.ts | 1 + 8 files changed, 2052 insertions(+), 124 deletions(-) create mode 100644 packages/db/drizzle/20260807133839_sudden_micromacro/migration.sql create mode 100644 packages/db/drizzle/20260807133839_sudden_micromacro/snapshot.json diff --git a/apps/worker/src/tasks/cleanup-attachments/processor.test.ts b/apps/worker/src/tasks/cleanup-attachments/processor.test.ts index 9ab6f2de..d47c09f5 100644 --- a/apps/worker/src/tasks/cleanup-attachments/processor.test.ts +++ b/apps/worker/src/tasks/cleanup-attachments/processor.test.ts @@ -1,141 +1,109 @@ import processor from "./processor.ts"; -import { db, postAttachments } from "@playfulprogramming/db"; +import { db, attachments } from "@playfulprogramming/db"; import { s3 } from "@playfulprogramming/s3"; const NOW = new Date("2025-05-05T12:00:00Z"); -const ONE_HOUR_MS = 60 * 60 * 1000; -const OUTSIDE_GRACE_PERIOD = new Date(NOW.getTime() - ONE_HOUR_MS - 1000); -const INSIDE_GRACE_PERIOD = new Date(NOW.getTime() - 30 * 60 * 1000); -test("Removes an attachment from S3 when no post_attachments row references it", async () => { +const deleteAttachmentReturning = db + .delete(attachments) + .where(expect.anything()).returning; +const insertAttachmentValues = db.insert(attachments).values; +const insertAttachmentOnConflictDoUpdate = db + .insert(attachments) + .values(expect.anything()).onConflictDoUpdate; + +test("Removes an attachment returned by the delete query from S3", async () => { vi.setSystemTime(NOW); - vi.mocked(s3.list).mockResolvedValue([ - { - key: "posts/example-post/en/content.md", - lastModified: OUTSIDE_GRACE_PERIOD, - }, - { - key: "posts/example-post/attachments/referenced-sha.pdf", - lastModified: OUTSIDE_GRACE_PERIOD, - }, - { - key: "posts/example-post/attachments/orphaned-sha.jpeg", - lastModified: OUTSIDE_GRACE_PERIOD, - }, - ]); - - vi.mocked(db.select).mockReturnValue({ - from: vi.fn().mockResolvedValue([ + vi.mocked(deleteAttachmentReturning) + .mockResolvedValueOnce([ { - attachmentKey: "posts/example-post/attachments/referenced-sha.pdf", + attachmentKey: "posts/example-post/attachments/orphaned-sha.jpeg", + sha: "orphaned-sha", + width: 100, + height: 100, }, - ]), - } as never); + ]) + .mockResolvedValueOnce([]); await processor({} as never); - expect(s3.remove).toBeCalledWith( + expect(s3.remove).toHaveBeenCalledWith( "example-bucket", "posts/example-post/attachments/orphaned-sha.jpeg", ); - expect(s3.remove).toBeCalledTimes(1); + expect(s3.remove).toHaveBeenCalledTimes(1); }); -test("Leaves an attachment alone when a post_attachments row still references it", async () => { +test("Does nothing when the delete query returns no rows", async () => { vi.setSystemTime(NOW); - vi.mocked(s3.list).mockResolvedValue([ - { - key: "posts/example-post/attachments/referenced-sha.pdf", - lastModified: OUTSIDE_GRACE_PERIOD, - }, - ]); - - vi.mocked(db.select).mockReturnValue({ - from: vi.fn().mockResolvedValue([ - { - attachmentKey: "posts/example-post/attachments/referenced-sha.pdf", - }, - ]), - } as never); + vi.mocked(deleteAttachmentReturning).mockResolvedValueOnce([]); await processor({} as never); - expect(s3.remove).not.toBeCalled(); + expect(s3.remove).not.toHaveBeenCalled(); }); -test("Does nothing when there are no attachment objects in S3", async () => { +test("Keeps deleting and removing until the delete query returns no more rows", async () => { vi.setSystemTime(NOW); - vi.mocked(s3.list).mockResolvedValue([ - { - key: "posts/example-post/en/content.md", - lastModified: OUTSIDE_GRACE_PERIOD, - }, - ]); - - await processor({} as never); - - expect(db.select).not.toBeCalled(); - expect(s3.remove).not.toBeCalled(); -}); - -test("Queries the full attachment table with no per-post filter", async () => { - vi.setSystemTime(NOW); - - vi.mocked(s3.list).mockResolvedValue([ - { - key: "posts/example-post/attachments/orphaned-sha.jpeg", - lastModified: OUTSIDE_GRACE_PERIOD, - }, - ]); - - const from = vi.fn().mockResolvedValue([]); - vi.mocked(db.select).mockReturnValue({ from } as never); + vi.mocked(deleteAttachmentReturning) + .mockResolvedValueOnce([ + { + attachmentKey: "posts/example-post/attachments/first.jpeg", + sha: "first", + width: 1, + height: 1, + }, + ]) + .mockResolvedValueOnce([ + { + attachmentKey: "posts/example-post/attachments/second.jpeg", + sha: "second", + width: 1, + height: 1, + }, + ]) + .mockResolvedValueOnce([]); await processor({} as never); - expect(from).toBeCalledWith(postAttachments); - expect(s3.remove).toBeCalledWith( + expect(s3.remove).toHaveBeenNthCalledWith( + 1, "example-bucket", - "posts/example-post/attachments/orphaned-sha.jpeg", + "posts/example-post/attachments/first.jpeg", ); + expect(s3.remove).toHaveBeenNthCalledWith( + 2, + "example-bucket", + "posts/example-post/attachments/second.jpeg", + ); + expect(s3.remove).toHaveBeenCalledTimes(2); }); -test("Fails the job when an S3 removal rejects, rather than continuing past the error", async () => { +test("Re-inserts the row and fails the job when S3 removal rejects, rather than leaking the object untracked", async () => { vi.setSystemTime(NOW); - vi.mocked(s3.list).mockResolvedValue([ - { - key: "posts/example-post/attachments/orphaned-sha.jpeg", - lastModified: OUTSIDE_GRACE_PERIOD, - }, - ]); - - vi.mocked(db.select).mockReturnValue({ - from: vi.fn().mockResolvedValue([]), - } as never); + const orphan = { + attachmentKey: "posts/example-post/attachments/orphaned-sha.jpeg", + sha: "orphaned-sha", + width: 100, + height: 100, + }; + vi.mocked(deleteAttachmentReturning).mockResolvedValueOnce([orphan]); const s3Error = new Error("S3 removal failed"); vi.mocked(s3.remove).mockRejectedValue(s3Error); await expect(processor({} as never)).rejects.toThrow(s3Error); -}); - -test("Leaves an unreferenced attachment alone when it's within the grace period", async () => { - vi.setSystemTime(NOW); - - vi.mocked(s3.list).mockResolvedValue([ - { - key: "posts/example-post/attachments/fresh-sha.jpeg", - lastModified: INSIDE_GRACE_PERIOD, - }, - ]); - - await processor({} as never); - // The grace period filters it out before the post_attachments query even runs - expect(db.select).not.toBeCalled(); - expect(s3.remove).not.toBeCalled(); + expect(insertAttachmentValues).toHaveBeenCalledWith({ + ...orphan, + lastModified: expect.any(Date), + }); + expect(insertAttachmentOnConflictDoUpdate).toHaveBeenCalledWith({ + target: attachments.attachmentKey, + set: { lastModified: expect.any(Date) }, + }); }); diff --git a/apps/worker/src/tasks/cleanup-attachments/processor.ts b/apps/worker/src/tasks/cleanup-attachments/processor.ts index 8a17c61c..91a5f7d8 100644 --- a/apps/worker/src/tasks/cleanup-attachments/processor.ts +++ b/apps/worker/src/tasks/cleanup-attachments/processor.ts @@ -1,39 +1,76 @@ import { env } from "@playfulprogramming/common"; import { Tasks } from "@playfulprogramming/bullmq"; -import { db, postAttachments } from "@playfulprogramming/db"; +import { db, attachments, postAttachments } from "@playfulprogramming/db"; import { s3 } from "@playfulprogramming/s3"; +import { and, eq, lt, notExists } from "drizzle-orm"; import { createProcessor } from "../../createProcessor.ts"; -const ATTACHMENTS_PREFIX = "posts/"; const GRACE_PERIOD_MS = 60 * 60 * 1000; export default createProcessor(Tasks.CLEANUP_ATTACHMENTS, async () => { const bucket = await s3.ensureBucket(env.S3_BUCKET); + const staleBefore = new Date(Date.now() - GRACE_PERIOD_MS); - const objects = await s3.list(bucket, ATTACHMENTS_PREFIX); - const now = Date.now(); + for (;;) { + // sync-post uploads an attachment to S3 and inserts its attachments row + // before the corresponding post_attachments row commits, so a very + // recent row may just be mid-flight rather than truly orphaned - skip + // anything younger than the grace period. Safety against a + // concurrently-committing post_attachments insert doesn't come from + // this being a single SQL statement - it comes from the + // post_attachments -> attachments foreign key (onDelete: "cascade" in + // posts.ts): inserting a post_attachments row takes a lock on the + // referenced attachments row, which serializes against this DELETE. If + // that FK is ever dropped, this safety goes with it. + const candidateKey = db + .select({ attachmentKey: attachments.attachmentKey }) + .from(attachments) + .where( + and( + notExists( + db + .select({ attachmentKey: postAttachments.attachmentKey }) + .from(postAttachments) + .where( + eq(postAttachments.attachmentKey, attachments.attachmentKey), + ), + ), + lt(attachments.lastModified, staleBefore), + ), + ) + .limit(1); - // sync-post uploads an attachment to S3 before its post_attachments row is - // committed, so a very recent object may just be mid-flight rather than - // truly orphaned - skip anything younger than the grace period. - const candidateKeys = objects - .filter((object) => object.key.includes("/attachments/")) - .filter((object) => now - object.lastModified.getTime() > GRACE_PERIOD_MS) - .map((object) => object.key); + const [deleted] = await db + .delete(attachments) + .where(eq(attachments.attachmentKey, candidateKey)) + .returning({ + attachmentKey: attachments.attachmentKey, + sha: attachments.sha, + width: attachments.width, + height: attachments.height, + }); - if (candidateKeys.length === 0) return; + if (!deleted) return; - const referencedRows = await db - .select({ attachmentKey: postAttachments.attachmentKey }) - .from(postAttachments); - const referencedKeys = new Set( - referencedRows.map((row) => row.attachmentKey), - ); + try { + await s3.remove(bucket, deleted.attachmentKey); + } catch (err) { + // The row is already claimed/deleted, so a failed removal here would + // otherwise permanently lose track of the S3 object - nothing would + // ever find it again. Re-insert it (with a fresh lastModified) so the + // next scheduled run picks it back up, then fail the job as a whole. + await db + .insert(attachments) + .values({ ...deleted, lastModified: new Date() }) + .onConflictDoUpdate({ + target: attachments.attachmentKey, + set: { lastModified: new Date() }, + }); + throw err; + } - for (const key of candidateKeys) { - if (referencedKeys.has(key)) continue; - - await s3.remove(bucket, key); - console.log(`Removed unreferenced attachment ${key} from S3`); + console.log( + `Removed unreferenced attachment ${deleted.attachmentKey} from S3`, + ); } }); diff --git a/apps/worker/src/tasks/sync-post/processor.test.ts b/apps/worker/src/tasks/sync-post/processor.test.ts index a85a3687..99a7006c 100644 --- a/apps/worker/src/tasks/sync-post/processor.test.ts +++ b/apps/worker/src/tasks/sync-post/processor.test.ts @@ -700,12 +700,14 @@ published: "2024-01-15T00:00:00Z" sha: "notes-sha", width: null, height: null, + lastModified: expect.any(Date), }); expect(db.insert(attachments).values).toHaveBeenCalledWith({ attachmentKey: "posts/attachment-post/attachments/banner-sha.jpeg", sha: "banner-sha", width: 1, height: 1, + lastModified: expect.any(Date), }); expect(db.insert(attachments).values).toHaveBeenCalledTimes(2); @@ -913,6 +915,7 @@ published: "2024-01-15T00:00:00Z" sha: "new-changed-sha", width: null, height: null, + lastModified: expect.any(Date), }); expect(db.insert(attachments).values).toHaveBeenCalledTimes(1); diff --git a/apps/worker/src/tasks/sync-post/processor.ts b/apps/worker/src/tasks/sync-post/processor.ts index 6a75a1e7..3587a234 100644 --- a/apps/worker/src/tasks/sync-post/processor.ts +++ b/apps/worker/src/tasks/sync-post/processor.ts @@ -310,6 +310,11 @@ export default createProcessor(Tasks.SYNC_POST, async (job, { signal }) => { ); console.log(`Uploaded attachment ${attachmentKey} to S3`); + // onConflictDoUpdate (not DoNothing) so a conflicting insert - e.g. the + // same sha reused across posts/branches - still refreshes lastModified. + // Otherwise a stale timestamp could let the cleanup sweep delete this row + // in the narrow window before its new post_attachments reference commits. + const attachmentLastModified = new Date(); await db .insert(attachments) .values({ @@ -317,8 +322,12 @@ export default createProcessor(Tasks.SYNC_POST, async (job, { signal }) => { sha, width, height, + lastModified: attachmentLastModified, }) - .onConflictDoNothing(); + .onConflictDoUpdate({ + target: attachments.attachmentKey, + set: { lastModified: attachmentLastModified }, + }); attachmentRows.push({ attachmentKey, diff --git a/packages/db/drizzle/20260807133839_sudden_micromacro/migration.sql b/packages/db/drizzle/20260807133839_sudden_micromacro/migration.sql new file mode 100644 index 00000000..93c1937c --- /dev/null +++ b/packages/db/drizzle/20260807133839_sudden_micromacro/migration.sql @@ -0,0 +1 @@ +ALTER TABLE "attachments" ADD COLUMN "last_modified" timestamp with time zone DEFAULT now() NOT NULL; \ No newline at end of file diff --git a/packages/db/drizzle/20260807133839_sudden_micromacro/snapshot.json b/packages/db/drizzle/20260807133839_sudden_micromacro/snapshot.json new file mode 100644 index 00000000..79d8e598 --- /dev/null +++ b/packages/db/drizzle/20260807133839_sudden_micromacro/snapshot.json @@ -0,0 +1,1906 @@ +{ + "version": "8", + "dialect": "postgres", + "id": "48611f58-9e82-489d-8592-4a2343177d29", + "prevIds": [ + "d3fee9ff-760a-4a38-bfc5-e8a035bb0a0e" + ], + "ddl": [ + { + "isRlsEnabled": false, + "name": "author_roles", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "profile_achievements", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "profiles", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "collection_authors", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "collection_data", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "collection_tags", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "collections", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "post_attachments", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "post_authors", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "post_groups", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "post_tags", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "posts", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "post_images", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "url_metadata", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "url_metadata_gist", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "url_metadata_gist_file", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "url_metadata_post", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "attachments", + "entityType": "tables", + "schema": "public" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "profile_slug", + "entityType": "columns", + "schema": "public", + "table": "author_roles" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "role", + "entityType": "columns", + "schema": "public", + "table": "author_roles" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "profile_slug", + "entityType": "columns", + "schema": "public", + "table": "profile_achievements" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "achievement_id", + "entityType": "columns", + "schema": "public", + "table": "profile_achievements" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "granted_at", + "entityType": "columns", + "schema": "public", + "table": "profile_achievements" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "slug", + "entityType": "columns", + "schema": "public", + "table": "profiles" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "profiles" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "''", + "generated": null, + "identity": null, + "name": "description", + "entityType": "columns", + "schema": "public", + "table": "profiles" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "profile_image", + "entityType": "columns", + "schema": "public", + "table": "profiles" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "published_at", + "entityType": "columns", + "schema": "public", + "table": "profiles" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "meta", + "entityType": "columns", + "schema": "public", + "table": "profiles" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "collection_slug", + "entityType": "columns", + "schema": "public", + "table": "collection_authors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "author_slug", + "entityType": "columns", + "schema": "public", + "table": "collection_authors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "slug", + "entityType": "columns", + "schema": "public", + "table": "collection_data" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "locale", + "entityType": "columns", + "schema": "public", + "table": "collection_data" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "title", + "entityType": "columns", + "schema": "public", + "table": "collection_data" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "''", + "generated": null, + "identity": null, + "name": "description", + "entityType": "columns", + "schema": "public", + "table": "collection_data" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "published_at", + "entityType": "columns", + "schema": "public", + "table": "collection_data" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "meta", + "entityType": "columns", + "schema": "public", + "table": "collection_data" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "cover_image", + "entityType": "columns", + "schema": "public", + "table": "collection_data" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "social_image", + "entityType": "columns", + "schema": "public", + "table": "collection_data" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "collection_slug", + "entityType": "columns", + "schema": "public", + "table": "collection_tags" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tag", + "entityType": "columns", + "schema": "public", + "table": "collection_tags" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "slug", + "entityType": "columns", + "schema": "public", + "table": "collections" + }, + { + "type": "uuid", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "post_id", + "entityType": "columns", + "schema": "public", + "table": "post_attachments" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "attachment_key", + "entityType": "columns", + "schema": "public", + "table": "post_attachments" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "attachment_name", + "entityType": "columns", + "schema": "public", + "table": "post_attachments" + }, + { + "type": "uuid", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "post_id", + "entityType": "columns", + "schema": "public", + "table": "post_authors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "author_slug", + "entityType": "columns", + "schema": "public", + "table": "post_authors" + }, + { + "type": "uuid", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "gen_random_uuid()", + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "post_groups" + }, + { + "type": "uuid", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "post_id", + "entityType": "columns", + "schema": "public", + "table": "post_tags" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tag", + "entityType": "columns", + "schema": "public", + "table": "post_tags" + }, + { + "type": "uuid", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "gen_random_uuid()", + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "slug", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "locale", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "branch", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "collection_slug", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "collection_order", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "uuid", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "group_id", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "''", + "generated": null, + "identity": null, + "name": "version_name", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "version_order", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "title", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "''", + "generated": null, + "identity": null, + "name": "description", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "word_count", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "social_image", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "banner_image", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "original_link", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "noindex", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "edited_at", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "published_at", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "meta", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "varchar(2048)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "slug", + "entityType": "columns", + "schema": "public", + "table": "post_images" + }, + { + "type": "varchar(2048)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "banner_key", + "entityType": "columns", + "schema": "public", + "table": "post_images" + }, + { + "type": "varchar(2048)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "link_preview_key", + "entityType": "columns", + "schema": "public", + "table": "post_images" + }, + { + "type": "varchar(2048)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "index_md5", + "entityType": "columns", + "schema": "public", + "table": "post_images" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "fetched_at", + "entityType": "columns", + "schema": "public", + "table": "post_images" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "error", + "entityType": "columns", + "schema": "public", + "table": "post_images" + }, + { + "type": "varchar(2048)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "url", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "varchar(2048)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "title", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "varchar(2048)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "icon_key", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "icon_width", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "icon_height", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "varchar(2048)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "banner_key", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "banner_width", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "banner_height", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "varchar(256)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "gist_id", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "varchar(256)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "post_id", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "varchar(2048)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "embed_src", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "embed_width", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "embed_height", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "embed_type", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "fetched_at", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "error", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "varchar(256)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "gist_id", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_gist" + }, + { + "type": "varchar(256)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "username", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_gist" + }, + { + "type": "varchar(2048)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "description", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_gist" + }, + { + "type": "varchar(256)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "gist_id", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_gist_file" + }, + { + "type": "varchar(256)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "filename", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_gist_file" + }, + { + "type": "varchar(256)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "content_key", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_gist_file" + }, + { + "type": "varchar(16)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "language", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_gist_file" + }, + { + "type": "varchar(256)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "post_id", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "varchar(2048)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "author_name", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "varchar(2048)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "author_handle", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "varchar(2048)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "content", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "varchar(2048)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "url", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "varchar(256)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "avatar_key", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "avatar_width", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "avatar_height", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "varchar(256)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "image_key", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "image_width", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "image_height", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "varchar(2048)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "image_alt_text", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "num_likes", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "num_reposts", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "num_replies", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "attachment_key", + "entityType": "columns", + "schema": "public", + "table": "attachments" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "sha", + "entityType": "columns", + "schema": "public", + "table": "attachments" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "width", + "entityType": "columns", + "schema": "public", + "table": "attachments" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "height", + "entityType": "columns", + "schema": "public", + "table": "attachments" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "last_modified", + "entityType": "columns", + "schema": "public", + "table": "attachments" + }, + { + "nameExplicit": false, + "columns": [ + "profile_slug" + ], + "schemaTo": "public", + "tableTo": "profiles", + "columnsTo": [ + "slug" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "author_roles_profile_slug_profiles_slug_fkey", + "entityType": "fks", + "schema": "public", + "table": "author_roles" + }, + { + "nameExplicit": false, + "columns": [ + "profile_slug" + ], + "schemaTo": "public", + "tableTo": "profiles", + "columnsTo": [ + "slug" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "profile_achievements_profile_slug_profiles_slug_fkey", + "entityType": "fks", + "schema": "public", + "table": "profile_achievements" + }, + { + "nameExplicit": false, + "columns": [ + "collection_slug" + ], + "schemaTo": "public", + "tableTo": "collections", + "columnsTo": [ + "slug" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "collection_authors_collection_slug_collections_slug_fk", + "entityType": "fks", + "schema": "public", + "table": "collection_authors" + }, + { + "nameExplicit": false, + "columns": [ + "author_slug" + ], + "schemaTo": "public", + "tableTo": "profiles", + "columnsTo": [ + "slug" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "collection_authors_author_slug_profiles_slug_fk", + "entityType": "fks", + "schema": "public", + "table": "collection_authors" + }, + { + "nameExplicit": false, + "columns": [ + "slug" + ], + "schemaTo": "public", + "tableTo": "collections", + "columnsTo": [ + "slug" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "collection_data_slug_collections_slug_fk", + "entityType": "fks", + "schema": "public", + "table": "collection_data" + }, + { + "nameExplicit": false, + "columns": [ + "collection_slug" + ], + "schemaTo": "public", + "tableTo": "collections", + "columnsTo": [ + "slug" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "collection_tags_collection_slug_collections_slug_fkey", + "entityType": "fks", + "schema": "public", + "table": "collection_tags" + }, + { + "nameExplicit": false, + "columns": [ + "post_id" + ], + "schemaTo": "public", + "tableTo": "posts", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "post_attachments_post_id_posts_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "post_attachments" + }, + { + "nameExplicit": false, + "columns": [ + "attachment_key" + ], + "schemaTo": "public", + "tableTo": "attachments", + "columnsTo": [ + "attachment_key" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "post_attachments_attachment_key_attachments_attachment_key_fkey", + "entityType": "fks", + "schema": "public", + "table": "post_attachments" + }, + { + "nameExplicit": false, + "columns": [ + "post_id" + ], + "schemaTo": "public", + "tableTo": "posts", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "post_authors_post_id_posts_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "post_authors" + }, + { + "nameExplicit": false, + "columns": [ + "author_slug" + ], + "schemaTo": "public", + "tableTo": "profiles", + "columnsTo": [ + "slug" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "post_authors_author_slug_profiles_slug_fkey", + "entityType": "fks", + "schema": "public", + "table": "post_authors" + }, + { + "nameExplicit": false, + "columns": [ + "post_id" + ], + "schemaTo": "public", + "tableTo": "posts", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "post_tags_post_id_posts_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "post_tags" + }, + { + "nameExplicit": false, + "columns": [ + "collection_slug" + ], + "schemaTo": "public", + "tableTo": "collections", + "columnsTo": [ + "slug" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "posts_collection_slug_collections_slug_fkey", + "entityType": "fks", + "schema": "public", + "table": "posts" + }, + { + "nameExplicit": false, + "columns": [ + "group_id" + ], + "schemaTo": "public", + "tableTo": "post_groups", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "posts_group_id_post_groups_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "posts" + }, + { + "nameExplicit": false, + "columns": [ + "gist_id" + ], + "schemaTo": "public", + "tableTo": "url_metadata_gist", + "columnsTo": [ + "gist_id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "url_metadata_gist_id_url_metadata_gist_gist_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "url_metadata" + }, + { + "nameExplicit": false, + "columns": [ + "post_id" + ], + "schemaTo": "public", + "tableTo": "url_metadata_post", + "columnsTo": [ + "post_id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "url_metadata_post_id_url_metadata_post_post_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "url_metadata" + }, + { + "nameExplicit": false, + "columns": [ + "gist_id" + ], + "schemaTo": "public", + "tableTo": "url_metadata_gist", + "columnsTo": [ + "gist_id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "url_metadata_gist_file_gist_id_url_metadata_gist_gist_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "url_metadata_gist_file" + }, + { + "columns": [ + "profile_slug", + "role" + ], + "nameExplicit": false, + "name": "author_roles_pkey", + "entityType": "pks", + "schema": "public", + "table": "author_roles" + }, + { + "columns": [ + "profile_slug", + "achievement_id" + ], + "nameExplicit": false, + "name": "profile_achievements_pkey", + "entityType": "pks", + "schema": "public", + "table": "profile_achievements" + }, + { + "columns": [ + "collection_slug", + "author_slug" + ], + "nameExplicit": false, + "name": "collection_authors_collection_slug_author_slug_pk", + "entityType": "pks", + "schema": "public", + "table": "collection_authors" + }, + { + "columns": [ + "slug", + "locale" + ], + "nameExplicit": false, + "name": "collection_data_slug_locale_pk", + "entityType": "pks", + "schema": "public", + "table": "collection_data" + }, + { + "columns": [ + "collection_slug", + "tag" + ], + "nameExplicit": false, + "name": "collection_tags_pkey", + "entityType": "pks", + "schema": "public", + "table": "collection_tags" + }, + { + "columns": [ + "post_id", + "attachment_key" + ], + "nameExplicit": false, + "name": "post_attachments_pkey", + "entityType": "pks", + "schema": "public", + "table": "post_attachments" + }, + { + "columns": [ + "post_id", + "author_slug" + ], + "nameExplicit": false, + "name": "post_authors_pkey", + "entityType": "pks", + "schema": "public", + "table": "post_authors" + }, + { + "columns": [ + "post_id", + "tag" + ], + "nameExplicit": false, + "name": "post_tags_pkey", + "entityType": "pks", + "schema": "public", + "table": "post_tags" + }, + { + "columns": [ + "gist_id", + "filename" + ], + "nameExplicit": false, + "name": "url_metadata_gist_file_pkey", + "entityType": "pks", + "schema": "public", + "table": "url_metadata_gist_file" + }, + { + "columns": [ + "slug" + ], + "nameExplicit": false, + "name": "profiles_pkey", + "schema": "public", + "table": "profiles", + "entityType": "pks" + }, + { + "columns": [ + "slug" + ], + "nameExplicit": false, + "name": "collections_pkey", + "schema": "public", + "table": "collections", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "post_groups_pkey", + "schema": "public", + "table": "post_groups", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "posts_pkey", + "schema": "public", + "table": "posts", + "entityType": "pks" + }, + { + "columns": [ + "slug" + ], + "nameExplicit": false, + "name": "post_images_pkey", + "schema": "public", + "table": "post_images", + "entityType": "pks" + }, + { + "columns": [ + "url" + ], + "nameExplicit": false, + "name": "url_metadata_pkey", + "schema": "public", + "table": "url_metadata", + "entityType": "pks" + }, + { + "columns": [ + "gist_id" + ], + "nameExplicit": false, + "name": "url_metadata_gist_pkey", + "schema": "public", + "table": "url_metadata_gist", + "entityType": "pks" + }, + { + "columns": [ + "post_id" + ], + "nameExplicit": false, + "name": "url_metadata_post_pkey", + "schema": "public", + "table": "url_metadata_post", + "entityType": "pks" + }, + { + "columns": [ + "attachment_key" + ], + "nameExplicit": false, + "name": "attachments_pkey", + "schema": "public", + "table": "attachments", + "entityType": "pks" + }, + { + "nameExplicit": false, + "columns": [ + "slug", + "locale", + "branch" + ], + "nullsNotDistinct": false, + "name": "posts_slug_locale_branch_unique", + "entityType": "uniques", + "schema": "public", + "table": "posts" + } + ], + "renames": [] +} \ No newline at end of file diff --git a/packages/db/src/schema/attachments.ts b/packages/db/src/schema/attachments.ts index bcfba9c3..c2477398 100644 --- a/packages/db/src/schema/attachments.ts +++ b/packages/db/src/schema/attachments.ts @@ -1,8 +1,11 @@ -import { pgTable, text, integer } from "drizzle-orm/pg-core"; +import { pgTable, text, integer, timestamp } from "drizzle-orm/pg-core"; export const attachments = pgTable("attachments", { attachmentKey: text("attachment_key").primaryKey(), sha: text("sha").notNull(), width: integer("width"), height: integer("height"), + lastModified: timestamp("last_modified", { withTimezone: true }) + .notNull() + .defaultNow(), }); diff --git a/packages/test-fixtures/src/db-mock.ts b/packages/test-fixtures/src/db-mock.ts index fe981522..58911b68 100644 --- a/packages/test-fixtures/src/db-mock.ts +++ b/packages/test-fixtures/src/db-mock.ts @@ -141,6 +141,7 @@ export function createDbMock() { sha: {}, width: {}, height: {}, + lastModified: {}, }, urlMetadata: {}, urlMetadataPost: {}, From ac901225d805eb94bb2e305140baea501c5fa829 Mon Sep 17 00:00:00 2001 From: Brian Bornino Date: Fri, 7 Aug 2026 08:11:36 -0700 Subject: [PATCH 5/5] fix(worker): address CodeRabbit findings on cleanup-attachments Switches the candidate-key comparison from eq() to inArray() per CodeRabbit's suggestion - confirmed via a live toSQL() check that eq() was already correctly inlining the subquery, so this is a clarity change rather than a correctness fix. Fixes abort-signal handling so a timed-out job stops looping instead of continuing to run after BullMQ has already marked it failed. Adds a covering test for the lastModified cutoff value and the onConflictDoUpdate call in sync-post, and adds an index on attachments.lastModified. --- .../cleanup-attachments/processor.test.ts | 20 + .../tasks/cleanup-attachments/processor.ts | 127 +- .../src/tasks/sync-post/processor.test.ts | 7 + .../20260807150244_fancy_blink/migration.sql | 1 + .../20260807150244_fancy_blink/snapshot.json | 1927 +++++++++++++++++ packages/db/src/schema/attachments.ts | 24 +- 6 files changed, 2034 insertions(+), 72 deletions(-) create mode 100644 packages/db/drizzle/20260807150244_fancy_blink/migration.sql create mode 100644 packages/db/drizzle/20260807150244_fancy_blink/snapshot.json diff --git a/apps/worker/src/tasks/cleanup-attachments/processor.test.ts b/apps/worker/src/tasks/cleanup-attachments/processor.test.ts index d47c09f5..688523c1 100644 --- a/apps/worker/src/tasks/cleanup-attachments/processor.test.ts +++ b/apps/worker/src/tasks/cleanup-attachments/processor.test.ts @@ -1,6 +1,13 @@ import processor from "./processor.ts"; import { db, attachments } from "@playfulprogramming/db"; import { s3 } from "@playfulprogramming/s3"; +import { lt } from "drizzle-orm"; +import type * as DrizzleOrm from "drizzle-orm"; + +vi.mock("drizzle-orm", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, lt: vi.fn(actual.lt) }; +}); const NOW = new Date("2025-05-05T12:00:00Z"); @@ -35,6 +42,19 @@ test("Removes an attachment returned by the delete query from S3", async () => { expect(s3.remove).toHaveBeenCalledTimes(1); }); +test("Uses a one-hour-old cutoff for the lastModified staleness check", async () => { + vi.setSystemTime(NOW); + + vi.mocked(deleteAttachmentReturning).mockResolvedValueOnce([]); + + await processor({} as never); + + expect(lt).toHaveBeenCalledWith( + attachments.lastModified, + new Date(NOW.getTime() - 60 * 60 * 1000), + ); +}); + test("Does nothing when the delete query returns no rows", async () => { vi.setSystemTime(NOW); diff --git a/apps/worker/src/tasks/cleanup-attachments/processor.ts b/apps/worker/src/tasks/cleanup-attachments/processor.ts index 91a5f7d8..87a989b4 100644 --- a/apps/worker/src/tasks/cleanup-attachments/processor.ts +++ b/apps/worker/src/tasks/cleanup-attachments/processor.ts @@ -2,75 +2,78 @@ import { env } from "@playfulprogramming/common"; import { Tasks } from "@playfulprogramming/bullmq"; import { db, attachments, postAttachments } from "@playfulprogramming/db"; import { s3 } from "@playfulprogramming/s3"; -import { and, eq, lt, notExists } from "drizzle-orm"; +import { and, eq, inArray, lt, notExists } from "drizzle-orm"; import { createProcessor } from "../../createProcessor.ts"; const GRACE_PERIOD_MS = 60 * 60 * 1000; -export default createProcessor(Tasks.CLEANUP_ATTACHMENTS, async () => { - const bucket = await s3.ensureBucket(env.S3_BUCKET); - const staleBefore = new Date(Date.now() - GRACE_PERIOD_MS); +export default createProcessor( + Tasks.CLEANUP_ATTACHMENTS, + async (_job, { signal }) => { + const bucket = await s3.ensureBucket(env.S3_BUCKET); + const staleBefore = new Date(Date.now() - GRACE_PERIOD_MS); - for (;;) { - // sync-post uploads an attachment to S3 and inserts its attachments row - // before the corresponding post_attachments row commits, so a very - // recent row may just be mid-flight rather than truly orphaned - skip - // anything younger than the grace period. Safety against a - // concurrently-committing post_attachments insert doesn't come from - // this being a single SQL statement - it comes from the - // post_attachments -> attachments foreign key (onDelete: "cascade" in - // posts.ts): inserting a post_attachments row takes a lock on the - // referenced attachments row, which serializes against this DELETE. If - // that FK is ever dropped, this safety goes with it. - const candidateKey = db - .select({ attachmentKey: attachments.attachmentKey }) - .from(attachments) - .where( - and( - notExists( - db - .select({ attachmentKey: postAttachments.attachmentKey }) - .from(postAttachments) - .where( - eq(postAttachments.attachmentKey, attachments.attachmentKey), - ), + while (!signal.aborted) { + // sync-post uploads an attachment to S3 and inserts its attachments row + // before the corresponding post_attachments row commits, so a very + // recent row may just be mid-flight rather than truly orphaned - skip + // anything younger than the grace period. Safety against a + // concurrently-committing post_attachments insert doesn't come from + // this being a single SQL statement - it comes from the + // post_attachments -> attachments foreign key (onDelete: "cascade" in + // posts.ts): inserting a post_attachments row takes a lock on the + // referenced attachments row, which serializes against this DELETE. If + // that FK is ever dropped, this safety goes with it. + const candidateKey = db + .select({ attachmentKey: attachments.attachmentKey }) + .from(attachments) + .where( + and( + notExists( + db + .select({ attachmentKey: postAttachments.attachmentKey }) + .from(postAttachments) + .where( + eq(postAttachments.attachmentKey, attachments.attachmentKey), + ), + ), + lt(attachments.lastModified, staleBefore), ), - lt(attachments.lastModified, staleBefore), - ), - ) - .limit(1); + ) + .limit(1); - const [deleted] = await db - .delete(attachments) - .where(eq(attachments.attachmentKey, candidateKey)) - .returning({ - attachmentKey: attachments.attachmentKey, - sha: attachments.sha, - width: attachments.width, - height: attachments.height, - }); + const [deleted] = await db + .delete(attachments) + .where(inArray(attachments.attachmentKey, candidateKey)) + .returning({ + attachmentKey: attachments.attachmentKey, + sha: attachments.sha, + width: attachments.width, + height: attachments.height, + }); - if (!deleted) return; + if (!deleted) return; - try { - await s3.remove(bucket, deleted.attachmentKey); - } catch (err) { - // The row is already claimed/deleted, so a failed removal here would - // otherwise permanently lose track of the S3 object - nothing would - // ever find it again. Re-insert it (with a fresh lastModified) so the - // next scheduled run picks it back up, then fail the job as a whole. - await db - .insert(attachments) - .values({ ...deleted, lastModified: new Date() }) - .onConflictDoUpdate({ - target: attachments.attachmentKey, - set: { lastModified: new Date() }, - }); - throw err; - } + try { + await s3.remove(bucket, deleted.attachmentKey); + } catch (err) { + // The row is already claimed/deleted, so a failed removal here would + // otherwise permanently lose track of the S3 object - nothing would + // ever find it again. Re-insert it (with a fresh lastModified) so the + // next scheduled run picks it back up, then fail the job as a whole. + await db + .insert(attachments) + .values({ ...deleted, lastModified: new Date() }) + .onConflictDoUpdate({ + target: attachments.attachmentKey, + set: { lastModified: new Date() }, + }); + throw err; + } - console.log( - `Removed unreferenced attachment ${deleted.attachmentKey} from S3`, - ); - } -}); + console.log( + `Removed unreferenced attachment ${deleted.attachmentKey} from S3`, + ); + } + }, +); diff --git a/apps/worker/src/tasks/sync-post/processor.test.ts b/apps/worker/src/tasks/sync-post/processor.test.ts index 99a7006c..49f4ae72 100644 --- a/apps/worker/src/tasks/sync-post/processor.test.ts +++ b/apps/worker/src/tasks/sync-post/processor.test.ts @@ -29,6 +29,9 @@ const selectPreviousAuthors = db const insertPostReturning = db .insert(posts) .values(expect.anything()).returning; +const insertAttachmentOnConflictDoUpdate = db + .insert(attachments) + .values(expect.anything()).onConflictDoUpdate; test("Syncs a standalone post successfully", async () => { const postId = ":test-post-uuid:"; @@ -710,6 +713,10 @@ published: "2024-01-15T00:00:00Z" lastModified: expect.any(Date), }); expect(db.insert(attachments).values).toHaveBeenCalledTimes(2); + expect(insertAttachmentOnConflictDoUpdate).toHaveBeenCalledWith({ + target: attachments.attachmentKey, + set: { lastModified: expect.any(Date) }, + }); expect(db.insert(postAttachments).values).toHaveBeenCalledExactlyOnceWith([ { diff --git a/packages/db/drizzle/20260807150244_fancy_blink/migration.sql b/packages/db/drizzle/20260807150244_fancy_blink/migration.sql new file mode 100644 index 00000000..9ea2fedc --- /dev/null +++ b/packages/db/drizzle/20260807150244_fancy_blink/migration.sql @@ -0,0 +1 @@ +CREATE INDEX "attachments_last_modified_idx" ON "attachments" ("last_modified"); \ No newline at end of file diff --git a/packages/db/drizzle/20260807150244_fancy_blink/snapshot.json b/packages/db/drizzle/20260807150244_fancy_blink/snapshot.json new file mode 100644 index 00000000..d6ca93ea --- /dev/null +++ b/packages/db/drizzle/20260807150244_fancy_blink/snapshot.json @@ -0,0 +1,1927 @@ +{ + "version": "8", + "dialect": "postgres", + "id": "65b68a81-c73c-4d93-8dcd-00f689b0b7cf", + "prevIds": [ + "48611f58-9e82-489d-8592-4a2343177d29" + ], + "ddl": [ + { + "isRlsEnabled": false, + "name": "author_roles", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "profile_achievements", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "profiles", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "collection_authors", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "collection_data", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "collection_tags", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "collections", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "post_attachments", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "post_authors", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "post_groups", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "post_tags", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "posts", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "post_images", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "url_metadata", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "url_metadata_gist", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "url_metadata_gist_file", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "url_metadata_post", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "attachments", + "entityType": "tables", + "schema": "public" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "profile_slug", + "entityType": "columns", + "schema": "public", + "table": "author_roles" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "role", + "entityType": "columns", + "schema": "public", + "table": "author_roles" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "profile_slug", + "entityType": "columns", + "schema": "public", + "table": "profile_achievements" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "achievement_id", + "entityType": "columns", + "schema": "public", + "table": "profile_achievements" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "granted_at", + "entityType": "columns", + "schema": "public", + "table": "profile_achievements" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "slug", + "entityType": "columns", + "schema": "public", + "table": "profiles" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "public", + "table": "profiles" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "''", + "generated": null, + "identity": null, + "name": "description", + "entityType": "columns", + "schema": "public", + "table": "profiles" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "profile_image", + "entityType": "columns", + "schema": "public", + "table": "profiles" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "published_at", + "entityType": "columns", + "schema": "public", + "table": "profiles" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "meta", + "entityType": "columns", + "schema": "public", + "table": "profiles" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "collection_slug", + "entityType": "columns", + "schema": "public", + "table": "collection_authors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "author_slug", + "entityType": "columns", + "schema": "public", + "table": "collection_authors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "slug", + "entityType": "columns", + "schema": "public", + "table": "collection_data" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "locale", + "entityType": "columns", + "schema": "public", + "table": "collection_data" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "title", + "entityType": "columns", + "schema": "public", + "table": "collection_data" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "''", + "generated": null, + "identity": null, + "name": "description", + "entityType": "columns", + "schema": "public", + "table": "collection_data" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "published_at", + "entityType": "columns", + "schema": "public", + "table": "collection_data" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "meta", + "entityType": "columns", + "schema": "public", + "table": "collection_data" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "cover_image", + "entityType": "columns", + "schema": "public", + "table": "collection_data" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "social_image", + "entityType": "columns", + "schema": "public", + "table": "collection_data" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "collection_slug", + "entityType": "columns", + "schema": "public", + "table": "collection_tags" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tag", + "entityType": "columns", + "schema": "public", + "table": "collection_tags" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "slug", + "entityType": "columns", + "schema": "public", + "table": "collections" + }, + { + "type": "uuid", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "post_id", + "entityType": "columns", + "schema": "public", + "table": "post_attachments" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "attachment_key", + "entityType": "columns", + "schema": "public", + "table": "post_attachments" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "attachment_name", + "entityType": "columns", + "schema": "public", + "table": "post_attachments" + }, + { + "type": "uuid", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "post_id", + "entityType": "columns", + "schema": "public", + "table": "post_authors" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "author_slug", + "entityType": "columns", + "schema": "public", + "table": "post_authors" + }, + { + "type": "uuid", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "gen_random_uuid()", + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "post_groups" + }, + { + "type": "uuid", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "post_id", + "entityType": "columns", + "schema": "public", + "table": "post_tags" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tag", + "entityType": "columns", + "schema": "public", + "table": "post_tags" + }, + { + "type": "uuid", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "gen_random_uuid()", + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "slug", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "locale", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "branch", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "collection_slug", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "collection_order", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "uuid", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "group_id", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "''", + "generated": null, + "identity": null, + "name": "version_name", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "version_order", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "title", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "''", + "generated": null, + "identity": null, + "name": "description", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "word_count", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "social_image", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "banner_image", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "original_link", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "noindex", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "edited_at", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "published_at", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "meta", + "entityType": "columns", + "schema": "public", + "table": "posts" + }, + { + "type": "varchar(2048)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "slug", + "entityType": "columns", + "schema": "public", + "table": "post_images" + }, + { + "type": "varchar(2048)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "banner_key", + "entityType": "columns", + "schema": "public", + "table": "post_images" + }, + { + "type": "varchar(2048)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "link_preview_key", + "entityType": "columns", + "schema": "public", + "table": "post_images" + }, + { + "type": "varchar(2048)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "index_md5", + "entityType": "columns", + "schema": "public", + "table": "post_images" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "fetched_at", + "entityType": "columns", + "schema": "public", + "table": "post_images" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "error", + "entityType": "columns", + "schema": "public", + "table": "post_images" + }, + { + "type": "varchar(2048)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "url", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "varchar(2048)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "title", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "varchar(2048)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "icon_key", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "icon_width", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "icon_height", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "varchar(2048)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "banner_key", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "banner_width", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "banner_height", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "varchar(256)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "gist_id", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "varchar(256)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "post_id", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "varchar(2048)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "embed_src", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "embed_width", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "embed_height", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "embed_type", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "fetched_at", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "error", + "entityType": "columns", + "schema": "public", + "table": "url_metadata" + }, + { + "type": "varchar(256)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "gist_id", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_gist" + }, + { + "type": "varchar(256)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "username", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_gist" + }, + { + "type": "varchar(2048)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "description", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_gist" + }, + { + "type": "varchar(256)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "gist_id", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_gist_file" + }, + { + "type": "varchar(256)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "filename", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_gist_file" + }, + { + "type": "varchar(256)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "content_key", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_gist_file" + }, + { + "type": "varchar(16)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "language", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_gist_file" + }, + { + "type": "varchar(256)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "post_id", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "varchar(2048)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "author_name", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "varchar(2048)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "author_handle", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "varchar(2048)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "content", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "varchar(2048)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "url", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "varchar(256)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "avatar_key", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "avatar_width", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "avatar_height", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "varchar(256)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "image_key", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "image_width", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "image_height", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "varchar(2048)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "image_alt_text", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "num_likes", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "num_reposts", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "num_replies", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "url_metadata_post" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "attachment_key", + "entityType": "columns", + "schema": "public", + "table": "attachments" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "sha", + "entityType": "columns", + "schema": "public", + "table": "attachments" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "width", + "entityType": "columns", + "schema": "public", + "table": "attachments" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "height", + "entityType": "columns", + "schema": "public", + "table": "attachments" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "last_modified", + "entityType": "columns", + "schema": "public", + "table": "attachments" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "last_modified", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "attachments_last_modified_idx", + "entityType": "indexes", + "schema": "public", + "table": "attachments" + }, + { + "nameExplicit": false, + "columns": [ + "profile_slug" + ], + "schemaTo": "public", + "tableTo": "profiles", + "columnsTo": [ + "slug" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "author_roles_profile_slug_profiles_slug_fkey", + "entityType": "fks", + "schema": "public", + "table": "author_roles" + }, + { + "nameExplicit": false, + "columns": [ + "profile_slug" + ], + "schemaTo": "public", + "tableTo": "profiles", + "columnsTo": [ + "slug" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "profile_achievements_profile_slug_profiles_slug_fkey", + "entityType": "fks", + "schema": "public", + "table": "profile_achievements" + }, + { + "nameExplicit": false, + "columns": [ + "collection_slug" + ], + "schemaTo": "public", + "tableTo": "collections", + "columnsTo": [ + "slug" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "collection_authors_collection_slug_collections_slug_fk", + "entityType": "fks", + "schema": "public", + "table": "collection_authors" + }, + { + "nameExplicit": false, + "columns": [ + "author_slug" + ], + "schemaTo": "public", + "tableTo": "profiles", + "columnsTo": [ + "slug" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "collection_authors_author_slug_profiles_slug_fk", + "entityType": "fks", + "schema": "public", + "table": "collection_authors" + }, + { + "nameExplicit": false, + "columns": [ + "slug" + ], + "schemaTo": "public", + "tableTo": "collections", + "columnsTo": [ + "slug" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "collection_data_slug_collections_slug_fk", + "entityType": "fks", + "schema": "public", + "table": "collection_data" + }, + { + "nameExplicit": false, + "columns": [ + "collection_slug" + ], + "schemaTo": "public", + "tableTo": "collections", + "columnsTo": [ + "slug" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "collection_tags_collection_slug_collections_slug_fkey", + "entityType": "fks", + "schema": "public", + "table": "collection_tags" + }, + { + "nameExplicit": false, + "columns": [ + "post_id" + ], + "schemaTo": "public", + "tableTo": "posts", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "post_attachments_post_id_posts_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "post_attachments" + }, + { + "nameExplicit": false, + "columns": [ + "attachment_key" + ], + "schemaTo": "public", + "tableTo": "attachments", + "columnsTo": [ + "attachment_key" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "post_attachments_attachment_key_attachments_attachment_key_fkey", + "entityType": "fks", + "schema": "public", + "table": "post_attachments" + }, + { + "nameExplicit": false, + "columns": [ + "post_id" + ], + "schemaTo": "public", + "tableTo": "posts", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "post_authors_post_id_posts_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "post_authors" + }, + { + "nameExplicit": false, + "columns": [ + "author_slug" + ], + "schemaTo": "public", + "tableTo": "profiles", + "columnsTo": [ + "slug" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "post_authors_author_slug_profiles_slug_fkey", + "entityType": "fks", + "schema": "public", + "table": "post_authors" + }, + { + "nameExplicit": false, + "columns": [ + "post_id" + ], + "schemaTo": "public", + "tableTo": "posts", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "post_tags_post_id_posts_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "post_tags" + }, + { + "nameExplicit": false, + "columns": [ + "collection_slug" + ], + "schemaTo": "public", + "tableTo": "collections", + "columnsTo": [ + "slug" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "posts_collection_slug_collections_slug_fkey", + "entityType": "fks", + "schema": "public", + "table": "posts" + }, + { + "nameExplicit": false, + "columns": [ + "group_id" + ], + "schemaTo": "public", + "tableTo": "post_groups", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "posts_group_id_post_groups_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "posts" + }, + { + "nameExplicit": false, + "columns": [ + "gist_id" + ], + "schemaTo": "public", + "tableTo": "url_metadata_gist", + "columnsTo": [ + "gist_id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "url_metadata_gist_id_url_metadata_gist_gist_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "url_metadata" + }, + { + "nameExplicit": false, + "columns": [ + "post_id" + ], + "schemaTo": "public", + "tableTo": "url_metadata_post", + "columnsTo": [ + "post_id" + ], + "onUpdate": "NO ACTION", + "onDelete": "SET NULL", + "name": "url_metadata_post_id_url_metadata_post_post_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "url_metadata" + }, + { + "nameExplicit": false, + "columns": [ + "gist_id" + ], + "schemaTo": "public", + "tableTo": "url_metadata_gist", + "columnsTo": [ + "gist_id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "url_metadata_gist_file_gist_id_url_metadata_gist_gist_id_fkey", + "entityType": "fks", + "schema": "public", + "table": "url_metadata_gist_file" + }, + { + "columns": [ + "profile_slug", + "role" + ], + "nameExplicit": false, + "name": "author_roles_pkey", + "entityType": "pks", + "schema": "public", + "table": "author_roles" + }, + { + "columns": [ + "profile_slug", + "achievement_id" + ], + "nameExplicit": false, + "name": "profile_achievements_pkey", + "entityType": "pks", + "schema": "public", + "table": "profile_achievements" + }, + { + "columns": [ + "collection_slug", + "author_slug" + ], + "nameExplicit": false, + "name": "collection_authors_collection_slug_author_slug_pk", + "entityType": "pks", + "schema": "public", + "table": "collection_authors" + }, + { + "columns": [ + "slug", + "locale" + ], + "nameExplicit": false, + "name": "collection_data_slug_locale_pk", + "entityType": "pks", + "schema": "public", + "table": "collection_data" + }, + { + "columns": [ + "collection_slug", + "tag" + ], + "nameExplicit": false, + "name": "collection_tags_pkey", + "entityType": "pks", + "schema": "public", + "table": "collection_tags" + }, + { + "columns": [ + "post_id", + "attachment_key" + ], + "nameExplicit": false, + "name": "post_attachments_pkey", + "entityType": "pks", + "schema": "public", + "table": "post_attachments" + }, + { + "columns": [ + "post_id", + "author_slug" + ], + "nameExplicit": false, + "name": "post_authors_pkey", + "entityType": "pks", + "schema": "public", + "table": "post_authors" + }, + { + "columns": [ + "post_id", + "tag" + ], + "nameExplicit": false, + "name": "post_tags_pkey", + "entityType": "pks", + "schema": "public", + "table": "post_tags" + }, + { + "columns": [ + "gist_id", + "filename" + ], + "nameExplicit": false, + "name": "url_metadata_gist_file_pkey", + "entityType": "pks", + "schema": "public", + "table": "url_metadata_gist_file" + }, + { + "columns": [ + "slug" + ], + "nameExplicit": false, + "name": "profiles_pkey", + "schema": "public", + "table": "profiles", + "entityType": "pks" + }, + { + "columns": [ + "slug" + ], + "nameExplicit": false, + "name": "collections_pkey", + "schema": "public", + "table": "collections", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "post_groups_pkey", + "schema": "public", + "table": "post_groups", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "posts_pkey", + "schema": "public", + "table": "posts", + "entityType": "pks" + }, + { + "columns": [ + "slug" + ], + "nameExplicit": false, + "name": "post_images_pkey", + "schema": "public", + "table": "post_images", + "entityType": "pks" + }, + { + "columns": [ + "url" + ], + "nameExplicit": false, + "name": "url_metadata_pkey", + "schema": "public", + "table": "url_metadata", + "entityType": "pks" + }, + { + "columns": [ + "gist_id" + ], + "nameExplicit": false, + "name": "url_metadata_gist_pkey", + "schema": "public", + "table": "url_metadata_gist", + "entityType": "pks" + }, + { + "columns": [ + "post_id" + ], + "nameExplicit": false, + "name": "url_metadata_post_pkey", + "schema": "public", + "table": "url_metadata_post", + "entityType": "pks" + }, + { + "columns": [ + "attachment_key" + ], + "nameExplicit": false, + "name": "attachments_pkey", + "schema": "public", + "table": "attachments", + "entityType": "pks" + }, + { + "nameExplicit": false, + "columns": [ + "slug", + "locale", + "branch" + ], + "nullsNotDistinct": false, + "name": "posts_slug_locale_branch_unique", + "entityType": "uniques", + "schema": "public", + "table": "posts" + } + ], + "renames": [] +} \ No newline at end of file diff --git a/packages/db/src/schema/attachments.ts b/packages/db/src/schema/attachments.ts index c2477398..d116acbb 100644 --- a/packages/db/src/schema/attachments.ts +++ b/packages/db/src/schema/attachments.ts @@ -1,11 +1,15 @@ -import { pgTable, text, integer, timestamp } from "drizzle-orm/pg-core"; +import { pgTable, text, integer, timestamp, index } from "drizzle-orm/pg-core"; -export const attachments = pgTable("attachments", { - attachmentKey: text("attachment_key").primaryKey(), - sha: text("sha").notNull(), - width: integer("width"), - height: integer("height"), - lastModified: timestamp("last_modified", { withTimezone: true }) - .notNull() - .defaultNow(), -}); +export const attachments = pgTable( + "attachments", + { + attachmentKey: text("attachment_key").primaryKey(), + sha: text("sha").notNull(), + width: integer("width"), + height: integer("height"), + lastModified: timestamp("last_modified", { withTimezone: true }) + .notNull() + .defaultNow(), + }, + (table) => [index("attachments_last_modified_idx").on(table.lastModified)], +);