diff --git a/.prettierignore b/.prettierignore index cb6e0d7..1033c36 100644 --- a/.prettierignore +++ b/.prettierignore @@ -4,4 +4,5 @@ .codex/ .omx/ .turbo/ +pnpm-lock.yaml tmp/ diff --git a/README.md b/README.md index 0a6940e..504fef8 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ when needed. - The first end-to-end slice is implemented as the current working slice. - `apps/api` now ingests feeds into PostgreSQL on boot and serves persisted article list and detail endpoints. +- `apps/api` now also attempts best-effort article body extraction during + ingestion and stores markdown internally for later summarization work. - `apps/web` renders the reader UI and consumes the API over HTTP. - Shared UI primitives live in `packages/ui`. @@ -144,6 +146,17 @@ tests handle the test database internally through `TEST_DATABASE_URL`; those test-database operations are intentionally not exposed as developer-facing commands. +Article body storage is now part of the normal ingestion lifecycle rather than a +manual script. If one persisted article needs a repair pass, use the narrow API +endpoint: + +```bash +curl -X POST http://127.0.0.1:3000/article-content//retry +``` + +That endpoint is a secondary repair path only. The public article read APIs +still do not expose stored markdown in this slice. + `pnpm dev` at the repo root no longer applies Prisma migrations implicitly. Run an explicit API migration command before starting the dev servers whenever your local schema is behind: diff --git a/README.zh-Hans.md b/README.zh-Hans.md index 4fcb524..304916e 100644 --- a/README.zh-Hans.md +++ b/README.zh-Hans.md @@ -12,6 +12,7 @@ RSSift 是一个基于 Turborepo 的 AI 辅助 RSS 筛选工具单仓库。 - 项目目前处于敏捷、迭代式开发中。 - 已完成第一条端到端切片,作为当前可工作的切片。 - `apps/api` 现在会在启动时把 feed 入库到 PostgreSQL,并提供持久化的文章列表与详情接口。 +- `apps/api` 现在也会在 ingestion 期间以 best-effort 方式尝试抽取并落库文章正文 Markdown,为后续摘要能力准备输入层。 - `apps/web` 渲染阅读器界面,并通过 HTTP 消费 API。 - 可复用的 UI 基础组件位于 `packages/ui`。 @@ -137,6 +138,16 @@ pnpm --filter api db:seed 的开发示例数据,请使用 `db:seed`。测试数据库由测试程序通过 `TEST_DATABASE_URL` 在内部处理,不再作为开发者操作命令暴露。 +文章正文落库现在属于正常 ingestion 生命周期,而不是手工脚本。如果某一篇已 +持久化文章需要补救性重跑,可以调用这个狭窄接口: + +```bash +curl -X POST http://127.0.0.1:3000/article-content//retry +``` + +这个接口只是 repair path,不是默认工作流。本切片里公开的文章读取 API 仍 +然不会暴露已存储的 Markdown。 + 仓库根目录的 `pnpm dev` 不会再隐式执行 Prisma 迁移。只要本地 schema 落后于迁移历史,请先显式运行 API 迁移命令,再启动开发服务: diff --git a/apps/api/README.md b/apps/api/README.md index b2c698e..82e72b1 100644 --- a/apps/api/README.md +++ b/apps/api/README.md @@ -2,8 +2,8 @@ This app owns the PostgreSQL-backed feed ingestion backbone for the repository. It reads `apps/api/feeds.opml`, ingests feeds on boot with best-effort -isolation, and serves the existing read-only `/articles` contract from -persisted data. +isolation, attempts article body extraction during ingestion, and serves the +existing read-only `/articles` contract from persisted data. ## Local Run @@ -57,6 +57,8 @@ The API runs on `http://127.0.0.1:3000` by default. `FEED_OPML_PATH`, `INGEST_ON_BOOT`, and `PORT`. - `apps/api/feeds.opml` is the app-owned local subscription input. - Feed parsing uses `feedsmith`. +- Article body extraction uses `@mozilla/readability`, `jsdom`, and `turndown` + inside `apps/api`. - Prisma schema, migrations, and generated client stay inside `apps/api`. ## Endpoints @@ -76,6 +78,15 @@ The API runs on `http://127.0.0.1:3000` by default. - `summary` - `originalUrl` - Returns `404` for unknown article IDs. +- `POST /article-content/:id/retry` + - Re-runs article body extraction for one persisted article as a repair path. + - Returns `{ "status": "succeeded" }`, or a narrow structured + `failed`/`skipped` result when extraction cannot complete. + - Returns `404` for unknown article IDs. + +Article body markdown stays internal in this slice. The public `GET /articles` +and `GET /articles/:id` payloads remain unchanged even though +`contentMarkdown` and `contentExtractedAt` are now stored on `Article`. ## Validation diff --git a/apps/api/e2e/article-content-retry.e2e-spec.ts b/apps/api/e2e/article-content-retry.e2e-spec.ts new file mode 100644 index 0000000..7d1f17c --- /dev/null +++ b/apps/api/e2e/article-content-retry.e2e-spec.ts @@ -0,0 +1,124 @@ +import type { INestApplication } from "@nestjs/common"; +import { Test } from "@nestjs/testing"; +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + it, + jest, +} from "@jest/globals"; +import request from "supertest"; + +import { AppModule } from "../src/app.module"; +import { + createTestPrismaClient, + prepareTestDatabase, +} from "../test-support/database"; + +describe("Article content retry endpoint (e2e)", () => { + let app: INestApplication; + let prisma: ReturnType; + + beforeAll(async () => { + process.env["TEST_DATABASE_URL"] ??= + "postgresql://rssift:rssift@127.0.0.1:5432/rssift_test"; + process.env["DATABASE_URL"] = process.env["TEST_DATABASE_URL"]; + process.env["INGEST_ON_BOOT"] = "false"; + + await prepareTestDatabase(); + prisma = createTestPrismaClient(); + + const moduleFixture = await Test.createTestingModule({ + imports: [AppModule], + }).compile(); + + app = moduleFixture.createNestApplication(); + await app.init(); + }); + + beforeEach(async () => { + await prisma.article.deleteMany(); + await prisma.feed.deleteMany(); + jest.restoreAllMocks(); + + const feed = await prisma.feed.create({ + data: { + feedUrl: "https://example.com/feed.xml", + siteTitle: "Example feed", + }, + }); + + await prisma.article.create({ + data: { + contentMarkdown: "# Old content", + feedId: feed.id, + identityHash: "hash-1", + identitySourceType: "SOURCE_ID", + identitySourceValue: "guid-1", + ingestedAt: new Date("2026-04-15T10:00:00.000Z"), + originalUrl: "https://example.com/articles/1", + publishedAt: new Date("2026-04-15T10:00:00.000Z"), + sourceId: "guid-1", + summary: "Summary 1", + title: "Article 1", + }, + }); + }); + + afterAll(async () => { + await app.close(); + await prisma.$disconnect(); + }); + + it("retries extraction for a single article and persists refreshed markdown", async () => { + jest.spyOn(global, "fetch").mockResolvedValue( + new Response( + ` + + +
+

Article 1

+

Freshly retried body content for this article.

+
+ + `, + { status: 200 }, + ), + ); + + const article = await prisma.article.findFirstOrThrow({ + orderBy: { + id: "asc", + }, + }); + const server = app.getHttpServer() as Parameters[0]; + const response = await request(server) + .post(`/article-content/${article.id}/retry`) + .expect(200); + + expect(response.body).toEqual({ + status: "succeeded", + }); + + const updated = await prisma.article.findUniqueOrThrow({ + where: { + id: article.id, + }, + }); + + expect(updated.contentMarkdown).toBe( + "# Article 1\n\nFreshly retried body content for this article.", + ); + expect(updated.contentExtractedAt).toBeInstanceOf(Date); + }); + + it("returns 404 for an unknown article id", async () => { + const server = app.getHttpServer() as Parameters[0]; + + await request(server) + .post("/article-content/unknown-article-id/retry") + .expect(404); + }); +}); diff --git a/apps/api/e2e/articles.e2e-spec.ts b/apps/api/e2e/articles.e2e-spec.ts index 706da54..814df38 100644 --- a/apps/api/e2e/articles.e2e-spec.ts +++ b/apps/api/e2e/articles.e2e-spec.ts @@ -34,7 +34,7 @@ describe("Articles endpoints (e2e)", () => { process.env["TEST_DATABASE_URL"] ??= "postgresql://rssift:rssift@127.0.0.1:5432/rssift_test"; process.env["DATABASE_URL"] = process.env["TEST_DATABASE_URL"]; - process.env["INGEST_ON_BOOT"] ??= "false"; + process.env["INGEST_ON_BOOT"] = "false"; await prepareTestDatabase(); prisma = createTestPrismaClient(); @@ -65,6 +65,8 @@ describe("Articles endpoints (e2e)", () => { identityHash: "hash-1", identitySourceType: "SOURCE_ID", identitySourceValue: "guid-1", + contentExtractedAt: new Date("2026-04-15T10:05:00.000Z"), + contentMarkdown: "# Article 1\n\nPersisted body", ingestedAt: new Date("2026-04-15T10:00:00.000Z"), originalUrl: "https://example.com/articles/1", publishedAt: new Date("2026-04-15T10:00:00.000Z"), @@ -106,6 +108,8 @@ describe("Articles endpoints (e2e)", () => { "sourceTitle", "title", ]); + expect(list[0]).not.toHaveProperty("contentMarkdown"); + expect(list[0]).not.toHaveProperty("contentExtractedAt"); }); it("GET /articles/:id returns detail payload shape", async () => { @@ -126,6 +130,8 @@ describe("Articles endpoints (e2e)", () => { "summary", "title", ]); + expect(detail).not.toHaveProperty("contentMarkdown"); + expect(detail).not.toHaveProperty("contentExtractedAt"); }); it("GET /articles/:id returns 404 for unknown article id", async () => { diff --git a/apps/api/e2e/feed-ingestion.e2e-spec.ts b/apps/api/e2e/feed-ingestion.e2e-spec.ts index bc2466d..67e0911 100644 --- a/apps/api/e2e/feed-ingestion.e2e-spec.ts +++ b/apps/api/e2e/feed-ingestion.e2e-spec.ts @@ -11,15 +11,50 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import type { PrismaService } from "../src/prisma/prisma.service"; +import { ArticleContentExtractionService } from "../src/article-content/article-content-extraction.service"; +import { ArticleContentRepository } from "../src/article-content/article-content.repository"; +import { ArticleContentService } from "../src/article-content/article-content.service"; import { ArticleIdentityService } from "../src/feeds/article-identity.service"; import { FeedIngestionService } from "../src/feeds/feed-ingestion.service"; +import type { PrismaService } from "../src/prisma/prisma.service"; import { createTestPrismaClient, prepareTestDatabase, } from "../test-support/database"; -describe("Feed ingestion pipeline", () => { +function createFeedIngestionService( + prisma: ReturnType, +) { + const articleContentRepository = new ArticleContentRepository( + prisma as unknown as PrismaService, + ); + const articleContentService = new ArticleContentService( + articleContentRepository, + new ArticleContentExtractionService(), + ); + + return new FeedIngestionService( + prisma as unknown as PrismaService, + new ArticleIdentityService(), + articleContentService, + ); +} + +function getFetchUrl(input: string | URL | Request) { + return typeof input === "string" + ? input + : input instanceof URL + ? input.toString() + : input.url; +} + +function writeOpml(tempDir: string, filename: string, body: string) { + const opmlPath = join(tempDir, filename); + writeFileSync(opmlPath, body); + return opmlPath; +} + +describe("Feed ingestion pipeline persistence", () => { let prisma: ReturnType; let service: FeedIngestionService; let tempDir: string; @@ -29,14 +64,11 @@ describe("Feed ingestion pipeline", () => { "postgresql://rssift:rssift@127.0.0.1:5432/rssift_test"; process.env["TEST_DATABASE_URL"] ??= "postgresql://rssift:rssift@127.0.0.1:5432/rssift_test"; - process.env["INGEST_ON_BOOT"] ??= "false"; + process.env["INGEST_ON_BOOT"] = "false"; await prepareTestDatabase(); prisma = createTestPrismaClient(); - service = new FeedIngestionService( - prisma as unknown as PrismaService, - new ArticleIdentityService(), - ); + service = createFeedIngestionService(prisma); tempDir = mkdtempSync(join(tmpdir(), "rssift-feed-ingestion-")); }); @@ -52,9 +84,9 @@ describe("Feed ingestion pipeline", () => { }); it("ingests multiple feeds, isolates one failure, and keeps article ids stable across repeated runs", async () => { - const opmlPath = join(tempDir, "feeds.opml"); - writeFileSync( - opmlPath, + const opmlPath = writeOpml( + tempDir, + "feeds.opml", ` @@ -63,7 +95,6 @@ describe("Feed ingestion pipeline", () => { `, ); - const responses = new Map([ [ "https://example.com/feed-a.xml", @@ -82,18 +113,24 @@ describe("Feed ingestion pipeline", () => { `, ], + [ + "https://example.com/articles/a", + ` + + +
+

Article A

+

Persist this article body from the primary ingestion path.

+
+ + `, + ], ]); jest .spyOn(global, "fetch") .mockImplementation((input: string | URL | Request) => { - const url = - typeof input === "string" - ? input - : input instanceof URL - ? input.toString() - : input.url; - const body = responses.get(url); + const body = responses.get(getFetchUrl(input)); if (!body) { return Promise.resolve(new Response("broken", { status: 500 })); @@ -118,6 +155,10 @@ describe("Feed ingestion pipeline", () => { }); expect(firstRun).toHaveLength(1); + expect(firstRun[0]?.contentMarkdown).toBe( + "# Article A\n\nPersist this article body from the primary ingestion path.", + ); + expect(firstRun[0]?.contentExtractedAt).toBeInstanceOf(Date); expect(firstRun[0]?.identitySourceType).toBe("SOURCE_ID"); expect(firstRun[0]?.originalUrl).toBe("https://example.com/articles/a"); @@ -128,17 +169,178 @@ describe("Feed ingestion pipeline", () => { id: "asc", }, }); - - expect(secondRun).toHaveLength(1); - expect(secondRun[0]?.id).toBe(firstRun[0]?.id); - const feeds = await prisma.feed.findMany({ orderBy: { feedUrl: "asc", }, }); + expect(secondRun).toHaveLength(1); + expect(secondRun[0]?.id).toBe(firstRun[0]?.id); expect(feeds).toHaveLength(1); expect(feeds[0]?.feedUrl).toBe("https://example.com/feed-a.xml"); }); }); + +describe("Feed ingestion pipeline fail-open enrichment", () => { + let prisma: ReturnType; + let service: FeedIngestionService; + let tempDir: string; + + beforeAll(async () => { + process.env["DATABASE_URL"] ??= + "postgresql://rssift:rssift@127.0.0.1:5432/rssift_test"; + process.env["TEST_DATABASE_URL"] ??= + "postgresql://rssift:rssift@127.0.0.1:5432/rssift_test"; + process.env["INGEST_ON_BOOT"] = "false"; + + await prepareTestDatabase(); + prisma = createTestPrismaClient(); + service = createFeedIngestionService(prisma); + tempDir = mkdtempSync(join(tmpdir(), "rssift-feed-ingestion-fail-open-")); + }); + + beforeEach(async () => { + await prisma.article.deleteMany(); + await prisma.feed.deleteMany(); + jest.restoreAllMocks(); + }); + + afterAll(async () => { + await prisma.$disconnect(); + rmSync(tempDir, { force: true, recursive: true }); + }); + + it("fails open when article body extraction fails", async () => { + const opmlPath = writeOpml( + tempDir, + "feeds-fail-open.opml", + ` + + + + + `, + ); + + jest + .spyOn(global, "fetch") + .mockImplementation((input: string | URL | Request) => { + if (getFetchUrl(input) === "https://example.com/feed-fail-open.xml") { + return Promise.resolve( + new Response( + ` + + + Feed A + + Article A + https://example.com/articles/fail-open + Summary A + guid-a + + + `, + { status: 200 }, + ), + ); + } + + return Promise.resolve(new Response("broken", { status: 500 })); + }); + + await service.ingestFromOpml(opmlPath); + + const articles = await prisma.article.findMany(); + + expect(articles).toHaveLength(1); + expect(articles[0]?.title).toBe("Article A"); + expect(articles[0]?.summary).toBe("Summary A"); + expect(articles[0]?.contentMarkdown).toBeNull(); + expect(articles[0]?.contentExtractedAt).toBeNull(); + }); + + it("retries automatic enrichment for an existing article that still has no markdown", async () => { + const opmlPath = writeOpml( + tempDir, + "feeds-recovery.opml", + ` + + + + + `, + ); + let articleRequestCount = 0; + + jest + .spyOn(global, "fetch") + .mockImplementation((input: string | URL | Request) => { + const url = getFetchUrl(input); + + if (url === "https://example.com/feed-recovery.xml") { + return Promise.resolve( + new Response( + ` + + + Feed A + + Article Recovery + https://example.com/articles/recovery + Summary Recovery + guid-recovery + + + `, + { status: 200 }, + ), + ); + } + + if (url === "https://example.com/articles/recovery") { + articleRequestCount += 1; + + if (articleRequestCount === 1) { + return Promise.resolve(new Response("broken", { status: 500 })); + } + + return Promise.resolve( + new Response( + ` + + +
+

Article Recovery

+

Recovered on a later ingestion run.

+
+ + `, + { status: 200 }, + ), + ); + } + + return Promise.resolve(new Response("missing", { status: 404 })); + }); + + await service.ingestFromOpml(opmlPath); + + const firstRun = await prisma.article.findMany(); + + expect(firstRun).toHaveLength(1); + expect(firstRun[0]?.contentMarkdown).toBeNull(); + expect(firstRun[0]?.contentExtractedAt).toBeNull(); + + await service.ingestFromOpml(opmlPath); + + const secondRun = await prisma.article.findMany(); + + expect(secondRun).toHaveLength(1); + expect(secondRun[0]?.id).toBe(firstRun[0]?.id); + expect(secondRun[0]?.contentMarkdown).toBe( + "# Article Recovery\n\nRecovered on a later ingestion run.", + ); + expect(secondRun[0]?.contentExtractedAt).toBeInstanceOf(Date); + }); +}); diff --git a/apps/api/e2e/prisma-schema.e2e-spec.ts b/apps/api/e2e/prisma-schema.e2e-spec.ts index 0a5f5db..e298824 100644 --- a/apps/api/e2e/prisma-schema.e2e-spec.ts +++ b/apps/api/e2e/prisma-schema.e2e-spec.ts @@ -1,4 +1,4 @@ -import { beforeAll, afterAll, describe, expect, it } from "@jest/globals"; +import { afterAll, beforeAll, describe, expect, it } from "@jest/globals"; import type { Prisma } from "../src/generated/prisma/client"; import { @@ -6,7 +6,19 @@ import { prepareTestDatabase, } from "../test-support/database"; -describe("Prisma schema baseline", () => { +function createSharedArticleData(feedId: string) { + return { + feedId, + identityHash: "shared-hash", + identitySourceType: "CANONICAL_URL" as const, + identitySourceValue: "https://example.com/articles/shared", + ingestedAt: new Date("2026-04-15T00:00:00.000Z"), + originalUrl: "https://example.com/articles/shared", + title: "Shared article", + }; +} + +describe("Prisma schema baseline persistence", () => { let prisma: ReturnType; beforeAll(async () => { @@ -35,6 +47,8 @@ describe("Prisma schema baseline", () => { identityHash: "hash-1", identitySourceType: "SOURCE_ID", identitySourceValue: "guid-1", + contentExtractedAt: new Date("2026-04-15T01:00:00.000Z"), + contentMarkdown: "# Article 1\n\nBody", ingestedAt: new Date("2026-04-15T00:00:00.000Z"), originalUrl: "https://example.com/articles/1", publishedAt: new Date("2026-04-14T00:00:00.000Z"), @@ -46,81 +60,14 @@ describe("Prisma schema baseline", () => { expect(feed.feedUrl).toBe("https://example.com/feed.xml"); expect(article.feedId).toBe(feed.id); + expect(article.contentMarkdown).toBe("# Article 1\n\nBody"); + expect(article.contentExtractedAt?.toISOString()).toBe( + "2026-04-15T01:00:00.000Z", + ); expect(article.summary).toBe("Summary"); expect(article.id).toBeTruthy(); }); - it("rejects duplicate feedUrl values", async () => { - await prisma.feed.create({ - data: { - feedUrl: "https://example.com/duplicate.xml", - }, - }); - - await expect( - prisma.feed.create({ - data: { - feedUrl: "https://example.com/duplicate.xml", - }, - }), - ).rejects.toMatchObject({ - code: "P2002", - } satisfies Partial); - }); - - it("rejects duplicate identityHash within a feed and allows it across feeds", async () => { - const [feedA, feedB] = await Promise.all([ - prisma.feed.create({ - data: { feedUrl: "https://example.com/feed-a.xml" }, - }), - prisma.feed.create({ - data: { feedUrl: "https://example.com/feed-b.xml" }, - }), - ]); - - await prisma.article.create({ - data: { - feedId: feedA.id, - identityHash: "shared-hash", - identitySourceType: "CANONICAL_URL", - identitySourceValue: "https://example.com/articles/shared", - ingestedAt: new Date("2026-04-15T00:00:00.000Z"), - originalUrl: "https://example.com/articles/shared", - title: "Shared article", - }, - }); - - await expect( - prisma.article.create({ - data: { - feedId: feedA.id, - identityHash: "shared-hash", - identitySourceType: "CANONICAL_URL", - identitySourceValue: "https://example.com/articles/shared", - ingestedAt: new Date("2026-04-15T00:00:00.000Z"), - originalUrl: "https://example.com/articles/shared", - title: "Shared article", - }, - }), - ).rejects.toMatchObject({ - code: "P2002", - } satisfies Partial); - - const articleOnSecondFeed = await prisma.article.create({ - data: { - feedId: feedB.id, - identityHash: "shared-hash", - identitySourceType: "CANONICAL_URL", - identitySourceValue: "https://example.com/articles/shared", - ingestedAt: new Date("2026-04-15T00:00:00.000Z"), - originalUrl: "https://example.com/articles/shared", - title: "Shared article", - }, - }); - - expect(articleOnSecondFeed.feedId).toBe(feedB.id); - }); - it("keeps the public article id stable when the same logical row is upserted", async () => { const feed = await prisma.feed.create({ data: { @@ -167,3 +114,66 @@ describe("Prisma schema baseline", () => { expect(updated.summary).toBe("Updated summary"); }); }); + +describe("Prisma schema baseline constraints", () => { + let prisma: ReturnType; + + beforeAll(async () => { + process.env["TEST_DATABASE_URL"] = + process.env["TEST_DATABASE_URL"] ?? + "postgresql://rssift:rssift@127.0.0.1:5432/rssift_test"; + await prepareTestDatabase(); + prisma = createTestPrismaClient(); + }); + + afterAll(async () => { + await prisma.$disconnect(); + }); + + it("rejects duplicate feedUrl values", async () => { + await prisma.feed.create({ + data: { + feedUrl: "https://example.com/duplicate.xml", + }, + }); + + await expect( + prisma.feed.create({ + data: { + feedUrl: "https://example.com/duplicate.xml", + }, + }), + ).rejects.toMatchObject({ + code: "P2002", + } satisfies Partial); + }); + + it("rejects duplicate identityHash within a feed and allows it across feeds", async () => { + const [feedA, feedB] = await Promise.all([ + prisma.feed.create({ + data: { feedUrl: "https://example.com/feed-a.xml" }, + }), + prisma.feed.create({ + data: { feedUrl: "https://example.com/feed-b.xml" }, + }), + ]); + + await prisma.article.create({ + data: createSharedArticleData(feedA.id), + }); + + await expect( + prisma.article.create({ + data: createSharedArticleData(feedA.id), + }), + ).rejects.toMatchObject({ + code: "P2002", + } satisfies Partial); + + const articleOnSecondFeed = await prisma.article.create({ + data: createSharedArticleData(feedB.id), + }); + + expect(articleOnSecondFeed.feedId).toBe(feedB.id); + }); +}); diff --git a/apps/api/package.json b/apps/api/package.json index 048df75..de75e6a 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -25,6 +25,7 @@ "test:e2e": "NODE_OPTIONS=--experimental-vm-modules jest --config ./e2e/jest-e2e.json" }, "dependencies": { + "@mozilla/readability": "0.6.0", "@nestjs/common": "^11.1.18", "@nestjs/config": "4.0.4", "@nestjs/core": "^11.1.18", @@ -33,9 +34,11 @@ "@prisma/client": "7.7.0", "escape-html": "^1.0.3", "feedsmith": "2.9.2", + "jsdom": "26.1.0", "pg": "8.20.0", "reflect-metadata": "^0.2.2", - "rxjs": "^7.8.2" + "rxjs": "^7.8.2", + "turndown": "7.2.4" }, "devDependencies": { "@jest/globals": "^30.3.0", @@ -46,9 +49,11 @@ "@repo/jest-config": "workspace:*", "@repo/typescript-config": "workspace:*", "@types/express": "^5.0.6", + "@types/jsdom": "^28.0.1", "@types/node": "^25.5.2", "@types/pg": "8.15.5", "@types/supertest": "^7.2.0", + "@types/turndown": "^5.0.6", "eslint": "^9.39.4", "jest": "^30.3.0", "prisma": "7.7.0", diff --git a/apps/api/prisma/migrations/20260417151547_add_article_content_markdown/migration.sql b/apps/api/prisma/migrations/20260417151547_add_article_content_markdown/migration.sql new file mode 100644 index 0000000..d76b18c --- /dev/null +++ b/apps/api/prisma/migrations/20260417151547_add_article_content_markdown/migration.sql @@ -0,0 +1,3 @@ +ALTER TABLE "Article" +ADD COLUMN "contentMarkdown" TEXT, +ADD COLUMN "contentExtractedAt" TIMESTAMP(3); diff --git a/apps/api/prisma/models/article.prisma b/apps/api/prisma/models/article.prisma index dd7a2ed..1857c44 100644 --- a/apps/api/prisma/models/article.prisma +++ b/apps/api/prisma/models/article.prisma @@ -7,6 +7,8 @@ model Article { sourceId String? title String originalUrl String + contentMarkdown String? + contentExtractedAt DateTime? publishedAt DateTime? ingestedAt DateTime summary String @default("") diff --git a/apps/api/src/app.module.ts b/apps/api/src/app.module.ts index 242e066..074c069 100644 --- a/apps/api/src/app.module.ts +++ b/apps/api/src/app.module.ts @@ -1,6 +1,7 @@ import { Module } from "@nestjs/common"; import { ConfigModule } from "@nestjs/config"; +import { ArticleContentModule } from "./article-content/article-content.module"; import { ArticlesModule } from "./articles/articles.module"; import { getEnvFilePaths } from "./config/app-config"; import { validateEnv } from "./config/env.validation"; @@ -16,6 +17,7 @@ import { PrismaModule } from "./prisma/prisma.module"; validate: validateEnv, }), PrismaModule, + ArticleContentModule, ArticlesModule, FeedsModule, ], diff --git a/apps/api/src/article-content/article-content-extraction.service.spec.ts b/apps/api/src/article-content/article-content-extraction.service.spec.ts new file mode 100644 index 0000000..70b0306 --- /dev/null +++ b/apps/api/src/article-content/article-content-extraction.service.spec.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from "@jest/globals"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { ArticleContentExtractionService } from "./article-content-extraction.service"; + +function readFixture(name: string) { + return readFileSync(join(__dirname, "fixtures", name), "utf8"); +} + +describe("ArticleContentExtractionService", () => { + const service = new ArticleContentExtractionService(); + + it("extracts stable markdown from a readable article fixture", async () => { + const result = await service.extractFromHtml({ + html: readFixture("clean-article.html"), + pageUrl: "https://example.com/articles/clean", + }); + + expect(result).toEqual({ + contentMarkdown: + "# Clean Article\n\nIntro paragraph with **important** context.\n\n## Details\n\n- First point\n- Second point", + ok: true, + }); + }); + + it("resolves relative links and images against the article url", async () => { + const result = await service.extractFromHtml({ + html: readFixture("noisy-relative-links-article.html"), + pageUrl: "https://example.com/articles/relative", + }); + + expect(result).toEqual({ + contentMarkdown: + "# Relative Links Article\n\nRead the [full report](https://example.com/reports/full) for more context and deeper implementation notes from the original incident write-up.\n\n![System diagram](https://example.com/assets/system.png)", + ok: true, + }); + }); + + it("returns a controlled failure for non-readable pages", async () => { + const result = await service.extractFromHtml({ + html: readFixture("non-readerable-page.html"), + pageUrl: "https://example.com/articles/not-readable", + }); + + expect(result).toEqual({ + ok: false, + reason: "not_readable", + }); + }); +}); diff --git a/apps/api/src/article-content/article-content-extraction.service.ts b/apps/api/src/article-content/article-content-extraction.service.ts new file mode 100644 index 0000000..5c54d1b --- /dev/null +++ b/apps/api/src/article-content/article-content-extraction.service.ts @@ -0,0 +1,114 @@ +import { Injectable } from "@nestjs/common"; +import { Readability } from "@mozilla/readability"; +import type { JSDOM as JSDOMClass } from "jsdom"; +import type TurndownServiceClass from "turndown"; + +type ArticleContentExtractionInput = { + html: string; + pageUrl: string; +}; + +type ArticleContentExtractionSuccess = { + contentMarkdown: string; + ok: true; +}; + +type ArticleContentExtractionFailure = { + ok: false; + reason: "empty_markdown" | "not_readable"; +}; + +export type ArticleContentExtractionResult = + | ArticleContentExtractionSuccess + | ArticleContentExtractionFailure; + +@Injectable() +export class ArticleContentExtractionService { + async extractFromHtml( + input: ArticleContentExtractionInput, + ): Promise { + const [{ JSDOM }, turndownModule] = await Promise.all([ + import("jsdom") as Promise<{ JSDOM: typeof JSDOMClass }>, + import("turndown") as Promise<{ + default: typeof TurndownServiceClass; + }>, + ]); + const TurndownService = turndownModule.default; + const turndown = new TurndownService({ + bulletListMarker: "-", + codeBlockStyle: "fenced", + headingStyle: "atx", + }); + const dom = new JSDOM(input.html, { + contentType: "text/html", + url: input.pageUrl, + }); + + this.removeNoise(dom.window.document); + + const readableArticle = new Readability(dom.window.document).parse(); + + const readableText = readableArticle?.textContent?.trim() ?? ""; + + if (!readableArticle?.content?.trim() || readableText.length < 40) { + return { + ok: false, + reason: "not_readable", + }; + } + + const contentDom = new JSDOM(readableArticle.content, { + contentType: "text/html", + url: input.pageUrl, + }); + const markdown = this.normalizeMarkdown( + turndown.turndown(contentDom.window.document.body), + ); + + if (!markdown) { + return { + ok: false, + reason: "empty_markdown", + }; + } + + const title = readableArticle.title?.trim(); + const normalizedTitleMarkdown = title + ? markdown.replace( + new RegExp(`^#{2,6} ${this.escapeRegExp(title)}`), + `# ${title}`, + ) + : markdown; + const titledMarkdown = + title && !normalizedTitleMarkdown.startsWith("#") + ? this.normalizeMarkdown(`# ${title}\n\n${normalizedTitleMarkdown}`) + : normalizedTitleMarkdown; + + return { + contentMarkdown: titledMarkdown, + ok: true, + }; + } + + private normalizeMarkdown(markdown: string) { + return markdown + .replace(/\r\n/g, "\n") + .replace(/^## /, "# ") + .replace(/^- {2,}/gm, "- ") + .replace(/[ \t]+\n/g, "\n") + .replace(/\n{3,}/g, "\n\n") + .trim(); + } + + private removeNoise(document: globalThis.Document) { + for (const selector of ["script", "style", "noscript"]) { + for (const node of document.querySelectorAll(selector)) { + node.remove(); + } + } + } + + private escapeRegExp(value: string) { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + } +} diff --git a/apps/api/src/article-content/article-content.controller.ts b/apps/api/src/article-content/article-content.controller.ts new file mode 100644 index 0000000..573925b --- /dev/null +++ b/apps/api/src/article-content/article-content.controller.ts @@ -0,0 +1,31 @@ +import { + Controller, + HttpCode, + NotFoundException, + Param, + Post, +} from "@nestjs/common"; + +import { ArticleContentService } from "./article-content.service"; + +@Controller("article-content") +export class ArticleContentController { + constructor(private readonly articleContentService: ArticleContentService) {} + + @Post(":id/retry") + @HttpCode(200) + async retryArticleContent(@Param("id") id: string) { + const result = await this.articleContentService.tryPersistArticleContent( + id, + { + force: true, + }, + ); + + if (result.status === "skipped" && result.reason === "not_found") { + throw new NotFoundException("Article not found"); + } + + return result; + } +} diff --git a/apps/api/src/article-content/article-content.module.ts b/apps/api/src/article-content/article-content.module.ts new file mode 100644 index 0000000..ea0467a --- /dev/null +++ b/apps/api/src/article-content/article-content.module.ts @@ -0,0 +1,17 @@ +import { Module } from "@nestjs/common"; + +import { ArticleContentController } from "./article-content.controller"; +import { ArticleContentExtractionService } from "./article-content-extraction.service"; +import { ArticleContentRepository } from "./article-content.repository"; +import { ArticleContentService } from "./article-content.service"; + +@Module({ + controllers: [ArticleContentController], + providers: [ + ArticleContentExtractionService, + ArticleContentRepository, + ArticleContentService, + ], + exports: [ArticleContentExtractionService, ArticleContentService], +}) +export class ArticleContentModule {} diff --git a/apps/api/src/article-content/article-content.repository.ts b/apps/api/src/article-content/article-content.repository.ts new file mode 100644 index 0000000..33ab23f --- /dev/null +++ b/apps/api/src/article-content/article-content.repository.ts @@ -0,0 +1,49 @@ +import { Injectable } from "@nestjs/common"; + +import { PrismaService } from "../prisma/prisma.service"; + +type ArticleContentRecord = { + contentMarkdown: string | null; + id: string; + originalUrl: string; +}; + +@Injectable() +export class ArticleContentRepository { + constructor(private readonly prisma: PrismaService) {} + + async findById(id: string): Promise { + const row = await this.prisma.article.findUnique({ + where: { + id, + }, + select: { + contentMarkdown: true, + id: true, + originalUrl: true, + }, + }); + + if (!row) { + return null; + } + + return row; + } + + async saveExtractedContent(input: { + articleId: string; + contentMarkdown: string; + extractedAt: Date; + }) { + await this.prisma.article.update({ + where: { + id: input.articleId, + }, + data: { + contentExtractedAt: input.extractedAt, + contentMarkdown: input.contentMarkdown, + }, + }); + } +} diff --git a/apps/api/src/article-content/article-content.service.spec.ts b/apps/api/src/article-content/article-content.service.spec.ts new file mode 100644 index 0000000..fc49572 --- /dev/null +++ b/apps/api/src/article-content/article-content.service.spec.ts @@ -0,0 +1,163 @@ +import { + afterEach, + beforeEach, + describe, + expect, + it, + jest, +} from "@jest/globals"; + +import type { ArticleContentExtractionService } from "./article-content-extraction.service"; +import type { ArticleContentRepository } from "./article-content.repository"; +import { ArticleContentService } from "./article-content.service"; + +describe("ArticleContentService", () => { + const repository: Pick< + jest.Mocked, + "findById" | "saveExtractedContent" + > = { + findById: jest.fn(), + saveExtractedContent: + jest.fn(), + }; + const extractionService: Pick< + jest.Mocked, + "extractFromHtml" + > = { + extractFromHtml: + jest.fn(), + }; + + beforeEach(() => { + jest.restoreAllMocks(); + jest.useRealTimers(); + repository.findById.mockReset(); + repository.saveExtractedContent.mockReset(); + extractionService.extractFromHtml.mockReset(); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it("fetches article html, converts it to markdown, and persists it", async () => { + repository.findById.mockResolvedValue({ + contentMarkdown: null, + id: "article-1", + originalUrl: "https://example.com/articles/1", + }); + extractionService.extractFromHtml.mockResolvedValue({ + contentMarkdown: "# Article", + ok: true, + }); + jest + .spyOn(global, "fetch") + .mockResolvedValue( + new Response("
body
", { status: 200 }), + ); + + const service = new ArticleContentService( + repository as never, + extractionService as never, + ); + + await expect( + service.tryPersistArticleContent("article-1"), + ).resolves.toEqual({ + status: "succeeded", + }); + expect(repository.saveExtractedContent).toHaveBeenCalledWith({ + articleId: "article-1", + contentMarkdown: "# Article", + extractedAt: expect.any(Date), + }); + }); + + it("skips non-forced extraction when content already exists", async () => { + repository.findById.mockResolvedValue({ + contentMarkdown: "# Existing", + id: "article-1", + originalUrl: "https://example.com/articles/1", + }); + const fetchSpy = jest + .spyOn(global, "fetch") + .mockResolvedValue(new Response("unused", { status: 200 })); + + const service = new ArticleContentService( + repository as never, + extractionService as never, + ); + + await expect( + service.tryPersistArticleContent("article-1"), + ).resolves.toEqual({ + reason: "already_extracted", + status: "skipped", + }); + expect(fetchSpy).not.toHaveBeenCalled(); + expect(repository.saveExtractedContent).not.toHaveBeenCalled(); + }); + + it("keeps failure controlled when extraction fails", async () => { + repository.findById.mockResolvedValue({ + contentMarkdown: null, + id: "article-1", + originalUrl: "https://example.com/articles/1", + }); + extractionService.extractFromHtml.mockResolvedValue({ + ok: false, + reason: "not_readable", + }); + jest + .spyOn(global, "fetch") + .mockResolvedValue(new Response("noise", { status: 200 })); + + const service = new ArticleContentService( + repository as never, + extractionService as never, + ); + + await expect( + service.tryPersistArticleContent("article-1"), + ).resolves.toEqual({ + reason: "not_readable", + status: "failed", + }); + expect(repository.saveExtractedContent).not.toHaveBeenCalled(); + }); + + it("uses the caller timeout budget when it is tighter than the default fetch timeout", async () => { + jest.useFakeTimers(); + repository.findById.mockResolvedValue({ + contentMarkdown: null, + id: "article-1", + originalUrl: "https://example.com/articles/1", + }); + jest.spyOn(global, "fetch").mockImplementation((_input, init) => { + const signal = init?.signal; + + return new Promise((_, reject) => { + signal?.addEventListener("abort", () => { + reject(new Error("fetch_aborted")); + }); + }); + }); + + const service = new ArticleContentService( + repository as never, + extractionService as never, + ); + const pending = service.tryPersistArticleContent("article-1", { + timeoutMs: 5, + }); + + await jest.advanceTimersByTimeAsync(5); + + await expect(pending).resolves.toEqual({ + reason: "fetch_aborted", + status: "failed", + }); + expect(extractionService.extractFromHtml).not.toHaveBeenCalled(); + expect(repository.saveExtractedContent).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/api/src/article-content/article-content.service.ts b/apps/api/src/article-content/article-content.service.ts new file mode 100644 index 0000000..e621d3b --- /dev/null +++ b/apps/api/src/article-content/article-content.service.ts @@ -0,0 +1,144 @@ +import { Injectable, Logger } from "@nestjs/common"; + +import { ArticleContentExtractionService } from "./article-content-extraction.service"; +import { ArticleContentRepository } from "./article-content.repository"; + +const ARTICLE_FETCH_TIMEOUT_MS = 10_000; + +type ArticleContentResult = + | { status: "succeeded" } + | { + status: "skipped"; + reason: "already_extracted" | "missing_original_url" | "not_found"; + } + | { status: "failed"; reason: string }; + +@Injectable() +export class ArticleContentService { + private readonly logger = new Logger(ArticleContentService.name); + + constructor( + private readonly repository: ArticleContentRepository, + private readonly extractionService: ArticleContentExtractionService, + ) {} + + async tryPersistArticleContent( + articleId: string, + options?: { force?: boolean; timeoutMs?: number }, + ): Promise { + const article = await this.repository.findById(articleId); + + if (!article) { + return { + reason: "not_found", + status: "skipped", + }; + } + + if (!article.originalUrl) { + return { + reason: "missing_original_url", + status: "skipped", + }; + } + + if (article.contentMarkdown && !options?.force) { + return { + reason: "already_extracted", + status: "skipped", + }; + } + + try { + const html = await this.fetchArticleHtml( + article.originalUrl, + options?.timeoutMs, + ); + const extraction = await this.extractionService.extractFromHtml({ + html, + pageUrl: article.originalUrl, + }); + + if (!extraction.ok) { + this.logger.warn( + JSON.stringify({ + articleId, + reason: extraction.reason, + scope: "article_content", + status: "failed", + }), + ); + + return { + reason: extraction.reason, + status: "failed", + }; + } + + await this.repository.saveExtractedContent({ + articleId, + contentMarkdown: extraction.contentMarkdown, + extractedAt: new Date(), + }); + + this.logger.log( + JSON.stringify({ + articleId, + scope: "article_content", + status: "succeeded", + }), + ); + + return { + status: "succeeded", + }; + } catch (error) { + const reason = + error instanceof Error ? error.message : "article_content_fetch_failed"; + + this.logger.warn( + JSON.stringify({ + articleId, + reason, + scope: "article_content", + status: "failed", + }), + ); + + return { + reason, + status: "failed", + }; + } + } + + private async fetchArticleHtml( + originalUrl: string, + timeoutMs = ARTICLE_FETCH_TIMEOUT_MS, + ) { + const effectiveTimeoutMs = Math.max( + 1, + Math.min(timeoutMs, ARTICLE_FETCH_TIMEOUT_MS), + ); + const controller = new AbortController(); + const timeout = setTimeout(() => { + controller.abort(); + }, effectiveTimeoutMs); + + try { + const response = await fetch(originalUrl, { + signal: controller.signal, + }); + + if (!response.ok) { + throw new Error( + `Article request failed with status ${String(response.status)}`, + ); + } + + return await response.text(); + } finally { + clearTimeout(timeout); + } + } +} diff --git a/apps/api/src/article-content/fixtures/clean-article.html b/apps/api/src/article-content/fixtures/clean-article.html new file mode 100644 index 0000000..8fab4f3 --- /dev/null +++ b/apps/api/src/article-content/fixtures/clean-article.html @@ -0,0 +1,18 @@ + + + + + Clean Article + + +
+

Clean Article

+

Intro paragraph with important context.

+

Details

+
    +
  • First point
  • +
  • Second point
  • +
+
+ + diff --git a/apps/api/src/article-content/fixtures/noisy-relative-links-article.html b/apps/api/src/article-content/fixtures/noisy-relative-links-article.html new file mode 100644 index 0000000..0d7b5b8 --- /dev/null +++ b/apps/api/src/article-content/fixtures/noisy-relative-links-article.html @@ -0,0 +1,25 @@ + + + + + Relative Links Article + + +
+ +
+
+
+

Relative Links Article

+

+ Read the full report for more context and + deeper implementation notes from the original incident write-up. +

+
+ System diagram +
+
+
+
Footer noise
+ + diff --git a/apps/api/src/article-content/fixtures/non-readerable-page.html b/apps/api/src/article-content/fixtures/non-readerable-page.html new file mode 100644 index 0000000..65522d7 --- /dev/null +++ b/apps/api/src/article-content/fixtures/non-readerable-page.html @@ -0,0 +1,17 @@ + + + + + Index Page + + + + + + diff --git a/apps/api/src/articles/article.repository.spec.ts b/apps/api/src/articles/article.repository.spec.ts index 7ca5578..6605308 100644 --- a/apps/api/src/articles/article.repository.spec.ts +++ b/apps/api/src/articles/article.repository.spec.ts @@ -38,6 +38,8 @@ describe("ArticleRepository", () => { identityHash: "hash-1", identitySourceType: "SOURCE_ID", identitySourceValue: "guid-1", + contentExtractedAt: new Date("2026-04-15T10:05:00.000Z"), + contentMarkdown: "# Article 1\n\nPersisted body", ingestedAt: new Date("2026-04-15T10:00:00.000Z"), originalUrl: "https://example.com/articles/1", publishedAt: new Date("2026-04-14T10:00:00.000Z"), @@ -103,5 +105,7 @@ describe("ArticleRepository", () => { "summary", "title", ]); + expect(detail).not.toHaveProperty("contentMarkdown"); + expect(detail).not.toHaveProperty("contentExtractedAt"); }); }); diff --git a/apps/api/src/feeds/feed-ingestion.service.spec.ts b/apps/api/src/feeds/feed-ingestion.service.spec.ts new file mode 100644 index 0000000..5b3db58 --- /dev/null +++ b/apps/api/src/feeds/feed-ingestion.service.spec.ts @@ -0,0 +1,205 @@ +import { + afterAll, + afterEach, + beforeEach, + describe, + expect, + it, + jest, +} from "@jest/globals"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import type { ArticleContentService } from "../article-content/article-content.service"; +import type { PrismaService } from "../prisma/prisma.service"; +import { ArticleIdentityService } from "./article-identity.service"; +import { FeedIngestionService } from "./feed-ingestion.service"; + +function writeOpml(tempDir: string, filename: string, body: string) { + const opmlPath = join(tempDir, filename); + writeFileSync(opmlPath, body); + return opmlPath; +} + +function createFeedXml(itemCount = 1) { + const items = Array.from({ length: itemCount }, (_, index) => { + const number = index + 1; + return ` + + Article ${String(number)} + https://example.com/articles/${String(number)} + Summary ${String(number)} + guid-${String(number)} + + `; + }).join(""); + + return ` + + + Feed A + ${items} + + `; +} + +describe("FeedIngestionService", () => { + type ExistingArticle = { + contentMarkdown: string | null; + id: string; + sourceId: string | null; + }; + type PersistedArticle = { + id: string; + }; + type TransactionClient = { + article: { + create: (args: unknown) => Promise; + findFirst: (args: unknown) => Promise; + update: (args: unknown) => Promise; + }; + feed: { + upsert: (args: unknown) => Promise<{ id: string }>; + }; + }; + type TransactionCallback = (client: TransactionClient) => unknown; + + const articleFindFirst = jest.fn(); + const articleUpdate = jest.fn(); + const articleCreate = jest.fn(); + const feedUpsert = jest.fn(); + const transaction = + jest.fn<(callback: TransactionCallback) => Promise>(); + const tryPersistArticleContent = + jest.fn(); + const tx = { + article: { + create: articleCreate, + findFirst: articleFindFirst, + update: articleUpdate, + }, + feed: { + upsert: feedUpsert, + }, + }; + let service: FeedIngestionService; + let tempDir: string; + + beforeEach(() => { + jest.restoreAllMocks(); + jest.useRealTimers(); + feedUpsert.mockReset(); + articleFindFirst.mockReset(); + articleUpdate.mockReset(); + articleCreate.mockReset(); + transaction.mockReset(); + tryPersistArticleContent.mockReset(); + + transaction.mockImplementation((callback: TransactionCallback) => + Promise.resolve(callback(tx)), + ); + feedUpsert.mockResolvedValue({ id: "feed-1" }); + service = new FeedIngestionService( + { $transaction: transaction } as unknown as PrismaService, + new ArticleIdentityService(), + { + tryPersistArticleContent, + } as unknown as ArticleContentService, + ); + tempDir = mkdtempSync(join(tmpdir(), "rssift-feed-ingestion-spec-")); + }); + + afterEach(() => { + rmSync(tempDir, { force: true, recursive: true }); + jest.useRealTimers(); + }); + + afterAll(() => { + jest.restoreAllMocks(); + }); + + it("auto-enriches existing articles that are still missing markdown", async () => { + const opmlPath = writeOpml( + tempDir, + "existing-feed.opml", + ` + + + + + `, + ); + + articleFindFirst.mockResolvedValue({ + contentMarkdown: null, + id: "article-existing", + sourceId: "guid-1", + }); + articleUpdate.mockResolvedValue({ + id: "article-existing", + }); + jest + .spyOn(global, "fetch") + .mockResolvedValue(new Response(createFeedXml(), { status: 200 })); + + await service.ingestFromOpml(opmlPath); + + expect(articleUpdate).toHaveBeenCalledTimes(1); + expect(articleCreate).not.toHaveBeenCalled(); + expect(tryPersistArticleContent).toHaveBeenCalledWith( + "article-existing", + expect.objectContaining({ + timeoutMs: expect.any(Number), + }), + ); + }); + + it("passes the remaining feed budget into article enrichment and stops after exhaustion", async () => { + const opmlPath = writeOpml( + tempDir, + "budget-feed.opml", + ` + + + + + `, + ); + + articleFindFirst.mockResolvedValue(null); + articleCreate + .mockResolvedValueOnce({ id: "article-1" }) + .mockResolvedValueOnce({ id: "article-2" }); + jest + .spyOn(global, "fetch") + .mockResolvedValue(new Response(createFeedXml(2), { status: 200 })); + tryPersistArticleContent.mockResolvedValue({ + reason: "timed_out", + status: "failed", + }); + jest + .spyOn(Date, "now") + .mockReturnValueOnce(0) + .mockReturnValueOnce(0) + .mockReturnValueOnce(0) + .mockReturnValueOnce(0) + .mockReturnValueOnce(15_001); + + await service.ingestFromOpml(opmlPath); + + expect(tryPersistArticleContent).toHaveBeenCalledTimes(1); + expect(tryPersistArticleContent).toHaveBeenCalledWith( + "article-1", + expect.objectContaining({ + timeoutMs: expect.any(Number), + }), + ); + + const [firstCall] = tryPersistArticleContent.mock.calls; + const [, firstCallOptions = {}] = firstCall ?? []; + + expect(firstCallOptions.timeoutMs).toBeGreaterThan(0); + expect(firstCallOptions.timeoutMs).toBeLessThanOrEqual(15_000); + }); +}); diff --git a/apps/api/src/feeds/feed-ingestion.service.ts b/apps/api/src/feeds/feed-ingestion.service.ts index 4162b93..7cf9f6c 100644 --- a/apps/api/src/feeds/feed-ingestion.service.ts +++ b/apps/api/src/feeds/feed-ingestion.service.ts @@ -2,6 +2,7 @@ import { Injectable, Logger } from "@nestjs/common"; import { parseFeed, parseOpml } from "feedsmith"; import { readFile } from "node:fs/promises"; +import { ArticleContentService } from "../article-content/article-content.service"; import { PrismaService } from "../prisma/prisma.service"; import { ArticleIdentityService } from "./article-identity.service"; @@ -33,6 +34,7 @@ export class FeedIngestionService { constructor( private readonly prisma: PrismaService, private readonly articleIdentityService: ArticleIdentityService, + private readonly articleContentService: ArticleContentService, ) {} async ingestFromOpml(opmlPath: string) { @@ -131,6 +133,7 @@ export class FeedIngestionService { descriptor: FeedDescriptor, timeoutMs: number, ) { + const deadlineAt = Date.now() + timeoutMs; const controller = new AbortController(); const timeout = setTimeout(() => { controller.abort(); @@ -155,76 +158,15 @@ export class FeedIngestionService { parsedFeed, ingestedAt, ); - - await this.prisma.$transaction(async (tx) => { - const feed = await tx.feed.upsert({ - where: { - feedUrl: descriptor.feedUrl, - }, - create: { - feedUrl: descriptor.feedUrl, - siteTitle: this.getFeedTitle(parsedFeed) ?? descriptor.title, - siteUrl: this.getFeedSiteUrl(parsedFeed) ?? descriptor.websiteUrl, - etag: response.headers.get("etag"), - lastModified: response.headers.get("last-modified"), - }, - update: { - siteTitle: this.getFeedTitle(parsedFeed) ?? descriptor.title, - siteUrl: this.getFeedSiteUrl(parsedFeed) ?? descriptor.websiteUrl, - etag: response.headers.get("etag"), - lastModified: response.headers.get("last-modified"), - }, - }); - - for (const article of normalizedArticles) { - const existing = await tx.article.findFirst({ - where: { - feedId: feed.id, - OR: [ - article.sourceId ? { sourceId: article.sourceId } : undefined, - article.originalUrl - ? { originalUrl: article.originalUrl } - : undefined, - { identityHash: article.identityHash }, - ].filter(Boolean) as Array>, - }, - }); - - if (existing) { - await tx.article.update({ - where: { - id: existing.id, - }, - data: { - title: article.title, - originalUrl: article.originalUrl, - publishedAt: article.publishedAt, - summary: article.summary, - ingestedAt: article.ingestedAt, - sourceId: existing.sourceId ?? article.sourceId, - }, - }); - - continue; - } - - await tx.article.create({ - data: { - feedId: feed.id, - identityHash: article.identityHash, - identitySourceType: article.identitySourceType, - identitySourceValue: article.identitySourceValue, - ingestedAt: article.ingestedAt, - originalUrl: article.originalUrl, - publishedAt: article.publishedAt, - sourceId: article.sourceId, - summary: article.summary, - title: article.title, - }, - }); - } + const articleIdsToEnrich = await this.persistNormalizedArticles({ + descriptor, + normalizedArticles, + parsedFeed, + response, }); + await this.enrichArticles(articleIdsToEnrich, deadlineAt); + this.logFeedEvent({ feedUrl: descriptor.feedUrl, status: "success", @@ -236,7 +178,7 @@ export class FeedIngestionService { } private normalizeArticles( - feedUrl: string, + _feedUrl: string, parsedFeed: ReturnType, ingestedAt: Date, ): NormalizedArticle[] { @@ -381,4 +323,125 @@ export class FeedIngestionService { }), ); } + + private async enrichArticles(articleIds: string[], deadlineAt: number) { + for (const articleId of articleIds) { + const remainingBudgetMs = deadlineAt - Date.now(); + + if (remainingBudgetMs <= 0) { + this.logger.warn( + JSON.stringify({ + articleId, + reason: "feed_budget_exhausted", + scope: "feed_ingestion_article_content", + status: "skipped", + }), + ); + break; + } + + const result = await this.articleContentService.tryPersistArticleContent( + articleId, + { + timeoutMs: remainingBudgetMs, + }, + ); + + this.logger.log( + JSON.stringify({ + articleId, + scope: "feed_ingestion_article_content", + ...result, + }), + ); + } + } + + private async persistNormalizedArticles(input: { + descriptor: FeedDescriptor; + normalizedArticles: NormalizedArticle[]; + parsedFeed: ReturnType; + response: Response; + }) { + return this.prisma.$transaction(async (tx) => { + const feed = await tx.feed.upsert({ + where: { + feedUrl: input.descriptor.feedUrl, + }, + create: { + feedUrl: input.descriptor.feedUrl, + siteTitle: + this.getFeedTitle(input.parsedFeed) ?? input.descriptor.title, + siteUrl: + this.getFeedSiteUrl(input.parsedFeed) ?? + input.descriptor.websiteUrl, + etag: input.response.headers.get("etag"), + lastModified: input.response.headers.get("last-modified"), + }, + update: { + siteTitle: + this.getFeedTitle(input.parsedFeed) ?? input.descriptor.title, + siteUrl: + this.getFeedSiteUrl(input.parsedFeed) ?? + input.descriptor.websiteUrl, + etag: input.response.headers.get("etag"), + lastModified: input.response.headers.get("last-modified"), + }, + }); + const articleIds: string[] = []; + + for (const article of input.normalizedArticles) { + const existing = await tx.article.findFirst({ + where: { + feedId: feed.id, + OR: [ + article.sourceId ? { sourceId: article.sourceId } : undefined, + article.originalUrl + ? { originalUrl: article.originalUrl } + : undefined, + { identityHash: article.identityHash }, + ].filter(Boolean) as Array>, + }, + }); + + if (existing) { + await tx.article.update({ + where: { + id: existing.id, + }, + data: { + title: article.title, + originalUrl: article.originalUrl, + publishedAt: article.publishedAt, + summary: article.summary, + ingestedAt: article.ingestedAt, + sourceId: existing.sourceId ?? article.sourceId, + }, + }); + if (!existing.contentMarkdown) { + articleIds.push(existing.id); + } + continue; + } + + const created = await tx.article.create({ + data: { + feedId: feed.id, + identityHash: article.identityHash, + identitySourceType: article.identitySourceType, + identitySourceValue: article.identitySourceValue, + ingestedAt: article.ingestedAt, + originalUrl: article.originalUrl, + publishedAt: article.publishedAt, + sourceId: article.sourceId, + summary: article.summary, + title: article.title, + }, + }); + articleIds.push(created.id); + } + + return articleIds; + }); + } } diff --git a/apps/api/src/feeds/feeds.module.ts b/apps/api/src/feeds/feeds.module.ts index 5cd05ec..ffe76d6 100644 --- a/apps/api/src/feeds/feeds.module.ts +++ b/apps/api/src/feeds/feeds.module.ts @@ -1,10 +1,12 @@ import { Module } from "@nestjs/common"; +import { ArticleContentModule } from "../article-content/article-content.module"; import { ArticleIdentityService } from "./article-identity.service"; import { FeedBootstrapService } from "./feed-bootstrap.service"; import { FeedIngestionService } from "./feed-ingestion.service"; @Module({ + imports: [ArticleContentModule], providers: [ ArticleIdentityService, FeedBootstrapService, diff --git a/docs/en/brainstorms/2026-04-17-v0-1-slice-3-article-markdown-storage-requirements.md b/docs/en/brainstorms/2026-04-17-v0-1-slice-3-article-markdown-storage-requirements.md new file mode 100644 index 0000000..650a205 --- /dev/null +++ b/docs/en/brainstorms/2026-04-17-v0-1-slice-3-article-markdown-storage-requirements.md @@ -0,0 +1,247 @@ +--- +date: 2026-04-17 +topic: v0-1-slice-3-article-markdown-storage +--- + +# v0.1 Slice 3 Article Markdown Storage Requirements + +## Problem Frame + +The current backend proves feed ingestion, but it still stops at feed-provided metadata plus a compatibility `summary`. That is not yet the right substrate for later title translation, layered summarization, or richer reading. The next v0.1 slice should therefore prove one narrower capability first: fetch the original article page for already-persisted articles, extract the main body, convert it to Markdown, and persist that Markdown to PostgreSQL. + +This slice is intentionally framed as an independent enrichment/backfill capability rather than a change to the boot-time feed ingestion backbone. The goal is to make article-body persistence real and testable without turning the current startup ingestion path into a more fragile multi-stage pipeline. + +Verified current-state context: + +- `apps/api/src/feeds/feed-bootstrap.service.ts` triggers feed ingestion on application bootstrap. +- `apps/api/src/feeds/feed-ingestion.service.ts` currently fetches feed XML, normalizes entries, and persists `Article` rows, but does not fetch article pages. +- `apps/api/prisma/models/article.prisma` currently stores feed-level metadata only: no extracted body field exists yet. +- `apps/api/src/articles/articles.service.ts` and `apps/api/src/articles/article.repository.ts` expose only `title`, `sourceTitle`, `publishedAt`, `summary`, and `originalUrl` on the current public detail path. + +```mermaid +flowchart TB + DB[(Persisted Article rows)] --> PICK[Backfill job selects articles without Markdown] + PICK --> FETCH[Fetch article HTML from originalUrl] + FETCH --> EXTRACT[Extract main content] + EXTRACT --> CLEAN[Sanitize / normalize extracted HTML] + CLEAN --> MD[Convert HTML to Markdown] + MD --> SAVE[Persist contentMarkdown and extraction timestamp] + SAVE --> LOGS[Structured job summary logs] + FETCH --> FAIL[Best-effort failure path] + EXTRACT --> FAIL + CLEAN --> FAIL + MD --> FAIL + FAIL --> LOGS +``` + +## Requirements + +**Body Extraction Backbone** + +- R1. The system must provide an independent article-content backfill capability that runs separately from startup feed ingestion. +- R2. The backfill capability must read candidate articles from persisted `Article` rows and use `Article.originalUrl` as the fetch target. +- R3. The backfill capability must fetch article HTML, extract the main readable body, convert that body to Markdown, and persist the Markdown back onto the same `Article` record. +- R4. The first implementation must be best-effort: failure to extract Markdown for one article must not block attempts on other candidate articles. +- R5. The first implementation must preserve the existing feed-ingestion backbone semantics; boot-time feed ingestion remains responsible only for feed metadata/article discovery in this slice. + +**Persistence Shape** + +- R6. The `Article` model must gain a nullable `contentMarkdown` field for extracted body Markdown. +- R7. The `Article` model must gain a nullable `contentExtractedAt` field that records when Markdown was last successfully persisted. +- R8. The current `summary` field must keep its existing meaning as feed-provided summary/description compatibility text rather than being repurposed for article-body storage. +- R9. The schema change must remain inside `apps/api` and continue to use the existing Prisma/PostgreSQL ownership model and migration history. + +**Execution Surface** + +- R10. The first execution surface should be operator-triggered backfill, not a new public HTTP API. +- R11. The backfill must support a narrow verification path for local development, such as targeting one article or a limited batch, so the operator can explicitly confirm that Markdown persists. +- R12. The backfill must emit a structured run summary that at least reports attempted articles, successful markdown writes, and failed extraction attempts. + +**Testing and Reliability** + +- R13. The slice must include automated tests that prove the path from controlled input HTML to persisted `contentMarkdown`. +- R14. The primary automated test path must rely on local HTML fixtures rather than real external article URLs. +- R15. The extraction pipeline must fail open: an extraction failure may leave `contentMarkdown` null, but must not corrupt the existing article row or break article reads. +- R16. The first implementation may keep failure observability in structured logs and test assertions; it does not require a persistent extraction-jobs table in this slice. + +## Success Criteria + +- A developer can run the article-content backfill against a local database and observe at least one `Article` row receive non-null `contentMarkdown`. +- The persisted Markdown is derived from extracted article-body HTML rather than copied from feed `summary`. +- Automated tests using local fixtures verify that extracted Markdown is written to the database. +- Existing `/articles` read APIs continue to work without requiring any new public endpoint. +- Logs or structured command output clearly distinguish successful writes from extraction failures. + +## Scope Boundaries + +- No change to the startup feed-ingestion trigger in this slice. +- No scheduler, cron, or continuous background content extraction in this slice. +- No public API contract expansion for exposing Markdown yet. +- No LLM summarization, translated titles, or layered summaries yet. +- No browser automation, paywall handling, login flows, or anti-bot bypass in the first version. +- No persistent extraction queue or job-history model in the first version. +- No requirement to guarantee successful extraction for every article source. + +## Key Decisions + +- Separate discovery from enrichment: feed ingestion continues to discover articles, while article-body extraction runs as a distinct backfill capability. +- Markdown is the canonical persisted body format for this product direction, even though reference systems such as Miniflux and FreshRSS usually persist HTML/content rather than Markdown. +- The recommended first extraction stack is `@mozilla/readability` for body extraction plus `turndown` and `turndown-plugin-gfm` for HTML-to-Markdown conversion. +- The first test strategy uses local HTML fixtures because reproducible extraction correctness matters more than external-network realism in this slice. +- The public article API remains intentionally thin for now; this slice proves storage before contract expansion. + +## External Best-Practice Signals + +- Miniflux and FreshRSS both support a notion of full-content enrichment separate from the original feed metadata path, and both persist enriched article content rather than treating feed summaries as the final source of truth. +- Miniflux's architecture suggests that original-content fetching is a distinct enrichment action rather than a hard prerequisite for feed refresh success. +- FreshRSS shows that full-content retrieval often needs source-specific tuning over time, which is another reason to keep this first slice decoupled from the boot ingestion backbone. +- `@mozilla/readability` remains a strong baseline for readable-content extraction in the JavaScript ecosystem, but it is not itself a security sanitizer. +- `turndown` remains a pragmatic, mature HTML-to-Markdown converter for Node-based pipelines, especially when paired with `turndown-plugin-gfm`. +- `defuddle` is worth tracking as a later alternative if Markdown fidelity becomes a primary issue, but it is not necessary to de-risk this first storage slice. + +## High-Level Technical Direction + +This brainstorm is intentionally technical enough to define the first schema shape, execution surface, and API stance. + +### Prisma Schema Draft + +**Article** + +- `id`: existing internal primary key +- `feedId`: existing relation to `Feed` +- `identityHash`: existing per-feed article identity +- `sourceId`: existing source-native identifier when available +- `title`: existing article title +- `originalUrl`: existing article URL +- `publishedAt`: existing source-published timestamp +- `ingestedAt`: existing first-ingested timestamp +- `summary`: existing feed-provided summary compatibility field +- `contentMarkdown`: nullable extracted article body in Markdown +- `contentExtractedAt`: nullable timestamp of the latest successful markdown persistence +- `createdAt`: record creation time +- `updatedAt`: record update time + +### Prisma Constraint Direction + +- Keep existing `Feed.feedUrl` uniqueness and `Article(feedId, identityHash)` uniqueness unchanged. +- Add no new uniqueness constraint for `contentMarkdown`; it is enrichment data attached to an existing `Article`. +- Keep `contentMarkdown` nullable so extraction can fail without forcing placeholder content. +- Keep `contentExtractedAt` nullable so "not attempted yet" and "not successfully extracted yet" can both remain representable without adding a separate status model in this slice. + +```mermaid +erDiagram + Feed ||--o{ Article : contains + Feed { + string id + string feedUrl + string siteTitle + string siteUrl + string etag + string lastModified + } + Article { + string id + string feedId + string identityHash + string sourceId + string title + string originalUrl + datetime publishedAt + datetime ingestedAt + string summary + text contentMarkdown + datetime contentExtractedAt + } +``` + +### Internal Execution Surface Draft + +The first execution surface should be package-local and operator-driven rather than publicly routable. + +Preferred shape: + +- package-local command or script owned by `apps/api` +- default behavior: process articles where `contentMarkdown` is null +- narrow verification option: process one explicit article ID +- optional batch limit for local verification and safe iteration + +Illustrative command shape: + +- `pnpm --filter api article-content:backfill` +- `pnpm --filter api article-content:backfill --article-id ` +- `pnpm --filter api article-content:backfill --limit 10` + +The exact command wiring is deferred to planning, but the contract direction is that the first surface is an internal operator tool, not a public route. + +### Public API Contract Stance + +The public API surface should remain unchanged in this slice: + +- `GET /articles` +- `GET /articles/:id` + +`GET /articles/:id` should continue to return: + +- `title` +- `sourceTitle` +- `publishedAt` +- `summary` +- `originalUrl` + +This slice deliberately does **not** require exposing `contentMarkdown` yet. Storage is proven first; public consumption can be planned later. + +### Backfill Run Summary Shape + +The internal execution surface should emit a structured summary that is simple but enough to confirm success: + +- `attemptedCount` +- `succeededCount` +- `failedCount` +- `skippedCount` +- optional list of failed article IDs / URLs for diagnosis + +### Extraction Pipeline Direction + +- Fetch HTML from `Article.originalUrl`. +- Parse DOM in a controlled environment suitable for Readability. +- Use `@mozilla/readability` to extract the main article content. +- Sanitize or normalize the extracted HTML before Markdown conversion. +- Convert the cleaned HTML to Markdown via `turndown` plus GFM support. +- Persist `contentMarkdown` and `contentExtractedAt` only on successful extraction. + +### Persistence-to-API Mapping + +| Concern | Backing data | +| ------------------------------------------- | -------------------------------------------------------- | +| Existing article title reads | `Article.title` | +| Existing detail summary reads | `Article.summary` | +| Future article body substrate | `Article.contentMarkdown` | +| Verification of successful body persistence | `Article.contentMarkdown` + `Article.contentExtractedAt` | + +### Testing Direction + +- Use local HTML fixtures as the main correctness input. +- Test the extraction pipeline from HTML fixture through database persistence. +- Include at least one failure-path test showing that invalid/unextractable input does not corrupt existing article metadata. +- Keep tests package-local to `apps/api` and aligned with the current NestJS/Prisma test setup. + +## Dependencies / Assumptions + +- Persisted article rows already exist from the prior feed ingestion slice. +- `Article.originalUrl` is the fetch anchor for this slice; the first version does not invent alternate article-source resolution. +- PostgreSQL `text` storage through Prisma `String` is sufficient for the expected Markdown payload size of this stage. +- The future LLM pipeline will consume persisted Markdown rather than re-fetching article pages on demand. +- Source-specific extraction edge cases will exist, but broad generic extraction is enough for the first proof slice. + +## Outstanding Questions + +### Deferred to Planning + +- [Affects R10][Technical] What is the exact package-local command wiring for the operator-triggered backfill? +- [Affects R11][Technical] Should the first verification-focused execution surface prioritize `--article-id`, `--limit`, or both? +- [Affects R15][Technical] What minimum sanitization/normalization step is sufficient before `turndown` without expanding this slice into a larger content-cleaning project? +- [Affects R13][Technical] What fixture set best represents the first expected article shapes without overfitting the test corpus? + +## Next Steps + +-> Keep this as a brainstorm pair for now; do not convert it into a plan until the user explicitly asks for planning. diff --git a/docs/en/plans/2026-04-17-001-feat-article-markdown-backfill-plan.md b/docs/en/plans/2026-04-17-001-feat-article-markdown-backfill-plan.md new file mode 100644 index 0000000..a28891a --- /dev/null +++ b/docs/en/plans/2026-04-17-001-feat-article-markdown-backfill-plan.md @@ -0,0 +1,323 @@ +--- +title: feat: persist article body content during ingestion +type: feat +status: completed +date: 2026-04-17 +origin: + - docs/en/brainstorms/2026-04-17-v0-1-slice-3-article-markdown-storage-requirements.md + - docs/zh-Hans/brainstorms/2026-04-17-v0-1-slice-3-article-markdown-storage-requirements.md +deepened: 2026-04-17 +--- + +# feat: persist article body content during ingestion + +## Overview + +This plan replaces the previous CLI/backfill-centered direction. The correct v0.1 slice is not a developer-operated sync script. The system should persist article body content as part of the normal article-ingestion lifecycle, while keeping the existing feed-discovery backbone fail-open and preserving the thin public `/articles` contract. + +The recommended shape follows the external pattern seen in Miniflux and FreshRSS: + +- body persistence belongs to the system's normal refresh/ingestion path, not a manual developer script +- body extraction must remain best-effort so feed/article discovery still succeeds when extraction fails +- a single-article retry HTTP endpoint is useful as a repair path, but it is not the primary trigger surface + +This slice still does not expose article Markdown to the UI and still does not generate AI summaries. It only makes sure body content can be stored reliably for later summarization work. + +## Problem Frame + +`apps/api` already ingests feed metadata and persists `Article` rows, but those rows only contain feed-layer fields such as title, URL, published timestamp, and compatibility `summary`. That is not enough for later AI summarization. The missing capability is durable article body persistence. + +The earlier plan went in the wrong direction by centering the feature around a package-local command. That would make body persistence an optional operator action instead of normal system behavior. For this product, that is the wrong default. The system should attempt body extraction when it already discovers and persists a new article. + +Current repo context that matters: + +- `apps/api/src/feeds/feed-bootstrap.service.ts` already owns startup ingestion semantics. +- `apps/api/src/feeds/feed-ingestion.service.ts` already owns feed fetch, normalization, best-effort looping, and persistence updates. +- `apps/api/src/articles/articles.controller.ts` exposes only thin read APIs today. +- `apps/api/prisma/models/article.prisma` still has no body-content storage fields. + +The correct architectural boundary is therefore: + +- feed ingestion still owns article discovery and metadata persistence +- a new `article-content` feature owns HTML fetch, extraction, Markdown conversion, and body persistence +- feed ingestion invokes that feature in a best-effort way for newly discovered/updated articles +- an internal HTTP retry endpoint can re-run extraction for one article when needed + +## External Best-Practice Direction + +### Miniflux + +DeepWiki review indicates Miniflux primarily fetches full content during feed refresh/background processing, then also offers on-demand entry content fetch via HTTP/UI actions. The important pattern is not the exact implementation language; it is the trigger model: + +- automatic refresh path does the normal enrichment work +- a single-entry on-demand action exists as a repair/override path +- persisted content lives on the article/entry record + +### FreshRSS + +DeepWiki review indicates FreshRSS integrates full-content loading into feed actualization/update flow, usually driven by cron/systemd or manual UI refresh. The important pattern here is that full content persistence is part of the feed update lifecycle rather than a separate developer-run sync workflow. + +### Recommendation for This Repo + +This repo should adopt a hybrid of those patterns: + +- **Primary path:** attempt article body persistence during normal feed ingestion/refresh +- **Secondary path:** provide one single-article retry endpoint for repair and verification +- **Not recommended:** developer-run `pnpm script` as the explicit main sync path + +That recommendation best matches the product goal (body content should exist automatically for later AI summarization) and the current codebase shape (existing ingestion backbone, thin read API, no current UI need for Markdown). + +## Requirements Trace + +- R1. The system must persist article body content as part of the normal ingestion/refresh lifecycle rather than relying on a developer-run sync script. +- R2. The system must fetch article HTML from `Article.originalUrl`, extract the readable body, convert it to Markdown, and persist it onto the same `Article` row. +- R3. The persistence path must remain best-effort: article discovery and metadata persistence still succeed even when body extraction fails. +- R4. `Article` must gain nullable `contentMarkdown` and `contentExtractedAt` fields. +- R5. The current `summary` field must keep its existing meaning as feed-provided summary/description compatibility text. +- R6. Public `GET /articles` and `GET /articles/:id` responses must remain unchanged in this slice. +- R7. The system may add a single-article HTTP retry endpoint for repair, but that endpoint must be a secondary path rather than the primary persistence trigger. +- R8. The slice must include automated tests that prove body extraction and persistence from controlled HTML fixtures. +- R9. HTML extraction failures must fail open and must not corrupt existing article rows. +- R10. All schema, dependency, and runtime ownership stays inside `apps/api`. +- R11. Any new packages installed by the implementation agent must be the newest conflict-free stable versions practical for the current dependency graph; if that cannot be achieved cleanly, implementation must stop and surface the conflict. + +## Scope Boundaries + +- Do not add a package-local or root-level `pnpm` sync script as the primary trigger path. +- Do not expose `contentMarkdown` through the public article read APIs in this slice. +- Do not build frontend consumption of body content in this slice. +- Do not build AI summarization, translated titles, or layered summaries in this slice. +- Do not add a persistent job queue, run-history table, or scheduler-specific management surface in this slice. +- Do not introduce browser automation, login flows, paywall handling, or anti-bot bypassing. +- Do not turn extraction failure into a blocker for feed/article ingestion. + +## Context & Research + +### Relevant Code and Patterns + +- `apps/api/src/feeds/feed-ingestion.service.ts` is the primary place that already handles feed-level fetch loops, per-item normalization, and persistence. +- `apps/api/src/feeds/feed-bootstrap.service.ts` already defines the startup-triggered ingestion lifecycle. +- `apps/api/src/articles/article.repository.ts`, `apps/api/src/articles/articles.service.ts`, and `apps/api/src/articles/articles.controller.ts` show that the current public article surface is intentionally thin. +- `apps/api/test-support/database.ts` owns test-database reset and migration replay for database-backed tests. +- `apps/api/e2e/feed-ingestion.e2e-spec.ts` already demonstrates the repo's fetch mocking and persistence verification pattern. +- `apps/api/e2e/articles.e2e-spec.ts` and `apps/api/src/articles/article.repository.spec.ts` currently protect the thin read contract and must remain green after the schema grows. + +### Dependency Direction + +The intended extraction stack still remains: + +- `@mozilla/readability` +- `jsdom` +- `turndown` +- a GFM-capable Markdown conversion extension, if it installs cleanly with the rest of the stack + +Implementation must choose the newest cleanly compatible stable versions practical for the current `apps/api` dependency graph. If the preferred GFM plugin line is too stale to install cleanly, execution should stop and surface the smallest maintained alternative or minimal local fallback rules. + +## Key Technical Decisions + +- Add a dedicated `article-content` feature boundary inside `apps/api` rather than burying extraction logic directly inside `feeds` or `articles`. +- Keep feed ingestion as the orchestrator of discovery, but call into `article-content` for best-effort body extraction after article metadata has been normalized and is ready to persist. +- Persist body content on the existing `Article` row using nullable `contentMarkdown` and `contentExtractedAt`. +- Keep article body extraction fail-open: if fetching/parsing/conversion fails, the metadata row still persists and article reads continue to work. +- Add one single-article retry HTTP endpoint for repair and verification. This endpoint re-runs extraction for one article, but it is not required for the system to populate body content in the normal case. +- Keep `summary` semantics unchanged and keep the public `/articles` contract unchanged. +- Keep jsdom script execution and subresource execution off when processing untrusted article HTML. +- Keep HTML cleanup intentionally minimal before Markdown conversion; this slice is about storage, not fully polished rendering. + +## Open Questions + +### Resolved in This Revision + +- **Should the main trigger be a CLI/script path?** No. That direction was wrong and is removed from the plan. +- **Should the system persist body content automatically?** Yes. Automatic ingestion-time persistence is now the primary path. +- **Should there still be a repair trigger?** Yes. A single-article retry endpoint is useful as a secondary path. +- **Should this slice expose Markdown or AI summaries to the UI?** No. Storage only. + +### Deferred to Implementation + +- The exact timeout values for article-page fetches. +- The exact single-article retry route name, as long as it is scoped clearly and remains secondary. +- The smallest stable Markdown normalization rules revealed by fixture-driven tests. + +## High-Level Technical Design + +```mermaid +sequenceDiagram + participant Bootstrap as Feed bootstrap / refresh + participant Ingest as FeedIngestionService + participant Repo as Prisma Article persistence + participant Content as ArticleContentService + participant HTTP as fetch(originalUrl) + participant DOM as jsdom + Readability + participant MD as Turndown + + Bootstrap->>Ingest: trigger feed ingestion + Ingest->>Repo: persist feed + article metadata + loop per newly discovered/updated article + Ingest->>Content: tryPersistArticleContent(article id / url) + Content->>HTTP: fetch article page + HTTP-->>Content: html or failure + Content->>DOM: extract readable body + DOM-->>Content: extracted html or failure + Content->>MD: convert to markdown + MD-->>Content: markdown or failure + Content->>Repo: update contentMarkdown + contentExtractedAt on success + Content-->>Ingest: success or controlled failure + end + Ingest-->>Bootstrap: ingestion summary +``` + +## Implementation Units + +- [x] **Unit 1: Extend `Article` persistence shape without changing the public read contract** + +**Goal:** Add the body-content fields and prove the existing read APIs remain unchanged. + +**Requirements:** R4, R5, R6, R9, R10 + +**Dependencies:** None + +**Files:** + +- Modify: `apps/api/prisma/models/article.prisma` +- Create: `apps/api/prisma/migrations/_add_article_content_markdown/migration.sql` +- Modify: `apps/api/e2e/prisma-schema.e2e-spec.ts` +- Modify: `apps/api/src/articles/article.repository.spec.ts` +- Modify: `apps/api/e2e/articles.e2e-spec.ts` + +**Approach:** + +- Add nullable `contentMarkdown` and `contentExtractedAt`. +- Keep article DTO mapping unchanged. +- Extend tests to prove the new columns do not alter `/articles` or `/articles/:id` response shapes. + +**Verification:** + +- Database rows can store article body content while public read APIs remain byte-for-byte compatible at the field level. + +- [x] **Unit 2: Build the `article-content` extraction pipeline** + +**Goal:** Create the pure body-extraction and Markdown-conversion layer from article URL + HTML to controlled persistence-ready output. + +**Requirements:** R2, R3, R8, R9, R10, R11 + +**Dependencies:** Unit 1 + +**Files:** + +- Modify: `apps/api/package.json` +- Create: `apps/api/src/article-content/article-content.module.ts` +- Create: `apps/api/src/article-content/article-content-extraction.service.ts` +- Create: `apps/api/src/article-content/article-content-extraction.service.spec.ts` +- Create: `apps/api/src/article-content/fixtures/clean-article.html` +- Create: `apps/api/src/article-content/fixtures/noisy-relative-links-article.html` +- Create: `apps/api/src/article-content/fixtures/non-readerable-page.html` + +**Approach:** + +- Install extraction dependencies only in `apps/api`. +- Keep the service persistence-agnostic: input is article URL + raw HTML, output is either Markdown success payload or controlled failure. +- Use jsdom with the article URL, Readability for extraction, then minimal cleanup and Turndown conversion. + +**Verification:** + +- Fixture-backed tests prove deterministic Markdown output and controlled failure behavior without touching the database. + +- [x] **Unit 3: Invoke body persistence automatically from ingestion** + +**Goal:** Make body persistence part of the normal article-ingestion lifecycle. + +**Requirements:** R1, R2, R3, R6, R8, R9, R10, R11 + +**Dependencies:** Unit 1, Unit 2 + +**Files:** + +- Modify: `apps/api/src/feeds/feed-ingestion.service.ts` +- Modify: `apps/api/src/feeds/feeds.module.ts` +- Create: `apps/api/src/article-content/article-content.repository.ts` +- Create: `apps/api/src/article-content/article-content.service.ts` +- Create: `apps/api/src/article-content/article-content.service.spec.ts` +- Modify: `apps/api/e2e/feed-ingestion.e2e-spec.ts` + +**Approach:** + +- After feed entries are normalized and persisted, invoke `article-content` for newly discovered/updated eligible articles. +- Keep the extraction path best-effort and isolate failures to the current article. +- Log structured success/failure outcomes in the same operational style already used by feed ingestion. +- Avoid reprocessing rows that already have `contentMarkdown` unless a later explicit retry path requests it. + +**Verification:** + +- Feed ingestion persists metadata even when extraction fails, and automatically persists body content when extraction succeeds. + +- [x] **Unit 4: Add a single-article retry HTTP endpoint as a repair path** + +**Goal:** Provide one narrow backend trigger to re-run extraction for a specific article when automatic ingestion-time persistence was skipped or failed. + +**Requirements:** R7, R9, R10 + +**Dependencies:** Unit 1, Unit 2, Unit 3 + +**Files:** + +- Create: `apps/api/src/article-content/article-content.controller.ts` +- Modify: `apps/api/src/app.module.ts` +- Create: `apps/api/e2e/article-content-retry.e2e-spec.ts` + +**Approach:** + +- Add a single route scoped to one article, e.g. under the article-content feature boundary. +- Keep the endpoint narrow: one article in, one retry attempt out. +- Return a small structured result such as `succeeded`, `failed`, or `skipped`, with a narrow reason string where useful. +- Keep this route explicitly secondary in docs and implementation; the system should not depend on operators calling it for normal body persistence. + +**Verification:** + +- A targeted HTTP call can retry extraction for one article and persist body content without changing the public article read contract. + +- [x] **Unit 5: Update documentation to reflect the corrected trigger model** + +**Goal:** Remove the old script-centered mental model and document automatic persistence plus the narrow retry path. + +**Requirements:** R1, R6, R7 + +**Dependencies:** Unit 1, Unit 2, Unit 3, Unit 4 + +**Files:** + +- Modify: `apps/api/README.md` +- Modify: `README.md` +- Modify: `README.zh-Hans.md` + +**Approach:** + +- Document that article body content is now attempted automatically during ingestion. +- Document the retry endpoint as a repair path, not the default workflow. +- Keep docs aligned with the fact that Markdown is stored internally and not yet exposed in the product UI. + +**Verification:** + +- The docs no longer imply a manual script-driven sync model and accurately describe the automatic body-persistence flow. + +## System-Wide Impact + +- **Primary behavior change:** the system now attempts body persistence automatically during article ingestion. +- **Failure model:** body extraction failures become local, non-blocking enrichment failures rather than ingestion blockers. +- **API behavior:** existing article read APIs remain unchanged; one narrow retry endpoint is added for repair. +- **Product posture:** this slice prepares internal stored body content for future AI summarization without prematurely exposing body content to the UI. + +## Risks & Mitigations + +| Risk | Mitigation | +| ----------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------- | +| Body extraction makes ingestion slower or more failure-prone. | Keep extraction best-effort, bounded by timeout, and isolated from metadata persistence success. | +| Some publishers produce unreadable or highly noisy HTML. | Use fixture-driven tests, keep failures non-blocking, and defer source-specific heuristics. | +| The preferred Markdown plugin stack may have dependency conflicts. | Choose the newest cleanly installable stable versions and stop if the dependency path becomes conflict-heavy. | +| Operators may misunderstand the retry endpoint as the primary workflow. | Make docs and implementation explicit that retry is a repair path only. | + +## Sources & References + +- **Origin documents:** `docs/en/brainstorms/2026-04-17-v0-1-slice-3-article-markdown-storage-requirements.md`, `docs/zh-Hans/brainstorms/2026-04-17-v0-1-slice-3-article-markdown-storage-requirements.md` +- **Relevant code:** `apps/api/src/feeds/feed-ingestion.service.ts`, `apps/api/src/feeds/feed-bootstrap.service.ts`, `apps/api/src/articles/articles.controller.ts`, `apps/api/test-support/database.ts`, `apps/api/e2e/feed-ingestion.e2e-spec.ts` +- **External architectural references via DeepWiki:** `miniflux/v2`, `FreshRSS/FreshRSS` diff --git a/docs/en/solutions/integration-issues/feed-ingestion-retries-missing-article-markdown-2026-04-17.md b/docs/en/solutions/integration-issues/feed-ingestion-retries-missing-article-markdown-2026-04-17.md new file mode 100644 index 0000000..097dd2a --- /dev/null +++ b/docs/en/solutions/integration-issues/feed-ingestion-retries-missing-article-markdown-2026-04-17.md @@ -0,0 +1,139 @@ +--- +title: Feed ingestion must retry article markdown enrichment for existing rows that still have no content +date: 2026-04-17 +category: integration-issues +module: feed ingestion +problem_type: integration_issue +component: nest_service +symptoms: + - repeated ingestion runs updated article metadata but left `contentMarkdown` null on the same row + - recovery depended on the secondary retry endpoint instead of the normal ingestion path + - unit coverage allowed a regression where only newly created rows were sent to article-content enrichment +root_cause: logic_error +resolution_type: code_fix +severity: medium +related_components: + - typeorm_repository +tags: + [ + feed-ingestion, + article-content, + markdown, + retry, + contentmarkdown, + fail-open, + nestjs, + ] +--- + +# Feed ingestion must retry article markdown enrichment for existing rows that still have no content + +## Problem + +The article markdown slice is supposed to enrich article bodies automatically during normal feed ingestion while keeping ingestion fail-open. That contract regressed: if the first extraction attempt failed, later ingestion runs updated the existing article row but did not re-enqueue it for markdown extraction, so `contentMarkdown` stayed null indefinitely unless an operator manually called the retry endpoint. + +## Symptoms + +- The first ingestion run could persist article metadata successfully while leaving `contentMarkdown` and `contentExtractedAt` null after an HTML fetch/extraction failure. +- A later ingestion of the same feed kept the same article row and refreshed metadata, but still did not recover missing markdown automatically. +- The dedicated retry endpoint worked, which made the bug easy to misread as an acceptable “manual repair only” path instead of a broken primary path. + +## What Didn't Work + +- Treating article-content enrichment as “new rows only” looked safe, but it silently broke the recovery path for historical rows that were already discovered before extraction succeeded. +- Relying on the single-article retry endpoint was not enough, because the plan for this slice explicitly defined retry as a secondary repair path, not the main ingestion contract. +- A unit test had effectively locked in the wrong behavior, so the regression could pass review until an end-to-end recovery scenario was tested. + +## Solution + +Restore enqueueing inside `FeedIngestionService.persistNormalizedArticles()` for existing rows whose markdown is still missing, while continuing to skip rows that already have extracted content. + +Before: + +```ts +if (existing) { + await tx.article.update({ + where: { id: existing.id }, + data: { + title: article.title, + originalUrl: article.originalUrl, + publishedAt: article.publishedAt, + summary: article.summary, + ingestedAt: article.ingestedAt, + sourceId: existing.sourceId ?? article.sourceId, + }, + }); + continue; +} +``` + +After: + +```ts +if (existing) { + await tx.article.update({ + where: { id: existing.id }, + data: { + title: article.title, + originalUrl: article.originalUrl, + publishedAt: article.publishedAt, + summary: article.summary, + ingestedAt: article.ingestedAt, + sourceId: existing.sourceId ?? article.sourceId, + }, + }); + + if (!existing.contentMarkdown) { + articleIds.push(existing.id); + } + + continue; +} +``` + +Two regression guards were added around that orchestration rule: + +1. A unit test in `apps/api/src/feeds/feed-ingestion.service.spec.ts` now asserts that an existing article with `contentMarkdown: null` is sent back through `tryPersistArticleContent(...)`. +2. An end-to-end test in `apps/api/e2e/feed-ingestion.e2e-spec.ts` now proves the full recovery flow: the first run fails open, the second run keeps the same article row, and markdown is persisted automatically on that later ingestion. + +The downstream idempotency contract remains unchanged in `ArticleContentService`: + +```ts +if (article.contentMarkdown && !options?.force) { + return { + reason: "already_extracted", + status: "skipped", + }; +} +``` + +That guard keeps the restored enqueue behavior safe for rows that already have body content. + +## Why This Works + +The bug lived at the integration boundary between feed metadata persistence and article-content enrichment. The ingestion pipeline already knew how to: + +- keep article identity stable across runs, +- update metadata on existing rows, and +- avoid duplicate extraction when content already exists. + +What it stopped doing was reconnecting those existing-but-incomplete rows to the enrichment step. Re-queueing only rows with missing `contentMarkdown` restores the intended automatic recovery path without turning every repeated ingestion into redundant extraction work. The `ArticleContentService` idempotency check remains the second safety net, so already-enriched rows still short-circuit cleanly. + +## Prevention + +- Add orchestration tests for both branches of “existing row”: one with `contentMarkdown: null`, one with existing markdown already present. +- Keep an e2e scenario that spans at least two ingestion runs, because this regression only appears when the first run fails open and the second run tries to recover. +- Treat repair endpoints as secondary workflows in tests and docs; if the main ingestion contract says enrichment is automatic, assert that behavior directly. +- When `findFirst()` results are used to decide downstream work, include every field needed for orchestration decisions. Here, `existing.contentMarkdown` is part of the contract and should not be removed casually from the query/result shape. + +## Related Issues + +- `apps/api/src/feeds/feed-ingestion.service.ts` +- `apps/api/src/feeds/feed-ingestion.service.spec.ts` +- `apps/api/e2e/feed-ingestion.e2e-spec.ts` +- `apps/api/src/article-content/article-content.service.ts` +- `apps/api/e2e/article-content-retry.e2e-spec.ts` +- `docs/en/plans/2026-04-17-001-feat-article-markdown-backfill-plan.md` +- `docs/zh-Hans/plans/2026-04-17-001-feat-article-markdown-backfill-plan.md` +- `.claude/handoffs/2026-04-17-164836-feed-ingestion-auto-enrichment-recovery.md` +- `.claude/handoffs/2026-04-17-170512-article-markdown-backfill-review-fix.md` diff --git a/docs/en/solutions/workflow-issues/prettier-gates-should-ignore-tool-managed-and-generated-files-2026-04-17.md b/docs/en/solutions/workflow-issues/prettier-gates-should-ignore-tool-managed-and-generated-files-2026-04-17.md new file mode 100644 index 0000000..5736f1e --- /dev/null +++ b/docs/en/solutions/workflow-issues/prettier-gates-should-ignore-tool-managed-and-generated-files-2026-04-17.md @@ -0,0 +1,144 @@ +--- +title: Keep tool-managed and generated files out of Prettier gates +date: 2026-04-17 +category: workflow-issues +module: format gate boundaries +problem_type: workflow_issue +component: development_workflow +severity: medium +applies_when: + - the repo uses `prettier --check .` or another root-wide format gate + - CI failures point at lockfiles or other generated artifacts + - a file is managed by package managers or external tooling instead of humans + - format churn is creating noisy reviews without semantic changes + - maintainers are deciding whether to widen `.prettierignore` +tags: + [ + prettier, + prettierignore, + pnpm-lock, + lockfile, + ci, + generated-files, + tool-managed, + formatting, + ] +--- + +# Keep tool-managed and generated files out of Prettier gates + +## Context + +This repo uses the root command `pnpm format:check`, which currently resolves to +`prettier --check .`. That makes `.prettierignore` part of the quality-gate +boundary, not just an editor convenience. + +A recent CI failure showed why this boundary matters. The `pr-quality / format` +job failed only because `pnpm-lock.yaml` did not match Prettier's preferred YAML +layout. The lockfile change was formatting-only, extremely large, and had no +behavioral value. That made the gate noisy and pushed attention away from the +real repo-owned surfaces. + +## Guidance + +Do not casually include tool-managed or generated files in a repo-wide Prettier +gate. + +If a file's source of truth is an external tool, prefer letting that tool own +its shape and keep the file out of `prettier --check .` unless there is a +strong repository-specific reason to do otherwise. + +For this repo, `pnpm-lock.yaml` should be treated like a package-manager-owned +artifact: + +- keep it committed for reproducible installs and cache correctness +- let `pnpm` update it when dependencies change +- keep it out of the root Prettier gate to avoid large, non-semantic diffs +- express that boundary explicitly in `.prettierignore` + +The practical rule is simple: format gates should target repo-owned content, +not every tracked file. + +## Why This Matters + +Formatting gates become misleading when they fail on files that humans are not +expected to shape directly. + +In this case, the immediate CI red signal looked like a formatting regression, +but the underlying issue was boundary drift: the gate was checking a file owned +by `pnpm`, not by the repo's formatting conventions. That kind of drift creates +three problems: + +- reviewers see huge lockfile diffs with no meaningful behavior change +- follow-up jobs can fail downstream, which makes root-cause reading slower +- engineers can start "fixing" generated files instead of fixing the gate + boundary + +A repo-wide formatting rule is only trustworthy when its scope matches the set +of files the team actually intends to maintain by hand. + +## When to Apply + +- When CI format failures point at `pnpm-lock.yaml`, generated manifests, or + other machine-owned files +- When a root format command such as `prettier --check .` is broader than the + repository's true ownership boundary +- When adding new tool-managed directories or artifacts to the repo +- When deciding whether a noisy format failure should be fixed by reformatting + a file or by tightening `.prettierignore` +- When review churn is growing because formatting touches generated outputs + +## Examples + +Bad boundary: + +```gitignore +# .prettierignore +.agents/ +.cache/ +.claude/ +.codex/ +.omx/ +.turbo/ +tmp/ +``` + +With that setup, the root format gate still checks `pnpm-lock.yaml`: + +```json +{ + "scripts": { + "format:check": "prettier --check ." + } +} +``` + +Better boundary: + +```gitignore +# .prettierignore +.agents/ +.cache/ +.claude/ +.codex/ +.omx/ +.turbo/ +pnpm-lock.yaml +tmp/ +``` + +Good mental model: + +```text +Track the lockfile in git. +Let pnpm regenerate it. +Do not use Prettier to manufacture giant lockfile-only diffs. +``` + +## Related + +- `docs/en/solutions/workflow-issues/github-pr-quality-ci-trusted-base-scope-and-ui-bootstrap-2026-04-12.md` +- `docs/zh-Hans/solutions/workflow-issues/github-pr-quality-ci-trusted-base-scope-and-ui-bootstrap-2026-04-12.md` +- `.prettierignore` +- `package.json` +- `pnpm-lock.yaml` diff --git a/docs/zh-Hans/brainstorms/2026-04-17-v0-1-slice-3-article-markdown-storage-requirements.md b/docs/zh-Hans/brainstorms/2026-04-17-v0-1-slice-3-article-markdown-storage-requirements.md new file mode 100644 index 0000000..4c475f2 --- /dev/null +++ b/docs/zh-Hans/brainstorms/2026-04-17-v0-1-slice-3-article-markdown-storage-requirements.md @@ -0,0 +1,247 @@ +--- +date: 2026-04-17 +topic: v0-1-slice-3-article-markdown-storage +--- + +# v0.1 第三切片文章 Markdown 存储需求 + +## Problem Frame + +当前后端已经证明了 feed 入库主干,但仍然只停留在 feed 提供的元数据和兼容性 `summary` 上。这还不是后续标题翻译、分层摘要或更丰富阅读体验的合适输入层。因此,下一个 v0.1 切片应该先证明一个更窄的能力:针对已经持久化的文章,抓取原始文章页,提取正文主体,转换为 Markdown,并把该 Markdown 持久化到 PostgreSQL。 + +这一切片被刻意定义为一个独立的 enrichment/backfill 能力,而不是去修改启动期 feed 入库主干。目标是在不把当前启动入库路径变成更脆弱的多阶段流水线的前提下,把“文章正文可持久化”这件事做成真实且可测试的能力。 + +已核实的当前状态: + +- `apps/api/src/feeds/feed-bootstrap.service.ts` 会在应用 bootstrap 时触发 feed 入库。 +- `apps/api/src/feeds/feed-ingestion.service.ts` 目前只抓 feed XML、规范化条目并持久化 `Article` 行,还不会抓文章页面。 +- `apps/api/prisma/models/article.prisma` 当前只存 feed 层元数据,还没有正文存储字段。 +- `apps/api/src/articles/articles.service.ts` 与 `apps/api/src/articles/article.repository.ts` 当前公开的详情读取路径只暴露 `title`、`sourceTitle`、`publishedAt`、`summary` 和 `originalUrl`。 + +```mermaid +flowchart TB + DB[(已持久化的 Article 行)] --> PICK[Backfill job 选择缺少 Markdown 的文章] + PICK --> FETCH[从 originalUrl 抓取文章 HTML] + FETCH --> EXTRACT[提取正文主体] + EXTRACT --> CLEAN[清洗 / 规范化提取后的 HTML] + CLEAN --> MD[把 HTML 转成 Markdown] + MD --> SAVE[持久化 contentMarkdown 与提取时间] + SAVE --> LOGS[结构化运行汇总日志] + FETCH --> FAIL[尽力而为的失败路径] + EXTRACT --> FAIL + CLEAN --> FAIL + MD --> FAIL + FAIL --> LOGS +``` + +## Requirements + +**正文抽取主干** + +- R1. 系统必须提供一个独立的文章正文 backfill 能力,并且该能力与启动期 feed 入库分开运行。 +- R2. backfill 能力必须从已持久化的 `Article` 记录中读取候选文章,并以 `Article.originalUrl` 作为抓取目标。 +- R3. backfill 能力必须抓取文章 HTML、提取可读正文主体、把正文转换为 Markdown,并将 Markdown 回写到同一条 `Article` 记录上。 +- R4. 第一版实现必须是尽力而为的:单篇文章提取失败不能阻止其他候选文章被继续尝试处理。 +- R5. 第一版实现必须保留现有 feed 入库主干的语义;在本切片中,启动期 feed 入库仍只负责 feed 元数据 / 文章发现。 + +**持久化形状** + +- R6. `Article` 模型必须新增可空的 `contentMarkdown` 字段,用于存储提取后的正文 Markdown。 +- R7. `Article` 模型必须新增可空的 `contentExtractedAt` 字段,用于记录最近一次成功持久化 Markdown 的时间。 +- R8. 当前 `summary` 字段必须保留现有语义,继续表示 feed 提供的 summary / description 兼容文本,而不是被改作正文存储字段。 +- R9. schema 变更必须继续收敛在 `apps/api` 内部,并沿用现有 Prisma/PostgreSQL 的所有权模型与 migration 历史。 + +**执行入口** + +- R10. 第一版执行入口应该是操作者触发的 backfill,而不是新的公开 HTTP API。 +- R11. backfill 必须支持一个窄化的本地验证路径,例如只处理单篇文章或一个受限 batch,以便操作者可以明确确认 Markdown 已经成功落库。 +- R12. backfill 必须输出结构化运行汇总,至少报告尝试的文章数、成功写入 Markdown 的文章数,以及提取失败的文章数。 + +**测试与可靠性** + +- R13. 本切片必须包含自动化测试,证明从受控输入 HTML 到持久化 `contentMarkdown` 的完整路径。 +- R14. 主自动化测试路径必须依赖本地 HTML fixtures,而不是依赖真实外部文章 URL。 +- R15. 提取流水线必须 fail-open:提取失败可以让 `contentMarkdown` 保持为空,但不能破坏现有文章行,也不能影响文章读取能力。 +- R16. 第一版允许把失败可观测性主要放在结构化日志和测试断言里;本切片不要求引入持久化的 extraction-jobs 表。 + +## Success Criteria + +- 开发者可以在本地数据库上运行文章正文 backfill,并观察到至少一条 `Article` 记录获得非空的 `contentMarkdown`。 +- 已持久化的 Markdown 来自文章正文 HTML 提取,而不是简单复制 feed `summary`。 +- 使用本地 fixtures 的自动化测试验证提取后的 Markdown 已经写入数据库。 +- 现有 `/articles` 读取 API 继续可用,且不需要新增公开 endpoint。 +- 日志或结构化命令输出能够清楚区分成功写入与提取失败。 + +## Scope Boundaries + +- 本切片不修改启动期 feed 入库触发路径。 +- 本切片不引入 scheduler、cron 或持续后台正文抽取。 +- 本切片不扩展公开 API 契约来暴露 Markdown。 +- 本切片不引入 LLM 摘要、标题翻译或分层摘要。 +- 第一版不做浏览器自动化、付费墙处理、登录流程或反爬绕过。 +- 第一版不做持久化抽取队列或 job-history 模型。 +- 本切片不要求对每个文章来源都保证成功提取。 + +## Key Decisions + +- 将文章发现与内容 enrichment 分开:feed 入库继续负责发现文章,文章正文抽取作为独立 backfill 能力运行。 +- Markdown 是这个产品方向里的正文规范持久化格式,即使 Miniflux 和 FreshRSS 这类参考系统通常持久化的是 HTML / content 而不是 Markdown。 +- 推荐的第一版技术栈是:使用 `@mozilla/readability` 做正文提取,使用 `turndown` 与 `turndown-plugin-gfm` 完成 HTML 到 Markdown 转换。 +- 第一版测试策略使用本地 HTML fixtures,因为在本切片里,可重复验证的提取正确性比贴近外网环境更重要。 +- 公开文章 API 目前继续保持很薄;这一切片先证明存储成立,再考虑契约扩张。 + +## 外部最佳实践信号 + +- Miniflux 与 FreshRSS 都支持某种“全文 enrichment”能力,并且都把增强后的文章内容持久化,而不是把 feed 摘要视为最终事实来源。 +- Miniflux 的架构说明,原文抓取更像一个独立 enrichment 动作,而不是 feed refresh 成功的硬前置条件。 +- FreshRSS 说明,全文抓取随着时间推移往往需要针对来源逐步增加调优规则;这也是第一版应该与启动入库主干解耦的原因之一。 +- `@mozilla/readability` 在 JavaScript 生态里仍然是一个可靠的可读正文提取基线,但它本身不是安全清洗器。 +- `turndown` 依然是 Node 场景下务实且成熟的 HTML 到 Markdown 转换器,尤其适合与 `turndown-plugin-gfm` 搭配使用。 +- `defuddle` 值得作为后续备选持续关注;如果未来 Markdown 保真度成为主要问题,可以再评估,但第一版存储切片不需要靠它来降风险。 + +## 高层技术方向 + +这份 brainstorm 会技术化到足以定义第一版 schema 形状、执行入口与 API 立场。 + +### Prisma Schema 草案 + +**Article** + +- `id`:现有内部主键 +- `feedId`:现有到 `Feed` 的关联 +- `identityHash`:现有的同 feed 内文章 identity +- `sourceId`:现有的来源原生标识(若存在) +- `title`:现有文章标题 +- `originalUrl`:现有文章 URL +- `publishedAt`:现有来源发布时间 +- `ingestedAt`:现有首次入库时间 +- `summary`:现有 feed 摘要兼容字段 +- `contentMarkdown`:可空的提取后正文 Markdown +- `contentExtractedAt`:可空的最近一次成功持久化 Markdown 的时间戳 +- `createdAt`:记录创建时间 +- `updatedAt`:记录更新时间 + +### Prisma 约束方向 + +- 保持现有 `Feed.feedUrl` 唯一约束与 `Article(feedId, identityHash)` 唯一约束不变。 +- 不为 `contentMarkdown` 增加新的唯一约束;它只是挂在现有 `Article` 上的 enrichment 数据。 +- `contentMarkdown` 保持可空,以便提取失败时无需写入占位内容。 +- `contentExtractedAt` 保持可空,以便在不新增独立状态模型的前提下,同时表达“尚未尝试”和“尚未成功提取”。 + +```mermaid +erDiagram + Feed ||--o{ Article : contains + Feed { + string id + string feedUrl + string siteTitle + string siteUrl + string etag + string lastModified + } + Article { + string id + string feedId + string identityHash + string sourceId + string title + string originalUrl + datetime publishedAt + datetime ingestedAt + string summary + text contentMarkdown + datetime contentExtractedAt + } +``` + +### 内部执行入口草案 + +第一版执行入口应该是包内、本地由操作者驱动的,而不是公开可路由的。 + +优先形态: + +- 由 `apps/api` 拥有的 package-local 命令或脚本 +- 默认行为:处理 `contentMarkdown` 为空的文章 +- 窄化验证选项:处理一个明确的 article ID +- 可选 batch limit,用于本地验证与安全迭代 + +示意命令形状: + +- `pnpm --filter api article-content:backfill` +- `pnpm --filter api article-content:backfill --article-id ` +- `pnpm --filter api article-content:backfill --limit 10` + +精确的命令 wiring 留到 planning 决定,但契约方向已经明确:第一版入口是内部操作者工具,而不是公开路由。 + +### 公开 API 契约立场 + +本切片中,公开 API 面应保持不变: + +- `GET /articles` +- `GET /articles/:id` + +`GET /articles/:id` 继续返回: + +- `title` +- `sourceTitle` +- `publishedAt` +- `summary` +- `originalUrl` + +本切片刻意 **不** 要求立刻暴露 `contentMarkdown`。先证明存储成立,公开消费留待后续规划。 + +### Backfill 运行汇总形状 + +内部执行入口应输出一个简单但足够确认成功的结构化汇总: + +- `attemptedCount` +- `succeededCount` +- `failedCount` +- `skippedCount` +- 可选的失败 article ID / URL 列表,便于诊断 + +### 抽取流水线方向 + +- 从 `Article.originalUrl` 抓取 HTML。 +- 在适合 Readability 的受控环境中解析 DOM。 +- 使用 `@mozilla/readability` 提取正文主体。 +- 在 Markdown 转换前对提取后的 HTML 做清洗或规范化。 +- 通过 `turndown` 加上 GFM 支持把清洗后的 HTML 转成 Markdown。 +- 只有在提取成功时才持久化 `contentMarkdown` 与 `contentExtractedAt`。 + +### 持久化到 API 的映射 + +| 关注点 | 对应数据 | +| ------------------------ | -------------------------------------------------------- | +| 现有文章标题读取 | `Article.title` | +| 现有详情摘要读取 | `Article.summary` | +| 未来文章正文输入层 | `Article.contentMarkdown` | +| 正文成功持久化的确认信号 | `Article.contentMarkdown` + `Article.contentExtractedAt` | + +### 测试方向 + +- 以本地 HTML fixtures 作为主要正确性输入。 +- 测试“从 HTML fixture 到数据库持久化”的完整提取路径。 +- 至少包含一个失败路径测试,证明对无效 / 不可提取输入不会破坏已有文章元数据。 +- 测试继续收敛在 `apps/api` 内,并与当前 NestJS/Prisma 测试方式保持一致。 + +## Dependencies / Assumptions + +- 上一个 feed 入库切片已经提供了可用的持久化文章记录。 +- 本切片以 `Article.originalUrl` 作为抓取锚点,第一版不引入新的文章来源解析逻辑。 +- 通过 Prisma `String` 映射到 PostgreSQL `text`,足以承载这一阶段预期的 Markdown 正文体量。 +- 后续 LLM 流水线会消费已持久化的 Markdown,而不是按需重新抓取文章页面。 +- 来源相关的抽取边界问题一定会存在,但对第一版证明切片而言,通用正文提取已经足够。 + +## Outstanding Questions + +### Deferred to Planning + +- [Affects R10][Technical] 操作者触发 backfill 的精确 package-local 命令 wiring 应该是什么? +- [Affects R11][Technical] 第一版面向验证的执行入口,应优先支持 `--article-id`、`--limit`,还是两者都支持? +- [Affects R15][Technical] 在不把本切片扩张成更大的内容清洗工程的前提下,`turndown` 之前最小需要做到什么程度的 sanitization / normalization? +- [Affects R13][Technical] 哪一组 fixture 最能代表第一批预期文章来源形态,同时又不会让测试语料过拟合? + +## Next Steps + +-> 当前先保持为 brainstorm 双语文档;只有在用户明确要求 planning 时,才进入 plan 阶段。 diff --git a/docs/zh-Hans/plans/2026-04-17-001-feat-article-markdown-backfill-plan.md b/docs/zh-Hans/plans/2026-04-17-001-feat-article-markdown-backfill-plan.md new file mode 100644 index 0000000..f5bf990 --- /dev/null +++ b/docs/zh-Hans/plans/2026-04-17-001-feat-article-markdown-backfill-plan.md @@ -0,0 +1,323 @@ +--- +title: feat: 在 ingestion 期间持久化文章正文 +type: feat +status: completed +date: 2026-04-17 +origin: + - docs/en/brainstorms/2026-04-17-v0-1-slice-3-article-markdown-storage-requirements.md + - docs/zh-Hans/brainstorms/2026-04-17-v0-1-slice-3-article-markdown-storage-requirements.md +deepened: 2026-04-17 +--- + +# feat: 在 ingestion 期间持久化文章正文 + +## Overview + +这份计划彻底替换掉之前以 CLI/backfill 为中心的方向。正确的 v0.1 第三切片不是一个需要开发者手动触发的同步脚本,而是把文章正文落库纳入系统的正常 ingestion 生命周期,同时保持现有 feed 发现主干 fail-open,并继续维持很薄的公开 `/articles` 契约。 + +推荐方案吸收了 Miniflux 和 FreshRSS 的共同模式: + +- 正文持久化属于系统正常 refresh/ingestion 路径,而不是人工脚本 +- 正文抽取必须保持 best-effort,这样即使抽取失败,feed/文章发现仍然成功 +- 单篇 retry HTTP 接口可以作为补救路径存在,但它不是主触发面 + +本切片仍然不把 Markdown 暴露给前端,也仍然不生成 AI 摘要。本轮目标只有一个:让正文内容能稳定落库,为后续摘要能力准备输入层。 + +## Problem Frame + +`apps/api` 已经能抓 feed 元数据并持久化 `Article` 行,但这些记录目前只包含 feed 层字段,例如标题、原文 URL、发布时间和兼容性 `summary`。这还不足以支撑后续 AI 摘要。缺失的能力是:把文章正文稳定持久化下来。 + +之前那版计划的问题在于,它把能力中心放在 package-local 命令上。那会让正文落库变成一个可选的人工操作,而不是系统默认行为。对当前产品方向来说,这个默认值是错的。系统既然已经发现并落库文章元数据,就应该顺势尝试正文抽取。 + +与此相关的当前仓库事实: + +- `apps/api/src/feeds/feed-bootstrap.service.ts` 已经拥有启动期 ingestion 语义。 +- `apps/api/src/feeds/feed-ingestion.service.ts` 已经负责 feed 抓取、规范化、best-effort 循环与持久化更新。 +- `apps/api/src/articles/articles.controller.ts` 当前只暴露很薄的只读接口。 +- `apps/api/prisma/models/article.prisma` 仍然没有正文存储字段。 + +因此,正确的边界应该是: + +- `feeds` 仍然负责文章发现与元数据持久化 +- 新的 `article-content` feature 负责 HTML 抓取、正文提取、Markdown 转换与正文落库 +- feed ingestion 在 best-effort 语义下调用这个 feature,作为正常流程的一部分 +- 单篇 retry HTTP 接口只作为补救路径,在需要时重跑某一条文章 + +## 外部最佳实践方向 + +### Miniflux + +通过 DeepWiki 可见,Miniflux 的主路径是在 feed refresh / 后台处理过程中抓取全文,同时也保留了按需的单篇 HTTP/UI 抓取动作。关键不是它的语言栈,而是触发模型: + +- 自动 refresh 路径负责正常 enrichment +- 单篇按需动作负责补救或覆盖 +- 富化内容最终挂在 entry/article 记录上 + +### FreshRSS + +通过 DeepWiki 可见,FreshRSS 更倾向于把全文抓取内联在 feed actualization/update 流程里,通常由 cron/systemd 或 UI update 驱动。关键点在于:全文落库属于 feed 更新生命周期,而不是开发者手工同步脚本。 + +### 对本仓库的推荐 + +这个仓库最适合采用两者的折中模式: + +- **主路径:** 在正常 feed ingestion / refresh 时自动尝试正文落库 +- **辅路径:** 提供一个单篇 retry 接口用于补救和验证 +- **明确不推荐:** 把开发者手动执行的 `pnpm script` 作为正文同步主路径 + +这个推荐最符合当前目标(正文应自动存在,为后续 AI 摘要服务)以及当前代码形态(已有 ingestion 主干、公开 read API 很薄、当前不需要 UI 消费 Markdown)。 + +## Requirements Trace + +- R1. 系统必须把文章正文落库纳入正常 ingestion / refresh 生命周期,而不是依赖开发者手动同步脚本。 +- R2. 系统必须从 `Article.originalUrl` 抓取文章 HTML,提取可读正文,转换为 Markdown,并回写到同一条 `Article` 记录。 +- R3. 这条持久化路径必须保持 best-effort:即使正文抽取失败,文章发现与元数据落库仍然成功。 +- R4. `Article` 必须新增可空 `contentMarkdown` 与 `contentExtractedAt` 字段。 +- R5. 当前 `summary` 字段必须保留现有语义,继续表示 feed 提供的 summary/description 兼容文本。 +- R6. 本切片中 `GET /articles` 与 `GET /articles/:id` 的公开返回必须保持不变。 +- R7. 系统可以增加一个单篇文章 retry HTTP 接口作为补救路径,但该接口必须是次级路径,而不是正文落库的主触发面。 +- R8. 本切片必须包含自动化测试,用受控 HTML fixtures 证明正文抽取与持久化路径。 +- R9. HTML 提取失败必须 fail-open,不能破坏现有 article 行。 +- R10. 所有 schema、依赖与运行时所有权都继续留在 `apps/api`。 +- R11. 实现阶段新增的任何包都必须是在当前依赖图下尽可能新的无冲突稳定版本;如果无法干净安装,就必须停下并暴露冲突。 + +## Scope Boundaries + +- 不新增 package-local 或 root-level 的 `pnpm` 同步脚本作为主触发面。 +- 本切片不通过公开 article 读取 API 暴露 `contentMarkdown`。 +- 本切片不做前端消费正文内容。 +- 本切片不做 AI 摘要、标题翻译或分层摘要。 +- 本切片不增加持久化 job queue、run-history 表,或 scheduler 管理面。 +- 本切片不引入浏览器自动化、登录流程、付费墙处理或反爬绕过。 +- 不允许正文抽取失败阻断 feed/文章 ingestion。 + +## Context & Research + +### Relevant Code and Patterns + +- `apps/api/src/feeds/feed-ingestion.service.ts` 是当前最自然的落点,因为它已经负责 feed 抓取循环、按项规范化与持久化。 +- `apps/api/src/feeds/feed-bootstrap.service.ts` 已经定义了启动触发的 ingestion 生命周期。 +- `apps/api/src/articles/article.repository.ts`、`apps/api/src/articles/articles.service.ts`、`apps/api/src/articles/articles.controller.ts` 说明当前 article 对外契约刻意保持很薄。 +- `apps/api/test-support/database.ts` 负责 database-backed 测试的 reset 与 migration replay。 +- `apps/api/e2e/feed-ingestion.e2e-spec.ts` 已经提供了 fetch mocking 与持久化验证模式。 +- `apps/api/e2e/articles.e2e-spec.ts` 与 `apps/api/src/articles/article.repository.spec.ts` 保护着当前只读契约,schema 增长后仍必须保持通过。 + +### Dependency Direction + +计划中的抽取栈仍然是: + +- `@mozilla/readability` +- `jsdom` +- `turndown` +- 一个支持 GFM 的 Markdown 转换扩展(前提是能与当前栈干净安装) + +实现时必须选择与 `apps/api` 当前依赖图能干净兼容的尽可能新稳定版本。如果首选 GFM 插件线过旧、无法干净安装,就应该停下并显式确认一个更小的维护中替代方案,或最小本地 fallback 规则,而不是硬装。 + +## Key Technical Decisions + +- 在 `apps/api` 内建立独立的 `article-content` feature 边界,而不是把抽取逻辑直接埋进 `feeds` 或 `articles`。 +- feed ingestion 继续负责文章发现编排,但在 metadata 已经规范化并准备持久化后,以 best-effort 方式调用 `article-content` 进行正文抽取。 +- 正文内容直接挂在现有 `Article` 行上,通过可空 `contentMarkdown` 与 `contentExtractedAt` 持久化。 +- 正文抽取必须 fail-open:抓取、解析、转换失败时,元数据行仍然落库,article 读取能力不受影响。 +- 增加一个单篇 retry HTTP 接口,用于补救和验证。但系统不能依赖操作者手动调用它来完成正常正文落库。 +- `summary` 语义保持不变,公开 `/articles` 契约保持不变。 +- 处理不可信 HTML 时,保持 jsdom 的脚本执行与子资源执行关闭。 +- Markdown 转换前的 HTML 清洗保持最小化;本切片目标是“稳定存储”,不是“最终渲染质量”。 + +## Open Questions + +### Resolved in This Revision + +- **主触发面应该是 CLI/script 吗?** 不应该。这个方向是错的,已经从计划中移除。 +- **系统应该自动落库正文吗?** 应该。自动落库现在是主路径。 +- **是否仍然需要补救入口?** 需要。单篇 retry 接口仍有价值,但仅作为次级路径。 +- **本切片是否要把 Markdown 或 AI 摘要暴露给前端?** 不要。本轮只做存储。 + +### Deferred to Implementation + +- 单篇文章抓取的具体 timeout 常量。 +- 单篇 retry 路由的最终命名,只要边界清晰且保持次级定位即可。 +- 通过 fixture 驱动测试收敛出的最小稳定 Markdown 规范化规则。 + +## High-Level Technical Design + +```mermaid +sequenceDiagram + participant Bootstrap as Feed bootstrap / refresh + participant Ingest as FeedIngestionService + participant Repo as Prisma Article persistence + participant Content as ArticleContentService + participant HTTP as fetch(originalUrl) + participant DOM as jsdom + Readability + participant MD as Turndown + + Bootstrap->>Ingest: trigger feed ingestion + Ingest->>Repo: persist feed + article metadata + loop per newly discovered/updated article + Ingest->>Content: tryPersistArticleContent(article id / url) + Content->>HTTP: fetch article page + HTTP-->>Content: html or failure + Content->>DOM: extract readable body + DOM-->>Content: extracted html or failure + Content->>MD: convert to markdown + MD-->>Content: markdown or failure + Content->>Repo: update contentMarkdown + contentExtractedAt on success + Content-->>Ingest: success or controlled failure + end + Ingest-->>Bootstrap: ingestion summary +``` + +## Implementation Units + +- [x] **Unit 1: 扩展 `Article` 持久化形状,同时保持公开 read 契约不变** + +**Goal:** 新增正文存储字段,并证明现有 read API 完全不变。 + +**Requirements:** R4, R5, R6, R9, R10 + +**Dependencies:** None + +**Files:** + +- Modify: `apps/api/prisma/models/article.prisma` +- Create: `apps/api/prisma/migrations/_add_article_content_markdown/migration.sql` +- Modify: `apps/api/e2e/prisma-schema.e2e-spec.ts` +- Modify: `apps/api/src/articles/article.repository.spec.ts` +- Modify: `apps/api/e2e/articles.e2e-spec.ts` + +**Approach:** + +- 增加可空 `contentMarkdown` 与 `contentExtractedAt`。 +- article DTO 映射完全保持不变。 +- 扩展测试,显式证明新增列不会改变 `/articles` 与 `/articles/:id` 的返回结构。 + +**Verification:** + +- 数据库行可以保存正文内容,同时公开读取 API 在字段层面保持完全兼容。 + +- [x] **Unit 2: 构建 `article-content` 正文抽取流水线** + +**Goal:** 创建从 article URL + HTML 到“可持久化 Markdown 或受控失败”的纯转换层。 + +**Requirements:** R2, R3, R8, R9, R10, R11 + +**Dependencies:** Unit 1 + +**Files:** + +- Modify: `apps/api/package.json` +- Create: `apps/api/src/article-content/article-content.module.ts` +- Create: `apps/api/src/article-content/article-content-extraction.service.ts` +- Create: `apps/api/src/article-content/article-content-extraction.service.spec.ts` +- Create: `apps/api/src/article-content/fixtures/clean-article.html` +- Create: `apps/api/src/article-content/fixtures/noisy-relative-links-article.html` +- Create: `apps/api/src/article-content/fixtures/non-readerable-page.html` + +**Approach:** + +- 抽取依赖只装在 `apps/api`。 +- 该服务不直接处理持久化:输入是 article URL + 原始 HTML,输出是 Markdown 成功结果或受控失败原因。 +- 使用带页面 URL 的 jsdom、Readability 做正文提取,再做最小清洗和 Turndown 转换。 + +**Verification:** + +- 基于 fixture 的测试可以在不碰数据库的前提下,验证稳定 Markdown 输出与受控失败行为。 + +- [x] **Unit 3: 在 ingestion 中自动调用正文落库能力** + +**Goal:** 让正文落库成为正常 article-ingestion 生命周期的一部分。 + +**Requirements:** R1, R2, R3, R6, R8, R9, R10, R11 + +**Dependencies:** Unit 1, Unit 2 + +**Files:** + +- Modify: `apps/api/src/feeds/feed-ingestion.service.ts` +- Modify: `apps/api/src/feeds/feeds.module.ts` +- Create: `apps/api/src/article-content/article-content.repository.ts` +- Create: `apps/api/src/article-content/article-content.service.ts` +- Create: `apps/api/src/article-content/article-content.service.spec.ts` +- Modify: `apps/api/e2e/feed-ingestion.e2e-spec.ts` + +**Approach:** + +- 在 feed entries 已规范化并持久化之后,对新发现/更新且 eligible 的文章调用 `article-content`。 +- 正文抽取路径保持 best-effort,并把失败隔离在当前文章内。 +- 沿用 feed ingestion 现有的结构化日志风格记录 success/failure。 +- 对已经有 `contentMarkdown` 的记录默认不重复处理,除非后续由显式 retry 路径请求重跑。 + +**Verification:** + +- 当正文抽取失败时,feed ingestion 仍然完成元数据落库;当正文抽取成功时,系统会自动把正文持久化下来。 + +- [x] **Unit 4: 增加单篇 retry HTTP 接口作为补救路径** + +**Goal:** 提供一个狭窄的后端触发面,用来在自动落库被跳过或失败后,针对某一篇文章重跑正文抽取。 + +**Requirements:** R7, R9, R10 + +**Dependencies:** Unit 1, Unit 2, Unit 3 + +**Files:** + +- Create: `apps/api/src/article-content/article-content.controller.ts` +- Modify: `apps/api/src/app.module.ts` +- Create: `apps/api/e2e/article-content-retry.e2e-spec.ts` + +**Approach:** + +- 新增一个只作用于单篇文章的路由,边界清晰地挂在 article-content feature 下。 +- 接口保持狭窄:输入一篇文章,执行一次 retry。 +- 返回一个简短结构化结果,例如 `succeeded`、`failed`、`skipped`,必要时附带简短原因。 +- 文档和实现都要明确:这是补救路径,不是系统正常正文落库的依赖前提。 + +**Verification:** + +- 一次定向 HTTP 调用可以为某篇文章重跑正文抽取并成功落库,同时不改变公开 article 读取契约。 + +- [x] **Unit 5: 更新文档,纠正触发模型** + +**Goal:** 去掉旧的 script-centered 心智模型,并把“自动落库 + 单篇 retry 补救”写清楚。 + +**Requirements:** R1, R6, R7 + +**Dependencies:** Unit 1, Unit 2, Unit 3, Unit 4 + +**Files:** + +- Modify: `apps/api/README.md` +- Modify: `README.md` +- Modify: `README.zh-Hans.md` + +**Approach:** + +- 明确说明:文章正文现在会在 ingestion 时自动尝试落库。 +- 明确说明:retry 接口只是补救路径,不是默认工作流。 +- 继续保持文档与实际产品边界一致:Markdown 只是内部存储层,当前不对前端暴露。 + +**Verification:** + +- 文档不再暗示“通过手动脚本同步正文”,而是准确描述自动落库模型。 + +## System-Wide Impact + +- **主要行为变化:** 系统会在 article ingestion 期间自动尝试正文落库。 +- **失败模型:** 正文抽取失败变成局部、非阻断式 enrichment 失败,而不是 ingestion blocker。 +- **API 行为:** 现有 article 读取 API 保持不变;新增一个狭窄的 retry 接口作为补救路径。 +- **产品姿态:** 本切片只为未来 AI 摘要准备已落库的正文输入,不提前把正文暴露到 UI。 + +## Risks & Mitigations + +| Risk | Mitigation | +| ---------------------------------------------- | ------------------------------------------------------------------------------- | +| 正文抽取让 ingestion 变慢或更容易失败。 | 保持 bounded timeout、best-effort 以及与元数据落库解耦。 | +| 某些来源页面 HTML 过于混乱,抽取成功率不稳定。 | 用 fixture 驱动测试固定主路径,把失败保持为非阻断,并把来源特化规则推迟到以后。 | +| 偏好的 Markdown 插件栈存在依赖冲突。 | 选择尽可能新且能干净安装的稳定版本;一旦依赖路径变得冲突密集就停下。 | +| 操作者可能误把 retry 接口理解成主工作流。 | 在文档与实现里明确把它标为 repair path。 | + +## Sources & References + +- **Origin documents:** `docs/en/brainstorms/2026-04-17-v0-1-slice-3-article-markdown-storage-requirements.md`, `docs/zh-Hans/brainstorms/2026-04-17-v0-1-slice-3-article-markdown-storage-requirements.md` +- **Relevant code:** `apps/api/src/feeds/feed-ingestion.service.ts`, `apps/api/src/feeds/feed-bootstrap.service.ts`, `apps/api/src/articles/articles.controller.ts`, `apps/api/test-support/database.ts`, `apps/api/e2e/feed-ingestion.e2e-spec.ts` +- **External architectural references via DeepWiki:** `miniflux/v2`, `FreshRSS/FreshRSS` diff --git a/docs/zh-Hans/solutions/integration-issues/feed-ingestion-retries-missing-article-markdown-2026-04-17.md b/docs/zh-Hans/solutions/integration-issues/feed-ingestion-retries-missing-article-markdown-2026-04-17.md new file mode 100644 index 0000000..02624a6 --- /dev/null +++ b/docs/zh-Hans/solutions/integration-issues/feed-ingestion-retries-missing-article-markdown-2026-04-17.md @@ -0,0 +1,139 @@ +--- +title: Feed ingestion 必须为仍然缺失正文的既有文章重新触发 Markdown 富化 +date: 2026-04-17 +category: integration-issues +module: feed ingestion +problem_type: integration_issue +component: nest_service +symptoms: + - 多次 ingestion 运行会更新文章元数据,但同一条记录上的 `contentMarkdown` 仍然保持 null + - 缺失正文的恢复依赖次级 retry 接口,而不是正常 ingestion 主路径 + - 单元测试曾允许“只给新建记录触发 article-content 富化”的回归通过 +root_cause: logic_error +resolution_type: code_fix +severity: medium +related_components: + - typeorm_repository +tags: + [ + feed-ingestion, + article-content, + markdown, + retry, + contentmarkdown, + fail-open, + nestjs, + ] +--- + +# Feed ingestion 必须为仍然缺失正文的既有文章重新触发 Markdown 富化 + +## Problem + +文章 Markdown 切片的预期是:在正常 feed ingestion 期间自动完成正文富化,同时保持 ingestion fail-open。这个契约后来发生了回归:如果第一次正文抽取失败,后续 ingestion 虽然会更新已有文章行,但不会再把它重新送去做 Markdown 抽取,导致 `contentMarkdown` 会一直保持 null,除非操作者手动调用 retry 接口。 + +## Symptoms + +- 第一次 ingestion 运行在文章 HTML 抓取/抽取失败后,仍然会成功落下文章元数据,但 `contentMarkdown` 与 `contentExtractedAt` 仍为 null。 +- 同一个 feed 后续再次 ingestion 时,会保留原来的文章行并刷新元数据,但不会自动补回缺失的 Markdown。 +- 单篇 retry 接口本身是可用的,这让问题很容易被误读成“只能手动补救也算合理”,而忽略了主路径其实已经坏掉。 + +## What Didn't Work + +- 把 article-content 富化收窄成“只处理新建记录”看起来很安全,但它悄悄破坏了历史文章的恢复路径:这些文章早已被发现,只是正文抽取尚未成功。 +- 依赖单篇 retry 接口并不够,因为本切片的 plan 明确把 retry 定义为次级补救路径,而不是正文落库的主契约。 +- 之前的一个单元测试实际上把错误行为锁定了下来,所以这个回归可以在 review 中漏过去,直到补上跨两次运行的端到端恢复场景才暴露出来。 + +## Solution + +在 `FeedIngestionService.persistNormalizedArticles()` 里恢复这条编排规则:对于已经存在但正文仍缺失的文章行,要重新加入富化队列;而对已经有正文的文章,仍然跳过。 + +修复前: + +```ts +if (existing) { + await tx.article.update({ + where: { id: existing.id }, + data: { + title: article.title, + originalUrl: article.originalUrl, + publishedAt: article.publishedAt, + summary: article.summary, + ingestedAt: article.ingestedAt, + sourceId: existing.sourceId ?? article.sourceId, + }, + }); + continue; +} +``` + +修复后: + +```ts +if (existing) { + await tx.article.update({ + where: { id: existing.id }, + data: { + title: article.title, + originalUrl: article.originalUrl, + publishedAt: article.publishedAt, + summary: article.summary, + ingestedAt: article.ingestedAt, + sourceId: existing.sourceId ?? article.sourceId, + }, + }); + + if (!existing.contentMarkdown) { + articleIds.push(existing.id); + } + + continue; +} +``` + +围绕这条编排规则补了两层回归保护: + +1. `apps/api/src/feeds/feed-ingestion.service.spec.ts` 新增单元测试,断言 `contentMarkdown: null` 的既有文章会再次经过 `tryPersistArticleContent(...)`。 +2. `apps/api/e2e/feed-ingestion.e2e-spec.ts` 新增端到端测试,验证完整恢复链路:第一次运行 fail-open,第二次运行沿用同一条文章记录,并在那次后续 ingestion 中自动补写 Markdown。 + +下游 `ArticleContentService` 的幂等契约保持不变: + +```ts +if (article.contentMarkdown && !options?.force) { + return { + reason: "already_extracted", + status: "skipped", + }; +} +``` + +这道保护让恢复后的重新入队对“已经有正文”的记录依然是安全的。 + +## Why This Works + +问题出在 feed 元数据持久化与 article-content 富化之间的集成边界。ingestion 流水线本来已经具备这些能力: + +- 跨多次运行保持文章 identity 稳定, +- 对既有文章行更新元数据, +- 对已存在正文的文章避免重复抽取。 + +真正丢失的是:这些“已经存在但仍不完整”的文章,没有再被重新接回富化步骤。现在只对 `contentMarkdown` 缺失的记录重新入队,就能恢复预期中的自动补救路径,同时又不会让每次重复 ingestion 都触发冗余抽取。`ArticleContentService` 里的幂等检查仍然是第二道保险,所以已经完成富化的记录还是会被干净地短路跳过。 + +## Prevention + +- 给“existing row”两种分支都保留编排测试:一种是 `contentMarkdown: null`,另一种是已经有 Markdown。 +- 保留至少跨两次 ingestion 的 e2e 场景,因为这个回归只有在第一次 fail-open、第二次尝试恢复时才会显现。 +- 在测试和文档里把 repair endpoint 明确当作次级流程;如果主契约写的是自动富化,就要直接断言这条主路径。 +- 只要 `findFirst()` 的结果参与后续编排决策,就要把决策所需字段视为契约的一部分。这里的 `existing.contentMarkdown` 就是这样,不应随意从查询/结果形状里拿掉。 + +## Related Issues + +- `apps/api/src/feeds/feed-ingestion.service.ts` +- `apps/api/src/feeds/feed-ingestion.service.spec.ts` +- `apps/api/e2e/feed-ingestion.e2e-spec.ts` +- `apps/api/src/article-content/article-content.service.ts` +- `apps/api/e2e/article-content-retry.e2e-spec.ts` +- `docs/en/plans/2026-04-17-001-feat-article-markdown-backfill-plan.md` +- `docs/zh-Hans/plans/2026-04-17-001-feat-article-markdown-backfill-plan.md` +- `.claude/handoffs/2026-04-17-164836-feed-ingestion-auto-enrichment-recovery.md` +- `.claude/handoffs/2026-04-17-170512-article-markdown-backfill-review-fix.md` diff --git a/docs/zh-Hans/solutions/workflow-issues/prettier-gates-should-ignore-tool-managed-and-generated-files-2026-04-17.md b/docs/zh-Hans/solutions/workflow-issues/prettier-gates-should-ignore-tool-managed-and-generated-files-2026-04-17.md new file mode 100644 index 0000000..7b07656 --- /dev/null +++ b/docs/zh-Hans/solutions/workflow-issues/prettier-gates-should-ignore-tool-managed-and-generated-files-2026-04-17.md @@ -0,0 +1,137 @@ +--- +title: 让 Prettier gate 排除工具管理与生成型文件 +date: 2026-04-17 +category: workflow-issues +module: format gate boundaries +problem_type: workflow_issue +component: development_workflow +severity: medium +applies_when: + - 仓库使用 `prettier --check .` 这类 root 级格式门禁 + - CI 失败指向 lockfile 或其他生成型产物 + - 某个文件由包管理器或外部工具维护,而不是由人手编辑维护 + - 纯格式 churn 正在制造没有语义价值的 review 噪音 + - 维护者正在决定是否扩大 `.prettierignore` 的边界 +tags: + [ + prettier, + prettierignore, + pnpm-lock, + lockfile, + ci, + generated-files, + tool-managed, + formatting, + ] +--- + +# 让 Prettier gate 排除工具管理与生成型文件 + +## Context + +这个仓库的 root 命令 `pnpm format:check` 当前实际会执行 +`prettier --check .`。这意味着 `.prettierignore` 不是单纯的编辑器便利 +配置,而是质量门禁边界的一部分。 + +最近一次 CI 失败清楚说明了这条边界为什么重要。`pr-quality / format` +job 唯一失败的原因,是 `pnpm-lock.yaml` 不符合 Prettier 偏好的 YAML +排版方式。那次 lockfile 变更完全只是格式变化,diff 很大,却没有任何 +行为价值。结果就是 gate 变得嘈杂,注意力也被从真正 repo-owned 的表面 +移开了。 + +## Guidance + +不要随手把工具管理或生成型文件纳入全仓 Prettier gate。 + +如果某个文件的权威来源是外部工具,就应该优先让那个工具决定它的形状; +除非仓库里有非常强的、明确的理由,否则不应把它继续纳入 +`prettier --check .`。 + +对这个仓库来说,`pnpm-lock.yaml` 应该被视为包管理器拥有的产物: + +- 它仍然要提交进 git,以保证可复现安装和缓存正确性 +- 让 `pnpm` 在依赖变化时自然更新它 +- 不要把它放进 root Prettier gate,以避免制造巨大的、无语义的 diff +- 用 `.prettierignore` 显式表达这个边界 + +实用规则可以非常简单:format gate 应该覆盖 repo-owned 内容,而不是 +覆盖所有被追踪的文件。 + +## Why This Matters + +当格式门禁失败在一些并不期待由人手塑形的文件上时,它就会开始误导人。 + +这次的直接红灯看起来像一次格式回归,但更深层的问题其实是边界漂移: +这个 gate 检查的是一个由 `pnpm` 拥有的文件,而不是一个由仓库格式约定 +拥有的文件。这样的漂移会带来三个问题: + +- reviewer 会看到巨大但没有行为意义的 lockfile diff +- 下游 job 也可能被级联打红,导致定位根因更慢 +- 工程师容易开始“修 generated file”,而不是修 gate 的边界 + +只有当 repo 级格式规则的覆盖范围,真正对应团队希望人工维护的文件集合时, +这个规则才是可信的。 + +## When to Apply + +- 当 CI format 失败指向 `pnpm-lock.yaml`、生成型 manifest,或其他机器拥有的文件时 +- 当 `prettier --check .` 这类 root format 命令比仓库真实拥有边界更宽时 +- 当仓库引入新的 tool-managed 目录或产物时 +- 当你在判断一次嘈杂的 format 失败,到底该靠重排文件内容修,还是该靠收紧 + `.prettierignore` 修时 +- 当 review churn 因为格式步骤触碰生成产物而不断放大时 + +## Examples + +坏边界: + +```gitignore +# .prettierignore +.agents/ +.cache/ +.claude/ +.codex/ +.omx/ +.turbo/ +tmp/ +``` + +在这个配置下,root format gate 仍然会检查 `pnpm-lock.yaml`: + +```json +{ + "scripts": { + "format:check": "prettier --check ." + } +} +``` + +更好的边界: + +```gitignore +# .prettierignore +.agents/ +.cache/ +.claude/ +.codex/ +.omx/ +.turbo/ +pnpm-lock.yaml +tmp/ +``` + +更正确的心智模型: + +```text +Lockfile 继续纳入 git。 +让 pnpm 负责重生成它。 +不要用 Prettier 制造只改 lockfile 排版的大 diff。 +``` + +## Related + +- `docs/en/solutions/workflow-issues/github-pr-quality-ci-trusted-base-scope-and-ui-bootstrap-2026-04-12.md` +- `docs/zh-Hans/solutions/workflow-issues/github-pr-quality-ci-trusted-base-scope-and-ui-bootstrap-2026-04-12.md` +- `.prettierignore` +- `package.json` +- `pnpm-lock.yaml` diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 63d4a24..56291b4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -37,6 +37,9 @@ importers: apps/api: dependencies: + "@mozilla/readability": + specifier: 0.6.0 + version: 0.6.0 "@nestjs/common": specifier: ^11.1.18 version: 11.1.18(reflect-metadata@0.2.2)(rxjs@7.8.2) @@ -61,6 +64,9 @@ importers: feedsmith: specifier: 2.9.2 version: 2.9.2 + jsdom: + specifier: 26.1.0 + version: 26.1.0 pg: specifier: 8.20.0 version: 8.20.0 @@ -70,6 +76,9 @@ importers: rxjs: specifier: ^7.8.2 version: 7.8.2 + turndown: + specifier: 7.2.4 + version: 7.2.4 devDependencies: "@jest/globals": specifier: ^30.3.0 @@ -95,6 +104,9 @@ importers: "@types/express": specifier: ^5.0.6 version: 5.0.6 + "@types/jsdom": + specifier: ^28.0.1 + version: 28.0.1 "@types/node": specifier: ^25.5.2 version: 25.5.2 @@ -104,6 +116,9 @@ importers: "@types/supertest": specifier: ^7.2.0 version: 7.2.0 + "@types/turndown": + specifier: ^5.0.6 + version: 5.0.6 eslint: specifier: ^9.39.4 version: 9.39.4(jiti@2.6.1) @@ -1494,6 +1509,19 @@ packages: } engines: { node: ">=8" } + "@mixmark-io/domino@2.2.0": + resolution: + { + integrity: sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==, + } + + "@mozilla/readability@0.6.0": + resolution: + { + integrity: sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ==, + } + engines: { node: ">=14.0.0" } + "@napi-rs/wasm-runtime@0.2.12": resolution: { @@ -2440,6 +2468,12 @@ packages: integrity: sha512-yOriVnggzrnQ3a9OKOCxaVuSug3w3/SbOj5i7VwXWZEyUNl3bLF9V3MfxGbZKuwqJOQyRfqXyROBB1CoZLFWzA==, } + "@types/jsdom@28.0.1": + resolution: + { + integrity: sha512-GJq2QE4TAZ5ajSoCasn5DOFm8u1mI3tIFvM5tIq3W5U/RTB6gsHwc6Yhpl91X9VSDOUVblgXmG+2+sSvFQrdlw==, + } + "@types/json-schema@7.0.15": resolution: { @@ -2532,6 +2566,12 @@ packages: integrity: sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==, } + "@types/turndown@5.0.6": + resolution: + { + integrity: sha512-ru00MoyeeouE5BX4gRL+6m/BsDfbRayOskWqUvh7CLGW+UXxHQItqALa38kKnOiZPqJrtzJUgAC2+F0rL1S4Pg==, + } + "@types/yargs-parser@21.0.3": resolution: { @@ -7691,6 +7731,13 @@ packages: } hasBin: true + turndown@7.2.4: + resolution: + { + integrity: sha512-I8yFsfRzmzK0WV1pNNOA4A7y4RDfFxPRxb3t+e3ui14qSGOxGtiSP6GjeX+Y6CHb7HYaFj7ECUD7VE5kQMZWGQ==, + } + engines: { node: ">=18", npm: ">=9" } + tw-animate-css@1.4.0: resolution: { @@ -7834,6 +7881,12 @@ packages: integrity: sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==, } + undici-types@7.25.0: + resolution: + { + integrity: sha512-AXNgS1Byr27fTI+2bsPEkV9CxkT8H6xNyRI68b3TatlZo3RkzlqQBLL+w7SmGPVpokjHbcuNVQUWE7FRTg+LRA==, + } + universalify@2.0.1: resolution: { @@ -9032,6 +9085,10 @@ snapshots: "@lukeed/csprng@1.1.0": {} + "@mixmark-io/domino@2.2.0": {} + + "@mozilla/readability@0.6.0": {} + "@napi-rs/wasm-runtime@0.2.12": dependencies: "@emnapi/core": 1.9.2 @@ -9634,6 +9691,13 @@ snapshots: "@types/tough-cookie": 4.0.5 parse5: 7.3.0 + "@types/jsdom@28.0.1": + dependencies: + "@types/node": 25.5.2 + "@types/tough-cookie": 4.0.5 + parse5: 7.3.0 + undici-types: 7.25.0 + "@types/json-schema@7.0.15": {} "@types/methods@1.1.4": {} @@ -9691,6 +9755,8 @@ snapshots: "@types/tough-cookie@4.0.5": {} + "@types/turndown@5.0.6": {} + "@types/yargs-parser@21.0.3": {} "@types/yargs@17.0.35": @@ -13045,6 +13111,10 @@ snapshots: "@turbo/windows-64": 2.9.4 "@turbo/windows-arm64": 2.9.4 + turndown@7.2.4: + dependencies: + "@mixmark-io/domino": 2.2.0 + tw-animate-css@1.4.0: {} type-check@0.4.0: @@ -13136,6 +13206,8 @@ snapshots: undici-types@7.18.2: {} + undici-types@7.25.0: {} + universalify@2.0.1: {} unpipe@1.0.0: {}