From 3b6cd32db41ffa22c3758298dd7b35d6c133871c Mon Sep 17 00:00:00 2001 From: Sarthak Agrawal Date: Mon, 10 Aug 2026 04:26:45 +0530 Subject: [PATCH 1/3] perf: avoid sanitizing plain RSS metadata --- .gitignore | 2 +- .../__tests__/rss-parser-performance.test.ts | 64 +++++++++++++++++++ src/lib/__tests__/rss-parser.test.ts | 13 ++++ src/lib/rss-parser.ts | 10 ++- 4 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 src/lib/__tests__/rss-parser-performance.test.ts diff --git a/.gitignore b/.gitignore index 5a83ae0..52a5f4e 100644 --- a/.gitignore +++ b/.gitignore @@ -76,5 +76,5 @@ docs-site/ # Local agent logs / scratch .agent-logs/ +.codevetter/ *.agent.log - diff --git a/src/lib/__tests__/rss-parser-performance.test.ts b/src/lib/__tests__/rss-parser-performance.test.ts new file mode 100644 index 0000000..cbbc5b4 --- /dev/null +++ b/src/lib/__tests__/rss-parser-performance.test.ts @@ -0,0 +1,64 @@ +import { createHash } from 'node:crypto'; +import { performance } from 'node:perf_hooks'; + +import { expect, test } from 'vitest'; + +import { parseFeed } from '../rss-parser'; + +const SIZES = [10, 100, 200]; +const ITERATIONS = 25; +const EXPECTED_HASHES = new Map([ + [10, 'e2c04d0f621e652d5cf5e180862d5ef0d3e420414fc6dd2766aa638cdd255d02'], + [100, 'bcc8798f918ca05af112d4a6c74f1defa2ae4073c4675bdcd65fb84dc91c2da9'], + [200, 'bb8a0c80775d61191a5ee3a23ea651a5b32b777250478821040a84cbce6148c0'], +]); + +test('RSS parsing scales across supported feed sizes', { timeout: 30_000 }, () => { + const metrics: string[] = []; + + for (const size of SIZES) { + const xml = buildFeed(size); + const expected = JSON.stringify(parseFeed(xml)); + const expectedHash = createHash('sha256').update(expected).digest('hex'); + expect(expectedHash).toBe(EXPECTED_HASHES.get(size)); + let durationMs = 0; + + for (let iteration = 0; iteration < ITERATIONS; iteration += 1) { + const startedAt = performance.now(); + const parsed = parseFeed(xml); + durationMs += performance.now() - startedAt; + expect(JSON.stringify(parsed)).toBe(expected); + expect(createHash('sha256').update(JSON.stringify(parsed)).digest('hex')).toBe(expectedHash); + } + + metrics.push(`size${size}=${(durationMs / ITERATIONS).toFixed(3)}ms/op`); + } + + console.log(`[benchmark] ${metrics.join(' ')} (${ITERATIONS} iterations)`); + console.log(`[resource] maximum_supported_entries=${SIZES.at(-1)}`); +}); + +function buildFeed(size: number): string { + const items = Array.from({ length: size }, (_, index) => { + const day = String((index % 28) + 1).padStart(2, '0'); + return ` + entry-${index} + Research update ${index} & notes + https://example.com/articles/${index}?source=reader + Author ${index % 20} + Mon, ${day} Jul 2026 00:00:00 GMT +

Finding ${index}

This is a useful research summary with + evidence, context, and follow-up details for the Reader library.

+

Additional paragraph ${index} with source.

+ + ]]> + `; + }).join(''); + + return ` + Reader performance feed + https://example.com/ + ${items} + `; +} diff --git a/src/lib/__tests__/rss-parser.test.ts b/src/lib/__tests__/rss-parser.test.ts index b00842d..333862c 100644 --- a/src/lib/__tests__/rss-parser.test.ts +++ b/src/lib/__tests__/rss-parser.test.ts @@ -47,6 +47,19 @@ describe('parseFeed', () => { expect(feed.entries[0].content).not.toContain(' { + const rss = ` + <![CDATA[Example <strong>RSS</strong>]]>https://example.com/ + post-1<![CDATA[First <em>entry</em>]]> + Summary + `; + + expect(parseFeed(rss)).toMatchObject({ + title: 'Example RSS', + entries: [{ title: 'First entry' }], + }); + }); + it('normalizes Atom entries and alternate links', () => { const atom = ` Example Atom diff --git a/src/lib/rss-parser.ts b/src/lib/rss-parser.ts index 949196d..e72b931 100644 --- a/src/lib/rss-parser.ts +++ b/src/lib/rss-parser.ts @@ -41,6 +41,12 @@ function cleanText(value: string | null | undefined, maxLength = 2_000): string return (document.body.textContent ?? '').replace(/\s+/g, ' ').trim().slice(0, maxLength); } +function normalizeText(value: string | null | undefined, maxLength = 2_000): string { + const text = value ?? ''; + if (text.includes('<')) return cleanText(text, maxLength); + return text.replace(/\s+/g, ' ').trim().slice(0, maxLength); +} + function cleanHtml(value: string | null | undefined): string | undefined { const cleaned = sanitizeHtml(value ?? '', { allowedTags: sanitizeHtml.defaults.allowedTags, @@ -56,7 +62,7 @@ function cleanHtml(value: string | null | undefined): string | undefined { function firstText(element: Element, names: string[]): string { for (const name of names) { const node = element.getElementsByTagName(name)[0]; - const value = cleanText(node?.textContent); + const value = normalizeText(node?.textContent); if (value) return value; } return ''; @@ -132,7 +138,7 @@ export function parseOpml(xml: string): OpmlSubscription[] { if (!feedUrl || seen.has(feedUrl)) continue; seen.add(feedUrl); subscriptions.push({ - title: cleanText( + title: normalizeText( outline.getAttribute('title') || outline.getAttribute('text') || new URL(feedUrl).hostname, 500 ), From 93bf33e90949628805a8ccee428587a5de530045 Mon Sep 17 00:00:00 2001 From: Sarthak Agrawal Date: Tue, 11 Aug 2026 00:59:44 +0530 Subject: [PATCH 2/3] feat: expose read-only ChatGPT data --- src/lib/__tests__/articles-db-search.test.ts | 15 +++- src/lib/articles-db.ts | 54 ++++++++++- src/lib/auth-api.test.ts | 43 +++++++++ src/lib/auth-api.ts | 17 +++- src/worker.ts | 2 + src/worker/routes/__tests__/mcp.test.ts | 94 ++++++++++++++++++++ src/worker/routes/mcp.ts | 81 +++++++++++++++++ 7 files changed, 303 insertions(+), 3 deletions(-) create mode 100644 src/lib/auth-api.test.ts create mode 100644 src/worker/routes/__tests__/mcp.test.ts create mode 100644 src/worker/routes/mcp.ts diff --git a/src/lib/__tests__/articles-db-search.test.ts b/src/lib/__tests__/articles-db-search.test.ts index 268ba38..94b73df 100644 --- a/src/lib/__tests__/articles-db-search.test.ts +++ b/src/lib/__tests__/articles-db-search.test.ts @@ -12,7 +12,7 @@ vi.mock('../db/client', () => ({ import { describe, expect, it } from 'vitest'; -import { searchArticles } from '../articles-db'; +import { searchArticles, searchArticleSummaries } from '../articles-db'; describe('searchArticles', () => { it('handles search terms with regex characters without throwing', async () => { @@ -44,4 +44,17 @@ describe('searchArticles', () => { await expect(searchArticles('user-1', 'a(')).resolves.toHaveLength(1); }); + + it('fails closed for a project identifier outside the owner virtual project', async () => { + mockedSelect.mockClear(); + await expect( + searchArticleSummaries('user-1', { + query: 'agent', + projectId: 'another-user_default', + limit: 10, + offset: 0, + }) + ).resolves.toEqual({ items: [], total: 0, nextOffset: null }); + expect(mockedSelect).not.toHaveBeenCalled(); + }); }); diff --git a/src/lib/articles-db.ts b/src/lib/articles-db.ts index 8869a11..3f03c9a 100644 --- a/src/lib/articles-db.ts +++ b/src/lib/articles-db.ts @@ -1,5 +1,5 @@ import crypto from 'crypto'; -import { and, desc, eq, inArray } from 'drizzle-orm'; +import { and, count, desc, eq, inArray, like, or, type SQL } from 'drizzle-orm'; import type { IOptions } from 'sanitize-html'; import sanitizeHtml from 'sanitize-html'; @@ -382,6 +382,58 @@ export async function fetchArticleSummaries( } } +export type ArticleSearchOptions = { + query?: string; + listId?: string; + projectId?: string; + type?: 'article' | 'link' | 'pdf'; + limit: number; + offset: number; +}; + +/** Bounded owner-scoped search for integration clients. */ +export async function searchArticleSummaries( + userId: string, + options: ArticleSearchOptions +): Promise<{ items: ArticleSummary[]; total: number; nextOffset: number | null }> { + if (options.projectId && options.projectId !== defaultProjectId(userId)) { + return { items: [], total: 0, nextOffset: null }; + } + const conditions: SQL[] = [eq(articles.userId, userId)]; + const query = options.query?.trim().replaceAll(/[%_]/g, '').slice(0, 200); + if (query) { + const pattern = `%${query}%`; + const search = or( + like(articles.title, pattern), + like(articles.url, pattern), + like(articles.byline, pattern), + like(articles.tags, pattern) + ); + if (search) conditions.push(search); + } + if (options.type) conditions.push(eq(articles.type, options.type)); + if (options.listId) { + conditions.push(like(articles.listIds, `%"${options.listId.replaceAll('"', '')}"%`)); + } + const where = and(...conditions); + const [rows, totals] = await Promise.all([ + db + .select() + .from(articles) + .where(where) + .orderBy(desc(articles.createdAt)) + .limit(options.limit) + .offset(options.offset), + db.select({ value: count() }).from(articles).where(where), + ]); + const total = totals[0]?.value ?? 0; + return { + items: rows.map(rowToSummary), + total, + nextOffset: options.offset + rows.length < total ? options.offset + rows.length : null, + }; +} + export async function fetchArticlesForSourceMap(userId: string): Promise { try { const rows = await db diff --git a/src/lib/auth-api.test.ts b/src/lib/auth-api.test.ts new file mode 100644 index 0000000..ddbc17b --- /dev/null +++ b/src/lib/auth-api.test.ts @@ -0,0 +1,43 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + verifyApiKey: vi.fn(), + createAuth: vi.fn(), +})); + +vi.mock('./api-keys', () => ({ + API_KEY_PREFIX: 'rdr_', + verifyApiKey: mocks.verifyApiKey, +})); +vi.mock('./auth', () => ({ createAuth: mocks.createAuth })); + +import { getApiKeyUserId } from './auth-api'; + +describe('Reader API-key-only authentication', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('accepts only an exact rdr bearer token', async () => { + mocks.verifyApiKey.mockResolvedValue('owner-1'); + await expect( + getApiKeyUserId(new Headers({ Authorization: 'Bearer rdr_valid-token' })) + ).resolves.toBe('owner-1'); + expect(mocks.verifyApiKey).toHaveBeenCalledWith('rdr_valid-token'); + expect(mocks.createAuth).not.toHaveBeenCalled(); + }); + + it('rejects cookies, JWTs, other token scopes, and ambiguous bearer values', async () => { + const inputs = [ + new Headers({ Cookie: 'better-auth.session_token=browser-session' }), + new Headers({ Authorization: 'Bearer header.payload.signature' }), + new Headers({ Authorization: 'Bearer calorie_read_wrong-scope' }), + new Headers({ Authorization: 'Bearer rdr_value extra' }), + ]; + for (const headers of inputs) { + await expect(getApiKeyUserId(headers)).resolves.toBeNull(); + } + expect(mocks.verifyApiKey).not.toHaveBeenCalled(); + expect(mocks.createAuth).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/auth-api.ts b/src/lib/auth-api.ts index 0ee1d8b..19a2c3a 100644 --- a/src/lib/auth-api.ts +++ b/src/lib/auth-api.ts @@ -1,6 +1,21 @@ import { API_KEY_PREFIX, verifyApiKey } from './api-keys'; import { createAuth, type AuthEnv } from './auth'; +/** Resolve only a dedicated long-lived Reader API key; never use browser auth. */ +export async function getApiKeyUserId(headers: Headers): Promise { + const authHeader = headers.get('authorization') ?? headers.get('Authorization'); + if (!authHeader) return null; + const [scheme, value, extra] = authHeader.trim().split(/\s+/, 3); + if ( + extra !== undefined || + scheme?.toLowerCase() !== 'bearer' || + !value?.startsWith(API_KEY_PREFIX) + ) { + return null; + } + return verifyApiKey(value); +} + /** * Resolve the authenticated user for an API request. * @@ -18,7 +33,7 @@ export async function getAuthenticatedUserId( if (authHeader) { const [scheme, value] = authHeader.split(' ', 2); if (scheme?.toLowerCase() === 'bearer' && value?.startsWith(API_KEY_PREFIX)) { - const userId = await verifyApiKey(value); + const userId = await getApiKeyUserId(headers); if (userId) return userId; return null; } diff --git a/src/worker.ts b/src/worker.ts index 56d3c96..9425cf2 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -10,6 +10,7 @@ import boardsRoutes from './worker/routes/boards'; import keysRoutes from './worker/routes/keys'; import listsRoutes from './worker/routes/lists'; import memoriesRoutes from './worker/routes/memories'; +import mcpReadRoutes from './worker/routes/mcp'; import miscRoutes from './worker/routes/misc'; import pdfRoutes from './worker/routes/pdf'; import rssRoutes from './worker/routes/rss'; @@ -60,6 +61,7 @@ api.route('/api/articles', articlesRoutes); api.route('/api/boards', boardsRoutes); api.route('/api/lists', listsRoutes); api.route('/api/memories', memoriesRoutes); +api.route('/api/mcp', mcpReadRoutes); api.route('/api/ai', aiRoutes); api.route('/api/keys', keysRoutes); api.route('/api/pdfs', pdfRoutes); diff --git a/src/worker/routes/__tests__/mcp.test.ts b/src/worker/routes/__tests__/mcp.test.ts new file mode 100644 index 0000000..ac1528d --- /dev/null +++ b/src/worker/routes/__tests__/mcp.test.ts @@ -0,0 +1,94 @@ +import { Hono } from 'hono'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + getApiKeyUserId: vi.fn(), + searchArticleSummaries: vi.fn(), + fetchArticleById: vi.fn(), + fetchLists: vi.fn(), +})); + +vi.mock('../../../lib/auth-api', () => ({ + getApiKeyUserId: mocks.getApiKeyUserId, +})); +vi.mock('../../../lib/articles-db', () => ({ + searchArticleSummaries: mocks.searchArticleSummaries, + fetchArticleById: mocks.fetchArticleById, +})); +vi.mock('../../../lib/lists-db', () => ({ fetchLists: mocks.fetchLists })); + +import mcpRoutes from '../mcp'; + +const app = new Hono(); +app.route('/api/mcp', mcpRoutes); + +describe('Reader MCP read projections', () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.getApiKeyUserId.mockResolvedValue('owner-1'); + mocks.searchArticleSummaries.mockResolvedValue({ items: [], total: 0, nextOffset: null }); + mocks.fetchLists.mockResolvedValue([]); + }); + + it('fails closed without an owner credential', async () => { + mocks.getApiKeyUserId.mockResolvedValue(null); + const response = await app.request('/api/mcp/reading?q=test'); + expect(response.status).toBe(401); + expect(mocks.searchArticleSummaries).not.toHaveBeenCalled(); + }); + + it('does not treat a browser session cookie as an MCP credential', async () => { + mocks.getApiKeyUserId.mockResolvedValue(null); + const response = await app.request('/api/mcp/reading?q=test', { + headers: { Cookie: 'better-auth.session_token=browser-session' }, + }); + expect(response.status).toBe(401); + expect(mocks.getApiKeyUserId).toHaveBeenCalledOnce(); + expect(mocks.searchArticleSummaries).not.toHaveBeenCalled(); + }); + + it('clamps search pagination and retains owner scope', async () => { + const response = await app.request('/api/mcp/reading?q=agents&limit=500&offset=2'); + expect(response.status).toBe(200); + expect(mocks.searchArticleSummaries).toHaveBeenCalledWith('owner-1', { + query: 'agents', + listId: undefined, + projectId: undefined, + type: undefined, + limit: 50, + offset: 2, + }); + }); + + it('passes the bounded virtual project filter to the owner-scoped query', async () => { + const response = await app.request( + '/api/mcp/reading?q=agents&projectId=owner-1_default&limit=10' + ); + expect(response.status).toBe(200); + expect(mocks.searchArticleSummaries).toHaveBeenCalledWith( + 'owner-1', + expect.objectContaining({ projectId: 'owner-1_default' }) + ); + }); + + it('projects item content without PDF download or credential fields', async () => { + mocks.fetchArticleById.mockResolvedValue({ + id: 'article-1', + url: 'https://example.com', + title: 'Example', + content: 'Readable content', + status: 'in_progress', + tags: [], + notes: [], + type: 'pdf', + listIds: [], + pdfUrl: '/api/pdfs/article-1', + aiChat: [{ role: 'user', content: 'private provider context' }], + }); + const response = await app.request('/api/mcp/reading/article-1'); + const body = await response.json(); + expect(response.status).toBe(200); + expect(body.item.pdfUrl).toBeUndefined(); + expect(body.item.aiChat).toBeUndefined(); + }); +}); diff --git a/src/worker/routes/mcp.ts b/src/worker/routes/mcp.ts new file mode 100644 index 0000000..9b5891b --- /dev/null +++ b/src/worker/routes/mcp.ts @@ -0,0 +1,81 @@ +import { Hono } from 'hono'; + +import { fetchArticleById, searchArticleSummaries } from '../../lib/articles-db'; +import { getApiKeyUserId } from '../../lib/auth-api'; +import { fetchLists } from '../../lib/lists-db'; +import type { WorkerEnv } from '../../lib/worker-env'; + +const mcpReads = new Hono<{ Bindings: WorkerEnv }>(); + +function pageValue(value: string | undefined, fallback: number, max: number) { + const parsed = Number(value); + return Number.isInteger(parsed) ? Math.min(Math.max(parsed, 0), max) : fallback; +} + +async function ownerId(headers: Headers) { + return getApiKeyUserId(headers); +} + +mcpReads.get('/reading', async (c) => { + const userId = await ownerId(c.req.raw.headers); + if (!userId) return c.json({ code: 'UNAUTHORIZED', message: 'Read credential required.' }, 401); + const type = c.req.query('type'); + const result = await searchArticleSummaries(userId, { + query: c.req.query('q'), + listId: c.req.query('listId'), + projectId: c.req.query('projectId'), + type: type === 'article' || type === 'link' || type === 'pdf' ? type : undefined, + limit: Math.max(1, pageValue(c.req.query('limit'), 10, 50)), + offset: pageValue(c.req.query('offset'), 0, 1_000_000), + }); + return c.json({ ...result, generatedAt: new Date().toISOString() }); +}); + +mcpReads.get('/reading/:id', async (c) => { + const userId = await ownerId(c.req.raw.headers); + if (!userId) return c.json({ code: 'UNAUTHORIZED', message: 'Read credential required.' }, 401); + const article = await fetchArticleById(c.req.param('id'), userId); + if (!article) return c.json({ code: 'NOT_FOUND', message: 'Saved item not found.' }, 404); + const item = { + id: article.id, + url: article.url, + title: article.title, + byline: article.byline, + content: article.content, + status: article.status, + tags: article.tags, + notes: article.notes, + summary: article.summary, + keyPoints: article.keyPoints, + type: article.type, + category: article.category, + createdAt: article.createdAt, + updatedAt: article.updatedAt, + listIds: article.listIds, + }; + return c.json({ item }); +}); + +mcpReads.get('/collections', async (c) => { + const userId = await ownerId(c.req.raw.headers); + if (!userId) return c.json({ code: 'UNAUTHORIZED', message: 'Read credential required.' }, 401); + const limit = Math.max(1, pageValue(c.req.query('limit'), 10, 50)); + const offset = pageValue(c.req.query('offset'), 0, 1_000_000); + const lists = await fetchLists(userId); + const items = lists.slice(offset, offset + limit).map((list) => ({ + id: list.id, + name: list.name, + color: list.color, + isDefault: list.isDefault, + createdAt: list.createdAt, + updatedAt: list.updatedAt, + })); + return c.json({ + generatedAt: new Date().toISOString(), + items, + total: lists.length, + nextOffset: offset + items.length < lists.length ? offset + items.length : null, + }); +}); + +export default mcpReads; From bda955e4d393bb5a59191e1661f643686c4f6a1d Mon Sep 17 00:00:00 2001 From: Sarthak Agrawal Date: Wed, 12 Aug 2026 18:23:37 +0530 Subject: [PATCH 3/3] perf: reduce repeated RSS parsing work --- src/lib/rss-parser.ts | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/lib/rss-parser.ts b/src/lib/rss-parser.ts index e72b931..0b60a7e 100644 --- a/src/lib/rss-parser.ts +++ b/src/lib/rss-parser.ts @@ -5,6 +5,7 @@ const MAX_OPML_BYTES = 1_000_000; export const MAX_FEED_BYTES = 2_000_000; const MAX_OPML_FEEDS = 500; const MAX_FEED_ENTRIES = 200; +const DOM_PARSER = new DOMParser(); export interface OpmlSubscription { title: string; @@ -34,10 +35,11 @@ function byteLength(value: string): number { function cleanText(value: string | null | undefined, maxLength = 2_000): string { const sanitized = sanitizeHtml(value ?? '', { allowedTags: [], allowedAttributes: {} }); - const document = new DOMParser().parseFromString( - `${sanitized}`, - 'text/html' - ); + return textFromSanitizedHtml(sanitized, maxLength); +} + +function textFromSanitizedHtml(value: string, maxLength: number): string { + const document = DOM_PARSER.parseFromString(`${value}`, 'text/html'); return (document.body.textContent ?? '').replace(/\s+/g, ' ').trim().slice(0, maxLength); } @@ -98,14 +100,18 @@ export function createEntryExternalId(parts: { title: string; publishedAt?: Date; }): string { - const explicitId = cleanText(parts.id, 1_000); + const rawId = parts.id ?? ''; + const explicitId = + rawId.includes('<') || rawId.includes('&') + ? cleanText(rawId, 1_000) + : rawId.replace(/\s+/g, ' ').trim().slice(0, 1_000); if (explicitId) return explicitId; const identity = [parts.url ?? '', parts.title, parts.publishedAt?.toISOString() ?? ''].join('|'); return `generated:${stableHash(identity)}`; } function parseXml(xml: string) { - const document = new DOMParser().parseFromString(xml, 'text/xml'); + const document = DOM_PARSER.parseFromString(xml, 'text/xml'); if (!document?.documentElement) throw new Error('Malformed XML document'); return document; } @@ -172,7 +178,7 @@ function normalizeEntry(element: Element, atom: boolean): NormalizedFeedEntry | atom ? ['content', 'summary'] : ['content:encoded', 'description'] ); const content = cleanHtml(rawContent); - const excerpt = cleanText(rawContent, 500) || undefined; + const excerpt = content ? textFromSanitizedHtml(content, 500) || undefined : undefined; const publishedAt = parseDate( firstRawText(element, atom ? ['published', 'updated'] : ['pubDate', 'dc:date', 'date']) );