diff --git a/src/collectors.ts b/src/collectors.ts index dac7725..fbd345b 100644 --- a/src/collectors.ts +++ b/src/collectors.ts @@ -43,7 +43,13 @@ function safeUnixIso(value: unknown): string | undefined { function sanitizeExcerpt(raw?: string): string | undefined { if (!raw) return undefined; - const stripped = stripHtml(raw) + const withoutChrome = raw + .replace(/<(script|style|nav|header|footer|aside|noscript|svg)\b[^>]*>[\s\S]*?<\/\1>/gi, ' ') + .replace(//g, ' '); + const main = withoutChrome.match(/]*>([\s\S]*?)<\/main>/i)?.[1]; + const article = withoutChrome.match(/]*>([\s\S]*?)<\/article>/i)?.[1]; + const scoped = main || article || withoutChrome; + const stripped = stripHtml(scoped) .replace(/```[\s\S]*?```/g, ' ') .replace(/`[^`]*`/g, ' ') .replace(/<(?:system|assistant|user|im_start|im_end)>[\s\S]*?<\/(?:system|assistant|user|im_start|im_end)>/gi, ' ') @@ -282,4 +288,4 @@ export async function collectSource(source: SourceRow): Promise { default: throw new Error(`Unsupported source type: ${source.type}`); } -} +} \ No newline at end of file diff --git a/src/daily.ts b/src/daily.ts index e1a223b..f52db41 100644 --- a/src/daily.ts +++ b/src/daily.ts @@ -1,7 +1,7 @@ import type { Env, ItemRow } from './types'; import { escapeHtml } from './utils'; import { getReportItems } from './db'; -import { pushDailyReport } from './telegram'; +import { buildChineseSummary, pushDailyReport } from './telegram'; export function beijingWindow(now = new Date()): { reportDate: string; start: string; end: string } { const shifted = new Date(now.getTime() + 8 * 60 * 60 * 1000); @@ -51,7 +51,8 @@ function itemCard(item: ItemRow & { source_name: string; last_verified_at: strin `发现：${formatBeijing(item.discovered_at)}`, `最后核验：${formatBeijing(item.last_verified_at)}`, ].filter(Boolean); - return `
${escapeHtml(kindLabel(item.kind))}${escapeHtml(status)}

${escapeHtml(item.title)}

${item.summary ? `

${escapeHtml(item.summary.slice(0, 900))}

` : ''}
    ${details.map((value) => `
  • ${escapeHtml(value)}
    • `).join('')}
${escapeHtml(item.source_name)}${item.url ? `查看原文` : ''}
`; + const summaryZh = buildChineseSummary(item, item.summary || undefined); + return `
${escapeHtml(kindLabel(item.kind))}${escapeHtml(status)}

${escapeHtml(item.title)}

${escapeHtml(summaryZh.slice(0, 900))}

    ${details.map((value) => `
  • ${escapeHtml(value)}
    • `).join('')}
${escapeHtml(item.source_name)}${item.url ? `查看原文` : ''}
`; } function section(title: string, items: (ItemRow & { source_name: string; last_verified_at: string | null })[], empty: string): string { diff --git a/src/db.ts b/src/db.ts index 0a6b4a6..826ea65 100644 --- a/src/db.ts +++ b/src/db.ts @@ -87,9 +87,10 @@ export async function noteSourceValue(db:D1Database,source:SourceRow,c:Classific export async function insertItem(db: D1Database, source: SourceRow, candidate: Candidate, c: Classification): Promise<{ inserted: boolean; id?: number }> { const fingerprint = await buildFingerprint(source,candidate); - const result = await db.prepare(`INSERT OR IGNORE INTO items(source_id,external_id,fingerprint,title,summary,url,published_at,kind,priority,score,source_confidence,verification_status,vendor,product,previous_price,current_price,currency,expires_at,raw_excerpt,ai_enriched) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14,?15,?16,?17,?18,?19,?20)`).bind(source.id,candidate.externalId||null,fingerprint,candidate.title,candidate.summary||null,candidate.url||null,candidate.publishedAt||null,c.kind,c.priority,c.score,c.sourceConfidence,c.verificationStatus,c.vendor||null,c.product||null,c.previousPrice??null,c.currentPrice??null,c.currency||null,c.expiresAt||null,candidate.rawExcerpt||null,c.aiEnriched?1:0).run(); + const displaySummary = c.summaryZh || candidate.summary || null; + const result = await db.prepare(`INSERT OR IGNORE INTO items(source_id,external_id,fingerprint,title,summary,url,published_at,kind,priority,score,source_confidence,verification_status,vendor,product,previous_price,current_price,currency,expires_at,raw_excerpt,ai_enriched) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14,?15,?16,?17,?18,?19,?20)`).bind(source.id,candidate.externalId||null,fingerprint,candidate.title,displaySummary,candidate.url||null,candidate.publishedAt||null,c.kind,c.priority,c.score,c.sourceConfidence,c.verificationStatus,c.vendor||null,c.product||null,c.previousPrice??null,c.currentPrice??null,c.currency||null,c.expiresAt||null,candidate.rawExcerpt||null,c.aiEnriched?1:0).run(); if (!result.meta.changes) return { inserted:false }; const row = await db.prepare('SELECT id FROM items WHERE fingerprint=?1').bind(fingerprint).first<{id:number}>(); return { inserted:true,id:row?.id }; } export async function markPushed(db: D1Database,id:number):Promise{await db.prepare('UPDATE items SET pushed_at=?1 WHERE id=?2').bind(isoNow(),id).run();} export async function getRecentItems(db:D1Database,limit=100):Promise<(ItemRow&{source_name:string;last_verified_at:string|null})[]>{const r=await db.prepare(`SELECT i.*,s.name AS source_name,s.last_success_at AS last_verified_at FROM items i JOIN sources s ON s.id=i.source_id ORDER BY i.discovered_at DESC LIMIT ?1`).bind(limit).all();return r.results||[];} -export async function getReportItems(db:D1Database,start:string,end:string):Promise<(ItemRow&{source_name:string;last_verified_at:string|null})[]>{const r=await db.prepare(`SELECT i.*,s.name AS source_name,s.last_success_at AS last_verified_at FROM items i JOIN sources s ON s.id=i.source_id WHERE i.discovered_at>=?1 AND i.discovered_at();return r.results||[];} +export async function getReportItems(db:D1Database,start:string,end:string):Promise<(ItemRow&{source_name:string;last_verified_at:string|null})[]>{const r=await db.prepare(`SELECT i.*,s.name AS source_name,s.last_success_at AS last_verified_at FROM items i JOIN sources s ON s.id=i.source_id WHERE i.discovered_at>=?1 AND i.discovered_at();return r.results||[];} \ No newline at end of file diff --git a/src/telegram.ts b/src/telegram.ts index f8a117c..ecb8a25 100644 --- a/src/telegram.ts +++ b/src/telegram.ts @@ -88,7 +88,7 @@ export function buildChineseSummary(item: ItemRow, aiSummary?: string): string { if (item.kind === 'discovered_model') { return `在信源中观测到模型「${name}」${createdDate ? `（创建/历史时间：${createdDate}）` : ''}，已作为信源发现记录收录，非近期新发布模型。`; } - return `检测到「${name}」重要更新，已进入 AI-Radar 高优先级队列，详情请查看原文。`; + return `检测到「${name}」页面或信源发生变化，当前尚未核实具体变化内容，建议查看原文。`; } export async function pushP1(env: Env, item: ItemRow & { source_name?: string }, summaryZh?: string): Promise { diff --git a/tests/chinese-summary-regression.test.ts b/tests/chinese-summary-regression.test.ts new file mode 100644 index 0000000..556800b --- /dev/null +++ b/tests/chinese-summary-regression.test.ts @@ -0,0 +1,72 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { collectSource } from '../src/collectors'; +import { renderReport } from '../src/daily'; +import type { ItemRow, SourceRow } from '../src/types'; + +const source: SourceRow = { + id: 99, + name: 'incoai/GLM-5.3-Flash-DFlash2 watch', + url: 'https://huggingface.co/incoai/GLM-5.3-Flash-DFlash2', + type: 'web', + trust_level: 'C', + enabled: 1, + interval_minutes: 360, + config_json: null, + etag: null, + last_modified: null, + content_hash: null, + next_fetch_at: null, + last_fetch_at: null, + last_success_at: null, + failure_count: 0, + status: 'ok', +}; + +const item: ItemRow & { source_name: string; last_verified_at: string | null } = { + id: 99, + source_id: 99, + title: 'incoai/GLM-5.3-Flash-DFlash2 watch changed', + summary: 'Hugging Face Models Datasets Spaces Buckets Docs Enterprise Pricing Website Tasks HuggingChat Collections', + url: source.url, + kind: 'other', + priority: 'P3', + score: 20, + source_confidence: 'low', + verification_status: 'unverified', + vendor: null, + product: null, + previous_price: null, + current_price: null, + currency: null, + expires_at: null, + discovered_at: '2026-09-04T02:01:02.000Z', + published_at: null, + pushed_at: null, + source_name: source.name, + last_verified_at: '2026-09-04T02:01:03.000Z', +}; + +test('daily report replaces raw English web text with a Chinese fallback summary', () => { + const html = renderReport('2026-09-04', [item]); + assert.match(html, /页面或信源发生变化/); + assert.doesNotMatch(html, /Hugging Face Models Datasets Spaces Buckets Docs/); +}); + +test('generic web collector prefers main content and strips global navigation chrome', async () => { + const originalFetch = globalThis.fetch; + try { + globalThis.fetch = async () => + new Response( + '

GLM-5.3-Flash-DFlash2

Model card updated with new inference details.

', + { status: 200 } + ); + const result = await collectSource(source); + const summary = result.candidates[0]?.summary || ''; + assert.match(summary, /Model card updated with new inference details/); + assert.doesNotMatch(summary, /Models Datasets Spaces Buckets Docs/); + assert.doesNotMatch(summary, /Terms Privacy/); + } finally { + globalThis.fetch = originalFetch; + } +});