From 6c6ebaeb1beef9dae9a0b5d7f80dc769a3bd2929 Mon Sep 17 00:00:00 2001 From: chitcommit <208086304+chitcommit@users.noreply.github.com> Date: Thu, 4 Jun 2026 13:15:56 +0000 Subject: [PATCH] chore: remove dead notion-sync-worker + monitor script These files are leftovers from an older notion-sync architecture: - src/workers/notion-sync-worker.js is not bound by any wrangler config (wrangler.jsonc, wrangler.hybrid.toml, wrangler-pages.toml all have zero Notion references) and is only referenced by itself. - scripts/monitor-notion-sync.js reads NOTION_SYNC_WORKER_URL which is not set anywhere in the repo or ecosystem and targets a worker that is not deployed. - No cross-repo references exist in CHITTYFOUNDATION/* or CHITTYOS/*. - Current Notion integration flows through src/services/notion-sync.js and src/api/notion-bridge.js, which are unaffected. Surfaced during chittyid PR #22 cleanup (orphan task 4b35b6e3-51a4-4c44-a8e4-ac43b9b17e68). Co-Authored-By: Claude Opus 4.7 --- scripts/monitor-notion-sync.js | 265 ------------- src/workers/notion-sync-worker.js | 602 ------------------------------ 2 files changed, 867 deletions(-) delete mode 100644 scripts/monitor-notion-sync.js delete mode 100644 src/workers/notion-sync-worker.js diff --git a/scripts/monitor-notion-sync.js b/scripts/monitor-notion-sync.js deleted file mode 100644 index e480705..0000000 --- a/scripts/monitor-notion-sync.js +++ /dev/null @@ -1,265 +0,0 @@ -#!/usr/bin/env node - -/** - * NotionSync Monitoring Script - * Monitors sync health and alerts on failures - */ - -const WORKER_URL = process.env.NOTION_SYNC_WORKER_URL; -if (!WORKER_URL) { - console.error('ERROR: NOTION_SYNC_WORKER_URL is not set.'); - console.error('Set it to the deployed notion-sync worker URL, e.g.:'); - console.error(' export NOTION_SYNC_WORKER_URL=https://notion-sync.chitty.cc'); - process.exit(1); -} -const ALERT_THRESHOLD = { - schema_mismatch: 0, - rate_limit_percentage: 2, - dlq_depth: 0, - sync_lag_seconds: 60 -}; - -class NotionSyncMonitor { - constructor() { - this.metrics = {}; - this.alerts = []; - } - - async checkSyncHealth() { - console.log('๐Ÿ” Checking NotionSync health...\n'); - - // 1. Verify configuration - await this.verifyConfig(); - - // 2. Check sync metrics - await this.checkMetrics(); - - // 3. Check DLQ depth - await this.checkDLQ(); - - // 4. Perform test sync - await this.testSync(); - - // 5. Report results - this.report(); - } - - async verifyConfig() { - try { - const response = await fetch(`${WORKER_URL}/sync/notion/verify`); - const result = await response.json(); - - if (result.valid) { - console.log('โœ… Notion configuration valid'); - console.log(` Database: ${result.database}`); - console.log(` Properties: ${result.properties.length} configured`); - } else { - console.error('โŒ Notion configuration invalid'); - console.error(` Error: ${result.error}`); - if (result.recommendation) { - console.log(` Fix: ${result.recommendation}`); - } - this.alerts.push({ - level: 'critical', - message: 'Notion configuration invalid', - details: result.error - }); - } - } catch (error) { - console.error('โŒ Failed to verify config:', error.message); - this.alerts.push({ - level: 'critical', - message: 'Cannot reach NotionSync worker', - details: error.message - }); - } - } - - async checkMetrics() { - try { - // Simulate fetching metrics from worker - const testSync = await fetch(`${WORKER_URL}/bridges/notion/facts:sync`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ limit: 1 }) - }); - - if (testSync.ok) { - const result = await testSync.json(); - this.metrics = result.metrics || {}; - - console.log('\n๐Ÿ“Š Sync Metrics:'); - console.log(` Successful syncs: ${this.metrics.notion_ok || 0}`); - console.log(` Rate limits (429): ${this.metrics.notion_429 || 0}`); - console.log(` Server errors (5xx): ${this.metrics.notion_5xx || 0}`); - console.log(` Schema mismatches: ${this.metrics.schema_mismatch || 0}`); - console.log(` Skipped upserts: ${this.metrics.upsert_skipped || 0}`); - console.log(` DLQ items: ${this.metrics.dlq_pushed || 0}`); - - // Check thresholds - if (this.metrics.schema_mismatch > ALERT_THRESHOLD.schema_mismatch) { - this.alerts.push({ - level: 'warning', - message: 'Schema mismatches detected', - details: `${this.metrics.schema_mismatch} mismatches found` - }); - } - - const totalRequests = this.metrics.notion_ok + this.metrics.notion_429 + this.metrics.notion_5xx; - if (totalRequests > 0) { - const rateLimitPercentage = (this.metrics.notion_429 / totalRequests) * 100; - if (rateLimitPercentage > ALERT_THRESHOLD.rate_limit_percentage) { - this.alerts.push({ - level: 'warning', - message: 'High rate limit percentage', - details: `${rateLimitPercentage.toFixed(2)}% of requests rate limited` - }); - } - } - } - } catch (error) { - console.error('โŒ Failed to check metrics:', error.message); - } - } - - async checkDLQ() { - try { - const response = await fetch(`${WORKER_URL}/sync/notion/dlq`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ limit: 0 }) // Just check count - }); - - if (response.ok) { - const result = await response.json(); - const dlqDepth = result.processed || 0; - - console.log(`\n๐Ÿ“ฆ DLQ Depth: ${dlqDepth}`); - - if (dlqDepth > ALERT_THRESHOLD.dlq_depth) { - this.alerts.push({ - level: 'warning', - message: 'DLQ has pending items', - details: `${dlqDepth} items in dead letter queue` - }); - - // Try to reprocess - console.log(' Attempting to reprocess DLQ items...'); - const reprocess = await fetch(`${WORKER_URL}/sync/notion/dlq`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ limit: 10 }) - }); - - if (reprocess.ok) { - const reprocessResult = await reprocess.json(); - console.log(` Reprocessed ${reprocessResult.processed} items`); - } - } - } - } catch (error) { - console.error('โŒ Failed to check DLQ:', error.message); - } - } - - async testSync() { - console.log('\n๐Ÿงช Running test sync...'); - - try { - const testFact = { - factId: `TEST-MONITOR-${Date.now()}`, - parentArtifactId: 'MONITOR-TEST', - factText: 'Monitoring test fact', - factType: 'STATUS', - classification: 'FACT', - weight: 1.0, - chainStatus: 'Pending' - }; - - // This would normally go through the full pipeline - console.log(' Creating test fact:', testFact.factId); - - // Simulate sync - const syncResponse = await fetch(`${WORKER_URL}/bridges/notion/facts:sync`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - facts: [testFact], - limit: 1 - }) - }); - - if (syncResponse.ok) { - const result = await syncResponse.json(); - if (result.summary.created > 0 || result.summary.updated > 0) { - console.log(' โœ… Test sync successful'); - } else if (result.summary.failed > 0) { - console.log(' โŒ Test sync failed'); - this.alerts.push({ - level: 'critical', - message: 'Test sync failed', - details: result.errors[0]?.error - }); - } - } - } catch (error) { - console.error(' โŒ Test sync error:', error.message); - this.alerts.push({ - level: 'critical', - message: 'Test sync failed', - details: error.message - }); - } - } - - report() { - console.log('\n' + '='.repeat(50)); - console.log('๐Ÿ“‹ NOTIONSYNC HEALTH REPORT'); - console.log('='.repeat(50)); - - if (this.alerts.length === 0) { - console.log('โœ… All systems operational'); - } else { - console.log(`โš ๏ธ ${this.alerts.length} alerts found:\n`); - this.alerts.forEach(alert => { - const icon = alert.level === 'critical' ? '๐Ÿ”ด' : '๐ŸŸก'; - console.log(`${icon} [${alert.level.toUpperCase()}] ${alert.message}`); - if (alert.details) { - console.log(` Details: ${alert.details}`); - } - }); - } - - console.log('\n๐Ÿ“ˆ Acceptance Criteria:'); - console.log(` โœ… 100% sync within 60s: ${this.checkSyncLag() ? 'PASS' : 'FAIL'}`); - console.log(` โœ… Zero schema mismatches: ${this.metrics.schema_mismatch === 0 ? 'PASS' : 'FAIL'}`); - console.log(` โœ… DLQ depth = 0: ${this.metrics.dlq_pushed === 0 ? 'PASS' : 'FAIL'}`); - console.log(` โœ… Idempotent replays: ${this.metrics.upsert_skipped >= 0 ? 'PASS' : 'UNKNOWN'}`); - - console.log('\n' + '='.repeat(50)); - } - - checkSyncLag() { - // In production, would check actual lag - return true; - } - - async runContinuous(intervalMs = 30000) { - console.log(`Starting continuous monitoring (interval: ${intervalMs / 1000}s)\n`); - - while (true) { - await this.checkSyncHealth(); - await new Promise(resolve => setTimeout(resolve, intervalMs)); - console.clear(); - } - } -} - -// Run monitoring -const monitor = new NotionSyncMonitor(); - -if (process.argv.includes('--continuous')) { - monitor.runContinuous(); -} else { - monitor.checkSyncHealth(); -} \ No newline at end of file diff --git a/src/workers/notion-sync-worker.js b/src/workers/notion-sync-worker.js deleted file mode 100644 index b593ee1..0000000 --- a/src/workers/notion-sync-worker.js +++ /dev/null @@ -1,602 +0,0 @@ -/** - * NotionSync Worker - Hardened synchronization for AtomicFacts - * Source: ChittyRouter โ†’ EvidenceEnvelope v1 โ†’ AtomicFacts โ†’ Notion - */ - -export class NotionSyncWorker { - constructor(env) { - this.env = env; - this.notionToken = env.NOTION_TOKEN; - this.databaseId = env.NOTION_DATABASE_ID_ATOMIC_FACTS; - this.notionVersion = '2022-06-28'; - this.baseUrl = 'https://api.notion.com/v1'; - - // Metrics counters - this.metrics = { - notion_ok: 0, - notion_429: 0, - notion_5xx: 0, - schema_mismatch: 0, - upsert_skipped: 0, - dlq_pushed: 0 - }; - } - - /** - * Field mapping: AtomicFact โ†’ Notion properties - */ - getFieldMapping() { - return { - factId: 'Fact ID', // title, required - parentArtifactId: 'Parent Document', // rich text - factText: 'Fact Text', // rich text - factType: 'Fact Type', // select - locationRef: 'Location in Document', // text - classification: 'Classification Level', // select - weight: 'Weight', // number - credibility: 'Credibility Factors', // multi-select - chainStatus: 'ChittyChain Status', // select - verifiedAt: 'Verification Date', // date - verificationMethod: 'Verification Method' // text - }; - } - - /** - * Valid select options for Notion properties - */ - getSelectOptions() { - return { - 'Fact Type': [ - 'DATE', 'AMOUNT', 'ADMISSION', 'IDENTITY', - 'LOCATION', 'RELATIONSHIP', 'ACTION', 'STATUS' - ], - 'Classification Level': [ - 'FACT', 'SUPPORTED_CLAIM', 'ASSERTION', - 'ALLEGATION', 'CONTRADICTION' - ], - 'ChittyChain Status': [ - 'Minted', 'Pending', 'Rejected' - ], - 'Credibility Factors': [ - 'Direct Evidence', 'Documentary', 'Witness Statement', - 'Expert Opinion', 'Circumstantial', 'Hearsay', - 'Blockchain Verified', 'AI Analyzed' - ] - }; - } - - /** - * Transform AtomicFact to Notion page properties - */ - transformToNotionPayload(atomicFact) { - const mapping = this.getFieldMapping(); - - const properties = { - [mapping.factId]: { - title: [{ - text: { content: atomicFact.factId || '' } - }] - }, - [mapping.parentArtifactId]: { - rich_text: [{ - text: { content: atomicFact.parentArtifactId || '' } - }] - }, - [mapping.factText]: { - rich_text: [{ - text: { - content: this.truncateText(atomicFact.factText || '', 2000) - } - }] - }, - [mapping.factType]: { - select: { - name: this.normalizeSelectValue(atomicFact.factType, 'Fact Type') - } - }, - [mapping.locationRef]: { - rich_text: [{ - text: { content: atomicFact.locationRef || '' } - }] - }, - [mapping.classification]: { - select: { - name: this.normalizeSelectValue(atomicFact.classification, 'Classification Level') - } - }, - [mapping.weight]: { - number: this.normalizeNumber(atomicFact.weight, 0, 1) - }, - [mapping.credibility]: { - multi_select: (atomicFact.credibility || []).map(factor => ({ - name: this.normalizeSelectValue(factor, 'Credibility Factors') - })) - }, - [mapping.chainStatus]: { - select: { - name: this.normalizeSelectValue(atomicFact.chainStatus || 'Pending', 'ChittyChain Status') - } - } - }; - - // Add optional date field - if (atomicFact.verifiedAt) { - properties[mapping.verifiedAt] = { - date: { - start: this.normalizeDate(atomicFact.verifiedAt) - } - }; - } - - // Add verification method if present - if (atomicFact.verificationMethod) { - properties[mapping.verificationMethod] = { - rich_text: [{ - text: { content: atomicFact.verificationMethod } - }] - }; - } - - return { properties }; - } - - /** - * Normalize select values to match Notion options - */ - normalizeSelectValue(value, propertyName) { - if (!value) return null; - - const options = this.getSelectOptions()[propertyName]; - if (!options) { - this.metrics.schema_mismatch++; - return value; - } - - // Case-insensitive match - const normalized = options.find(opt => - opt.toUpperCase() === value.toUpperCase() - ); - - if (!normalized) { - this.metrics.schema_mismatch++; - console.warn(`Unknown select value: ${value} for ${propertyName}`); - return value; // Return as-is, Notion might create it - } - - return normalized; - } - - /** - * Normalize number within bounds - */ - normalizeNumber(value, min, max) { - if (value === undefined || value === null) return null; - const num = parseFloat(value); - if (isNaN(num)) return null; - return Math.max(min, Math.min(max, num)); - } - - /** - * Normalize date to ISO string - */ - normalizeDate(date) { - if (!date) return null; - try { - return new Date(date).toISOString().split('T')[0]; - } catch { - return null; - } - } - - /** - * Truncate text to Notion limits - */ - truncateText(text, maxLength) { - if (!text || text.length <= maxLength) return text; - return text.substring(0, maxLength - 3) + '...'; - } - - /** - * Lookup existing page by factId - */ - async lookupByFactId(factId) { - const mapping = this.getFieldMapping(); - - try { - const response = await fetch(`${this.baseUrl}/databases/${this.databaseId}/query`, { - method: 'POST', - headers: this.getHeaders(), - body: JSON.stringify({ - filter: { - property: mapping.factId, - title: { - equals: factId - } - }, - page_size: 1 - }) - }); - - if (!response.ok) { - throw new Error(`Lookup failed: ${response.status}`); - } - - const data = await response.json(); - return data.results[0] || null; - } catch (error) { - console.error(`Lookup error for ${factId}:`, error); - return null; - } - } - - /** - * Upsert fact to Notion with idempotency - */ - async upsertFact(atomicFact) { - const factId = atomicFact.factId; - if (!factId) { - console.error('Missing factId, skipping'); - this.metrics.upsert_skipped++; - return { status: 'skipped', reason: 'missing_factId' }; - } - - try { - // Check if exists - const existingPage = await this.lookupByFactId(factId); - - if (existingPage) { - // Update existing page - return await this.updatePage(existingPage.id, atomicFact); - } else { - // Create new page - return await this.createPage(atomicFact); - } - } catch (error) { - console.error(`Upsert failed for ${factId}:`, error); - await this.pushToDLQ(atomicFact, error.message); - return { status: 'failed', error: error.message }; - } - } - - /** - * Create new Notion page - */ - async createPage(atomicFact) { - const payload = { - parent: { database_id: this.databaseId }, - ...this.transformToNotionPayload(atomicFact) - }; - - const response = await this.fetchWithRetry(`${this.baseUrl}/pages`, { - method: 'POST', - headers: this.getHeaders(atomicFact.factId), - body: JSON.stringify(payload) - }); - - if (response.ok) { - this.metrics.notion_ok++; - return { status: 'created', pageId: (await response.json()).id }; - } - - throw new Error(`Create failed: ${response.status} ${await response.text()}`); - } - - /** - * Update existing Notion page - */ - async updatePage(pageId, atomicFact) { - const payload = this.transformToNotionPayload(atomicFact); - - const response = await this.fetchWithRetry(`${this.baseUrl}/pages/${pageId}`, { - method: 'PATCH', - headers: this.getHeaders(atomicFact.factId), - body: JSON.stringify(payload) - }); - - if (response.ok) { - this.metrics.notion_ok++; - return { status: 'updated', pageId }; - } - - throw new Error(`Update failed: ${response.status} ${await response.text()}`); - } - - /** - * Fetch with exponential backoff and retry - */ - async fetchWithRetry(url, options, retryCount = 0, maxRetries = 5) { - try { - const response = await fetch(url, options); - - if (response.status === 429) { - this.metrics.notion_429++; - if (retryCount < maxRetries) { - const delay = this.getBackoffDelay(retryCount); - console.log(`Rate limited, retrying in ${delay}ms`); - await this.sleep(delay); - return this.fetchWithRetry(url, options, retryCount + 1, maxRetries); - } - } - - if (response.status >= 500) { - this.metrics.notion_5xx++; - if (retryCount < maxRetries) { - const delay = this.getBackoffDelay(retryCount); - console.log(`Server error, retrying in ${delay}ms`); - await this.sleep(delay); - return this.fetchWithRetry(url, options, retryCount + 1, maxRetries); - } - } - - return response; - } catch (error) { - if (retryCount < maxRetries) { - const delay = this.getBackoffDelay(retryCount); - await this.sleep(delay); - return this.fetchWithRetry(url, options, retryCount + 1, maxRetries); - } - throw error; - } - } - - /** - * Calculate exponential backoff with jitter - */ - getBackoffDelay(retryCount) { - const baseDelay = Math.pow(2, retryCount) * 1000; - const jitter = Math.random() * 1000; - return Math.min(baseDelay + jitter, 30000); // Max 30 seconds - } - - /** - * Sleep helper - */ - sleep(ms) { - return new Promise(resolve => setTimeout(resolve, ms)); - } - - /** - * Get Notion API headers - */ - getHeaders(idempotencyKey = null) { - const headers = { - 'Authorization': `Bearer ${this.notionToken}`, - 'Content-Type': 'application/json', - 'Notion-Version': this.notionVersion - }; - - if (idempotencyKey) { - headers['X-Idempotency-Key'] = idempotencyKey; - } - - return headers; - } - - /** - * Push failed fact to DLQ - */ - async pushToDLQ(atomicFact, errorReason) { - const dlqItem = { - fact: atomicFact, - error: errorReason, - timestamp: new Date().toISOString(), - retry_at: new Date(Date.now() + 3600000).toISOString() // Retry in 1 hour - }; - - await this.env.DLQ.put(`dlq:fact:${atomicFact.factId}`, JSON.stringify(dlqItem)); - this.metrics.dlq_pushed++; - } - - /** - * Process batch of facts with rate limiting - */ - async processBatch(facts, batchSize = 10, delayMs = 200) { - const results = []; - - for (let i = 0; i < facts.length; i += batchSize) { - const batch = facts.slice(i, i + batchSize); - - const batchResults = await Promise.all( - batch.map(fact => this.upsertFact(fact)) - ); - - results.push(...batchResults); - - // Rate limit between batches - if (i + batchSize < facts.length) { - await this.sleep(delayMs); - } - } - - return results; - } - - /** - * Main sync endpoint - */ - async sync(request) { - const { since, limit = 100 } = await request.json(); - - try { - // Fetch AtomicFacts from upstream - const facts = await this.fetchAtomicFacts(since, limit); - - // Process in batches - const results = await this.processBatch(facts); - - // Count results - const summary = { - created: results.filter(r => r.status === 'created').length, - updated: results.filter(r => r.status === 'updated').length, - skipped: results.filter(r => r.status === 'skipped').length, - failed: results.filter(r => r.status === 'failed').length, - errors: results.filter(r => r.status === 'failed').map(r => ({ - factId: r.factId, - error: r.error - })) - }; - - // Log metrics - await this.logMetrics(); - - return new Response(JSON.stringify({ - success: true, - summary, - metrics: this.metrics - }), { - headers: { 'Content-Type': 'application/json' } - }); - - } catch (error) { - console.error('Sync error:', error); - return new Response(JSON.stringify({ - success: false, - error: error.message, - metrics: this.metrics - }), { - status: 500, - headers: { 'Content-Type': 'application/json' } - }); - } - } - - /** - * Fetch AtomicFacts from upstream - */ - async fetchAtomicFacts(since, limit) { - // TODO: Fetch from ChittyLedger/ChittyAssets when available - console.warn('fetchAtomicFacts not yet implemented โ€” requires ChittyLedger integration'); - return []; - } - - /** - * Log metrics to analytics - */ - async logMetrics() { - if (this.env.CHITTY_ANALYTICS) { - await this.env.CHITTY_ANALYTICS.writeDataPoint({ - blobs: ['notion_sync'], - doubles: [ - this.metrics.notion_ok, - this.metrics.notion_429, - this.metrics.notion_5xx, - this.metrics.schema_mismatch, - this.metrics.upsert_skipped, - this.metrics.dlq_pushed - ], - indexes: ['chittyid'] - }); - } - } - - /** - * Reprocess DLQ items - */ - async reprocessDLQ(request) { - const { limit = 10 } = await request.json(); - - // List DLQ items - const list = await this.env.DLQ.list({ prefix: 'dlq:fact:', limit }); - const results = []; - - for (const item of list.keys) { - const dlqData = JSON.parse(await this.env.DLQ.get(item.name)); - - // Check if ready for retry - if (new Date(dlqData.retry_at) > new Date()) { - continue; - } - - // Retry the fact - const result = await this.upsertFact(dlqData.fact); - - if (result.status !== 'failed') { - // Remove from DLQ on success - await this.env.DLQ.delete(item.name); - } - - results.push(result); - } - - return new Response(JSON.stringify({ - success: true, - processed: results.length, - results - }), { - headers: { 'Content-Type': 'application/json' } - }); - } - - /** - * Verify Notion configuration - */ - async verifyConfig() { - try { - // Test database access - const response = await fetch(`${this.baseUrl}/databases/${this.databaseId}`, { - headers: this.getHeaders() - }); - - if (!response.ok) { - return { - valid: false, - error: `Database access failed: ${response.status}` - }; - } - - const database = await response.json(); - - // Verify required properties exist - const mapping = this.getFieldMapping(); - const requiredProps = Object.values(mapping); - const dbProps = Object.keys(database.properties); - - const missing = requiredProps.filter(prop => !dbProps.includes(prop)); - - if (missing.length > 0) { - return { - valid: false, - error: `Missing properties: ${missing.join(', ')}`, - recommendation: 'Create these properties in Notion database' - }; - } - - return { - valid: true, - database: database.title[0]?.plain_text || 'Unnamed', - properties: dbProps - }; - - } catch (error) { - return { - valid: false, - error: error.message - }; - } - } -} - -// Export for Cloudflare Workers -export default { - async fetch(request, env) { - const worker = new NotionSyncWorker(env); - const url = new URL(request.url); - - if (url.pathname === '/bridges/notion/facts:sync' && request.method === 'POST') { - return worker.sync(request); - } - - if (url.pathname === '/sync/notion/dlq' && request.method === 'POST') { - return worker.reprocessDLQ(request); - } - - if (url.pathname === '/sync/notion/verify' && request.method === 'GET') { - const result = await worker.verifyConfig(); - return new Response(JSON.stringify(result), { - headers: { 'Content-Type': 'application/json' } - }); - } - - return new Response('NotionSync Worker Ready', { status: 200 }); - } -}; \ No newline at end of file