diff --git a/.github/workflows/index-meilisearch.yml b/.github/workflows/index-meilisearch.yml new file mode 100644 index 00000000000..a617d6044cd --- /dev/null +++ b/.github/workflows/index-meilisearch.yml @@ -0,0 +1,35 @@ +name: index-meilisearch + +on: + push: + branches: + - master + workflow_dispatch: + +jobs: + index-meilisearch: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "22" + - uses: actions/cache@v4 + with: + path: | + ${{ github.workspace }}/**/.cache + ${{ github.workspace }}/website/node_modules + key: | + ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + + - name: Install dependencies + run: cd website && yarn + + - name: Index to MeiliSearch + run: cd website && yarn run meilisearch:build-index + env: + MEILI_HOST: ${{ secrets.MEILI_HOST }} + MEILI_MASTER_KEY: ${{ secrets.MEILI_MASTER_KEY }} + MEILI_INDEX_NAME: ${{ secrets.MEILI_INDEX_NAME }} diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000000..6ab682576c9 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,6 @@ +{ + "name": "docs", + "lockfileVersion": 3, + "requires": true, + "packages": {} +} diff --git a/website/.env.example b/website/.env.example index 9d4ff263177..09669a1e4a2 100644 --- a/website/.env.example +++ b/website/.env.example @@ -9,4 +9,10 @@ REACT_APP_NEAR_TOWN_HALL_CALENDAR_ID= # Analytics REACT_APP_PUBLIC_POSTHOG_KEY= -REACT_APP_PUBLIC_POSTHOG_HOST= \ No newline at end of file +REACT_APP_PUBLIC_POSTHOG_HOST= + +# MeiliSearch +MEILI_HOST=http://localhost:7700 +MEILI_MASTER_KEY=masterKey123 +MEILI_SEARCH_KEY= +MEILI_INDEX_NAME=near-docs \ No newline at end of file diff --git a/website/docker-compose.yml b/website/docker-compose.yml new file mode 100644 index 00000000000..2d45e7376e9 --- /dev/null +++ b/website/docker-compose.yml @@ -0,0 +1,16 @@ +version: '3.8' +services: + meilisearch: + image: getmeili/meilisearch:v1.34 + container_name: near-docs-search + ports: + - "7700:7700" + environment: + - MEILI_MASTER_KEY=${MEILI_MASTER_KEY:-masterKey123} + - MEILI_ENV=development + volumes: + - meilisearch_data:/meili_data + +volumes: + meilisearch_data: + diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 455634a878a..3115c9b1476 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -30,6 +30,11 @@ const config = { REACT_APP_GOOGLE_CALENDAR_API_KEY: process.env.REACT_APP_GOOGLE_CALENDAR_API_KEY, REACT_APP_LUMA_NEAR_CALENDAR_ID: process.env.REACT_APP_LUMA_NEAR_CALENDAR_ID, REACT_APP_DEVHUB_GOOGLE_CALENDAR_ID: process.env.REACT_APP_DEVHUB_GOOGLE_CALENDAR_ID, + meilisearch: { + host: process.env.MEILI_HOST || 'http://localhost:7700', + apiKey: process.env.MEILI_SEARCH_KEY || process.env.MEILI_MASTER_KEY || 'masterKey123', + indexName: process.env.MEILI_INDEX_NAME || 'near-docs', + }, }, themes: ['@saucelabs/theme-github-codeblock', '@docusaurus/theme-mermaid'], onBrokenLinks: 'throw', @@ -278,29 +283,6 @@ const config = { src: 'img/near_logo.svg', }, }, - algolia: { - // The application ID provided by Algolia - appId: '0LUM67N2P2', - // Public API key: it is safe to commit it - apiKey: '41e2feb6ffa0d3450ca9d0a0c1826c1c', - indexName: 'docs', - askAi: { - assistantId: 'ck1jQv3AzZ5R', - indexName: 'docs.md', - apiKey: '41e2feb6ffa0d3450ca9d0a0c1826c1c', - appId: '0LUM67N2P2', - }, - // Optional: see doc section below - contextualSearch: true, - // Optional: Algolia search parameters - searchParameters: { - clickAnalytics: true, - analytics: true, - }, - //... other Algolia params - placeholder: 'Search the Docs...', - insights: true, - }, }, }; diff --git a/website/package.json b/website/package.json index 0365e439f95..c357a03dbf6 100644 --- a/website/package.json +++ b/website/package.json @@ -16,6 +16,7 @@ "build:dev": "./node_modules/.bin/docusaurus build --dev && yarn run process-markdown", "build:preview": "yarn run process-markdown && ./node_modules/.bin/docusaurus build --locale en && yarn run process-markdown", "process-markdown": "node ./scripts/copy-md-to-static.js", + "meilisearch:build-index": "node scripts/index-meilisearch.mjs", "swizzle": "docusaurus swizzle", "docusaurus": "docusaurus" }, @@ -49,6 +50,7 @@ "gleap": "^13.7.3", "lodash": "^4.17.21", "lucide-react": "^0.555.0", + "meilisearch": "^0.55.0", "monaco-editor": "^0.55.1", "near-api-js": "^7.0.0", "near-connect-hooks": "^1.0.2", diff --git a/website/scripts/index-meilisearch.mjs b/website/scripts/index-meilisearch.mjs new file mode 100644 index 00000000000..70dfe493869 --- /dev/null +++ b/website/scripts/index-meilisearch.mjs @@ -0,0 +1,334 @@ +import 'dotenv/config'; +import { MeiliSearch } from 'meilisearch'; +import fs from 'fs'; +import path from 'path'; +import { fileURLToPath } from 'url'; +import { createHash } from 'crypto'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +const MEILI_HOST = process.env.MEILI_HOST || 'http://localhost:7700'; +const MEILI_MASTER_KEY = process.env.MEILI_MASTER_KEY || 'masterKey123'; +const MEILI_INDEX_NAME = process.env.MEILI_INDEX_NAME || 'near-docs'; +const DOCS_PATH = path.resolve(__dirname, '../../docs'); +const BATCH_SIZE = 100; +const TASK_TIMEOUT = 300000; // 5 minutes timeout for tasks with embedders + +const CATEGORY_MAP = { + 'protocol': 'Protocol', + 'chain-abstraction': 'Multi-Chain', + 'ai': 'AI & Agents', + 'smart-contracts': 'Smart Contracts', + 'web3-apps': 'Web3 Apps', + 'primitives': 'Primitives', + 'data-infrastructure': 'Data Infrastructure', + 'tools': 'Tools', + 'api': 'API', +}; + +function generateId(content) { + return createHash('md5').update(content).digest('hex').substring(0, 12); +} + +function extractFrontmatter(content) { + const frontmatterRegex = /^---\n([\s\S]*?)\n---/; + const match = content.match(frontmatterRegex); + + if (!match) return { frontmatter: {}, body: content }; + + const frontmatterStr = match[1]; + const body = content.slice(match[0].length).trim(); + + const frontmatter = {}; + const lines = frontmatterStr.split('\n'); + + for (const line of lines) { + const colonIndex = line.indexOf(':'); + if (colonIndex > 0) { + const key = line.slice(0, colonIndex).trim(); + let value = line.slice(colonIndex + 1).trim(); + // Remove quotes if present + if ((value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'"))) { + value = value.slice(1, -1); + } + frontmatter[key] = value; + } + } + + return { frontmatter, body }; +} + +function extractHeadings(content) { + const headings = []; + const headingRegex = /^#{1,6}\s+(.+)$/gm; + let match; + + while ((match = headingRegex.exec(content)) !== null) { + headings.push(match[1].replace(/[*_`]/g, '').trim()); + } + + return headings; +} + +function cleanContent(content) { + return content + // Remove code blocks + .replace(/```[\s\S]*?```/g, '') + // Remove inline code + .replace(/`[^`]+`/g, '') + // Remove links but keep text + .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1') + // Remove images + .replace(/!\[[^\]]*\]\([^)]+\)/g, '') + // Remove HTML tags + .replace(/<[^>]+>/g, '') + // Remove import statements + .replace(/^import\s+.*$/gm, '') + // Remove markdown emphasis + .replace(/[*_]{1,2}([^*_]+)[*_]{1,2}/g, '$1') + // Normalize whitespace + .replace(/\n{3,}/g, '\n\n') + .trim(); +} + +function getUrlPath(filePath, frontmatter = {}) { + // If slug is explicitly set in frontmatter, use it directly + if (frontmatter.slug) { + const slug = frontmatter.slug.startsWith('/') ? frontmatter.slug : '/' + frontmatter.slug; + return slug; + } + + const relativePath = path.relative(DOCS_PATH, filePath); + const pathParts = relativePath.replace(/\\/g, '/').split('/'); + const fileName = pathParts.pop().replace(/\.mdx?$/, ''); + + // Get document ID: from frontmatter.id, or filename (without numeric prefix) + const docId = frontmatter.id || fileName.replace(/^\d+-/, ''); + + // Remove numeric prefixes from path parts + const cleanPathParts = pathParts.map(part => part.replace(/^\d+-/, '')); + + // Build the URL path + let urlPath; + + if (docId === 'index') { + // index files: /path/path/index -> /path/path + urlPath = cleanPathParts.join('/'); + } else { + // Check if docId matches the parent folder name + const parentFolder = cleanPathParts[cleanPathParts.length - 1]; + if (docId === parentFolder) { + // /primitives/nft/nft -> /primitives/nft + urlPath = cleanPathParts.join('/'); + } else { + // Normal case: /path/to/file/ + urlPath = [...cleanPathParts, docId].join('/'); + } + } + + return '/' + urlPath; +} + +function getCategoryFromPath(filePath) { + const relativePath = path.relative(DOCS_PATH, filePath); + const firstFolder = relativePath.split(path.sep)[0]; + return CATEGORY_MAP[firstFolder] || 'General'; +} + +function getHierarchy(filePath) { + const relativePath = path.relative(DOCS_PATH, filePath); + const parts = relativePath.split(path.sep); + + // Remove file name + parts.pop(); + + const hierarchy = { + lvl0: getCategoryFromPath(filePath), + lvl1: '', + lvl2: '', + }; + + if (parts.length > 0) { + hierarchy.lvl1 = parts[0] + .replace(/-/g, ' ') + .replace(/^\d+\s*/, '') + .replace(/\b\w/g, c => c.toUpperCase()); + } + + if (parts.length > 1) { + hierarchy.lvl2 = parts.slice(1).join(' > ') + .replace(/-/g, ' ') + .replace(/^\d+\s*/g, '') + .replace(/\b\w/g, c => c.toUpperCase()); + } + + return hierarchy; +} + +function getAllMarkdownFiles(dir) { + const files = []; + + function walk(currentDir) { + const entries = fs.readdirSync(currentDir, { withFileTypes: true }); + + for (const entry of entries) { + const fullPath = path.join(currentDir, entry.name); + + if (entry.isDirectory()) { + // Skip hidden directories + if (!entry.name.startsWith('.')) { + walk(fullPath); + } + } else if (entry.isFile() && /\.mdx?$/.test(entry.name)) { + files.push(fullPath); + } + } + } + + walk(dir); + return files; +} + +async function indexDocuments() { + console.log('Starting MeiliSearch indexation...'); + console.log(`Host: ${MEILI_HOST}`); + console.log(`Index: ${MEILI_INDEX_NAME}`); + + // Initialize client + const client = new MeiliSearch({ + host: MEILI_HOST, + apiKey: MEILI_MASTER_KEY, + }); + + // Check connection + try { + const health = await client.health(); + console.log('MeiliSearch status:', health.status); + } catch (error) { + console.error('Failed to connect to MeiliSearch:', error.message); + console.error('Make sure MeiliSearch is running at', MEILI_HOST); + process.exit(1); + } + + // Get or create index + let index; + try { + index = await client.getIndex(MEILI_INDEX_NAME); + console.log('Using existing index:', MEILI_INDEX_NAME); + } catch { + console.log('Creating new index:', MEILI_INDEX_NAME); + const task = await client.createIndex(MEILI_INDEX_NAME, { primaryKey: 'id' }); + // Wait for index creation to complete + await client.tasks.waitForTask(task.taskUid, { timeout: TASK_TIMEOUT }); + index = client.index(MEILI_INDEX_NAME); + } + + // Configure index settings + console.log('Configuring index settings...'); + await index.updateSettings({ + searchableAttributes: ['title', 'content', 'section', 'hierarchy_lvl0', 'hierarchy_lvl1', 'hierarchy_lvl2'], + filterableAttributes: ['category', 'version', 'hierarchy_lvl0'], + sortableAttributes: ['timestamp'], + rankingRules: ['words', 'typo', 'proximity', 'attribute', 'sort', 'exactness'], + distinctAttribute: 'path', + embedders: { + default: { + source: 'huggingFace', + model: 'sentence-transformers/all-MiniLM-L6-v2', + documentTemplate: '{{doc.title}} {{doc.content}}', + }, + }, + }); + + // Get all markdown files + const files = getAllMarkdownFiles(DOCS_PATH); + console.log(`Found ${files.length} markdown files`); + + // Process files into documents + const documents = []; + + for (const filePath of files) { + try { + const content = fs.readFileSync(filePath, 'utf-8'); + const { frontmatter, body } = extractFrontmatter(content); + const headings = extractHeadings(body); + const cleanedContent = cleanContent(body); + + const urlPath = getUrlPath(filePath); + const title = frontmatter.title || + frontmatter.sidebar_label || + headings[0] || + path.basename(filePath, path.extname(filePath)).replace(/-/g, ' '); + + const hierarchy = getHierarchy(filePath); + + const doc = { + id: generateId(urlPath), + title, + content: cleanedContent, // Limit content size + path: urlPath, + section: frontmatter.sidebar_label || title, + category: hierarchy.lvl0, + version: 'current', + hierarchy_lvl0: hierarchy.lvl0, + hierarchy_lvl1: hierarchy.lvl1, + hierarchy_lvl2: hierarchy.lvl2, + timestamp: Date.now(), + }; + + documents.push(doc); + } catch (error) { + console.warn(`Warning: Failed to process ${filePath}:`, error.message); + } + } + + console.log(`Prepared ${documents.length} documents for indexing`); + + // Delete all existing documents + console.log('Clearing existing documents...'); + const deleteTask = await index.deleteAllDocuments(); + await client.tasks.waitForTask(deleteTask.taskUid, { timeout: TASK_TIMEOUT }); + + // Upload documents in batches + console.log('Uploading documents...'); + const uploadTasks = []; + for (let i = 0; i < documents.length; i += BATCH_SIZE) { + const batch = documents.slice(i, i + BATCH_SIZE); + const task = await index.addDocuments(batch); + uploadTasks.push(task.taskUid); + console.log(`Uploaded batch ${Math.floor(i / BATCH_SIZE) + 1}/${Math.ceil(documents.length / BATCH_SIZE)} (Task: ${task.taskUid})`); + } + + // Wait for all indexing tasks to complete (takes longer with embedders) + console.log('Waiting for indexing to complete (this may take a few minutes with embedders)...'); + for (const taskUid of uploadTasks) { + await client.tasks.waitForTask(taskUid, { timeout: TASK_TIMEOUT }); + } + + // Get final stats + const stats = await index.getStats(); + console.log('\nIndexing complete!'); + console.log(`Total documents indexed: ${stats.numberOfDocuments}`); + console.log(`Index is indexing: ${stats.isIndexing}`); + + // Generate search API key if master key is provided + if (MEILI_MASTER_KEY && MEILI_MASTER_KEY !== 'masterKey123') { + try { + const keys = await client.getKeys(); + const searchKey = keys.results.find(k => k.actions.includes('search')); + if (searchKey) { + console.log(`\nSearch API Key: ${searchKey.key}`); + console.log('Add this to your .env file as MEILI_SEARCH_KEY'); + } + } catch (error) { + console.log('Note: Could not retrieve search key. Use the dashboard to get it.'); + } + } +} + +// Run indexation +indexDocuments().catch(error => { + console.error('Indexation failed:', error); + process.exit(1); +}); diff --git a/website/src/theme/Icon/Search/index.tsx b/website/src/theme/Icon/Search/index.tsx new file mode 100644 index 00000000000..58097c91c11 --- /dev/null +++ b/website/src/theme/Icon/Search/index.tsx @@ -0,0 +1,35 @@ +import React from 'react'; + +interface IconProps { + width?: number; + height?: number; + className?: string; +} + +export function SearchIcon({ width = 20, height = 20, className }: IconProps): JSX.Element { + return ( + + + + + ); +} diff --git a/website/src/theme/SearchBar/index.tsx b/website/src/theme/SearchBar/index.tsx new file mode 100644 index 00000000000..5d1fde107c4 --- /dev/null +++ b/website/src/theme/SearchBar/index.tsx @@ -0,0 +1,287 @@ +import React, { useState, useEffect, useCallback, useRef } from 'react'; +import { useHistory } from '@docusaurus/router'; +import useDocusaurusContext from '@docusaurus/useDocusaurusContext'; +import { MeiliSearch } from 'meilisearch'; +import { trackSearch, trackSearchResultClick, trackSearchNoResults } from '../../utils/searchAnalytics'; +import { SearchIcon } from '../Icon/Search'; +import styles from './styles.module.css'; + +interface SearchHit { + id: string; + title: string; + content: string; + path: string; + section: string; + category: string; + hierarchy_lvl0: string; + hierarchy_lvl1: string; + hierarchy_lvl2: string; + _formatted?: { + title?: string; + content?: string; + }; +} + +interface SearchResult { + hits: SearchHit[]; + query: string; + processingTimeMs: number; + estimatedTotalHits: number; +} + +const CATEGORIES = [ + { id: 'all', label: 'All' }, + { id: 'Protocol', label: 'Protocol' }, + { id: 'Multi-Chain', label: 'Multi-Chain' }, + { id: 'AI & Agents', label: 'AI' }, + { id: 'Smart Contracts', label: 'Contracts' }, + { id: 'Web3 Apps', label: 'Web3 Apps' }, + { id: 'Primitives', label: 'Tokens & Primitives' }, + { id: 'Data Infrastructure', label: 'Data Infrastructure' }, + { id: 'Tools', label: 'Tools' }, + { id: 'API', label: 'API' }, +]; + +export default function SearchBar(): JSX.Element { + const { siteConfig } = useDocusaurusContext(); + const history = useHistory(); + + const [isOpen, setIsOpen] = useState(false); + const [query, setQuery] = useState(''); + const [results, setResults] = useState([]); + const [loading, setLoading] = useState(false); + const [selectedIndex, setSelectedIndex] = useState(0); + const [selectedCategory, setSelectedCategory] = useState('all'); + const [client, setClient] = useState(null); + + const inputRef = useRef(null); + const resultsRef = useRef(null); + + useEffect(() => { + const config = siteConfig.customFields?.meilisearch as { + host?: string; + apiKey?: string; + indexName?: string; + } | undefined; + + if (config?.host) { + const meiliClient = new MeiliSearch({ + host: config.host, + apiKey: config.apiKey || '', + }); + setClient(meiliClient); + } + }, [siteConfig]); + + useEffect(() => { + const handleKeyDown = (e: KeyboardEvent) => { + if ((e.metaKey || e.ctrlKey) && e.key === 'k') { + e.preventDefault(); + setIsOpen(true); + } + if (e.key === 'Escape') { + setIsOpen(false); + } + }; + + document.addEventListener('keydown', handleKeyDown); + return () => document.removeEventListener('keydown', handleKeyDown); + }, []); + + useEffect(() => { + if (isOpen && inputRef.current) { + setTimeout(() => inputRef.current?.focus(), 100); + } + }, [isOpen]); + + const search = useCallback(async (searchQuery: string, category: string) => { + if (!client || !searchQuery.trim()) { + setResults([]); + return; + } + + const config = siteConfig.customFields?.meilisearch as { indexName?: string } | undefined; + const indexName = config?.indexName || 'near-docs'; + + setLoading(true); + try { + const index = client.index(indexName); + const filter = category !== 'all' ? `category = "${category}"` : undefined; + + const searchResult: SearchResult = await index.search(searchQuery, { + limit: 10, + attributesToHighlight: ['title', 'content'], + highlightPreTag: '', + highlightPostTag: '', + filter, + hybrid: { + semanticRatio: 0.6, + embedder: 'default' + }, + }); + + setResults(searchResult.hits); + setSelectedIndex(0); + + trackSearch(searchQuery, searchResult.hits.length, category); + + if (searchResult.hits.length === 0) { + trackSearchNoResults(searchQuery); + } + } catch (error) { + console.error('Search error:', error); + setResults([]); + } finally { + setLoading(false); + } + }, [client, siteConfig]); + + useEffect(() => { + const timer = setTimeout(() => { + search(query, selectedCategory); + }, 300); + + return () => clearTimeout(timer); + }, [query, selectedCategory, search]); + + const handleKeyDown = (e: React.KeyboardEvent) => { + if (e.key === 'ArrowDown') { + e.preventDefault(); + setSelectedIndex(prev => Math.min(prev + 1, results.length - 1)); + } else if (e.key === 'ArrowUp') { + e.preventDefault(); + setSelectedIndex(prev => Math.max(prev - 1, 0)); + } else if (e.key === 'Enter' && results[selectedIndex]) { + e.preventDefault(); + navigateToResult(results[selectedIndex], selectedIndex); + } + }; + + const navigateToResult = (hit: SearchHit, index: number) => { + trackSearchResultClick(query, index, hit.path); + setIsOpen(false); + setQuery(''); + history.push(hit.path); + }; + + useEffect(() => { + if (resultsRef.current && results.length > 0) { + const selectedElement = resultsRef.current.children[selectedIndex] as HTMLElement; + selectedElement?.scrollIntoView({ block: 'nearest' }); + } + }, [selectedIndex, results.length]); + + const renderHighlight = (text: string | undefined, fallback: string) => { + if (!text) return fallback; + return ; + }; + + return ( + <> + + + {isOpen && ( +
+