diff --git a/README.md b/README.md index 48c70f7..51084a1 100644 --- a/README.md +++ b/README.md @@ -25,11 +25,11 @@ The build pipeline performs the following steps: 1. Compiles TypeScript and builds the client production assets via Vite. 2. Runs the Open Graph image generator (`scripts/og.ts`) to pre-render cards for all routes and patches the output HTML files with page-specific titles, descriptions, and JSON-LD breadcrumb schemas. -3. Runs the sitemap generator (`scripts/sitemap.ts`) to scan the build output directory and automatically generate a fresh `sitemap.xml` for all static routes. +3. Runs the sitemap generator (`scripts/gen-sitemap.mjs`) to generate a fresh `sitemap.xml` covering all static and dynamic routes. ## SEO & Metadata -- **`robots.txt`**: Located in `public/robots.txt` and copied to the build root. It allows indexing on all paths and points crawlers to the sitemap location. +- **`robots.txt`**: Located in `public/robots.txt` and copied to the build root. It allows search engine crawling while excluding staging, preview, admin, and 404 routes, and points crawlers to the sitemap location. - **`sitemap.xml`**: Dynamically compiled during build time into `dist/sitemap.xml` and `public/sitemap.xml`. - **JSON-LD**: Embedded in the site's ``: - Global `Organization` schema representing Wraith Protocol. diff --git a/package.json b/package.json index af17847..ae768a9 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "dev": "vite", "build": "tsc -b && vite build && node scripts/inline-css.js && tsx scripts/og.ts && tsx scripts/sitemap.ts && node scripts/gen-rss.mjs", "og:generate": "tsx scripts/og.ts", + "generate:sitemap": "node scripts/gen-sitemap.mjs", "preview": "vite preview", "test": "vitest run", "test:e2e": "playwright test", diff --git a/public/robots.txt b/public/robots.txt index 6a918b6..d39166b 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,4 +1,13 @@ User-agent: * Allow: / +# Exclude staging, preview, administrative, and 404 routes +Disallow: /staging/ +Disallow: /staging +Disallow: /_staging/ +Disallow: /preview/ +Disallow: /admin/ +Disallow: /404 +Disallow: /api/ + Sitemap: https://usewraith.xyz/sitemap.xml diff --git a/scripts/gen-sitemap.mjs b/scripts/gen-sitemap.mjs new file mode 100644 index 0000000..bd778dc --- /dev/null +++ b/scripts/gen-sitemap.mjs @@ -0,0 +1,112 @@ +import { writeFileSync, readFileSync, readdirSync, statSync, existsSync } from 'fs'; +import { join, dirname } from 'path'; +import { fileURLToPath } from 'url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const rootDir = join(__dirname, '..'); +const distDir = join(rootDir, 'dist'); +const publicDir = join(rootDir, 'public'); +const siteUrl = 'https://usewraith.xyz'; + +/** + * Known static application routes + */ +const knownRoutes = [ + '/', + '/faq', + '/privacy', + '/use-cases', + '/roadmap', + '/case-studies', + '/stellar', + '/careers', + '/press', +]; + +/** + * Dynamically extract routes from case studies data + */ +function getCaseStudyRoutes() { + const routes = []; + const csPath = join(rootDir, 'src', 'data', 'case-studies.json'); + if (existsSync(csPath)) { + try { + const data = JSON.parse(readFileSync(csPath, 'utf8')); + if (Array.isArray(data.entries)) { + for (const entry of data.entries) { + if (entry.slug) { + routes.push(`/case-studies/${entry.slug}`); + } + } + } + } catch (err) { + console.warn('Could not read case-studies.json:', err.message); + } + } + return routes; +} + +/** + * Discover html routes from dist build directory if available + */ +function getDistRoutes(dir, base = '') { + const routes = []; + if (!existsSync(dir)) return routes; + + const files = readdirSync(dir); + if (files.includes('index.html') && base) { + routes.push(base); + } + + for (const file of files) { + if (file === 'og' || file === '404' || file.startsWith('.')) continue; + const fullPath = join(dir, file); + if (statSync(fullPath).isDirectory()) { + routes.push(...getDistRoutes(fullPath, `${base}/${file}`)); + } + } + + return routes; +} + +function generateSitemap() { + try { + const csRoutes = getCaseStudyRoutes(); + const distRoutes = existsSync(distDir) ? getDistRoutes(distDir) : []; + + const allRoutes = Array.from( + new Set([...knownRoutes, ...csRoutes, ...distRoutes]) + ).filter((r) => r && r !== '/404' && !r.includes('/staging') && !r.includes('/preview')); + + const today = new Date().toISOString().split('T')[0]; + + const sitemapXml = ` + +${allRoutes + .map((r) => { + const loc = `${siteUrl}${r === '/' ? '' : r}`; + const priority = r === '/' ? '1.0' : r.startsWith('/case-studies/') ? '0.7' : '0.8'; + const changefreq = r === '/' ? 'daily' : 'weekly'; + return ` + ${loc} + ${today} + ${changefreq} + ${priority} + `; + }) + .join('\n')} + +`; + + if (existsSync(distDir)) { + writeFileSync(join(distDir, 'sitemap.xml'), sitemapXml, 'utf8'); + } + writeFileSync(join(publicDir, 'sitemap.xml'), sitemapXml, 'utf8'); + console.log(`sitemap.xml generated successfully: ${allRoutes.length} routes found.`); + } catch (error) { + console.error('Failed to generate sitemap.xml:', error); + process.exit(1); + } +} + +generateSitemap(); diff --git a/scripts/sitemap.ts b/scripts/sitemap.ts index a9da622..8c665d8 100644 --- a/scripts/sitemap.ts +++ b/scripts/sitemap.ts @@ -1,4 +1,4 @@ -import { writeFileSync, readdirSync, statSync, existsSync } from 'fs'; +import { writeFileSync, readFileSync, readdirSync, statSync, existsSync } from 'fs'; import { join, dirname } from 'path'; import { fileURLToPath } from 'url'; @@ -8,18 +8,51 @@ const distDir = join(rootDir, 'dist'); const publicDir = join(rootDir, 'public'); const siteUrl = 'https://usewraith.xyz'; +const knownRoutes = [ + '/', + '/faq', + '/privacy', + '/use-cases', + '/roadmap', + '/case-studies', + '/stellar', + '/careers', + '/press', +]; + +function getCaseStudyRoutes(): string[] { + const routes: string[] = []; + const csPath = join(rootDir, 'src', 'data', 'case-studies.json'); + if (existsSync(csPath)) { + try { + const data = JSON.parse(readFileSync(csPath, 'utf8')); + if (Array.isArray(data.entries)) { + for (const entry of data.entries) { + if (entry.slug) { + routes.push(`/case-studies/${entry.slug}`); + } + } + } + } catch { + // ignore + } + } + return routes; +} + function getRoutes(dir: string, base = ''): string[] { const routes: string[] = []; if (!existsSync(dir)) { return routes; } const files = readdirSync(dir); - if (files.includes('index.html')) { - routes.push(base || '/'); + if (files.includes('index.html') && base) { + routes.push(base); } for (const file of files) { + if (file === 'og' || file === '404' || file.startsWith('.')) continue; const path = join(dir, file); - if (statSync(path).isDirectory() && file !== 'og') { + if (statSync(path).isDirectory()) { routes.push(...getRoutes(path, `${base}/${file}`)); } } @@ -27,29 +60,34 @@ function getRoutes(dir: string, base = ''): string[] { } try { - if (!existsSync(distDir)) { - console.error('dist/ directory not found. Please run `pnpm build` first.'); - process.exit(1); - } + const csRoutes = getCaseStudyRoutes(); + const distRoutes = existsSync(distDir) ? getRoutes(distDir) : []; + const allRoutes = Array.from( + new Set([...knownRoutes, ...csRoutes, ...distRoutes]) + ).filter((r) => r && r !== '/404' && !r.includes('/staging') && !r.includes('/preview')); + + const today = new Date().toISOString().split('T')[0]; - const routes = getRoutes(distDir); const sitemap = ` -${routes +${allRoutes .map( (r) => ` ${siteUrl}${r === '/' ? '' : r} - ${new Date().toISOString().split('T')[0]} - daily - ${r === '/' ? '1.0' : '0.8'} + ${today} + ${r === '/' ? 'daily' : 'weekly'} + ${r === '/' ? '1.0' : r.startsWith('/case-studies/') ? '0.7' : '0.8'} `, ) .join('\n')} -`; + +`; - writeFileSync(join(distDir, 'sitemap.xml'), sitemap, 'utf8'); + if (existsSync(distDir)) { + writeFileSync(join(distDir, 'sitemap.xml'), sitemap, 'utf8'); + } writeFileSync(join(publicDir, 'sitemap.xml'), sitemap, 'utf8'); - console.log(`sitemap.xml generated successfully: ${routes.length} routes found.`); + console.log(`sitemap.xml generated successfully: ${allRoutes.length} routes found.`); } catch (error) { console.error('Failed to generate sitemap.xml:', error); process.exit(1);