runner.js

import scrapeIndeed from './scraper/indeed.js';
import scrapeRemoteOK from './scraper/remoteok.js';
import scrapeWeWorkRemotely from './scraper/weworkremotely.js';
import saveJobs from './db/saveJobs.js';
import dedupeJobs from './utils/dedupe.js';
import { scrapeJobDetail } from './src/services/jobDetailScraper.service.js';
import pool from './db.js';

(async () => {
  const indeedJobs = await scrapeIndeed();
  const remoteOKJobs = await scrapeRemoteOK();
  const wwrJobs = await scrapeWeWorkRemotely();

  const allJobs = [
    ...indeedJobs,
    ...remoteOKJobs,
    ...wwrJobs
  ];

  console.log('Before dedupe:', allJobs.length);

  const uniqueJobs = dedupeJobs(allJobs);

  console.log('After dedupe:', uniqueJobs.length);
  console.log(uniqueJobs);

  try {
    const { newJobs } = await saveJobs(uniqueJobs);

    if (newJobs && newJobs.length > 0) {
      console.log(`\n--- Automating JD Scraping for ${newJobs.length} new jobs ---`);
      for (const [index, job] of newJobs.entries()) {
        console.log(`[${index + 1}/${newJobs.length}] Scraping details for: ${job.link}`);
        try {
          const scrapeResult = await scrapeJobDetail(job);
          if (scrapeResult && scrapeResult.description) {
            await pool.query(
              'UPDATE jobs SET description = $1 WHERE id = $2',
              [scrapeResult.description, job.id]
            );
            console.log(`✓ Saved description (${scrapeResult.description.length} chars)`);
          } else {
            console.warn(`! No description found for: ${job.link}`);
          }
        } catch (scrapeErr) {
          console.error(`✗ Scraping failed for ${job.link}:`, scrapeErr.message);
        }
      }
      console.log('--- Automated Scraping Complete ---\n');
    }

  } catch (err) {
    console.error('Runner: failed to save jobs:', err && err.stack ? err.stack : err);
    process.exitCode = 1;
  } finally {
    // Close pool to allow process to exit
    await pool.end();
  }
})();