corpus.php

<?php
/**
 * Corpus queries over the `token` index.
 *
 * Shared by the public concordance and (later) the poem-page panels. Every
 * query is bound and every one is a single indexed lookup — no Python, no
 * shelling out, no megabyte of plotting library.
 *
 * Expects $mysql (mysqli) and the helpers in sql_queries.php.
 */

// The folding rules must match admin/tokenize.php exactly, or a search will
// not find what the indexer stored. Kept as small pure functions so the two
// can be compared side by side.
if (!function_exists('corpusNormalize')) {

    /** Fold a search term the same way the indexer folded the corpus. */
    function corpusNormalize(string $w): string
    {
        // Narrow no-break, thin and hair spaces are typographic gaps inside
        // a word that has been set with its letters spaced out. See
        // admin/tokenize.php::YI_GAP. Stripped so the spaced form matches
        // the ordinary spelling.
        $w = preg_replace('/[\x{202F}\x{2009}\x{200A}]/u', '', $w) ?? $w;
        $w = strtr($w, [
            "\u{05F0}" => "\u{05D5}\u{05D5}",   // װ -> וו
            "\u{05F1}" => "\u{05D5}\u{05D9}",   // ױ -> וי
            "\u{05F2}" => "\u{05D9}\u{05D9}",   // ײ -> יי
            "'"        => "\u{05F3}",
            "\u{2019}" => "\u{05F3}",
            '"'        => "\u{05F4}",
        ]);
        return strtr($w, [
            "\u{05DA}" => "\u{05DB}", "\u{05DD}" => "\u{05DE}",
            "\u{05DF}" => "\u{05E0}", "\u{05E3}" => "\u{05E4}",
            "\u{05E5}" => "\u{05E6}",
        ]);
    }

    /** Same, plus every point removed. Lossy — fallback only. */
    function corpusBare(string $w): string
    {
        return preg_replace(
            '/[\x{0591}-\x{05BD}\x{05BF}\x{05C1}\x{05C2}\x{05C4}\x{05C5}\x{05C7}]/u',
            '', corpusNormalize($w)) ?? '';
    }

    /** Strip anything that is not a Yiddish letter, mark or joiner. */
    function corpusCleanTerm(string $w): string
    {
        // Gaps are allowed through so a spaced-out word can be pasted into
        // the search box; corpusNormalize then removes them.
        $w = preg_replace('/[^\x{05D0}-\x{05EA}\x{05F0}-\x{05F2}'
            . '\x{0591}-\x{05BD}\x{05BF}\x{05C1}\x{05C2}\x{05C4}\x{05C5}\x{05C7}'
            . '\x{05BE}\x{05F3}\x{05F4}\x{2019}\'\-'
            . '\x{202F}\x{2009}\x{200A}]/u', '', trim($w)) ?? '';
        return mb_substr($w, 0, 64);
    }
}

/**
 * Concordance (keyword in context).
 *
 * Returns ['mode' => 'exact'|'unpointed'|'none', 'total' => int,
 *          'poems' => int, 'hits' => [...]]
 *
 * `mode` matters: an unpointed match may have merged distinct words, and the
 * page must say so rather than presenting it as a clean result.
 */
function corpusConcordance(mysqli $db, string $term, string $genre,
                           int $width = 6, int $limit = 100, int $offset = 0): array
{
    $term = corpusCleanTerm($term);
    if ($term === '') {
        return ['mode' => 'none', 'total' => 0, 'poems' => 0, 'hits' => [], 'term' => ''];
    }

    // Exact first. Only fall back to unpointed if that finds nothing.
    foreach ([['norm', corpusNormalize($term), 'exact'],
              ['bare', corpusBare($term),      'unpointed']] as [$col, $key, $mode]) {

        $count = db_row($db,
            "SELECT COUNT(*) n, COUNT(DISTINCT t.poem) p
               FROM token t JOIN poem pm ON pm.poem = t.poem
              WHERE t.`$col` = ? AND pm.public IS TRUE AND pm.genre = ?",
            'ss', $key, $genre);
        $total = (int)($count['n'] ?? 0);
        if ($total === 0) {
            continue;
        }

        $hits = db_all($db,
            "SELECT t.poem, t.line_no, t.pos, t.surface,
                    pm.title_y, pm.poet, YEAR(pm.date) yr
               FROM token t JOIN poem pm ON pm.poem = t.poem
              WHERE t.`$col` = ? AND pm.public IS TRUE AND pm.genre = ?
              ORDER BY YEAR(pm.date), pm.poet, t.poem, t.pos
              LIMIT ? OFFSET ?",
            'ssii', $key, $genre, $limit, $offset);

        // Context: one query per poem, then slice in PHP. Poems average 154
        // tokens, so fetching a whole poem is cheaper than a range query per
        // hit — and a page of 100 hits usually spans far fewer poems.
        $byPoem = [];
        foreach ($hits as $h) {
            $byPoem[$h['poem']] = true;
        }
        $words = [];
        foreach (array_keys($byPoem) as $pm) {
            foreach (db_all($db,
                "SELECT pos, surface FROM token WHERE poem = ? ORDER BY pos",
                's', $pm) as $t) {
                $words[$pm][(int)$t['pos']] = $t['surface'];
            }
        }
        foreach ($hits as &$h) {
            $w = $words[$h['poem']] ?? [];
            $p = (int)$h['pos'];
            $before = $after = [];
            for ($i = $p - $width; $i < $p; $i++) {
                if (isset($w[$i])) $before[] = $w[$i];
            }
            for ($i = $p + 1; $i <= $p + $width; $i++) {
                if (isset($w[$i])) $after[] = $w[$i];
            }
            $h['before'] = implode(' ', $before);
            $h['after']  = implode(' ', $after);
        }
        unset($h);

        return ['mode' => $mode, 'total' => $total,
                'poems' => (int)($count['p'] ?? 0), 'hits' => $hits, 'term' => $term];
    }

    return ['mode' => 'none', 'total' => 0, 'poems' => 0, 'hits' => [], 'term' => $term];
}

/** Every spelling that folds to the same key — shown so the reader can see the merge. */
function corpusSpellings(mysqli $db, string $term, bool $unpointed = false): array
{
    $col = $unpointed ? 'bare' : 'norm';
    $key = $unpointed ? corpusBare($term) : corpusNormalize($term);
    return array_column(db_all($db,
        "SELECT DISTINCT surface FROM token WHERE `$col` = ? ORDER BY surface",
        's', $key), 'surface');
}

/** Occurrences per year, for the frequency chart. */
function corpusByYear(mysqli $db, string $term, string $genre, bool $unpointed = false): array
{
    $col = $unpointed ? 'bare' : 'norm';
    $key = $unpointed ? corpusBare($term) : corpusNormalize($term);
    return db_all($db,
        "SELECT YEAR(pm.date) yr, COUNT(*) n, COUNT(DISTINCT t.poem) poems
           FROM token t JOIN poem pm ON pm.poem = t.poem
          WHERE t.`$col` = ? AND pm.public IS TRUE AND pm.genre = ?
                AND YEAR(pm.date) > 0
          GROUP BY yr ORDER BY yr",
        'ss', $key, $genre);
}

/** Occurrences per poet, with each poet's corpus size so rates are comparable. */
function corpusByPoet(mysqli $db, string $term, string $genre, bool $unpointed = false): array
{
    $col = $unpointed ? 'bare' : 'norm';
    $key = $unpointed ? corpusBare($term) : corpusNormalize($term);
    return db_all($db,
        "SELECT pm.poet, COUNT(*) n, COUNT(DISTINCT t.poem) poems,
                (SELECT COALESCE(SUM(s.tokens), 0)
                   FROM poem_stat s JOIN poem p2 ON p2.poem = s.poem
                  WHERE p2.poet = pm.poet AND p2.public IS TRUE
                        AND p2.genre = ?) AS poet_tokens
           FROM token t JOIN poem pm ON pm.poem = t.poem
          WHERE t.`$col` = ? AND pm.public IS TRUE AND pm.genre = ?
          GROUP BY pm.poet ORDER BY n DESC",
        'sss', $genre, $key, $genre);
}

/** Statistics for one poem. */
function corpusPoemStat(mysqli $db, string $code): ?array
{
    return db_row($db, "SELECT * FROM poem_stat WHERE poem = ?", 's', $code);
}

/**
 * The spelling a reader should see for a match key.
 *
 * `norm` is a lookup string, not a word. Falls back to the key only if `form` is missing.
 */
function corpusDisplay(mysqli $db, string $key): string
{
    $r = db_row($db, "SELECT disp FROM `form` WHERE norm = ?", 's', $key);
    return (string)($r['disp'] ?? $key);
}

/**
 * The commonest words in one poem, as spellings rather than keys.
 *
 * Counted over `norm` so the several spellings of one word count together,
 * displayed from `form` so the reader sees a real word.
 */
function corpusTopWords(mysqli $db, string $code, int $limit = 8,
                        bool $skipStopWords = false): array
{
    if (!$skipStopWords) {
        return db_all($db,
            "SELECT COALESCE(f.disp, t.norm) AS word, COUNT(*) AS n
               FROM token t LEFT JOIN `form` f ON f.norm = t.norm
              WHERE t.poem = ?
              GROUP BY t.norm, f.disp
              ORDER BY n DESC, word ASC
              LIMIT ?", 'si', $code, $limit);
    }

    // include, not require: a missing file must degrade to "no filtering",
    // never to a fatal. This function is called from poem.php above the
    // script tags, and a compile error there takes the whole page's
    // JavaScript down with it — and a failed require is NOT catchable.
    if (!function_exists('yiStopWords')) {
        @include_once __DIR__ . '/stopwords.php';
    }
    if (!function_exists('yiStopWords')) {
        error_log('stopwords.php missing — frequency list not filtered');
        return corpusTopWords($db, $code, $limit, false);
    }
    $stop = array_keys(yiStopWords());
    if (!$stop) {
        return corpusTopWords($db, $code, $limit, false);
    }
    // Bound placeholders rather than an interpolated IN list — the words are
    // ours, but building SQL by concatenation is how the old code got into
    // trouble and there is no reason to reintroduce the habit.
    $ph    = implode(',', array_fill(0, count($stop), '?'));
    $types = 's' . str_repeat('s', count($stop)) . 'i';
    return db_all($db,
        "SELECT COALESCE(f.disp, t.norm) AS word, COUNT(*) AS n
           FROM token t LEFT JOIN `form` f ON f.norm = t.norm
          WHERE t.poem = ? AND t.norm NOT IN ($ph)
          GROUP BY t.norm, f.disp
          ORDER BY n DESC, word ASC
          LIMIT ?", $types, $code, ...[...$stop, $limit]);
}

/**
 * Where a poem's vocabulary richness sits among comparable poems.
 *
 * Returns null when the poem is too short to score. Poems under 100 words
 * have no `sttr`, and inventing one for them would reintroduce exactly the
 * length bias the measure exists to remove.
 */
function corpusSttrRank(mysqli $db, string $code, string $genre): ?array
{
    $me = db_row($db, "SELECT sttr FROM poem_stat WHERE poem = ?", 's', $code);
    if ($me === null || $me['sttr'] === null) {
        return null;
    }
    $mine = (float)$me['sttr'];
    $r = db_row($db,
        "SELECT COUNT(*) n, SUM(s.sttr < ?) below
           FROM poem_stat s JOIN poem p ON p.poem = s.poem
          WHERE s.sttr IS NOT NULL AND p.public IS TRUE AND p.genre = ?",
        'ds', $mine, $genre);
    $n = (int)($r['n'] ?? 0);
    if ($n < 2) {
        return null;
    }
    $med = db_row($db,
        "SELECT s.sttr FROM poem_stat s JOIN poem p ON p.poem = s.poem
          WHERE s.sttr IS NOT NULL AND p.public IS TRUE AND p.genre = ?
          ORDER BY s.sttr LIMIT 1 OFFSET ?",
        'si', $genre, intdiv($n - 1, 2));
    return [
        'sttr'       => $mine,
        'percentile' => (int)round(100 * (int)($r['below'] ?? 0) / $n),
        'median'     => (float)($med['sttr'] ?? 0),
        'n'          => $n,
    ];
}

/** Corpus totals, for the stats page. */
function corpusTotals(mysqli $db, string $genre): array
{
    return db_row($db,
        "SELECT COUNT(*) poems, COALESCE(SUM(s.tokens), 0) tokens,
                COALESCE(SUM(s.lines_n), 0) lines_n
           FROM poem p JOIN poem_stat s ON s.poem = p.poem
          WHERE p.public IS TRUE AND p.genre = ?", 's', $genre) ?? [];
}

/** Is the index present and populated? Pages should degrade quietly if not. */
function corpusReady(mysqli $db): bool
{
    try {
        $r = db_row($db, "SELECT COUNT(*) n FROM token");
        return (int)($r['n'] ?? 0) > 0;
    } catch (Throwable $e) {
        return false;
    }
}