diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..d9bdc490 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,72 @@ +# VCS +.git +.github/ +.gitattributes +.gitignore + +# OS/editor junk +.DS_Store +Thumbs.db +*.swp +*.swo +*~ +*.log +.idea/ +.vscode/ + +# Local build outputs +bin/ + +src/**/*.o +src/**/*.a + +src/anal/cruncher +src/anal/deverbal +src/anal/findbase +src/anal/morpheus +src/anal/pname + +src/gener/checkstype +src/gener/do_conj +src/gener/gener + +src/gkdict/combitype +src/gkdict/conj1 +src/gkdict/fixgend +src/gkdict/fixhesc +src/gkdict/indexcomps +src/gkdict/indexnoms +src/gkdict/indexvbs +src/gkdict/latnom +src/gkdict/latvb +src/gkdict/newlems +src/gkdict/newlems2 +src/gkdict/setquant +src/gkdict/splitlat +src/gkdict/splitlems + +src/gkends/buildderiv +src/gkends/buildend +src/gkends/buildword +src/gkends/indderivtables +src/gkends/indendtables + +# update.sh outputs +stemlib/Latin/conjfile +stemlib/Latin/oddfile + +stemlib/**/endtables/ascii/** +stemlib/**/endtables/indices/** +stemlib/**/endtables/out/** + +stemlib/**/derivs/ascii/ +stemlib/**/derivs/indices/ +stemlib/**/derivs/out/ + +stemlib/**/steminds/ + +stemlib/Latin/stemsrc/nom.irreg +stemlib/Latin/stemsrc/vbs.irreg + +# Optional: repo docs not needed for the build +doc/ \ No newline at end of file diff --git a/.gitignore b/.gitignore index 469fce39..7fe4344a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ bin src/anal/cruncher src/anal/deverbal src/anal/findbase +src/anal/morpheus src/anal/pname src/gener/checkstype src/gener/do_conj diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..80d7e22d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +FROM ubuntu:24.04 + +ARG DEBIAN_FRONTEND=noninteractive + +# Install dependencies for compiling and for the data generation scripts +RUN apt-get update -qq && apt-get install -qq -y \ + build-essential \ + flex \ + perl + +ADD . /morpheus +WORKDIR /morpheus + +# fix line endings / make executable for the perl script (still needed) +RUN chmod +x stemlib/Greek/getentities.pl && \ + sed -i 's/\r$//' stemlib/Greek/getentities.pl + +RUN make && \ + echo "salve" | MORPHLIB=stemlib bin/cruncher -L \ No newline at end of file diff --git a/README_DOCKER.md b/README_DOCKER.md new file mode 100644 index 00000000..2a182b72 --- /dev/null +++ b/README_DOCKER.md @@ -0,0 +1,62 @@ +Docker +====== + +### Build the image + +```bash +docker build -t [IMAGE_ID] . +``` +[1] + +### Run the image + +You can run the tools either interactively or directly via a one-shot command. + +#### Option A: Interactively + +``` bash +docker run -it --rm [IMAGE_ID] bash +``` + +Inside the container: + +- **cruncher** (always available): + + ```bash + echo "rosa" | MORPHLIB=stemlib bin/cruncher -L + ``` + [2] + +- **morpheus** XML output [3] + + ```bash + echo "rosa" | MORPHLIB=stemlib bin/morpheus -L + ``` + +#### Option B: One-shot command + +You can also execute a run directly from your host without entering an +interactive shell: + +```bash +docker run --rm [IMAGE_ID] /bin/sh -c 'echo "rosa" | MORPHLIB=stemlib bin/cruncher -L' +``` + +### Footnotes + +
    +
  1. + IMAGE_ID stands for any name you want to give + ↩ +
    2. +
  2. + "-L" selects Latin; drop it for Greek (the default). See the main + README for the rest of the flags + ↩ +
    3. +
  3. + Only if this build includes it; check first + with "ls bin/morpheus" inside the container, or "docker run --rm [IMAGE_ID] ls bin/" from outside + ↩ +
    4. +
\ No newline at end of file diff --git a/README_XML.md b/README_XML.md new file mode 100644 index 00000000..88839cbd --- /dev/null +++ b/README_XML.md @@ -0,0 +1,207 @@ +morpheus XML output +==================== + +This branch adds `morpheus`, a second binary alongside `cruncher` that shares +the same analysis pipeline (`AnalyzeString`, `SortAnals`, `GoodAnals`) but +emits Perseus-style XML instead of the ``-line format. It is unconditional: +morpheus always emits XML, there is no flag to switch it off. + +Why XML +------- + +`cruncher`'s Perseus format represents each analysis as a single positional +text line: abbreviated vocabulary (`nom`, `masc`, `pres`), a single +part-of-speech prefix (`P/N/V/E/I`), and multi-value fields on one line +(e.g. `nom/voc/acc masc/fem pl`). morpheus renders the same analyses as a +structured document instead: full-word vocabulary, a refined part of speech +per analysis, and multi-value fields expanded into a cartesian product of +`` elements (one case/gender/etc. combination per element), so +downstream code doesn't have to re-parse a slash-separated field. + +Because both binaries run through the same `SortAnals`/`GoodAnals` filtering, +the *set* of analyses is guaranteed identical between the two. XML is a +different serialization of the same data, not a different analysis. + +Building +-------- + +`morpheus` is built as part of the normal top-level build: +```bash +make +``` + +It installs to `bin/morpheus` alongside `cruncher`. + +Architecture / rebase isolation +-------------------------------- + +This is meant to be a long-lived branch that stays rebasable against +upstream. The XML support is kept isolated to two files: + +- `src/anal/morpheus.c` — dedicated main loop (arg parsing, word intake). +- `src/anal/prntalph.c` — XML formatting and vocabulary-mapping tables. + +The core analysis pipeline and dictionary compilation (`checkstring`, +`AnalyzeString`, `gener/`, `gkdict/`) are untouched. If you rebase and the +dictionary changes, `morpheus`'s output changes exactly as much as +`cruncher`'s does. No separate mapping logic to keep in sync. + +Running morpheus +----------------- +```bash +MORPHLIB=stemlib bin/morpheus < wordlist +``` + +Words can also be given directly as arguments instead of on stdin: + +```bash +MORPHLIB=stemlib bin/morpheus lo/gos timh/ +``` + +For Latin, add -L: + +```bash +echo "rosa" | MORPHLIB=stemlib bin/morpheus -L +``` + +```bash +MORPHLIB=stemlib bin/morpheus -L rosa dominus +``` + +The `echo | ...` form pipes the word in on stdin; the +`bin/morpheus -L rosa dominus` form passes words as argv arguments instead. +Both produce identical output for a single word — see above for how the two +input paths differ. + +Each argv word is analyzed as its own word (not split further); after argv words +are consumed, morpheus exits rather than falling through to read stdin. + +### Flags + +Shares most analysis flags with `cruncher`: + +`-L` Latin instead of Greek. `-I` Italian. `-S` disables strict case matching +(allows capitalized/sentence-initial words). `-n` accent-insensitive retry. +`-V` verb forms only. `-i` adds a `` block per analysis with +low-level internal fields (raw stem/suffix/preverb pieces, morphflag bytes). + +`-c`, `-p`, `-x` are accepted (for `getopt` parity with `cruncher`'s flag set) +but currently have no effect on morpheus's output: it always produces XML +regardless. + +### Output format + +Root element is `...`. Comment-only lines in the input +(starting with `#`) pass through as XML comments; literal `--` inside a +comment is neutralized to `- ` so it can't collide with ``. + +Each analyzed word is a `` element containing a `
` (the input +form) and one `` per distinct lemma. Each `` has a `` +block (`` = lemma, `` = part of speech with an `order` attribute, +`` for nouns) followed by one `` per case/gender/mood/tense/etc. +combination that analysis covers. + +Unrecognized words are emitted inline as `word` +rather than being dropped to stderr the way `cruncher` handles them. + +If there are no words at all (empty stdin, no argv words, no comments), +morpheus emits `No words provided` with no `` wrapper +and exits `2`. + +### Intentional divergences from cruncher + +Deliberate, not bugs. See `tests/run_xml_tests.sh` (tests N1–N8) for the +checks that pin them down: + +- Long-form vocabulary (`nominative`, `singular`) instead of cruncher's + abbreviations (`nom`, `sg`). +- `` is a refined category (pronoun/article/preposition/etc., not just + cruncher's coarse `P/N/V/E/I` prefix), with a sentinel `order="0"` for + categories not in the ordering table. +- Case/gender combinations expand into a cartesian product of `` + elements (one per combination) rather than a single combined cruncher line. +- A line with multiple whitespace-separated words only analyzes the first + token (matches cruncher's batch behavior; diverges from the original + upstream alpheios code, which analyzed every token on the line). + +Testing +------- + +```bash +bash tests/run_xml_tests.sh +``` + +This is intentionally independent of `make test` / `tests/run_tests.sh` and +is not wired into the Makefile. Run it by hand. To (re)generate golden +baselines: + +```bash +bash tests/run_xml_tests.sh --update +``` + +Only regenerate a baseline after you've deliberately changed something that +affects that fixture's output *and* manually confirmed the new output is +correct. A baseline diff on its own is not a reason to `--update`; it's the +suite's signal to go look at what changed, not to silently accept it. + +### Test ID legend + +IDs are grouped by what they assert: + +- **P*n*** — Positive: morpheus **should** agree with cruncher on the same + word (same known/unknown status, same lemmas, same stemtypes, etc.), or + otherwise behave as documented (flag parsing, argv handling). A failure + here means morpheus disagrees with cruncher, or with its own documented + behavior, where it shouldn't. +- **N*n*** — Negative / divergence: morpheus is **deliberately** different + from cruncher here (richer vocabulary, expanded fields, inline ``, + etc.). A failure here means morpheus accidentally matches cruncher's old + behavior where it was supposed to have changed it, or produced something + outside the intended divergence. + +Plain-English meaning of each: + +| ID | What it checks | +|----------------|-------------------------------------------------------------------------| +| flag-n-arg | `-n` doesn't swallow the next argv word as an option-argument | +| argv-cleanup | argv words get the same digit-trimming cleanup as stdin words | +| P1 | known/unknown word partition matches cruncher | +| P2 | lemma set and first-occurrence order match cruncher | +| P3 | stemtype set matches cruncher | +| P4 | (soft) feature-token coverage (approximate, see comparator notes) | +| P5 | (soft) case/gender cartesian-expansion count (approximate) | +| P6 | determinism: same input twice gives byte-identical output | +| P7 | exit codes, and the empty-input error document is well-formed | +| P8 | after consuming argv words, morpheus exits instead of hanging on stdin | +| N1 | long-form vocabulary (`nominative`) instead of cruncher's abbreviations | +| N2 | refined `` categories, with a sentinel `order="0"` fallback | +| N3 | runtime XML escaping (literal `&`) instead of a pre-escaped table | +| N4 | unknown words appear inline as `` instead of going to stderr | +| N5 | dialect table isn't truncated early; `prose`/`Homeric`/etc. reachable | +| N6 | no literal `(null)` text leaks into gender/pofs output | +| N7 | comment lines produce a single well-formed `` document | +| N8 | a multi-token input line only analyzes the first token | + +### Result levels + +- `PASS`/`FAIL` — hard checks: build/link smoke tests, well-formedness, + flag/argv-handling checks (flag-n-arg, argv-cleanup, P8), known/unknown-word + partition parity (P1), lemma set/order parity (P2), stemtype set parity + (P3), determinism (P6), exit codes (P7), and the N1–N8 divergence checks. + A `FAIL` here blocks the suite (`exit 1`). +- `WARN` — soft checks: golden-baseline text diffs (informational — see + above), and feature-token coverage (P4) / case-gender expansion counts (P5) + from the comparator script, which use an approximate normalization map and + can produce false positives on multi-value ``/`` tags. Don't + block on these without reading the diff first. + + P4 deliberately excludes anything rendered via `` as gender is + already validated via P1–P3 and N1, and P4 checking it too would just be + redundant duplication of a stricter check. +- `XFAIL`/`XPASS` — reserved for a known, already-diagnosed defect that's + expected to fail until fixed. None currently active in this suite; if you + introduce one, add the `xfail`/`xpass` wrapper and a comment explaining + what's broken, and convert it back to plain `pass`/`fail` once fixed. + +Requires `python3` (or `xmllint`) for well-formedness checks and +`bin/cruncher` for the parity suite. Both are skipped gracefully if absent. \ No newline at end of file diff --git a/src/anal/checkstring.c b/src/anal/checkstring.c index 2144afa6..674f42a7 100755 --- a/src/anal/checkstring.c +++ b/src/anal/checkstring.c @@ -52,6 +52,26 @@ int checkstring(char *string, PrntFlags prntflags, FILE *fout) int nanals = 0; int nlems = 0; + Gkword = AnalyzeString(string,prntflags); + if( ! Gkword ) return(0); + + if( prntflags & LEMCOUNT ) { + nlems = cntlems(Gkword); + FreeGkword( Gkword ); + return(nlems); + } + + if( prntflags && (nanals=totanal_of(Gkword)) > 0 ) { + PrntAnalyses(Gkword,prntflags,fcurout); + } + FreeGkword( Gkword ); + return(nanals); +} + +gk_word * AnalyzeString(char *string, PrntFlags prntflags) +{ + gk_word * Gkword = NULL; + if( is_blank(string) ) return(0); if( strlen(string) >= MAXWORDSIZE ) return(0); @@ -66,20 +86,9 @@ int checkstring(char *string, PrntFlags prntflags, FILE *fout) checkstring1(Gkword); - if( prntflags & LEMCOUNT ) { - nlems = cntlems(Gkword); - FreeGkword( Gkword ); - return(nlems); - } - - if( prntflags && (nanals=totanal_of(Gkword)) > 0 ) { - PrntAnalyses(Gkword,prntflags,fcurout); - } - FreeGkword( Gkword ); - return(nanals); + return(Gkword); } - int cntlems(gk_word *Gkword ) { int i; diff --git a/src/anal/checkstring.proto.h b/src/anal/checkstring.proto.h index 53670ce0..26b053bc 100755 --- a/src/anal/checkstring.proto.h +++ b/src/anal/checkstring.proto.h @@ -3,6 +3,7 @@ /* checkstring.c */ +gk_word * AnalyzeString(char *, PrntFlags); int checkstring(char *, PrntFlags, FILE *); void checkstring1(gk_word *); int checkstring2(gk_word *); diff --git a/src/anal/makefile b/src/anal/makefile index 81a10a5e..14aa4fda 100755 --- a/src/anal/makefile +++ b/src/anal/makefile @@ -37,7 +37,7 @@ ${ANALLIB}: ${ANALOBJ} ar rv ${ANALLIB} ${ANALOBJ} ranlib ${ANALLIB} -ALL=cruncher pname findbase deverbal ${ANALLIB} +ALL=cruncher morpheus pname findbase deverbal ${ANALLIB} all: ${ALL} @@ -51,6 +51,9 @@ lcnt: lcnt.o ${ANALLIB} ${LIBS} cruncher:stdiomorph.o ${ANALLIB} ${LIBS} ${CC} -o cruncher stdiomorph.o ../gener/genwd.o ${ANALLIB} ${LIBS} +morpheus:morpheus.o prntalph.o ${ANALLIB} ${LIBS} + ${CC} -g -o morpheus morpheus.o prntalph.o ../gener/genwd.o ${ANALLIB} ${LIBS} + morphcheck:morphcheck.o ${LSJDIR}/MDBlib.o ${LSJDIR}/flen.o ${ANALOBJ} ${LIBS} ${CC} -o morphcheck morphcheck.o ${LSJDIR}/MDBlib.o ${LSJDIR}/flen.o ../gener/genwd.o ${ANALOBJ} ${LIBS} ${LEXLIB} @@ -64,7 +67,7 @@ deverbal: deverb.o ${ANALOBJ} ${LIBS} ${CC} -o deverbal deverb.o ${ANALOBJ} ${LIBS} clean: - rm -f cruncher *.o + rm -f cruncher morpheus *.o proclems: proclems.o ${CC} -o proclems proclems.o diff --git a/src/anal/morpheus.c b/src/anal/morpheus.c new file mode 100644 index 00000000..a5c026b0 --- /dev/null +++ b/src/anal/morpheus.c @@ -0,0 +1,161 @@ +#include +#include +#include +#include +#include +#include + +#include "../greeklib/xstrings.proto.h" +#include "../greeklib/stripbreath.proto.h" +#include "../greeklib/addbreath.proto.h" +#include "../morphlib/morphpath.proto.h" +#include "../morphlib/setlang.proto.h" +#include "../morphlib/trimwhite.proto.h" +#include "../morphlib/gkstring.proto.h" +#include "prntalph.h" +#include "checkstring.proto.h" + +#define ARGS "ILcixSVpn" + +int quickflag = 0; + +void trimdigit(char *s) +{ + char *p = s; + while (*s) s++; + s--; + while (isdigit((unsigned char)*s) && s > p) *s-- = 0; +} + +void print_safe_comment(const char *s) +{ + fputs("\n", stdout); +} + +int main(int argc, char** argv) +{ + char line[BUFSIZ*4]; + char word[BUFSIZ*4]; + PrntFlags flags = (PERSEUS_FORMAT|STRICT_CASE); + int c, errflg = 0; + + /* process arguments */ + while (!errflg && (c = getopt(argc, argv, ARGS)) != -1) + { + switch (c) + { + case 'c': flags |= CHECK_PREVERB; break; + case 'I': set_lang(ITALIAN); break; + case 'L': set_lang(LATIN); break; + case 'i': flags |= SHOW_FULL_INFO; break; + case 'x': flags |= LEXICON_OUTPUT; break; + case 'V': flags |= VERBS_ONLY; break; + case 'S': flags &= ~(STRICT_CASE); break; + case 'n': flags |= IGNORE_ACCENTS; break; + case 'p': flags |= PARSE_FORMAT; break; + default: errflg++; break; + } + } + + int had_args = (optind < argc); + int started = 0; + int nwords = 0; + int nunknown = 0; + + while (1) + { + if (optind < argc) + { + Xstrncpy(word, argv[optind++], sizeof(word)); + trimdigit(word); + } + else + { + if (had_args) break; + + if (!fgets(line, sizeof(line), stdin)) break; + + trimwhite(line); + if (isspace((unsigned char)line[0]) || !line[0]) continue; + + if (line[0] == '#') + { + if (!started) { printf("\n"); started = 1; } + print_safe_comment(line + 1); + continue; + } + + trimdigit(line); + char *p = line; + while (*p && !isspace((unsigned char)*p)) p++; + if (p == line) continue; + *p = 0; + + Xstrncpy(word, line, sizeof(word)); + } + + if (!started) { printf("\n"); started = 1; } + + gk_word *gkword = AnalyzeString(word, flags); + int rval = gkword ? totanal_of(gkword) : 0; + + if (cur_lang() != LATIN && cur_lang() != ITALIAN && !rval && (flags & IGNORE_ACCENTS)) + { + char tmpform[BUFSIZ]; + if (gkword) FreeGkword(gkword); + + Xstrncpy(tmpform, word, sizeof(tmpform)); + stripbreath(tmpform); + addbreath(tmpform, ')'); + gkword = AnalyzeString(tmpform, flags); + rval = gkword ? totanal_of(gkword) : 0; + + if (!rval) + { + if (gkword) FreeGkword(gkword); + stripbreath(tmpform); + addbreath(tmpform, '('); + gkword = AnalyzeString(tmpform, flags); + rval = gkword ? totanal_of(gkword) : 0; + } + } + + if (rval > 0) + { + alpheiosPrintWord(gkword, flags, stdout); + } + else + { + printf("", get_xml_lang()); + xml_write_text(stdout, word); + printf("\n"); + ++nunknown; + } + + if (gkword) FreeGkword(gkword); + ++nwords; + } + + if (!started) { + printf("No words provided\n"); + return 2; + } else { + printf("\n"); + fprintf(stderr, "%d word%s analyzed, %d unknown\n", + nwords, nwords == 1 ? "" : "s", nunknown); + } + + return 0; +} \ No newline at end of file diff --git a/src/anal/prntalph.c b/src/anal/prntalph.c new file mode 100644 index 00000000..d485bf46 --- /dev/null +++ b/src/anal/prntalph.c @@ -0,0 +1,1066 @@ +#include +#include "prntalph.h" +#include "prntanal.proto.h" +#include "../morphlib/morphflags.proto.h" +#include "../morphlib/setlang.proto.h" + +typedef struct +{ + const char* d_name; + long d_flags; +} MorphEntry; + +typedef struct +{ + const char* d_name; + const char* d_value; +} AttributeEntry; + +/* + Tables mapping flags for morphological categories to textual names + Note: Entries are tested by ANDing flags with table entries, and + some entries correspond to flag combinations. Entries are tested + in order until a match is found, so entries with multiple flag + values must appear in the table after values with fewer flags. + */ + +/* +#define SUBMASK 0777 +MorphEntry alpheiosPofsNames[] = +{ + {"adjective", ADJSTEM|SUBMASK}, + {"noun", NOUNSTEM|SUBMASK}, + {"verb", PPARTMASK}, + {"verb", VERBSTEM}, + {"numeral", NUMERAL}, + {"preposition", PREPOSITION}, + {"article", ARTICLE}, + {"pronoun", PRONOUN}, + {"pronoun", INDEF_PRON}, + {"pronoun", PERS_PRON}, + {"pronoun", REL_PRON}, + {"pronoun", INDEF_REL_PRON}, + {"particle", PARTICLE}, + {"conjunction", CONJUNCT}, + {"adverb", ADVERB}, + {NULL, 0} +}; +*/ + +AttributeEntry alpheiosPofsOrder[] = +{ + {"adverb", "7"}, + {"preposition", "6"}, + {"pronoun", "5"}, + {"numeral", "4"}, + {"noun", "3"}, + {"adjective", "2"}, + {"verb", "1"}, + {NULL, "0"} +}; + +MorphEntry alpheiosDeclNames[] = +{ + {"1st", DECL1}, + {"2nd", DECL2}, + {"3rd", DECL3}, + {"4th", DECL4}, + {"5th", DECL5}, + {"1st & 2nd", DECL1|DECL2}, + {"1st & 3rd", DECL1|DECL3}, + {NULL, 0} +}; + +MorphEntry alpheiosCaseNames[] = +{ + {"nominative", NOMINATIVE}, + {"genitive", GENITIVE}, + {"ablative", ABLATIVE}, + {"dative", DATIVE}, + {"accusative", ACCUSATIVE}, + {"vocative", VOCATIVE}, + {"genitive/dative", GENITIVE|DATIVE}, + {"ablative/dative", ABLATIVE|DATIVE}, + {"nominative/accusative", NOMINATIVE|ACCUSATIVE}, + {"nominative/vocative", NOMINATIVE|VOCATIVE}, + {"nominative/vocative/accusative", NOMINATIVE|VOCATIVE|ACCUSATIVE}, + {NULL, 0} +}; + +AttributeEntry alpheiosCaseOrder[] = +{ + {"nominative", "7"}, + {"genitive", "6"}, + {"dative", "5"}, + {"accusative", "4"}, + {"ablative", "3"}, + {"locative", "2"}, + {"vocative", "1"}, + {NULL, "0"} +}; + +MorphEntry alpheiosComparisonNames[] = +{ + {"comparative", COMPARATIVE}, + {"superlative", SUPERLATIVE}, + {NULL, 0} +}; + +MorphEntry alpheiosGenderNames[] = +{ + {"masculine", MASCULINE}, + {"feminine", FEMININE}, + {"neuter", NEUTER}, + {"adverbial", ADVERBIAL}, + {"masculine/neuter", MASCULINE|NEUTER}, + {"masculine/feminine", MASCULINE|FEMININE}, + {"masculine/feminine/neuter", MASCULINE|FEMININE|NEUTER}, + {"common", MASCULINE|FEMININE|NEUTER}, + {NULL, 0} +}; + +MorphEntry alpheiosMoodNames[] = { + {"indicative", INDICATIVE}, + {"subjunctive", SUBJUNCTIVE}, + {"imperative", IMPERATIVE}, + {"supine", SUPINE}, + {"optative", OPTATIVE}, + {"infinitive", INFINITIVE}, + {"participle", PARTICIPLE}, + {"conditional", CONDITIONAL}, + {"gerundive", GERUNDIVE}, + {NULL, 0} +}; + +MorphEntry alpheiosNumberNames[] = +{ + {"singular", SINGULAR}, + {"plural", PLURAL}, + {"dual", DUAL}, + {NULL, 0} +}; + +MorphEntry alpheiosPersonNames[] = +{ + {"1st", PERS1}, + {"2nd", PERS2}, + {"3rd", PERS3}, + {NULL, 0} +}; + +MorphEntry alpheiosTenseNames[] = +{ + {"present", PRESENT}, + {"future", FUTURE}, + {"aorist", AORIST}, + {"perfect", PERFECT}, + {"imperfect", IMPERF}, + {"pluperfect", PLUPERF}, + {"future perfect", FUTPERF}, + {"past absolute", PASTABSOLUTE}, + {NULL, 0} +}; + +MorphEntry alpheiosVoiceNames[] = +{ + {"active", ACTIVE}, + {"passive", PASSIVE}, + {"middle", MIDDLE}, + {"mediopassive", MEDIO_PASS}, + {"deponent", ACTIVE|MIDDLE}, + {NULL, 0} +}; + +MorphEntry alpheiosStemNames[] = +{ + {"pp_pr", PP_PR}, + {"pp_fu", PP_FU}, + {"pp_ao", PP_AO}, + {"pp_pf", PP_PF}, + {"pp_pp", PP_PP}, + {"pp_ap", PP_AP}, + {"pp_fp", PP_FP}, + {"pp_p4", PP_SU}, + {"pp_va", PP_VA}, + {"pp_vn", PP_VN}, + {"verbstem", VERBSTEM}, + {"indecl", INDECL}, + {"adj3", ADJSTEM|DECL3}, + {"noun1", NOUNSTEM|DECL1}, + {"noun2", NOUNSTEM|DECL2}, + {"noun3", NOUNSTEM|DECL3}, + {"noun4", NOUNSTEM|DECL4}, + {"noun5", NOUNSTEM|DECL5}, + {"prim_deriv", VERBSTEM|PRIM_CONJ}, + {"reg_deriv", VERBSTEM|REG_CONJ}, + {"adj1", ADJSTEM|DECL1|DECL2}, + {"adj2", ADJSTEM|DECL1|DECL2}, + {"indecl1", INDECL|NOUNSTEM|DECL1}, + {"indecl2", INDECL|NOUNSTEM|DECL2}, + {"indecl3", INDECL|NOUNSTEM|DECL3}, + {"pron3", INDECL|NOUNSTEM|DECL3}, + {"pron1", INDECL|NOUNSTEM|DECL1|DECL2}, + {NULL, 0} +}; + +MorphEntry alpheiosFlagNames[] = +{ + {"syll_augment", SYLL_AUGMENT}, + {"comp_only", COMP_ONLY}, + {"not_in_comp", NOT_IN_COMPOSITION}, + {"enclitic", ENCLITIC}, + {"proclitic", PROCLITIC}, + {"iterative", ITERATIVE}, + {"ant_acc", ANT_ACC}, + {"stem_acc", STEM_ACC}, + {"pen_acc", STEM_ACC}, + {"suff_acc", SUFF_ACC}, + {"ult_acc", SUFF_ACC}, + {"rec_acc", REC_ACC}, + {"needs_acc", NEEDS_ACCENT}, + {"contracted", CONTRACTED}, + {"uncontr_end", UNCONTR_END}, + {"uncontracted", UNCONTR_END}, + {"uncontr_stem", UNCONTR_STEM}, + {"pers_name", PERS_NAME}, + {"prevb_augment", PREVB_AUGMENT}, + {"double_augment", DOUBLE_AUGMENT}, + {"no_comp", NO_COMP}, + {"irreg_comp", IRREG_COMP}, + {"irreg_superl", IRREG_SUPERL}, + {"short_pen", SHORT_PEN}, + {"long_pen", LONG_PEN}, + {"r_e_i_alpha", R_E_I_ALPHA}, + {"unaugmented", UNAUGMENTED}, + {"apocope", APOCOPE}, + {"has_augment", HAS_AUGMENT}, + {"nu_movable", NU_MOVABLE}, + {"interv_s_to_h", INTERV_S_TO_H}, + {"poetic", POETIC}, + {"dissimilation", DISSIMILATION}, + {"metathesis", METATHESIS}, + {"elide_preverb", ELIDE_PREVERB}, + {"root_preverb", ROOT_PREVERB}, + {"diminutive", DIMINUTIVE}, + {"early", EARLY}, + {"late", LATE}, + {"rare", RARE}, + {"raw_preverb", RAW_PREVERB}, + {"short_subj", SHORT_SUBJ}, + {"unasp_preverb", UNASP_PREVERB}, + {"redupl", REDUPL}, + {"attic_redupl", ATTIC_REDUPL}, + {"is_deriv", IS_DERIV}, + {"no_redupl", NO_REDUPL}, + {"n_infix", N_INFIX}, + {"syncope", SYNCOPE}, + {"impersonal", IMPERSONAL}, + {"indeclform", INDECLFORM}, + {"needs_rbreath", NEEDS_RBREATH}, + {"no_circumflex", NO_CIRCUMFLEX}, + {"causal", CAUSAL}, + {"intrans", INTRANS}, + {"tmesis", TMESIS}, + {"raw_sonant", RAW_SONANT}, + {"prodelision", PRODELISION}, + {"frequentative", FREQUENTAT}, + {"desiderative", DESIDERATIVE}, + {"impersonal", IMPERSONAL}, + {"later", LATER}, + {"double_redupl", DOUBLE_REDUPL}, + {"pres_redupl", PRES_REDUPL}, + {"ends_in_digamma", ENDS_IN_DIGAMMA}, + {"geog_name", GEOG_NAME}, + {"doubled_cons", DOUBLED_CONS}, + {"iota_intens", IOTA_INTENS}, + {"sig_to_ci", SIG_TO_CI}, + {"short_eis", SHORT_EIS}, + {"pros_to_poti", PROS_TO_POTI}, + {"pros_to_proti", PROS_TO_PROTI}, + {"meta_to_peda", META_TO_PEDA}, + {"upo_to_upai", UPO_TO_UPAI}, + {"para_to_parai", PARA_TO_PARAI}, + {"uper_to_upeir", UPER_TO_UPEIR}, + {"en_to_eni", EN_TO_ENI}, + {"a_priv", A_PRIV}, + {"a_copul", A_COPUL}, + {"metrical_long", METRICAL_LONG}, + {NULL, 0} +}; + +/* dialects */ +MorphEntry alpheiosDialectNames[] = +{ + {"Attic", ATTIC}, + {"epic", EPIC}, + {"Homeric", HOMERIC}, + {"non-Homeric epic", NON_HOMERIC_EPIC}, + {"Doric", DORIC}, + {"Ionic", IONIC}, + {"Aeolic", AEOLIC}, + {"paradigm form", PARADIGM}, + // {"all", ALL_DIAL}, commented out to represent unrestricted dialect by no dialect flags + {"need_not_aug", HOMERIC}, + {"prose", PROSE}, +/* + {"eo_ou_dial", ATTIC}, + {"laconian", LACONIAN}, +*/ + {"Ionic/Homeric", IONIC|HOMERIC}, +/* + {"eo_eu_dial", IONIC|HOMERIC}, + {"no_contr_fut", HOMERIC|IONIC}, + {"a_no_contr", HOMERIC|DORIC}, + {"ee_ee_dial", (~ATTIC)}, + {"eo_eo_dial", (~ATTIC)}, + {"eou_eou_dial", (~ATTIC)}, + {"ew_ew_dial", (~ATTIC)}, + {"un_contr", (~ATTIC)}, +*/ + {NULL, 0} +}; + +/* Geographical Regions */ +MorphEntry alpheiosGeoNames[] = +{ + {"Phocis", PHOCIS}, + {"Locris", LOCRIS}, + {"Elis", ELIS}, + {"Laconia", LACONIA}, + {"Heraclea", HERACLEA}, + {"Megarid", MEGARID}, + {"Argolid", ARGOLID}, + {"Rhodes", RHODES}, + {"Cos", COS}, + {"Thera", THERA}, + {"Cyrene", CYRENE}, + {"Crete", CRETE}, + {"Arcadia", ARCADIA}, + {"Cyprus", CYPRUS}, + {"Boeotia", BOEOTIA}, + {NULL, 0} +}; + +const char *get_xml_lang(void) +{ + if (cur_lang() == GREEK) return "grc-x-beta"; + if (cur_lang() == LATIN) return "lat"; + if (cur_lang() == ITALIAN) return "it"; + return "und"; +} + +void xml_write_text(FILE *f, const char *s) +{ + if (!s) return; + for (; *s; ++s) + { + switch (*s) + { + case '&': fputs("&", f); break; + case '<': fputs("<", f); break; + case '>': fputs(">", f); break; + case '"': fputs(""", f); break; + default: fputc((unsigned char)*s, f); break; + } + } +} + +void alpheiosDumpWord(gk_word* gkword, PrntFlags prntflags, FILE* fout); +void alpheiosDumpAnalysis(gk_analysis* analysis, FILE* fout); +const char* alpheiosDumpPartOfSpeech(gk_analysis* analysis, + FILE* fout, + int nopart); +void alpheiosDumpMorphology(word_form a_wf, FILE* a_fout); +void alpheiosDumpString(const char* a_label, + const char* a_indent, + gk_string* a_string, + FILE* a_fout); +void alpheiosDumpFlag(const char* a_tag, + const MorphEntry* a_table, + long a_flags, + FILE* a_fout); +void alpheiosDumpFlags(const char* a_tag, + const MorphEntry* a_table, + long a_flags, + FILE* a_fout); +const char* alpheiosMorphLookup(const MorphEntry* a_table, long a_flags); +const char* alpheiosAttributeLookup(const AttributeEntry* a_table, + const char* a_name); +bool isEmptyForm(word_form); + +/* print out info on a word */ +int alpheiosPrintWord(gk_word* gkword, PrntFlags prntflags, FILE* fout) +{ + int nanals = totanal_of(gkword); + SortAnals(analysis_of(gkword), nanals); + + if (prntflags & PERSEUS_FORMAT) + { + alpheiosDumpWord(gkword, prntflags, fout); + return nanals; + } + + return 0; +} + +/* dump out info on a single word */ +void alpheiosDumpWord(gk_word* gkword, PrntFlags prntflags, FILE* fout) +{ + int nanals = totanal_of(gkword); + int goodanals = GoodAnals(gkword,0); + char curlem[MAXWORDSIZE]; + *curlem = '\0'; + + /* start word */ + if (nanals > 0) + { + fprintf(fout, "\n"); + fprintf(fout, "", get_xml_lang()); + xml_write_text(fout, rawword_of(gkword)); + fprintf(fout, "\n"); + } + + /* for each analysis */ + gk_analysis* nxtAnalysis = analysis_of(gkword); + gk_analysis* endAnalysis = nxtAnalysis + nanals; + for (; nxtAnalysis != endAnalysis; ++nxtAnalysis) + { + if (prntflags & SHOW_FULL_INFO) + { + fprintf(fout, "\n"); + alpheiosDumpString("self", " ", (gk_string*) nxtAnalysis, fout); + if (*(nxtAnalysis->st_dictform)) + { + fprintf(fout, " "); + xml_write_text(fout, nxtAnalysis->st_dictform); + fprintf(fout, "\n"); + } + if (*(nxtAnalysis->st_engform)) + { + fprintf(fout, " "); + xml_write_text(fout, nxtAnalysis->st_engform); + fprintf(fout, "\n"); + } + alpheiosDumpString("preverb", " ", &nxtAnalysis->gs_preverb, fout); + alpheiosDumpString("aug1", " ", &nxtAnalysis->gs_aug1, fout); + alpheiosDumpString("stem", " ", &nxtAnalysis->gs_stem, fout); + alpheiosDumpString("suffix", " ", &nxtAnalysis->gs_suffix, fout); + alpheiosDumpString("end", " ", &nxtAnalysis->gs_endstring, fout); + if (*(nxtAnalysis->st_rawprvb)) + { + fprintf(fout, " "); + xml_write_text(fout, nxtAnalysis->st_rawprvb); + fprintf(fout, "\n"); + } + if (*(nxtAnalysis->st_rawword)) + { + fprintf(fout, " "); + xml_write_text(fout, nxtAnalysis->st_rawword); + fprintf(fout, "\n"); + } + if (*(nxtAnalysis->st_workword)) + { + fprintf(fout, " "); + xml_write_text(fout, nxtAnalysis->st_workword); + fprintf(fout, "\n"); + } + if (*(nxtAnalysis->st_crasis)) + { + fprintf(fout, " "); + xml_write_text(fout, nxtAnalysis->st_crasis); + fprintf(fout, "\n"); + } + if (*(nxtAnalysis->z)) + { + fprintf(fout, " "); + xml_write_text(fout, nxtAnalysis->z); + fprintf(fout, "\n"); + } + fprintf(fout, "\n"); + } + + /* if there are no good analyses or this is a good one */ + /* (lemma does not contain hyphen) */ + if (!goodanals || !strchr(lemma_of(nxtAnalysis), '-')) + { + /* if this is a new lemma */ + if (strcmp(curlem, lemma_of(nxtAnalysis))) + { + /* terminate last entry */ + if (*curlem) + fprintf(fout, "\n"); + + strcpy(curlem, lemma_of(nxtAnalysis)); + + /* start new entry */ + fprintf(fout, "\n"); + + /* put out info on lemma */ + fprintf(fout, "\n"); + fprintf(fout, "", get_xml_lang()); + xml_write_text(fout, curlem); + fprintf(fout, "\n"); + + /* put out part of speech for first instance */ + /* as part of speech for lemma */ + /* (should we be doing this at all?) */ + const char* pofs = alpheiosDumpPartOfSpeech(nxtAnalysis, + fout, + 1); + + /* put out gender for noun */ + if (pofs && strcmp(pofs, "noun") == 0) + { + const char *gendName = alpheiosMorphLookup(alpheiosGenderNames, gender_of(forminfo_of(nxtAnalysis))); + if (gendName) + { + fprintf(fout, ""); + xml_write_text(fout, gendName); + fprintf(fout, "\n"); + } + } + + fprintf(fout, "\n"); + } + + alpheiosDumpAnalysis(nxtAnalysis, fout); + } + continue; + } + + /* terminate last entry */ + if (*curlem) + fprintf(fout, "\n"); + + /* terminate word */ + if (nanals > 0) + fprintf(fout, "
\n"); +} + +void alpheiosDumpAnalysis( +gk_analysis* analysis, +FILE* fout) +{ + /* + Note: The lookup tables for gender and case may + return a multi-valued string (with values separated by "/") + so we need to iterate and produce an inflection element for + each combination of gender and case. + */ + + /* calculate term (stem + suffix) to display */ + char stem[BUFSIZ]; + char suffix[BUFSIZ]; + char temp[BUFSIZ]; + int stemlen = 0; + int suffixlen = 0; + *stem = '\0'; + *suffix = '\0'; + + /* build stem from preverb, aug1, stem, with colons between pieces */ + const char* part = preverb_of(analysis); + if (part && *part) + { + strncat(stem, part, BUFSIZ - 1); + stemlen = strlen(stem); + } + /* aug1's containing > seem to indicate form changes already present */ + /* in other parts; those without represent a new piece */ + part = aug1_of(analysis); + if (part && *part && !strchr(part, '>')) + { + if (stemlen > 0) + strncat(stem, ":", BUFSIZ - stemlen - 1); + strncat(stem, part, BUFSIZ - strlen(stem) - 1); + stemlen = strlen(stem); + } + part = stem_of(analysis); + if (part && *part) + { + if (stemlen > 0) + strncat(stem, ":", BUFSIZ - stemlen - 1); + strncat(stem, part, BUFSIZ - strlen(stem) - 1); + stemlen = strlen(stem); + } + + /* build suffix from suffix and endstring */ + part = suffix_of(analysis); + if (part && *part) + { + strncat(suffix, part, BUFSIZ - 1); + suffixlen = strlen(suffix); + } + part = endstring_of(analysis); + if (part && *part) + { + if (suffixlen > 0) + strncat(suffix, ":", BUFSIZ - suffixlen - 1); + strncat(suffix, part, BUFSIZ - strlen(suffix) - 1); + suffixlen = strlen(suffix); + } + + /* get case(s), initialize ptrs to first case */ + word_form wf = forminfo_of(analysis); + const char* caseNames = alpheiosMorphLookup(alpheiosCaseNames, case_of(wf)); + const char* nextCase; + const char* endCase; + if (caseNames) + { + nextCase = caseNames; + endCase = strchr(nextCase, '/'); + if (!endCase) + endCase = nextCase + strlen(nextCase); + } + else + { + nextCase = endCase = ""; + } + + /* for each case (using empty string if none exist) */ + while (nextCase) + { + /* get gender(s), initialize ptrs to first gender */ + const char* genderNames = alpheiosMorphLookup(alpheiosGenderNames, + gender_of(wf)); + const char* nextGender; + const char* endGender; + if (genderNames) + { + nextGender = genderNames; + endGender = strchr(nextGender, '/'); + if (!endGender) + endGender = nextGender + strlen(nextGender); + } + else + { + nextGender = endGender = ""; + } + + /* for each gender (using empty string if none exist) */ + while (nextGender) + { + fprintf(fout, "\n"); + + /* put out term */ + fprintf(fout, "", get_xml_lang()); + if (stemlen > 0) + { + fprintf(fout, ""); + xml_write_text(fout, stem); + fprintf(fout, ""); + } + if (suffixlen > 0) + { + fprintf(fout, ""); + xml_write_text(fout, suffix); + fprintf(fout, ""); + } + fprintf(fout, "\n"); + + /* put out part of speech */ + alpheiosDumpPartOfSpeech(analysis, fout, 0); + + /* dump case and gender (if any) and other morphological info */ + int caseLen = endCase - nextCase; + int genderLen = endGender - nextGender; + if (caseLen) + { + strncpy(temp, nextCase, caseLen); + temp[caseLen] = '\0'; + const char *ord = alpheiosAttributeLookup(alpheiosCaseOrder, temp); + fprintf(fout, ""); + xml_write_text(fout, temp); + fprintf(fout, "\n"); + } + if (genderLen) + { + strncpy(temp, nextGender, genderLen); + temp[genderLen] = '\0'; + fprintf(fout, ""); + xml_write_text(fout, temp); + fprintf(fout, "\n"); + } + alpheiosDumpMorphology(wf, fout); + + /* other info: geographic region, dialect, types, etc. */ + alpheiosDumpFlags("geo", + alpheiosGeoNames, + geogregion_of(analysis), + fout); + alpheiosDumpFlags("dial", + alpheiosDialectNames, + dialect_of(analysis), + fout); + const char* val = NameOfStemtype(stemtype_of(analysis)); + if (val && *val) + { + fprintf(fout, ""); + xml_write_text(fout, val); + fprintf(fout, "\n"); + } + + val = NameOfDerivtype(derivtype_of(analysis)); + if (val && *val) + { + fprintf(fout, ""); + xml_write_text(fout, val); + fprintf(fout, "\n"); + } + + *temp = '\0'; + MorphNames(morphflags_of(analysis), temp, " ", 1); + if (*temp) + { + fprintf(fout, ""); + xml_write_text(fout, temp); + fprintf(fout, "\n"); + } + + fprintf(fout, "\n"); + + /* advance to next gender */ + if (*endGender == '/') + { + nextGender = endGender + 1; + endGender = strchr(nextGender, '/'); + if (!endGender) + endGender = nextGender + strlen(nextGender); + } + else + { + nextGender = NULL; + } + } + + /* advance to next case */ + if (*endCase == '/') + { + nextCase = endCase + 1; + endCase = strchr(nextCase, '/'); + if (!endCase) + endCase = nextCase + strlen(nextCase); + } + else + { + nextCase = NULL; + } + } +} + +/* dump part of speech */ +const char* alpheiosDumpPartOfSpeech( +gk_analysis* analysis, +FILE* fout, +int nopart) +{ + /* check various part of speech forms */ + const char* pofs = NULL; + if (Is_participle(analysis)) + { + /* if not looking for participles, say it's a verb */ + pofs = (nopart ? "verb" : "verb participle"); + } + else if (Is_nounform(analysis)) + { + pofs = "noun"; + } + else if (Is_adjform(analysis)) + { + pofs = "adjective"; + } + else if (Is_verbform(analysis)) + { + pofs = "verb"; + } + + /* check stemtype and adjust part of speech */ + const char* stemType = NameOfStemtype(stemtype_of(analysis)); + if (stemType && *stemType) + { + if (strstr(stemType, "pron") || + !strcmp(stemType, "indef") || + !strcmp(stemType, "relative") || + !strcmp(stemType, "demonstr") || + !strcmp(stemType, "art_adj")) + pofs = "pronoun"; + else if (strstr(stemType, "_adj")) + pofs = "adjective"; + else if (!strcmp(stemType, "adverb") || + !strcmp(stemType, "article") || + !strcmp(stemType, "particle") || + !strcmp(stemType, "numeral")) + pofs = stemType; + else if (!strcmp(stemType, "conj")) + pofs = "conjunction"; + else if (!strcmp(stemType, "exclam")) + pofs = "exclamation"; + else if (!strcmp(stemType, "indecl")) + pofs = "irregular"; + else if (!strcmp(stemType, "prep")) + pofs = "preposition"; + } + + /* if part of speech found */ + if (pofs) + { + const char *ord = alpheiosAttributeLookup(alpheiosPofsOrder, pofs); + fprintf(fout, ""); + xml_write_text(fout, pofs); + fprintf(fout, "\n"); + + /* if noun or adjective, look for declension */ + if ((strcmp(pofs, "noun") == 0) || + (strcmp(pofs, "adjective") == 0)) + { + alpheiosDumpFlag("decl", + alpheiosDeclNames, + stemtype_of(analysis) & DECL_MASK, + fout); + } + } + + return pofs; +} + +/* dump morphological values (except case and gender) */ +void alpheiosDumpMorphology(word_form a_wf, FILE* a_fout) +{ + alpheiosDumpFlag("comp", alpheiosComparisonNames, degree_of(a_wf), a_fout); + alpheiosDumpFlag("mood", alpheiosMoodNames, mood_of(a_wf), a_fout); + alpheiosDumpFlag("num", alpheiosNumberNames, number_of(a_wf), a_fout); + alpheiosDumpFlag("pers", alpheiosPersonNames, person_of(a_wf), a_fout); + alpheiosDumpFlag("tense", alpheiosTenseNames, tense_of(a_wf), a_fout); + alpheiosDumpFlag("voice", alpheiosVoiceNames, voice_of(a_wf), a_fout); +} + +void alpheiosDumpString( +const char* a_label, +const char* a_indent, +gk_string* a_string, +FILE* a_fout) +{ + /* if no content, don't do anything */ + int i; + for (i = 0; i < MORPHFLAG_BYTES; ++i) + { + if (a_string->gs_morphflags[i]) + break; + } + if ((i == MORPHFLAG_BYTES) && + isEmptyForm(a_string->gs_forminfo) && + !a_string->gs_steminfo && + !a_string->gs_derivtype && + !a_string->gs_dialect && + !a_string->gs_geogregion && + !*(a_string->st_domains) && + !*(a_string->gs_gkstring)) + { + return; + } + + fprintf(a_fout, "%s<%s>\n", a_indent, a_label); + if (!isEmptyForm(a_string->gs_forminfo)) + { + unsigned formval = 0; + memcpy(&formval, &a_string->gs_forminfo, sizeof formval); + fprintf(a_fout, "%s
0%o
\n", a_indent, formval); + if (a_string->gs_forminfo.f_voice) + { + fprintf(a_fout, "%s 0%o\n", + a_indent, + a_string->gs_forminfo.f_voice); + } + if (a_string->gs_forminfo.f_mood) + { + fprintf(a_fout, "%s 0%o\n", + a_indent, + a_string->gs_forminfo.f_mood); + } + if (a_string->gs_forminfo.f_tense) + { + fprintf(a_fout, "%s 0%o\n", + a_indent, + a_string->gs_forminfo.f_tense); + } + if (a_string->gs_forminfo.f_person) + { + fprintf(a_fout, "%s 0%o\n", + a_indent, + a_string->gs_forminfo.f_person); + } + if (a_string->gs_forminfo.f_number) + { + fprintf(a_fout, "%s 0%o\n", + a_indent, + a_string->gs_forminfo.f_number); + } + if (a_string->gs_forminfo.f_case) + { + fprintf(a_fout, "%s 0%o\n", + a_indent, + a_string->gs_forminfo.f_case); + } + if (a_string->gs_forminfo.f_degree) + { + fprintf(a_fout, "%s 0%o\n", + a_indent, + a_string->gs_forminfo.f_degree); + } + if (a_string->gs_forminfo.f_gender) + { + fprintf(a_fout, "%s 0%o\n", + a_indent, + a_string->gs_forminfo.f_gender); + } + } + if (a_string->gs_steminfo) + { + const char *name = NameOfStemtype(stemtype_of(a_string)); + fprintf(a_fout, "%s 0%o ", + a_indent, + a_string->gs_steminfo); + xml_write_text(a_fout, name ? name : ""); + fprintf(a_fout, "\n"); + } + if (a_string->gs_derivtype) + { + fprintf(a_fout, "%s 0%o\n", + a_indent, + a_string->gs_derivtype); + } + if (a_string->gs_dialect) + { + fprintf(a_fout, "%s 0%o\n", + a_indent, + a_string->gs_dialect); + } + if (a_string->gs_geogregion) + { + fprintf(a_fout, "%s 0%o\n", + a_indent, + a_string->gs_geogregion); + } + if (i < MORPHFLAG_BYTES) + { + fprintf(a_fout, "%s ", a_indent); + for (i = 0; i < MORPHFLAG_BYTES; ++i) + { + if (i > 0) + fprintf(a_fout, ","); + fprintf(a_fout, "%d", a_string->gs_morphflags[i]); + } + fprintf(a_fout, "\n"); + } + if (*(a_string->st_domains)) + { + fprintf(a_fout, "%s ", a_indent); + xml_write_text(a_fout, a_string->st_domains); + fprintf(a_fout, "\n"); + } + if (*(a_string->gs_gkstring)) + { + fprintf(a_fout, "%s ", a_indent); + xml_write_text(a_fout, a_string->gs_gkstring); + fprintf(a_fout, "\n"); + } + fprintf(a_fout, "%s\n", a_indent, a_label); +} + +void alpheiosDumpFlag( +const char* a_tag, +const MorphEntry* a_table, +long a_flags, +FILE* a_fout) +{ + const char* name = alpheiosMorphLookup(a_table, a_flags); + if (name && *name) + { + fprintf(a_fout, "<%s>", a_tag); + xml_write_text(a_fout, name); + fprintf(a_fout, "\n", a_tag); + } +} + +void alpheiosDumpFlags( +const char* a_tag, +const MorphEntry* a_table, +long a_flags, +FILE* a_fout) +{ + if (!a_flags || !a_table) + return; + + char temp[BUFSIZ]; + *temp = '\0'; + + const MorphEntry* nextEntry; + for (nextEntry = a_table; nextEntry->d_flags != 0; ++nextEntry) + { + /* if this entry is contained in flags */ + if ((nextEntry->d_flags & a_flags) == nextEntry->d_flags) + { + /* mask out used flags and add to output */ + a_flags &= ~(nextEntry->d_flags); + if (*temp) + strcat(temp, " "); + strncat(temp, nextEntry->d_name, BUFSIZ - strlen(temp) - 1); + } + } + + if (*temp) + { + fprintf(a_fout, "<%s>", a_tag); + xml_write_text(a_fout, temp); + fprintf(a_fout, "\n", a_tag); + } +} + +const char* alpheiosMorphLookup( +const MorphEntry* a_table, +long a_flags) +{ + if (!a_flags || !a_table) + return NULL; + + const MorphEntry* nextEntry; + for (nextEntry = a_table; nextEntry->d_flags != 0; ++nextEntry) + { + if ((nextEntry->d_flags & a_flags) == a_flags) + return nextEntry->d_name; + } + + return NULL; +} + +const char* alpheiosAttributeLookup( +const AttributeEntry* a_table, +const char* a_name) +{ + if (!a_name || !a_table) + return NULL; + + const AttributeEntry* nextEntry; + for (nextEntry = a_table; nextEntry->d_name != NULL; ++nextEntry) + { + if (strcmp(nextEntry->d_name, a_name) == 0) + break; + } + + return nextEntry->d_value; +} + +bool isEmptyForm(word_form a_wf) +{ + return !a_wf.f_voice && + !a_wf.f_mood && + !a_wf.f_tense && + !a_wf.f_person && + !a_wf.f_number && + !a_wf.f_case && + !a_wf.f_degree && + !a_wf.f_gender; +} \ No newline at end of file diff --git a/src/anal/prntalph.h b/src/anal/prntalph.h new file mode 100644 index 00000000..a5de5c5e --- /dev/null +++ b/src/anal/prntalph.h @@ -0,0 +1,11 @@ +#ifndef PRNTALPH_H +#define PRNTALPH_H + +#include +#include + +int alpheiosPrintWord(gk_word *gkword, PrntFlags prntflags, FILE *fout); +void xml_write_text(FILE *f, const char *s); +const char *get_xml_lang(void); + +#endif /* PRNTALPH_H */ \ No newline at end of file diff --git a/src/anal/prntanal.proto.h b/src/anal/prntanal.proto.h index 8e81b78d..7abed7e5 100755 --- a/src/anal/prntanal.proto.h +++ b/src/anal/prntanal.proto.h @@ -3,6 +3,7 @@ /* prntanal.c */ +int GoodAnals(gk_word *, int); void PrntAnalyses(gk_word *, PrntFlags, FILE *); char *anal_buf(void); void PrntOneAnalysis(gk_analysis *, PrntFlags, FILE *); diff --git a/src/includes/stemtype.h b/src/includes/stemtype.h index fa840d89..7b0b9a2a 100755 --- a/src/includes/stemtype.h +++ b/src/includes/stemtype.h @@ -21,6 +21,7 @@ typedef unsigned int Stemtype; #define VERBSTEM (0100) */ +#define DECL_MASK (DECL1 | DECL2 | DECL3 | DECL4 | DECL5) #define DECL1 (0100) #define DECL2 (0200) #define DECL3 (0400) diff --git a/tests/greek_words.txt b/tests/greek_words.txt index c567b18b..eb5fe876 100644 --- a/tests/greek_words.txt +++ b/tests/greek_words.txt @@ -263,3 +263,4 @@ xyzzy lo/gos42 'qa/non cu/ndesmos +e)ti/qh diff --git a/tests/run_xml_tests.sh b/tests/run_xml_tests.sh new file mode 100644 index 00000000..467ed765 --- /dev/null +++ b/tests/run_xml_tests.sh @@ -0,0 +1,506 @@ +#!/bin/bash +# +# XML output test suite for morpheus +# +# This is intentionally INDEPENDENT of tests/run_tests.sh and is NOT wired +# into the Makefile. +# Run it by hand: +# +# bash tests/run_xml_tests.sh --update (re)generate golden baselines +# bash tests/run_xml_tests.sh run tests against saved baselines +# +# Fixtures reused from the existing (tracked) test suite -- not modified: +# tests/greek_words.txt tests/latin_words.txt tests/greek_probe.txt +# tests/greek_probe_upper.txt tests/latin_probe_upper.txt +# tests/greek_probe_noaccent.txt +# +# Golden baselines this script creates all end in "_expected.txt", which is +# already covered by the existing .gitignore rule (tests/*_expected.txt) -- +# no .gitignore changes needed. +# +# Conventions (matching tests/run_tests.sh): +# - A golden-baseline text diff is reported but does NOT fail the run. +# - A crash, a malformed-XML document, or a violated parity contract +# DOES fail the run. +# - Known, already-diagnosed defects get a dedicated XFAIL test so that +# fixing them shows up as a visible XPASS instead of silently doing +# nothing. +# +set -u + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +MORPHEUS="$PROJECT_DIR/bin/morpheus" +CRUNCHER="$PROJECT_DIR/bin/cruncher" +COMPARATOR="$SCRIPT_DIR/xml_vs_nl.py" +PY=python3 + +UPDATE=0 +[ "${1:-}" = "--update" ] && UPDATE=1 + +export MORPHLIB="$PROJECT_DIR/stemlib" +export PATH="$PROJECT_DIR/bin:$PATH" + +PASS=0; FAIL=0; WARN=0; SKIP=0; XFAIL=0; XPASS=0 + +pass() { echo "PASS: $*"; PASS=$((PASS+1)); } +fail() { echo "FAIL: $*"; FAIL=$((FAIL+1)); } +warn() { echo "WARN: $*"; WARN=$((WARN+1)); } +skip() { echo "SKIP: $*"; SKIP=$((SKIP+1)); } +xfail() { echo "XFAIL: $* (known issue; expected to fail until fixed)"; XFAIL=$((XFAIL+1)); } +xpass() { echo "XPASS: $* -- unexpectedly passing now; remove the XFAIL marker for this test"; XPASS=$((XPASS+1)); } + +hr() { echo; echo "--- $* ---"; } + +print_summary() { + echo + echo "================================================================" + echo " PASS=$PASS FAIL=$FAIL WARN=$WARN SKIP=$SKIP XFAIL=$XFAIL XPASS=$XPASS" + echo "================================================================" +} + +finish() { + print_summary + [ "$FAIL" -gt 0 ] && exit 1 + exit 0 +} + +# --------------------------------------------------------------------------- +# Well-formedness helper (xmllint if present, else python3 stdlib, else SKIP) +# --------------------------------------------------------------------------- +have_wf_checker() { + command -v xmllint >/dev/null 2>&1 || command -v "$PY" >/dev/null 2>&1 +} + +check_wellformed() { + # returns 0 if well-formed, 1 otherwise. Caller must call have_wf_checker first. + local f="$1" + if command -v xmllint >/dev/null 2>&1; then + xmllint --noout "$f" >/dev/null 2>&1 + return $? + fi + "$PY" -c ' +import sys, xml.etree.ElementTree as ET +try: + ET.parse(sys.argv[1]) +except Exception: + sys.exit(1) +' "$f" + return $? +} + +wf_check() { + local label="$1" file="$2" + if ! have_wf_checker; then + skip "$label wellformed (no xmllint or python3 available)" + return + fi + if check_wellformed "$file"; then + pass "$label wellformed" + else + fail "$label wellformed" + fi +} + +# --------------------------------------------------------------------------- +# 1. Build/link smoke tests (5.4) +# --------------------------------------------------------------------------- +hr "Build/link smoke tests" + +if [ ! -x "$MORPHEUS" ]; then + fail "bin/morpheus exists and is executable (build it with 'make')" + echo + echo "bin/morpheus is not available; skipping the rest of the XML test suite." + finish +fi +pass "bin/morpheus exists and is executable" + +"$MORPHEUS" /tmp/morpheus_smoke_$$ 2>/dev/null +rc=$? +rm -f /tmp/morpheus_smoke_$$ +if [ "$rc" -eq 2 ]; then + pass "morpheus and several tests will fail." + fi +done + +# --------------------------------------------------------------------------- +# 2. Golden baseline tests (5.1) +# --------------------------------------------------------------------------- +hr "Golden baseline tests" + +run_morpheus_golden() { + local label="$1" flags="$2" input="$3" baseline="$4" + local out err + out=$(mktemp); err=$(mktemp) + "$MORPHEUS" $flags < "$input" > "$out" 2>"$err" + local rc=$? + + if [ "$rc" -ge 128 ]; then + fail "$label (morpheus crashed / killed, exit=$rc)" + sed 's/^/ stderr: /' "$err" + rm -f "$out" "$err" + return + fi + rm -f "$err" + + if [ "$UPDATE" -eq 1 ]; then + cp "$out" "$baseline" + echo "updated: $baseline" + rm -f "$out" + return + fi + + if [ ! -f "$baseline" ]; then + skip "$label (no baseline; run 'bash $0 --update')" + elif diff -u "$baseline" "$out" > /tmp/golden_diff_$$ 2>&1; then + pass "$label matches baseline" + else + warn "$label differs from baseline (diff below; not a hard failure)" + head -40 /tmp/golden_diff_$$ + rm -f /tmp/golden_diff_$$ + fi + + wf_check "$label" "$out" + rm -f "$out" +} + +GP="$SCRIPT_DIR/greek_probe.txt" + +run_morpheus_golden "Greek words (default)" "" "$SCRIPT_DIR/greek_words.txt" "$SCRIPT_DIR/greek_xml_expected.txt" +run_morpheus_golden "Latin words (-L)" "-L" "$SCRIPT_DIR/latin_words.txt" "$SCRIPT_DIR/latin_xml_expected.txt" +run_morpheus_golden "Greek -i (dump_analysis)" "-i" "$GP" "$SCRIPT_DIR/greek_xml_probe_i_expected.txt" +run_morpheus_golden "Greek -S (upper case)" "-S" "$SCRIPT_DIR/greek_probe_upper.txt" "$SCRIPT_DIR/greek_xml_probe_S_expected.txt" +run_morpheus_golden "Latin -LS (upper case)" "-LS" "$SCRIPT_DIR/latin_probe_upper.txt" "$SCRIPT_DIR/latin_xml_probe_S_expected.txt" +run_morpheus_golden "Greek -n (no accent)" "-n" "$SCRIPT_DIR/greek_probe_noaccent.txt" "$SCRIPT_DIR/greek_xml_probe_n_expected.txt" + +if [ "$UPDATE" -eq 1 ]; then + finish +fi + +# --------------------------------------------------------------------------- +# 3. Regression tests for specific bugs of the original implementation for Alpheios +# --------------------------------------------------------------------------- +hr "Flag parsing and argv-handling tests" + +# ARGS has "...n:" so bare -n incorrectly swallows the next argv token +# as its option-argument. Detect this without needing a real argv word: feed +# -n a word via argv with /dev/null stdin. If -n eats it, the program falls +# straight through to empty stdin and exits 2 ("No words provided"). If -n +# is fixed to take no argument, the word is processed from argv and it exits 0. +test_flag_n_no_arg() { + "$MORPHEUS" -n 'lo/gos' /tmp/flag_n_$$ 2>/dev/null + local rc=$? + rm -f /tmp/flag_n_$$ + if [ "$rc" -eq 0 ]; then + pass "flag-n-arg: -n does not consume the following argv word as its optarg" + else + fail "flag-n-arg: -n incorrectly requires an argument (ARGS has 'n:'); exit=$rc, expected 0" + fi +} +test_flag_n_no_arg + +# P8: after consuming argv words, morpheus must not fall through and +# block on stdin. Use a pipe that is held open (never closed, never written +# to) so a real attempt to read stdin would hang until `timeout` kills it. +test_p8() { + if ! command -v timeout >/dev/null 2>&1; then + skip "P8: morpheus does not hang after argv words (timeout(1) unavailable)" + return + fi + local out + out=$(mktemp) + timeout 5 "$MORPHEUS" 'lo/gos' < <(sleep 30) > "$out" 2>/dev/null + local rc=$? + if [ "$rc" -eq 124 ]; then + fail "P8: morpheus hung reading stdin after processing an argv word (timed out)" + elif [ "$rc" -ne 0 ]; then + fail "P8: unexpected exit code $rc for argv-word invocation" + else + local nwords + nwords=$(grep -c '' "$out") + if [ "$nwords" -eq 1 ]; then + pass "P8: morpheus with an argv word exits promptly with exactly one " + else + fail "P8: expected exactly one element, got $nwords" + fi + fi + rm -f "$out" +} +test_p8 + +# argv words skip the same cleaning (trimdigit) that stdin words get. +test_argv_cleanup() { + local w='lo/gos2' + local out_argv out_stdin + out_argv=$(mktemp); out_stdin=$(mktemp) + "$MORPHEUS" "$w" "$out_argv" 2>/dev/null + printf '%s\n' "$w" | "$MORPHEUS" >"$out_stdin" 2>/dev/null + if grep -q '' "$out_stdin"; then + if grep -q '' "$out_argv"; then + pass "argv-cleanup: argv word '$w' is cleaned the same way as the stdin word" + else + fail "argv-cleanup: argv word '$w' not cleaned like the stdin word (stdin=known, argv=unknown)" + fi + else + skip "argv-cleanup check for '$w': the stdin path itself did not recognize it" + fi + rm -f "$out_argv" "$out_stdin" +} +test_argv_cleanup + +# --------------------------------------------------------------------------- +# 4. Positive parity tests (P1-P5 via comparator, plus P6/P7 directly) +# --------------------------------------------------------------------------- +hr "Positive parity tests (P1-P5, P6, P7)" + +run_parity_suite() { + local label="$1" wordlist="$2" cflags="$3" mflags="$4" + if [ ! -x "$CRUNCHER" ]; then + skip "$label (bin/cruncher missing)" + return + fi + if ! command -v "$PY" >/dev/null 2>&1; then + skip "$label (python3 unavailable)" + return + fi + local out + out=$(mktemp) + "$PY" "$COMPARATOR" "$wordlist" \ + --cruncher "$CRUNCHER" --cruncher-flags="$cflags" \ + --morpheus "$MORPHEUS" --morpheus-flags="$mflags" \ + > "$out" + local rc=$? + cat "$out" + PASS=$((PASS + $(grep -c '^PASS:' "$out"))) + FAIL=$((FAIL + $(grep -c '^FAIL:' "$out"))) + WARN=$((WARN + $(grep -c '^WARN:' "$out"))) + if [ "$rc" -ne 0 ]; then + echo "--- $label: comparator reported hard parity failures (see above) ---" + else + echo "--- $label: all hard parity checks (P1-P3) passed ---" + fi + rm -f "$out" +} + +run_parity_suite "Greek parity vs cruncher" "$SCRIPT_DIR/greek_words.txt" "" "" +run_parity_suite "Latin parity vs cruncher" "$SCRIPT_DIR/latin_words.txt" "-L" "-L" + +# P6: determinism +test_p6() { + local out1 out2 + out1=$(mktemp); out2=$(mktemp) + "$MORPHEUS" < "$SCRIPT_DIR/greek_words.txt" > "$out1" 2>/dev/null + "$MORPHEUS" < "$SCRIPT_DIR/greek_words.txt" > "$out2" 2>/dev/null + if cmp -s "$out1" "$out2"; then + pass "P6: morpheus output is byte-identical across repeated runs" + else + fail "P6: morpheus output differs between repeated runs (non-determinism)" + fi + rm -f "$out1" "$out2" +} +test_p6 + +# P7: exit codes + well-formedness of the empty-input error document +test_p7() { + "$MORPHEUS" /dev/null 2>/dev/null + local rc_empty=$? + [ "$rc_empty" -eq 2 ] && pass "P7: empty stdin exits 2" \ + || fail "P7: empty stdin expected exit 2, got $rc_empty" + + printf 'lo/gos\n' | "$MORPHEUS" >/dev/null 2>/dev/null + local rc_ok=$? + [ "$rc_ok" -eq 0 ] && pass "P7: normal input exits 0" \ + || fail "P7: normal input expected exit 0, got $rc_ok" + + if ! have_wf_checker; then + skip "P7: empty-input error document wellformed (no checker)" + return + fi + local out + out=$(mktemp) + "$MORPHEUS" < /dev/null > "$out" 2>/dev/null + if check_wellformed "$out"; then + pass "P7: empty-input error document is well-formed XML" + else + fail "P7: empty-input error document is well-formed XML" + fi + rm -f "$out" +} +test_p7 + +# --------------------------------------------------------------------------- +# 5. Negative tests -- intentional divergences from cruncher (N1-N8) +# --------------------------------------------------------------------------- +hr "Negative tests (intentional divergences from cruncher)" + +# N1: vocabulary (long-form case/number, not abbreviations) +test_n1() { + local out + out=$(mktemp) + printf 'lo/gos\n' | "$MORPHEUS" >"$out" 2>/dev/null + if grep -q 'nominative' "$out" \ + && grep -q 'singular' "$out" \ + && ! grep -q '>nom<' "$out"; then + pass "N1: XML uses long-form vocabulary (nominative/singular), not cruncher's abbreviations" + else + fail "N1: expected long-form case/number vocabulary for lo/gos" + fi + wf_check "N1 (lo/gos)" "$out" + rm -f "$out" +} +test_n1 + +# N2: pofs refinement (article, sentinel order=0) +test_n2() { + local out + out=$(mktemp) + printf 'o(\n' | "$MORPHEUS" >"$out" 2>/dev/null + if grep -q 'article' "$out"; then + pass "N2: XML refines article pofs with sentinel order=0 (cruncher just says 'I')" + else + fail "N2: expected article for o(" + fi + wf_check "N2 (o()" "$out" + rm -f "$out" +} +test_n2 + +# N3: runtime escaping (raw & in , replacing alpheios's pre-escaped table) +test_n3() { + local out + out=$(mktemp) + printf 'bonus\n' | "$MORPHEUS" -L >"$out" 2>/dev/null + if grep -q '1st & 2nd' "$out"; then + pass "N3: runtime XML escaping produces literal & in " + else + fail "N3: expected literal '&' escaping in for bonus" + fi + wf_check "N3 (bonus)" "$out" + rm -f "$out" +} +test_n3 + +# N4: unknown word handling (inline , both languages) +test_n4() { + local out + out=$(mktemp) + printf 'xyzzy\n' | "$MORPHEUS" >"$out" 2>/dev/null + if grep -q 'xyzzy' "$out"; then + pass "N4: unknown Greek word emitted inline as inside " + else + fail "N4: expected inline for xyzzy" + fi + wf_check "N4 (xyzzy, greek)" "$out" + rm -f "$out" + + out=$(mktemp) + printf 'xyzzy\n' | "$MORPHEUS" -L >"$out" 2>/dev/null + if grep -q 'xyzzy' "$out"; then + pass "N4b: unknown Latin word emitted inline as " + else + fail "N4b: expected inline for xyzzy" + fi + wf_check "N4b (xyzzy, latin)" "$out" + rm -f "$out" +} +test_n4 + +# N5: dialect table change (removal of {"all", ALL_DIAL} unblocks 'prose' etc.) +test_n5() { + if [ ! -f "$SCRIPT_DIR/greek_xml_expected.txt" ]; then + skip "N5: dialect table PROSE check (run --update first to create a baseline)" + return + fi + if grep -q '[^<]*[Pp]rose' "$SCRIPT_DIR/greek_xml_expected.txt"; then + pass "N5: contains 'prose' somewhere in the Greek fixture output" + else + warn "N5: no 'prose' dialect observed anywhere in the Greek fixture; add/confirm a word that exercises it" + fi +} +test_n5 + +# N6: NULL-safety (no "(null)" leaking into gender/pofs output) +test_n6() { + local out + out=$(mktemp) + "$MORPHEUS" < "$SCRIPT_DIR/greek_words.txt" >> "$out" 2>/dev/null + "$MORPHEUS" -L < "$SCRIPT_DIR/latin_words.txt" >> "$out" 2>/dev/null + if grep -qi '(null)' "$out"; then + fail "N6: literal '(null)' text found in XML output (NULL-safety regression)" + else + pass "N6: no '(null)' text in XML output (gender/pofs NULL guards hold)" + fi + rm -f "$out" +} +test_n6 + +# N7: comment handling +test_n7() { + if ! have_wf_checker; then + skip "N7: comment handling wellformedness checks (no checker)" + return + fi + + local out + out=$(mktemp) + printf '#hello\nlo/gos\n' | "$MORPHEUS" >"$out" 2>/dev/null + local nopen nclose nword + nopen=$(grep -c '' "$out") + nclose=$(grep -c '' "$out") + nword=$(grep -c '' "$out") + if [ "$nopen" -eq 1 ] && [ "$nclose" -eq 1 ] && [ "$nword" -eq 1 ] \ + && check_wellformed "$out"; then + pass "N7a: comment-then-word yields a single well-formed document" + else + fail "N7a: comment-then-word yields a single well-formed document (open=$nopen close=$nclose word=$nword)" + fi + rm -f "$out" + + out=$(mktemp) + printf '#just a comment\n' | "$MORPHEUS" >"$out" 2>/dev/null + if check_wellformed "$out"; then + pass "N7b: comment-only input produces a well-formed document" + else + fail "N7b: comment-only input produces a well-formed document" + fi + rm -f "$out" + + out=$(mktemp) + printf '#a--b\n#a-->b\nlo/gos\n' | "$MORPHEUS" >"$out" 2>/dev/null + if check_wellformed "$out"; then + pass "N7c: pathological comment content (--, -->) stays well-formed" + else + fail "N7c: pathological comment content (--, -->) stays well-formed" + fi + rm -f "$out" +} +test_n7 + +# N8: first-token-only semantics (cruncher parity, diverges from alpheios original) +test_n8() { + local out + out=$(mktemp) + printf 'lo/gos timh/\n' | "$MORPHEUS" >"$out" 2>/dev/null + local nform + nform=$(grep -c '
lo/gos<' "$out" && ! grep -q '>timh/<' "$out"; then + pass "N8: multi-token line processes only the first token (cruncher parity)" + else + fail "N8: expected exactly one for 'lo/gos' only (got $nform)" + fi + wf_check "N8 (lo/gos timh/)" "$out" + rm -f "$out" +} +test_n8 + +# --------------------------------------------------------------------------- +finish diff --git a/tests/xml_vs_nl.py b/tests/xml_vs_nl.py new file mode 100644 index 00000000..8b1fc974 --- /dev/null +++ b/tests/xml_vs_nl.py @@ -0,0 +1,380 @@ +#!/usr/bin/env python3 +""" +xml_vs_nl.py -- parity comparator between cruncher's Perseus-format +output and morpheus's XML output. + +Implements a best-effort version of invariants P1-P5 from the morpheus XML +test plan: + + P1 known/unknown-word partition parity + P2 lemma set + first-occurrence order parity + P3 stemtype set parity (whole word, deduped) + P4 feature-value coverage under a normalization map (soft: WARN) + P5 case/gender cartesian-expansion sanity (soft: WARN) + +Design notes +------------ +Both binaries are invoked *once per word*, feeding the single word on +stdin. This sidesteps the fact that cruncher prints *nothing at all* to +stdout for an unknown word (so there is no way to realign a multi-word +batch run against the input list). It is slower than a single batch +call, but for the small curated fixture lists used by the test suite this +is a non-issue, and it is far more robust than trying to re-derive block +boundaries from cruncher's flat stream. + +Exit status is 1 if any *hard* check (P1-P3) failed for any word, 0 +otherwise. P4/P5 mismatches are printed as WARN and never affect the +exit status, because the mapping between cruncher's single-line combined +notation and morpheus's cartesian-expanded elements is not fully +specified anywhere and this script's normalization is necessarily +approximate. + +stdlib only (per the project's testing conventions). +""" + +import argparse +import re +import subprocess +import sys +import xml.etree.ElementTree as ET + +# -------------------------------------------------------------------------- +# cruncher morphkeys.h abbreviation -> long-form vocabulary used by morpheus +# -------------------------------------------------------------------------- +NORM = { + "nom": "nominative", + "gen": "genitive", + "dat": "dative", + "acc": "accusative", + "voc": "vocative", + "abl": "ablative", + "masc": "masculine", + "fem": "feminine", + "neut": "neuter", + "sg": "singular", + "pl": "plural", + "dual": "dual", + "1st": "1st", + "2nd": "2nd", + "3rd": "3rd", + "pres": "present", + "imperf": "imperfect", + "fut": "future", + "aor": "aorist", + "perf": "perfect", + "plup": "pluperfect", + "futperf": "future perfect", + "ind": "indicative", + "subj": "subjunctive", + "opt": "optative", + "imperat": "imperative", + "inf": "infinitive", + "part": "participle", + "gerundive": "gerundive", + "supine": "supine", + "act": "active", + "mid": "middle", + "pass": "passive", + "mp": "mediopassive", + "dep": "deponent", + "attic": "Attic", + "ionic": "Ionic", + "doric": "Doric", + "aeolic": "Aeolic", + "epic": "epic", + "homeric": "Homeric", + "parad_form": "paradigm form", + "comp": "comparative", + "comparative": "comparative", + "superl": "superlative", + "superlative": "superlative", +} + +CASE_ABBRS = ("nom", "gen", "dat", "acc", "voc", "abl") +GEND_ABBRS = ("masc", "fem", "neut") + +NL_RE = re.compile(r"(.*?)", re.S) + + +def norm_tok(tok): + return NORM.get(tok, tok) + + +def run_cmd(binary, flags, word, timeout=15): + args = [binary] + list(flags) + try: + p = subprocess.run( + args, + check=False, + capture_output=True, + input=word + "\n", + text=True, + timeout=timeout, + ) + return p.stdout, p.returncode + except (OSError, subprocess.SubprocessError): + return "", -1 + + +def parse_cruncher(stdout): + """Return a list of dicts: type, workword, lemma, features, stemtype, derivtype.""" + entries = [] + for m in NL_RE.finditer(stdout): + toks = m.group(1).split() + if len(toks) < 2: + continue + typ = toks[0] + lemma_field = toks[1] + if "," in lemma_field: + workword, lemma = lemma_field.split(",", 1) + else: + workword, lemma = None, lemma_field + rest = toks[2:] + stemfield = rest[-1] if rest else "" + features = rest[:-1] + if "," in stemfield: + stemtype, derivtype = stemfield.split(",", 1) + else: + stemtype, derivtype = stemfield, "" + entries.append( + dict( + type=typ, + workword=workword, + lemma=lemma, + features=features, + stemtype=stemtype, + derivtype=derivtype, + ) + ) + return entries + + +def parse_morpheus(stdout): + """Return (known, entries, parse_error). + + entries: list of {hdwd, stemtypes:set, infls:list[dict]} + known: True/False, or None if the document did not parse at all. + """ + + try: + root = ET.fromstring(stdout) + except ET.ParseError as e: + return None, [], str(e) + + word_el = root.find("word") + if word_el is None: + return False, [], None + + entries = [] + for entry in word_el.findall("entry"): + d = entry.find("dict") + hdwd_el = d.find("hdwd") if d is not None else None + hdwd = hdwd_el.text if hdwd_el is not None else None + stemtypes = set() + infls = [] + for infl in entry.findall("infl"): + info = {} + for tag in ( + "case", + "gend", + "mood", + "tense", + "voice", + "pers", + "num", + "comp", + "dial", + "stemtype", + "morph", + "pofs", + ): + el = infl.find(tag) + if el is not None and el.text: + info[tag] = el.text + st = infl.find("stemtype") + if st is not None and st.text: + stemtypes.add(st.text) + infls.append(info) + entries.append(dict(hdwd=hdwd, stemtypes=stemtypes, infls=infls)) + return True, entries, None + + +def is_case_combo(tok): + return all(p in CASE_ABBRS for p in tok.split("/")) + + +def is_gend_combo(tok): + return all(p in GEND_ABBRS for p in tok.split("/")) + + +def compare_word(word, cbin, cflags, mbin, mflags): + lines = [] + hard_fail = False + + cout, _ = run_cmd(cbin, cflags, word) + mout, _ = run_cmd(mbin, mflags, word) + + c_entries = parse_cruncher(cout) + c_known = len(c_entries) > 0 + + m_known, m_entries, perr = parse_morpheus(mout) + if m_known is None: + lines.append(f"FAIL: [{word}] morpheus output is not well-formed XML: {perr}") + return lines, True + + # ---- P1: known/unknown partition ------------------------------------- + if c_known == m_known: + lines.append( + f"PASS: [{word}] P1 known/unknown parity " + f"({'known' if c_known else 'unknown'})" + ) + else: + lines.append( + f"FAIL: [{word}] P1 known/unknown mismatch: " + f"cruncher={c_known} morpheus={m_known}" + ) + hard_fail = True + + if not c_known or not m_known: + return lines, hard_fail + + # ---- P2: lemma set + first-occurrence order -------------------------- + c_lemmas = [] + for e in c_entries: + if e["lemma"] not in c_lemmas: + c_lemmas.append(e["lemma"]) + m_lemmas = [] + for e in m_entries: + if e["hdwd"] and e["hdwd"] not in m_lemmas: + m_lemmas.append(e["hdwd"]) + + if set(c_lemmas) == set(m_lemmas): + lines.append(f"PASS: [{word}] P2 lemma set parity {sorted(set(c_lemmas))}") + if c_lemmas == m_lemmas: + lines.append(f"PASS: [{word}] P2 lemma order parity") + else: + lines.append( + f"FAIL: [{word}] P2 lemma order mismatch: " + f"cruncher={c_lemmas} morpheus={m_lemmas}" + ) + hard_fail = True + else: + lines.append( + f"FAIL: [{word}] P2 lemma set mismatch: " + f"cruncher={sorted(set(c_lemmas))} morpheus={sorted(set(m_lemmas))}" + ) + hard_fail = True + + # ---- P3: stemtype set parity (whole word, deduped) -------------------- + c_stemtypes = {e["stemtype"] for e in c_entries if e["stemtype"]} + m_stemtypes = set() + for e in m_entries: + m_stemtypes |= e["stemtypes"] + + if c_stemtypes == m_stemtypes: + lines.append(f"PASS: [{word}] P3 stemtype set parity {sorted(c_stemtypes)}") + else: + lines.append( + f"FAIL: [{word}] P3 stemtype set mismatch: " + f"cruncher={sorted(c_stemtypes)} morpheus={sorted(m_stemtypes)}" + ) + hard_fail = True + + # ---- P4 (soft): normalized feature-token coverage ---------------------- + + # Gender values that don't participate in masc/fem/neut slash-combos, so + # is_gend_combo() alone won't catch them, but are still rendered via + # rather than anything P4 scans. Excluded here rather than documented as a + # permanent WARN, since a check that can never pass shouldn't fire at all. + NON_COMBO_GEND_TOKENS = {"adverbial"} + + for e in c_entries: + plain_tokens = [ + t + for t in e["features"] + if not is_case_combo(t) + and not is_gend_combo(t) + and t not in NON_COMBO_GEND_TOKENS + ] + wanted = {norm_tok(t) for t in plain_tokens} + pool = [i for me in m_entries if me["hdwd"] == e["lemma"] for i in me["infls"]] + if not pool: + continue + seen_raw = [ + v + for info in pool + for k, v in info.items() + if k not in ("case", "gend", "stemtype", "pofs") + ] + missing = {w for w in wanted if not any(w in v for v in seen_raw)} + if missing: + lines.append( + f"WARN: [{word}] P4 feature coverage: lemma={e['lemma']} " + f"tokens not observed in any matching " + f"(normalization is approximate): " + f"{sorted(m for m in missing if m is not None)}" + ) + + # ---- P5 (soft): case/gender cartesian-expansion sanity ----------------- + for e in c_entries: + case_tok = next((t for t in e["features"] if is_case_combo(t)), None) + gend_tok = next((t for t in e["features"] if is_gend_combo(t)), None) + if not case_tok and not gend_tok: + continue + cases = [norm_tok(c) for c in (case_tok.split("/") if case_tok else [None])] + genders = [norm_tok(g) for g in (gend_tok.split("/") if gend_tok else [None])] + expected = len(cases) * len(genders) + pool = [i for me in m_entries if me["hdwd"] == e["lemma"] for i in me["infls"]] + combos = set() + for info in pool: + c = info.get("case") + g = info.get("gend") + case_ok = case_tok is None or c in cases + gend_ok = gend_tok is None or g in genders + if case_ok and gend_ok: + combos.add((c, g)) + if len(combos) < expected: + lines.append( + f"WARN: [{word}] P5 expansion: lemma={e['lemma']} " + f"case={case_tok} gend={gend_tok}: expected >= {expected} distinct " + f"case/gender combos, found {len(combos)}" + ) + + return lines, hard_fail + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("wordlist") + ap.add_argument("--cruncher", required=True) + ap.add_argument("--cruncher-flags", default="") + ap.add_argument("--morpheus", required=True) + ap.add_argument("--morpheus-flags", default="") + args = ap.parse_args() + + cflags = args.cruncher_flags.split() + mflags = args.morpheus_flags.split() + + with open(args.wordlist, encoding="utf-8") as f: + words = [w.strip() for w in f if w.strip() and not w.startswith("#")] + + any_hard_fail = False + for word in words: + try: + lines, hard_fail = compare_word( + word, args.cruncher, cflags, args.morpheus, mflags + ) + # don't let one bad word kill the whole run + except Exception as e: # pylint: disable=broad-except + print(f"FAIL: [{word}] comparator raised an exception: {e}") + any_hard_fail = True + continue + for line in lines: + print(line) + any_hard_fail = any_hard_fail or hard_fail + + sys.exit(1 if any_hard_fail else 0) + + +if __name__ == "__main__": + main()