From 734580d0c3c30ff9ccfbe4be101e6c6c5cd55931 Mon Sep 17 00:00:00 2001
From: linearcombination <4829djaskdfj@gmail.com>
Date: Thu, 18 Jun 2026 10:45:55 -0700
Subject: [PATCH 01/17] Make it possible to specify languages that prefer
non-ulb source text
Prior to this commit, ulb was always preferred for any language. We
learned that at least one language, fr, should prefer non-ulb usfm
text if available. This change makes it possible to specify additional
languages besides fr that may need the same treatment.
---
backend/doc/config.py | 2 ++
backend/passages/domain/document_generator.py | 17 +++++++---
backend/stet/domain/document_generator.py | 31 ++++++++++++++-----
3 files changed, 38 insertions(+), 12 deletions(-)
diff --git a/backend/doc/config.py b/backend/doc/config.py
index f2bf1cfc..a5c63522 100755
--- a/backend/doc/config.py
+++ b/backend/doc/config.py
@@ -20,6 +20,8 @@ class Settings(BaseSettings):
DATA_API_URL: HttpUrl
+ LANGUAGES_WHERE_NON_ULB_PREFERRED: Sequence[str] = ["fr"]
+
# This is only used to see if a lang_code is in the collection
# otherwise it is a heart language. Eventually the graphql data api may
# provide gateway/heart boolean value.
diff --git a/backend/passages/domain/document_generator.py b/backend/passages/domain/document_generator.py
index 4a79e56d..7d310667 100644
--- a/backend/passages/domain/document_generator.py
+++ b/backend/passages/domain/document_generator.py
@@ -101,6 +101,9 @@ def get_usfm_books_and_usfm_resource_type(
bible_references_with_availability: list[BibleReferenceWithAvailability],
lang_code: str,
usfm_resource_types: Sequence[str] = settings.USFM_RESOURCE_TYPES,
+ languages_where_non_ulb_preferred: Sequence[
+ str
+ ] = settings.LANGUAGES_WHERE_NON_ULB_PREFERRED,
) -> tuple[list[USFMBook], str]:
# Invariant: book codes are only those that were available from USFM resources
book_codes = list(
@@ -128,10 +131,16 @@ def get_usfm_books_and_usfm_resource_type(
)
usfm_books = []
usfm_resource_type = ""
- if ulb_usfm_resource_types: # Prefer ulb if available
- usfm_resource_type = ulb_usfm_resource_types[0]
- elif usfm_resource_types:
- usfm_resource_type = usfm_resource_types[0]
+ if lang_code not in languages_where_non_ulb_preferred:
+ if ulb_usfm_resource_types: # Prefer ulb if available
+ usfm_resource_type = ulb_usfm_resource_types[0]
+ elif usfm_resource_types:
+ usfm_resource_type = usfm_resource_types[0]
+ else:
+ if usfm_resource_types: # Prefer non-ulb if available
+ usfm_resource_type = usfm_resource_types[0]
+ elif ulb_usfm_resource_types:
+ usfm_resource_type = ulb_usfm_resource_types[0]
if usfm_resource_type:
usfm_book = None
for book_code in book_codes:
diff --git a/backend/stet/domain/document_generator.py b/backend/stet/domain/document_generator.py
index b45feb79..3b1c04a9 100644
--- a/backend/stet/domain/document_generator.py
+++ b/backend/stet/domain/document_generator.py
@@ -61,6 +61,9 @@ def generate_docx_document(
resource_type_codes_and_names: Mapping[
str, str
] = settings.RESOURCE_TYPE_CODES_AND_NAMES,
+ languages_where_non_ulb_preferred: Sequence[
+ str
+ ] = settings.LANGUAGES_WHERE_NON_ULB_PREFERRED,
) -> str:
"""
Generate the scriptural terms evaluation document.
@@ -116,14 +119,26 @@ def generate_docx_document(
target_usfm_books = []
lang0_usfm_resource_type = ""
lang1_usfm_resource_type = ""
- if lang0_ulb_usfm_resource_types: # Prefer ulb if available
- lang0_usfm_resource_type = lang0_ulb_usfm_resource_types[0]
- elif lang0_usfm_resource_types:
- lang0_usfm_resource_type = lang0_usfm_resource_types[0]
- if lang1_ulb_usfm_resource_types: # Prefer ulb if available
- lang1_usfm_resource_type = lang1_ulb_usfm_resource_types[0]
- elif lang1_usfm_resource_types:
- lang1_usfm_resource_type = lang1_usfm_resource_types[0]
+ if lang0_code not in languages_where_non_ulb_preferred:
+ if lang0_ulb_usfm_resource_types: # Prefer ulb if available
+ lang0_usfm_resource_type = lang0_ulb_usfm_resource_types[0]
+ elif lang0_usfm_resource_types:
+ lang0_usfm_resource_type = lang0_usfm_resource_types[0]
+ else:
+ if lang0_usfm_resource_types: # Prefer non-ulb if available
+ lang0_usfm_resource_type = lang0_usfm_resource_types[0]
+ elif lang0_ulb_usfm_resource_types:
+ lang0_usfm_resource_type = lang0_ulb_usfm_resource_types[0]
+ if lang1_code not in languages_where_non_ulb_preferred:
+ if lang1_ulb_usfm_resource_types: # Prefer ulb if available
+ lang1_usfm_resource_type = lang1_ulb_usfm_resource_types[0]
+ elif lang1_usfm_resource_types:
+ lang1_usfm_resource_type = lang1_usfm_resource_types[0]
+ else:
+ if lang1_usfm_resource_types: # Prefer non-ulb if available
+ lang1_usfm_resource_type = lang1_usfm_resource_types[0]
+ elif lang1_ulb_usfm_resource_types:
+ lang1_usfm_resource_type = lang1_ulb_usfm_resource_types[0]
if lang0_usfm_resource_type and lang1_usfm_resource_type:
source_usfm_book = None
target_usfm_book = None
From ba9668269322883ae0657c3090c846285bc9162a Mon Sep 17 00:00:00 2001
From: linearcombination <4829djaskdfj@gmail.com>
Date: Thu, 18 Jun 2026 10:50:40 -0700
Subject: [PATCH 02/17] Improve splitting of chapters into verses
A test with fr, f10, revealed that its chapter source was not being
properly split into verses because of the extra markup formatting that
is in f10. This commit switches to using BeautifulSoup to ensure
proper matching of nested beginning and ending tags so that chapters
are properly split into discrete verses. It may be more robust for
other languages that we have not tested yet too which also have
similar markup characteristics.
---
backend/doc/domain/parsing.py | 71 ++++++-------------
backend/passages/domain/document_generator.py | 6 +-
backend/stet/domain/document_generator.py | 20 ++++--
3 files changed, 38 insertions(+), 59 deletions(-)
diff --git a/backend/doc/domain/parsing.py b/backend/doc/domain/parsing.py
index 2423f68c..09a9dd1c 100644
--- a/backend/doc/domain/parsing.py
+++ b/backend/doc/domain/parsing.py
@@ -2,6 +2,7 @@
This module provides an API for parsing content.
"""
+from bs4 import BeautifulSoup
from re import (
compile,
escape,
@@ -1408,50 +1409,10 @@ def lookup_verse_text(usfm_book: USFMBook, chapter_num: int, verse_ref: str) ->
return verse
-# Used by STET and PASSAGES apps
-def split_chapter_into_verses(chapter: USFMChapter) -> dict[str, str]:
- # Sample HTML content with multiple verse elements
- # html_content = '''
- #
- # 19
- # For through the law I died to the law, so that I might live for God. I have been crucified with Christ.
- #
- #
- #
- #
- # 20
- # I have been crucified with Christ and I no longer live, but Christ lives in me. The life I now live in the body, I live by faith in the Son of God, who loved me and gave himself for me.
- #
- #
- #
- # '''
- verse_dict = {}
- # Find all verse spans
- verse_spans = findall(r'(.*?)', chapter.content, DOTALL)
- for verse_span in verse_spans:
- # Extract the verse number from the versemarker
- verse_number = search(r'(\d+)', verse_span)
- if verse_number:
- verse_number_ = verse_number.group(1)
- # Remove versemarker
- verse_text = sub(r'.*?', "", verse_span)
- # Remove footnotes numbers
- verse_text = sub(r'.*?', "", verse_text)
- # Fix spacing issue when div class="poetry-*" type divs
- # are used, e.g., yielding 'heartsas' for Hebrews 3:8
- verse_text = sub(
- r'(.*?)
',
- r" \1",
- verse_text,
- )
- # Add to the dictionary with verse number as the key and verse text as the value
- verse_dict[verse_number_] = verse_text
- return verse_dict
-
-
def handle_split_chapter_into_verses(
usfm_book: USFMBook,
usfm_chapter: USFMChapter,
+ remove_versemarker: bool = False,
resource_type_codes_and_names: Mapping[
str, str
] = settings.RESOURCE_TYPE_CODES_AND_NAMES,
@@ -1460,13 +1421,18 @@ def handle_split_chapter_into_verses(
usfm_book.lang_code == "fr"
and usfm_book.resource_type_name == resource_type_codes_and_names["f10"]
):
- return split_chapter_into_verses_with_formatting_for_f10(usfm_chapter)
+ return split_chapter_into_verses_with_formatting_for_f10(
+ usfm_chapter, remove_versemarker
+ )
else:
- return split_chapter_into_verses_with_formatting(usfm_chapter)
+ return split_chapter_into_verses_with_formatting(
+ usfm_chapter, remove_versemarker
+ )
def split_chapter_into_verses_with_formatting(
chapter: USFMChapter,
+ remove_versemarker: bool = False,
empty_paragraph: str = "",
sectionhead5_element: str = '',
) -> dict[VerseRef, str]:
@@ -1500,20 +1466,24 @@ def split_chapter_into_verses_with_formatting(
"""
+ soup = BeautifulSoup(chapter.content, "html.parser")
# TODO What to do about footnote targets? Perhaps have the value be a
# tuple with first element of the verse HTML (which includes the
# footnote callers) and the second element the target footnotes HTML?
verse_dict = {}
# Find all verse spans
- verse_spans = findall(r'(.*?)', chapter.content, DOTALL)
- for verse_span in verse_spans:
+ for verse_span in soup.find_all("span", class_="verse"):
# Extract the verse number from the versemarker
- verse_number = search(r'(\d+)', verse_span)
+ sup = verse_span.find("sup", class_="versemarker")
+ if not sup or not sup.string:
+ continue
+ verse_number = sup.string.strip()
+ if remove_versemarker:
+ sup.decompose()
if verse_number:
- verse_number_ = verse_number.group(1)
# Add to the dictionary with verse number as the key and verse text as the value
- verse_dict[verse_number_] = (
- verse_span.strip()
+ verse_dict[verse_number] = (
+ str(verse_span)
.replace(empty_paragraph, "")
.replace(sectionhead5_element, "")
)
@@ -1522,6 +1492,7 @@ def split_chapter_into_verses_with_formatting(
def split_chapter_into_verses_with_formatting_for_f10(
chapter: USFMChapter,
+ remove_versemarker: bool = False,
empty_paragraph: str = "",
sectionhead5_element: str = '',
) -> dict[str, str]:
@@ -1539,6 +1510,8 @@ def split_chapter_into_verses_with_formatting_for_f10(
if not sup or not sup.string:
continue
verse_number = sup.string.strip()
+ if remove_versemarker:
+ sup.decompose()
# unwrap all word-entry spans: replace X
# with X (preserving whitespace/punctuation)
for we in verse_span.find_all("span", class_="word-entry"):
diff --git a/backend/passages/domain/document_generator.py b/backend/passages/domain/document_generator.py
index 7d310667..c1214833 100644
--- a/backend/passages/domain/document_generator.py
+++ b/backend/passages/domain/document_generator.py
@@ -8,7 +8,7 @@
from doc.domain.bible_books import BOOK_NAMES
from doc.domain.email_utils import send_email_with_attachment, should_send_email
from doc.domain.model import Attachment, USFMBook
-from doc.domain.parsing import split_chapter_into_verses, usfm_book_content
+from doc.domain.parsing import handle_split_chapter_into_verses, usfm_book_content
from doc.domain.resource_lookup import (
book_codes_for_lang_from_usfm_only,
prepare_resource_filepath,
@@ -159,8 +159,8 @@ def get_usfm_books_and_usfm_resource_type(
False,
)
for chapter_num_, chapter_ in usfm_book.chapters.items():
- usfm_book.chapters[chapter_num_].verses = split_chapter_into_verses(
- chapter_
+ usfm_book.chapters[chapter_num_].verses = (
+ handle_split_chapter_into_verses(usfm_book, chapter_, True)
)
usfm_books.append(usfm_book)
return usfm_books, usfm_resource_type
diff --git a/backend/stet/domain/document_generator.py b/backend/stet/domain/document_generator.py
index 3b1c04a9..69355c97 100644
--- a/backend/stet/domain/document_generator.py
+++ b/backend/stet/domain/document_generator.py
@@ -10,7 +10,7 @@
from doc.domain.model import Attachment
from doc.domain.parsing import (
lookup_verse_text,
- split_chapter_into_verses,
+ handle_split_chapter_into_verses,
usfm_book_content,
)
from doc.domain.resource_lookup import (
@@ -161,9 +161,12 @@ def generate_docx_document(
lang0_resource_dir,
False,
)
- for chapter_num_, chapter_ in source_usfm_book.chapters.items():
- source_usfm_book.chapters[chapter_num_].verses = (
- split_chapter_into_verses(chapter_)
+ for (
+ chapter_num_,
+ chapter_,
+ ) in source_usfm_book.chapters.items():
+ chapter_.verses = handle_split_chapter_into_verses(
+ source_usfm_book, chapter_, True
)
source_usfm_books.append(source_usfm_book)
lang1_resource_lookup_dto_ = resource_lookup_dto(
@@ -181,9 +184,12 @@ def generate_docx_document(
lang1_resource_dir,
False,
)
- for chapter_num_, chapter_ in target_usfm_book.chapters.items():
- target_usfm_book.chapters[chapter_num_].verses = (
- split_chapter_into_verses(chapter_)
+ for (
+ chapter_num_,
+ chapter_,
+ ) in target_usfm_book.chapters.items():
+ chapter_.verses = handle_split_chapter_into_verses(
+ target_usfm_book, chapter_, True
)
target_usfm_books.append(target_usfm_book)
# Count total occurrences per reference (using source_reference as key)
From 1c584794be65a4a969aad32e1b53c306a1c88ae7 Mon Sep 17 00:00:00 2001
From: linearcombination <4829djaskdfj@gmail.com>
Date: Thu, 18 Jun 2026 10:54:26 -0700
Subject: [PATCH 03/17] Remove reviewer's guide resource from consideration in
DOC
We already have disabled rg from the ui by request of the PO, but here
is a place where it was still potentially being used in the
consideration of what books to consider available for a particular
language.
---
backend/doc/domain/resource_lookup.py | 16 ----------------
1 file changed, 16 deletions(-)
diff --git a/backend/doc/domain/resource_lookup.py b/backend/doc/domain/resource_lookup.py
index f3da9d4c..fe42dd31 100644
--- a/backend/doc/domain/resource_lookup.py
+++ b/backend/doc/domain/resource_lookup.py
@@ -289,22 +289,6 @@ def get_resource_types(
]
elif resource_type in usfm_resource_types:
book_assets = find_usfm_files(resource_filepath)
- elif resource_type == "rg":
- between_texts, bible_reference_strs = find_bible_references(
- join(en_rg, docx_file_path)
- )
- bible_references = [
- parse_bible_reference(bible_reference)
- for bible_reference in bible_reference_strs
- ]
- book_codes_ = {
- bible_reference.book_code
- for bible_reference in bible_references
- if bible_reference
- }
- book_assets = [
- book_code for book_code in book_codes if book_code in book_codes_
- ]
if book_assets or resource_type == "tw":
resource_types.append(
(
From 128bf04cd92f284e4f1b001386e1c9035216c438 Mon Sep 17 00:00:00 2001
From: linearcombination <4829djaskdfj@gmail.com>
Date: Thu, 18 Jun 2026 10:57:48 -0700
Subject: [PATCH 04/17] Two variables renamed for consistency
---
backend/passages/domain/document_generator.py | 22 +++++++++----------
1 file changed, 11 insertions(+), 11 deletions(-)
diff --git a/backend/passages/domain/document_generator.py b/backend/passages/domain/document_generator.py
index c1214833..274ac10e 100644
--- a/backend/passages/domain/document_generator.py
+++ b/backend/passages/domain/document_generator.py
@@ -206,22 +206,22 @@ def generate_docx_document(
)
)
current_task.update_state(state="Assembling content")
- passages_lang0 = get_passages(
+ lang0_passages = get_passages(
bible_references_with_availability_lang0,
usfm_resource_type_lang0,
usfm_books_lang0,
)
- passages_lang1 = []
+ lang1_passages = []
if lang1_code:
- passages_lang1 = get_passages(
+ lang1_passages = get_passages(
bible_references_with_availability_lang1,
usfm_resource_type_lang1,
usfm_books_lang1,
)
current_task.update_state(state="Converting to Docx")
generate_docx(
- passages_lang0,
- passages_lang1,
+ lang0_passages,
+ lang1_passages,
docx_filepath_,
lang0_code,
lang0_name,
@@ -232,8 +232,8 @@ def generate_docx_document(
def generate_docx(
- passages_lang0: list[Passage],
- passages_lang1: list[Passage],
+ lang0_passages: list[Passage],
+ lang1_passages: list[Passage],
docx_filepath: str,
lang0_code: str,
lang0_name: str,
@@ -254,9 +254,9 @@ def generate_docx(
html_to_docx = HtmlToDocx()
has_lang1 = lang1_code is not None and lang1_name is not None
if has_lang1:
- assert len(passages_lang0) == len(passages_lang1), (
+ assert len(lang0_passages) == len(lang1_passages), (
f"Passage count mismatch: "
- f"{len(passages_lang0)} vs {len(passages_lang1)}"
+ f"{len(lang0_passages)} vs {len(lang1_passages)}"
)
columns: list[str] = ["lang0"]
if has_lang1:
@@ -280,9 +280,9 @@ def generate_docx(
table.columns[i].width = w
col_index = {name: i for i, name in enumerate(columns)}
pairs = (
- zip(passages_lang0, passages_lang1)
+ zip(lang0_passages, lang1_passages)
if has_lang1
- else ((p, None) for p in passages_lang0)
+ else ((p, None) for p in lang0_passages)
)
for p0, p1 in pairs:
row = table.add_row()
From 5deea9ebeccbea59235fe436e5746b0a333d056d Mon Sep 17 00:00:00 2001
From: linearcombination <4829djaskdfj@gmail.com>
Date: Thu, 18 Jun 2026 11:49:03 -0700
Subject: [PATCH 05/17] Move function from DOC to STET
Only STET uses it
---
backend/doc/domain/resource_lookup.py | 49 -----------------------
backend/stet/domain/resource_lookup.py | 55 ++++++++++++++++++++++++++
backend/stet/entrypoints/routes.py | 2 +-
3 files changed, 56 insertions(+), 50 deletions(-)
create mode 100644 backend/stet/domain/resource_lookup.py
diff --git a/backend/doc/domain/resource_lookup.py b/backend/doc/domain/resource_lookup.py
index fe42dd31..ee7ade7d 100644
--- a/backend/doc/domain/resource_lookup.py
+++ b/backend/doc/domain/resource_lookup.py
@@ -55,15 +55,6 @@
)
-# List of languages which do not have USFM available for any books. We use this
-# to filter these out of STET's list of source and target
-# languages so that the user doesn't have the frustrating experience of
-# selecting a language which might have non-USFM resources available but
-# not USFM so that when their resulting doc is generated no scripture is
-# present. It makes it seem like a bug in STET and is bad UX.
-LANG_CODES_WITH_NO_USFM: frozenset[str] = frozenset(["ru"])
-
-
@cached(fetch_source_data_cache)
def fetch_source_data(
data_api_url: HttpUrl = settings.DATA_API_URL,
@@ -175,46 +166,6 @@ def lang_codes_and_names(
return sorted(unique_values, key=lambda value: value[1])
-def lang_codes_and_names_having_usfm(
- lang_code_filter_list: frozenset[str] = LANG_CODES_WITH_NO_USFM,
- gateway_languages: frozenset[str] = settings.GATEWAY_LANGUAGES,
-) -> Sequence[tuple[str, str, bool]]:
- """
- >>> from doc.domain import resource_lookup
- >>> ();result = resource_lookup.lang_codes_and_names_having_usfm();() # doctest: +ELLIPSIS
- (...)
- >>> result[0]
- ('abz', 'Abui', False)
- >>> heart_lang_codes = [lang_code_and_name[0] for lang_code_and_name in resource_lookup.lang_codes_and_names_having_usfm() if not lang_code_and_name[2]]
- >>> sorted(heart_lang_codes)[0]
- 'aao'
- """
- data = fetch_source_data()
- values = []
- if data is None or not data.git_repo:
- logger.info("Data API is down or no git_repo found!")
- return []
- try:
- for repo_info in data.git_repo:
- language_info = repo_info.content
- language = language_info.language
- ietf_code = language.ietf_code
- english_name = language.english_name if language.english_name else ""
- localized_name = language.national_name
- is_gateway = ietf_code in gateway_languages
- if ietf_code not in lang_code_filter_list:
- if english_name in localized_name:
- values.append((ietf_code, localized_name, is_gateway))
- else:
- values.append(
- (ietf_code, f"{localized_name} ({english_name})", is_gateway)
- )
- except Exception:
- logger.exception("Failed due to the following exception.")
- unique_values = unique_tuples(values)
- return sorted(unique_values, key=lambda value: value[1])
-
-
def repos_to_clone(
lang_code: str,
augmented_repos_info: list[RepoEntry],
diff --git a/backend/stet/domain/resource_lookup.py b/backend/stet/domain/resource_lookup.py
new file mode 100644
index 00000000..d8db861e
--- /dev/null
+++ b/backend/stet/domain/resource_lookup.py
@@ -0,0 +1,55 @@
+from typing import Sequence
+
+from doc.config import settings
+from doc.domain.resource_lookup import fetch_source_data
+from doc.utils.list_utils import unique_tuples
+
+# List of languages which do not have USFM available for NT books. We use this
+# to filter these out of STET's list of source and target
+# languages so that the user doesn't have the frustrating experience of
+# selecting a language which might have OT USFM resources available but
+# not NT USFM so that when their resulting doc is generated no scripture is
+# present. It makes it seem like a bug in STET and is bad UX.
+LANG_CODES_WITH_NO_NT_USFM: frozenset[str] = frozenset(["ru"])
+
+logger = settings.logger(__name__)
+
+
+def lang_codes_and_names_having_usfm(
+ lang_code_filter_list: frozenset[str] = LANG_CODES_WITH_NO_NT_USFM,
+ gateway_languages: frozenset[str] = settings.GATEWAY_LANGUAGES,
+) -> Sequence[tuple[str, str, bool]]:
+ """
+ >>> from doc.domain import resource_lookup
+ >>> ();result = resource_lookup.lang_codes_and_names_having_usfm();() # doctest: +ELLIPSIS
+ (...)
+ >>> result[0]
+ ('abz', 'Abui', False)
+ >>> heart_lang_codes = [lang_code_and_name[0] for lang_code_and_name in resource_lookup.lang_codes_and_names_having_usfm() if not lang_code_and_name[2]]
+ >>> sorted(heart_lang_codes)[0]
+ 'aao'
+ """
+ data = fetch_source_data()
+ values = []
+ if data is None or not data.git_repo:
+ logger.info("Data API is down or no git_repo found!")
+ return []
+ try:
+ for repo_info in data.git_repo:
+ language_info = repo_info.content
+ language = language_info.language
+ ietf_code = language.ietf_code
+ english_name = language.english_name if language.english_name else ""
+ localized_name = language.national_name
+ is_gateway = ietf_code in gateway_languages
+ if ietf_code not in lang_code_filter_list:
+ if english_name in localized_name:
+ values.append((ietf_code, localized_name, is_gateway))
+ else:
+ values.append(
+ (ietf_code, f"{localized_name} ({english_name})", is_gateway)
+ )
+ except Exception:
+ logger.exception("Failed due to the following exception.")
+ unique_values = unique_tuples(values)
+ return sorted(unique_values, key=lambda value: value[1])
diff --git a/backend/stet/entrypoints/routes.py b/backend/stet/entrypoints/routes.py
index ce0501d9..5a2f5e97 100644
--- a/backend/stet/entrypoints/routes.py
+++ b/backend/stet/entrypoints/routes.py
@@ -4,7 +4,7 @@
import celery.states
from celery.result import AsyncResult
from doc.config import settings
-from doc.domain import resource_lookup
+from stet.domain import resource_lookup
from docx import Document
from fastapi import APIRouter, Request, HTTPException, status
from fastapi.responses import JSONResponse
From 966efb113126d67e7748ef6b2a6237a68ce91fcc Mon Sep 17 00:00:00 2001
From: linearcombination <4829djaskdfj@gmail.com>
Date: Thu, 18 Jun 2026 11:50:51 -0700
Subject: [PATCH 06/17] Add new STET input docs
Add gu, ru, and bulgarian. Note that ru does not provide NT books,
only OT books, so it has no utility in STET where all references are
to NT. Thus ru is filtered out of the list of source languages
currently displayed by STET.
---
backend/stet/data/stet_gu.docx | Bin 0 -> 107079 bytes
backend/stet/data/stet_rmn-x-yerliroman.docx | Bin 0 -> 110841 bytes
backend/stet/data/stet_ru.docx | Bin 0 -> 103234 bytes
backend/stet/domain/strings.py | 22 +++++++++++++++++++
4 files changed, 22 insertions(+)
create mode 100644 backend/stet/data/stet_gu.docx
create mode 100644 backend/stet/data/stet_rmn-x-yerliroman.docx
create mode 100644 backend/stet/data/stet_ru.docx
diff --git a/backend/stet/data/stet_gu.docx b/backend/stet/data/stet_gu.docx
new file mode 100644
index 0000000000000000000000000000000000000000..7e3601cd719721ede355453ab9e2fc37be11b4fe
GIT binary patch
literal 107079
zcmeFX<9B7z+a(&?wsm5jq+;7Sv5kt&s@S$|JC&qjt76+mg%#iY-hTUz+vE0s(7iwG
zvBw_!%U)~E=b6u(Q&|oY3JVMd3=Rwoj2!HMDni~591IKq0|tf(1_z-l=3wt?ZtrRc
z@^UnH(P#Fwvn4Bpf}qO>gZTRX|6czGuRwG1gv|gen#5DcN6>8Rplk3OFoH?Nt
z!Y2JK5Begab@s@%#|2d>7A>j?oA0qy`u-X3X!=huyUo=eB_4Eel!k;_*lEC*@n
zkUf`A*S}nG`bha}FNFmRJ*
z2jf{ENtP)L!I?k3*%38HpG=m`>XE0*I0~XD4qMAJ2*H)-XYBTAugp!7p}jwc&1!jw
zif>qjAJeY9xQ{?mz%zr_yIhHD^?$x?XSIVs#o}YD6!PZP@N}T!!GJfk~@qYJrt7qL4tvOenNsN|1YQ|Ng!yu
z{yK_^U!X+%Lam{*xvdKe^MCID7p?yXL-hai>(z;UUuZ=Vz7Bd1n(b0q>%%KjWHFst
z$6JMk)0L6OTwk$S{&?qKS%J_$HI|x~n@^hYcFC4>+ey~HAxKq6j_iV8dNvr)ef{kT
zmJ;&2gw1owaSti$=+WYRoKglb9u%dG9XU+^lkhJ*eX1ApkY4=punf_f0x-Q`O4U@9
z>!iN?l>UPwKZ9*?$wpe&2ewi`qC1qh;}8B11Ph^tG!CZtxc;q1{Xx)K(l7QH8^8l2
zfh{Wwb#{VDH3GrS@7Xi&qMov$h!95j0{94)S6>S~44seT$Rm6q;5E5+v*E4-VYa@=
z-B<7W|Kp>vHWXEd7aXiKfgB9|Yf(HLoXuFw98BHpzFOOVdfJ7bt^=V|>(wjxpPyv+
zjh2DY>Q7fR?!RX>Ed$3wcmG{@!QBeW*~zVA0F+&`CzJML^k7_IyjC22(B3XM5T-CX$6V7Z>#8vg0(~!flhM0jp4mIjkl}m?)pO;COHA^F2KRue3
z{VYRrDH&>hcP{B}Ul{mU9)9Zm`LSQu><*iAe1Wb
za@c)~xH*!%qw_M0@B4MNEfNgh<>KOION?)4laIZROU?HBr9N@C?zvsvW39oa-h`&Z
z`e1ooGhZL>?SIaxzP-ogzXt+S`7w@N6(`LrbB
zwJ=p5vjHmr82WDVBB`)FS`yisvrb<2c%DV!A$?+enOytgOqMc5ZVW!&M30Vyf3pd1
z^D(aC*{yL1o({CvI6r@{ojZ8I4As4VuUgOfcp&8K0l;jXA_41c?d?zmnTDxsIOQ#y
zeBQA%+U>~7^av2}HSN+~7!TUkVo7ej{-bGz%;vyljJ|c`bF%RbyxxBt3O!(ez|RM
z{`iu^pgTm2eYQ@|S#ORvBZMfjgnDHC5#gmjq4aQYminXdM^TBw-uQDr{O>v4PmmBw
zt*tsL?(1UGcALEogPLdZP+i^?R^C!O+euCJnb7C4&fmR}9MQ-ng!$~J3C*ae!o6HX
zhlRJLb^I=w?N5zb*tPcoK8A7r)zj+3#gp${H3n=w;u#-?C@m?5#MvKb3|)QsZGJE3
zuUX5$(trP^uC9)}Kdzb~-$txqwFE?mQfjszyqBf+(a|#z>+ByrVAH&z$3UMQixofk
z{+Vv9cQn-C;uQ778L&E>K5oTCmJEG3e5!m%qFxyrrZeAjFF(wGA2MZ=oCwExKm>3a
zdR~|VDyqFQ0LEMTvHS#%Gb|rEj6auHu-ioX<;O?oJxmQcR#*%VDp(?ME3B?vUvjq3
ze4_W#eMFSIb$B8m-|5qSoKWXJw|e_eEIwGw+|G|XG%uZ^2(AG1#HxTNoz+|XS`0s9
ze^g;j_`WVlw=D^sCU3W|zxwqwsh!QQz2@8!6v+;84e~_DRe%^Jnuhf#t
z_`HL?L?u5N=cX8%SR%;}A<|LD4foww{(QA1Qc`|4Ni@9NSRo+tdmVxYx>sgB8{M30
z2o~-HO`IN1c>U~rv)55VxCDHxwCk|m{fT7VcKoik?&5wkQ9`(uO@j9;vSU*yI@CO+
z`09gvi}eZ$^=%w2hNO)4Un$VN~}i?Jpf{Cgr?*ObJZLf4q~U&wHA7m1w(-5R-8
zFTCF#2hXy${w+sfi7G*h{O6Rcn7#=esm|7K;h|QiglolO!KAZxNaO{__wLJm{WH4`
z3%D1Rnq-__SJ3gjdLgMR&;AnmI(tw#2U`(M*jupi2$h`T9tag4nnHy9*}(knkI%P`9wte}2Pi>Z;>Iu7Q_+
zi{|01xNG**fFtZ=VITX3HcT@}+rSNo@jk8lGk|Qt@DkbZ;{ASg>*hPihe32&Ds}`j
zoGifc?=|l5>|}GCzc&Tzl1RKC7aUv{EAuF%>tZ!r%0%8PoRR>)M>L_XR*`Oayt;ok
z;F3!`c*`Dq^_72RY2IZ`>8Geu<;ZEAh0berD$*WxN1S2i{90-B2VW5OSe_v7TB5`%C)@lP-`KbMpImk_U7
zXd8!r+hpblnzC{wKYyt@gH`AO3Xt2**NE>Es&5#_&}$IrrhmcG!V!vF9a2CYH8c!w
z<1fF|9UbfQ_uD{RxNwa81nR`{L-B40-O#Z4E`d|6>^2E2#^j~fpr;$4$E!Z*ty*0q&Qm6tS=VX2XAh^Z=)8JvQ;wIbS+vBFM)n_mw7s(gY|SQIXV5?v%Y<@e~yLvBHs`9?-w_X!D#?8Tz%QvkDsG|05S
zRHKjvq6&|srv>!IO---EsSAJ%(}}@A+=%?jn(xByD>JIxIyO6UI%uf`(4#W`j%C}C
z^Y8D~;#f2c(r#JrC*
zg7$c4Dzq{eCC1gWA`40siz2!_y$leB&_6QCeyhPh>_V}hQY1{G%nufUhGy+u&;5o#Jz8n~OT<15wpY-?=i*ND
zyFMF6Rd~qbO9rxz`lq?#5^l2KC6Z!?c9Mh?NA!Znk3g
zd6j5tdw}1VK@?TWu55eU*A8-Y19h@pDiQ^64v4|FoET;YSq*uqO
z^6uZP;Ob0&vN<0xFAEJ_k8$q;h%K$Gu@JA+#3?G2btxEq+TWurht`C#UX699n^QFv-rYHzt1^*&xZEE>5yrRlLTS{M)TEj
zylW4bpqMv{s8)Py;ulRy>Lhueam(&J=1Mp&c0b-yiL5zHc7->d;$-;-z9{+Wkki10
zj|Wi?fQOMvu=Zr3{IY^Lj!475PQv9o9CbSlO{Y7!7?KIkMET?-NdjCY(_b1$^4eJ#
zX{#I{3X~=Q{R&0QT|q+Yp25#1z7;wK-#egN21OT-JV#-?qLnMj!Z}Mil3w{F)L)tA
zazS|Mk`&pcj^7E_SMIIeo0Tu!$Gx;qkSug?5`w$hYZF=PU46r4Osf{m8mS@Z5$vcH
zP*R7T&nZ8C%ySMp?x?^R6DY?b%KFoLv9XW=j%3S*;E=-$hYB$wg1I$Q#o61ol*JK-
z6C)HkpgTp}arko!uff>`OkPw!oNDzq4BFBqtv5L<4KqaeC;@c@kyS&2V`3*kVe7}6Nn4r&{n?_SIs+#(^_~i#-uU>#y8)m
zHz{U&8LaSd#d#U$;qeFF*05XY8}};Dd~qSDLVJgjOt0!k-?4QGr=tkYh}@X&wtVj7DN2O286eh`&R*nNEw>v=HZzRup9Arr?-@P
zpk7fc6meNrtq8Pt=MnzT7iz$Fpm`as-iwz$^ByuVIP_%!j<`nZc2fRx#dZV&uhpu>
zG>A!ua2H%hU#y3{$u}0{MM3=+A@}BLzgDGIF#mDl&nHT?jxj{Pmo7={4e6&QB}-Cv
zR62A90xQt%8xQyH{_o4xZyvI@WECJ4-k}Wm#2=x>hG%wZu`BdQ}!#n2X{GouXPE41O_xfCA|hER@R%UY6K9VS^|i{yf@m#JUbds
zT)`CCg^0PU;5VeEpD}K{JH!;5DbsEu3eH#SqNpj*?O_kXa~)0!9_kbCj~M6AdkxqX
zM)hYq!QPqLjgoafgT+$(w?~K*)V;V$tT*0y?j>Z{v(la__5RzXD>?tJ2C?V
z=gSEEIq8zF_7%g>uk^`XEuzCZW3%Q&w+fdO@aD`EYSa)M5d6?S0H=!shY=}cSox$#
z*_iI$2`&ImpJ{^J6e}4zmtFcTO$AhHZyln%C>Am+5CcP4|q0W{;F8TnM!en}#j*@C2D3*tAHw)-}9v0{A5ngS;r|8=H6|&R;Po?Md8-)P{
z?Fi1#h7Y)pEQLl(sCZ(qUo3>pU-Y=C6P;URYBzL>=rR-fM}<;asVamZG6A{3*g8fz
zv>I;LWE;OS&5_VLC#b+yih@!(v)U>nh3RNw{W7t2LrA#X4sL^@FPU7pj681FJAq93
zWt?u+(vxD2v+2D8)Y7@X{W
zOfCqkvz-}z+u3(X5U|f4kpxA!$^^d;yEv^l#C0qZt*+ne)TLDAY;(cjd*&5F8OkrZ
z?^GDlp$6-=%OOM!Uz=LcG`2Qb!AXvrTg--|2Qfm+7vs);hDhP~C-{$rNSR!_%oQL`
zE`UMD(bwSQL{|1HQwBpxI#oes5H=$lH3@c+M&Hg>I&dM`QiMi?O-473yUiykMu|A%
z9#27yp7t>;L$)Xw)?3rMTi%4Jv*K5GI&QpH`cmCwel&4PwhPxO5eNSBppt?gjR06`
zLlu}%X-bDwaUEWEriFtRKA&IV=Z3h65b!T&++(Q6FlfYGZgKWU7qYp?QJ4Y$P^9T_
zb$4jWS^4$p(Pju&V-{}}xb|b9n;sNfR_WMO^dTbR5>v00aUU;iw*S)=426Q0b+yWT
zJ$)cC378+$x@lgIPEYkz;Y?g0p!NP3XfDa!+IkzAKd+dbwdwYr*3SmmvcS2%j8S~j
zVFNYj+Y=;KRmG_twqjseN97RIUw-fpsw6FF}U+yUa^P%gF1d;
z+G6;Eb()`t4ltF5^0HO-vT^>4kdJdzZ&t6t0CxqRO*S}z*$8*IMC4-F((oas;E^-L
z4X%zyft7`Zu|*8aAH7|8e;S}_Q^D#|1-ZG3#~Ih9te~sOb|ZJvB|O?QYKBh!me&&h
zp~q=%_V`YdN{WLrx6zaUZA_)&tKwv`Kd-W}&U~;tisVY&vvz*G__QXZCBUD=1N(zV
zp0*5y{+mXtLSV)AQMVd4_IL%Qt
z)rB%u{rtt11o&T>*7CuOWpxFVJK^)S8j{@rNh-vCKz~8(K!eVaX-7+0Zobh-PNpkNA!L+7$B+
z*0W1KT`~5?h}t#Z^S`>m`{1V=Y3
zyV{9FBqRW(X9O2dqv=}V!nyGea>I-g{8ped1j41MfM`03JCUeZxeqB=y_dm*=Fo~(
zC-fWu=XI6t7h2;^GgfB|t7E^`x#{IYz%cT4e>8cn9$38pFeAc_?f4#q(Kj40)~-(%
z;iIkflXub}G;dlIp55xG4j_|g$4BAnSx-=fp;i1=@ZntDj5A9BT?(}p$u+5X>d<#I
zrsT4Da*(3)Pi_imGK#b1<6(QF#p><{hNkfyb^NqIx(iuPIL{d82k;g{_YaSJ0BH-#
z^}
zwJ5xx0t{>;4=&ER=3%JL{+5PA
z+d*Or!09%Kp9L9W;|+Y|(1&};r6z${*E6`3-6$zSAvuB!JpD!rHRRaC5s%V4GfkRzQ~rBv1A)LyW}G%slPUgn
zxxQ6O_;;J`kka;BHJ?sxb>ZR?qn96Q>I|J6xZC6K3yCH1LC}N17vp{*VWuJ?aml^il`159Y=x@L1bE
zhHe*9X`{bv{i7;TdU|JXrwfKJ
zuqYWsPfkg~W?VRC1v&X>g(##S$YRb~{i|Eb3^UYQh0+WLeNme@Pj?(1TqTM%!vj?Eb6}TDW)vY&
zeaL~~QY57o!ZdRga!j~$Y7p}!U>Mes&cCcE;W4pik*4TD82ZlNK|5X)sV4`*l(RkJ)xjODY6NFG+8*`ijDraA9&Dl*TwY#vEsn9xNG&$z@IDf5kfM&}w$fg7I?-UebAp;Qy&zz5(7A>`J*={~{V2
zCZdK++&(D7@D*5(*oI5B9;UR03qWw<|JA)U;`lB_zTkXLb1m&K{&`Mz^eiEWc)Cc^T
zk8COcbbv(9)1=YFyy%9NMYFPHJsnjTiX57@Bts9Zulu*$wX61v=7h3Y#NAPN(j)}-
zp88_b2aGHw>uFnK$tofer94?CPkvqkJ}uf|F3|}z1%oUiAUYuWiJnD634apo0)IP*
z>wCoZ#P1XrMW@gOK=^Cwx_)sjtoCCDdkX-r?zs1C));_{0UsuPga@H$O@R>^@P?ML
zkfAcp^qD-Ai%TD?&9!62S)tO*l>HJYlS++6_`Tn>;X>nL;V_kcdV-?4Tv!HVr@owX
z0vE-Ps7hd>h^(xRrI-^bK<-tHwY2O%MnP$JC0YdcPF5gpDNYuXZ^gM<`lWoBXCctc
z$5TXo9f3v#VwWCs498uz19YOe{hTf0^&p?f(g24>*m%WJ*KwfRZc?!d#XO#XrlW5o
zn_!4(jZkK2Meo56PgfGfE04-rdJ3FjJ<1>HW73{7c=%Mu<5WjMN_`Li-C<{cugz?0
z2_@rRZaY90sq+u*wkTrE4Jy&Rz%iUouu{G@;HG05fNy1M(3#}HM4*=4JrRDrXkGI$
z=?7QD(lK<9z<&xsC<4$IPo~c;v%hkHVYIp99mSI5{5
zgP@11AfduEk397Nda^Q=tGFvW+OBMH0Mz0dj*T#-4T)u;dz0bsBNXFiD63~@g3*{o
zvI_|862z{BPEJbztd@p&8TLrkO{?(
z?rqR|6iBfqwDhN~$}Z<(@#gzPKNU
zfmx~3<9)uoPq_8~m83kRjV7N^9|<)rrSjB{#F>TyQYAnKsBNgiN>SVCPa;Y;%bhn0
z?oveHkIkF}QYNa4J9Gcki0rkTTo~e!H%oqDf4q-0?-n;vNU3mL!U{NeFdzRaVRU7M
ze8tWz>2!G@9(|c?W&NHlB>zJYeQ(xxs?mORWJS8WX&kvBoX3hIfz!^u6VhZ_IZYY-XX
zCY5^r0q7dC#H$LlZxI+NC;@U*MCXCnS~x`B(Ojq(%6)ZSVB)1D7sx%LPJzU`4Q~#sU$8p)fT)xOsjs~QL`cHU
zb%?I@3(N8z8C7ntctYG;2%e{`POuy*l?R8_CPNkX!mMr#XWl<;=zB-CEO(@>LC-ZQ
z>@nAP`t@ntCe{{>L@tUGTyN#o^e=x$s~*24(n|U&M
z^E@r=u^=;dX1z#BqY;Ps@4GB0Y=7ZW7t=h$w`xQ)lVIv>6SDcSokgQm&72ry!=Np)i#)@43lk
ztXdB*|5~`N0@il5F}71xu*-SU3lFGXS{-5?q7tlQZgv3G93Z?a|L-OEJl}bn(aZUw
z5UF?eg0h-MrXJ{cacsLQaKiB<%{)Tx0pTd&Ly7Pc?WzlPla;5aG2gJQ%Qb_vR%!{f
z{@-EQYFr@=eNBT4;!6}KL;6OE5ycrq>GZn>-0l^{HZpoR+2q;P)0X7QrdN*ugoYyp
zNYesh5v5?-A?taA0ZPC3sxHXQm^qi>C(ZNZ8^n{P&HUnDC=Bdq(V{d)g9eCnL5+MJ
zb7q>!%|9A-eC*AGAv4k9qAyxp#QX)n722a(6a=s>SHxk63LbP>k{EISFq3Y_Ht87jQB4
z;eT@~`u(6AwKNbnS&syi|F|CR*cvr=<3HE5m}wUEq$rXd-rwsJ!=bGJ3^~B`mSTU&
zHs*(2gK-e-tKbJzk}ksm9u8XbkwKuE_0rncDz>hMO(i!dO$R~salf+!z@xFH{OR|V
zWQv=vFWNZWK^Z$fsU0(t$X$5zHi*eMEQ#fU-S9F*KPicoo@d*4dZ$u50T~V#5RSyJ
zV*_dGN&s!cGY%KHx&#F#U337bPYe$ENEMB(7uBWYf2ID8PL3$5HE_if`~%2-k5%*F
zBy0>3LY!5^Or=BhT5Pyf@%giZgJjQa;BDiS!?a_df+V&cpvH)_U1pVVRJ5rb1JR((JOW_G}e4PU%_Q7v}@Y{-dfDjX|1L
z*XTK8{iZ7voduxqCJaB%{4;PUS}!b~(x1aLONUa)G}*x6AsA>eF)Ce4=A9{5Wl{7D
zccJ7Vus**%a(AfzTY)m^JSKz2UXU#zs0tr5F84h0tRXeR+gz@Do2HsLqN@0hR~&|)
zY#b_lGDQxlG#h<=2%#j@?tL6dl)mI9WiJUoP;ZDGk%IWt6s9+R9O32&3?O2KCZD3L
z{8i6ldrkOx^0slO&C;NWDzm9uzQuOxr>-bPTBcnl&fgvb_yP
z!PL?HkSPtnQS;uncq$(T8S=A^2i_LEDD;GQH`dyUbD3+?22!KOrLZT5cyJBbxDT_3
z?>1Q4aj@|(A%Bsz1}3t(doTuKZfk<2u;u~C9orUx$LFdC`z+aCq@`-M+*5YrgL~Jo
z7|6x{UT5U-m7;IAnSG-2KbFN>Na1^Sgjw1iEgD$*Vr||nByAVA;2yNhDF&Kb`OWFI)4yXmSC^Uv}j$7
zq=hz|BoXRV=d;mMjG;f-1bioavWLie;8F
zm3Rj;xv_h|Ryy6wh2V%$cGE>f!|Td0$4L+pU8rzBXM)ptsL8l8Bv6a#{OO*3FBvSj
zK9?lOD;49i8V47Loy%9sdu^H@2~)_)V)6P-x7cQY7`;*o&Qp*IEcn9;CGjU)?A+!%
zE3V^7?C7Is0JWMch95OIM|(amQaZq&?1)P17AH)tjTLz-3_R!N=qe3Txy
z$|xqC6#F#AZtG=Hx2HBCNQSutlt4xDcmy+9;&_ml$cW?_>J6VRcRE^foRmIiZL?37
zc6X%u%~=L7B3v5a44)=9t)we$ZHNkjRBCBaB8M