diff --git a/README.md b/README.md index 2f4057c..4b55cd1 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,19 @@ Ayah-by-ayah translations (835,624 rows — 6,236 ayahs × 134 editions). | `start_ayah_id` | First ayah in this hizb | | `end_ayah_id` | Last ayah in this hizb | +### `pages` (SQLite & PostgreSQL — added by the converters) + +The 604-page Madinah Mushaf navigation map. It is derived at import time from +the `ayahs.page` metadata supplied in the versioned source dump, so its ayah +boundaries stay aligned with the imported text. + +| Column | Description | +| --- | --- | +| `id` | Page ID (1–604) | +| `page_number` | Mushaf page number (1–604) | +| `start_ayah_id` | First ayah assigned to the page | +| `end_ayah_id` | Last ayah assigned to the page | + ## Setup ### Project commands @@ -340,7 +353,7 @@ We welcome contributions! Here's the planned roadmap for this project. Pick any - [x] Add proper indexes for faster queries - [x] Add `juz` (parts) table with ayah ranges - [x] Add `hizb` and `rub` (quarter) divisions -- [ ] Add `pages` table (Mushaf page mapping) +- [x] Add `pages` table (Mushaf page mapping) - [ ] Add word-by-word breakdown table (Arabic root, morphology) - [x] Add sajdah (prostration) markers - [x] Support PostgreSQL and SQLite exports diff --git a/convert_to_postgres.py b/convert_to_postgres.py index 9c9f6e1..e3b9e69 100644 --- a/convert_to_postgres.py +++ b/convert_to_postgres.py @@ -47,6 +47,7 @@ def create_postgres_schema(db: Connection) -> None: cur.execute(""" DROP TABLE IF EXISTS ayah_edition CASCADE; DROP TABLE IF EXISTS editions CASCADE; + DROP TABLE IF EXISTS pages CASCADE; DROP TABLE IF EXISTS ayahs CASCADE; DROP TABLE IF EXISTS hizbs CASCADE; DROP TABLE IF EXISTS juzs CASCADE; @@ -98,6 +99,14 @@ def create_postgres_schema(db: Connection) -> None: updated_at TIMESTAMP ); + CREATE TABLE pages ( + id INTEGER PRIMARY KEY CHECK(id BETWEEN 1 AND 604), + page_number INTEGER NOT NULL UNIQUE CHECK(page_number BETWEEN 1 AND 604), + start_ayah_id INTEGER NOT NULL REFERENCES ayahs(id), + end_ayah_id INTEGER NOT NULL REFERENCES ayahs(id), + CHECK(start_ayah_id <= end_ayah_id) + ); + CREATE TABLE editions ( id INTEGER PRIMARY KEY, identifier TEXT NOT NULL UNIQUE, @@ -400,6 +409,37 @@ def populate_lookup_tables(db: Connection) -> None: cur.close() +def populate_pages(db: Connection) -> None: + """Derive the 604-page Mushaf lookup from the source ayahs.page values.""" + cur: Cursor = db.cursor() + try: + cur.execute(""" + INSERT INTO pages (id, page_number, start_ayah_id, end_ayah_id) + SELECT page, page, MIN(id), MAX(id) + FROM ayahs + GROUP BY page + ORDER BY page + """) + cur.execute("SELECT COUNT(*) FROM ayahs") + ayah_count: tuple[int] | None = cur.fetchone() + assert ayah_count is not None + if ayah_count != (6236,): + return + + cur.execute(""" + SELECT COUNT(*), MIN(page_number), MAX(page_number), COUNT(DISTINCT page_number) + FROM pages + """) + mapping: tuple[int, int, int, int] | None = cur.fetchone() + if mapping != (604, 1, 604, 604): + raise ValueError( + "expected a complete 604-page Mushaf mapping from ayahs.page, " + f"got {mapping}" + ) + finally: + cur.close() + + TABLE_ORDER: list[str] = ["surahs", "ayahs", "editions", "ayah_edition"] # Columns the enriched model renames on the way in. Stating them here is what @@ -486,6 +526,9 @@ def convert() -> None: ): print(f" {table}: {table_counts[table]} rows...") + print(" Populating pages lookup table...") + populate_pages(db) + cur.execute(""" CREATE OR REPLACE VIEW surah_stats AS SELECT s.id, s.name_ar, s.name_en, s.name_en_translation, s.type, @@ -528,9 +571,14 @@ def convert() -> None: hizb_result: tuple[int] | None = cur.fetchone() assert hizb_result is not None + cur.execute("SELECT COUNT(*) FROM pages") + page_result: tuple[int] | None = cur.fetchone() + assert page_result is not None + print( f" Sajdah ayahs: {sajdah_result[0]}, " - f"Juzs: {juz_result[0]}, Hizbs: {hizb_result[0]}" + f"Juzs: {juz_result[0]}, Hizbs: {hizb_result[0]}, " + f"Pages: {page_result[0]}" ) db.commit() diff --git a/convert_to_sqlite.py b/convert_to_sqlite.py index 55ffb63..bb42e9d 100644 --- a/convert_to_sqlite.py +++ b/convert_to_sqlite.py @@ -63,6 +63,14 @@ def create_sqlite_schema(db): updated_at TEXT ); + CREATE TABLE IF NOT EXISTS pages ( + id INTEGER PRIMARY KEY CHECK(id BETWEEN 1 AND 604), + page_number INTEGER NOT NULL UNIQUE CHECK(page_number BETWEEN 1 AND 604), + start_ayah_id INTEGER NOT NULL REFERENCES ayahs(id), + end_ayah_id INTEGER NOT NULL REFERENCES ayahs(id), + CHECK(start_ayah_id <= end_ayah_id) + ); + CREATE TABLE IF NOT EXISTS editions ( id INTEGER PRIMARY KEY, identifier TEXT NOT NULL UNIQUE, @@ -343,6 +351,37 @@ def populate_lookup_tables(db): db.executemany("INSERT INTO hizbs VALUES (?,?,?,?,?,?)", hizbs_data) +def populate_pages(db): + """Derive the 604-page Mushaf lookup from the source ayahs.page values.""" + db.execute(""" + INSERT INTO pages (id, page_number, start_ayah_id, end_ayah_id) + SELECT page, page, MIN(id), MAX(id) + FROM ayahs + GROUP BY page + ORDER BY page + """) + + +def validate_page_mapping(db): + """Validate the complete source dump's 604-page Mushaf mapping. + + Small fixture imports deliberately carry only a subset of ayahs, so they + retain their useful partial page map without claiming it is production. + """ + ayah_count = db.execute("SELECT COUNT(*) FROM ayahs").fetchone()[0] + if ayah_count != 6236: + return + mapping = db.execute(""" + SELECT COUNT(*), MIN(page_number), MAX(page_number), COUNT(DISTINCT page_number) + FROM pages + """).fetchone() + if mapping != (604, 1, 604, 604): + raise ValueError( + "expected a complete 604-page Mushaf mapping from ayahs.page, " + f"got {mapping}" + ) + + TABLE_ORDER = ['surahs', 'ayahs', 'editions', 'ayah_edition'] # Columns the enriched model renames on the way in. Stating them here is what @@ -409,8 +448,10 @@ def convert(): db.commit() # Populate lookup tables after data - print(" Populating juzs and hizbs lookup tables...") + print(" Populating juzs, hizbs, and pages lookup tables...") populate_lookup_tables(db) + populate_pages(db) + validate_page_mapping(db) # Create views db.execute(""" @@ -451,7 +492,11 @@ def convert(): sajdah_count = db.execute("SELECT COUNT(*) FROM ayahs WHERE sajda = 1").fetchone()[0] juz_count = db.execute("SELECT COUNT(*) FROM juzs").fetchone()[0] hizb_count = db.execute("SELECT COUNT(*) FROM hizbs").fetchone()[0] - print(f" Sajdah ayahs: {sajdah_count}, Juzs: {juz_count}, Hizbs: {hizb_count}") + page_count = db.execute("SELECT COUNT(*) FROM pages").fetchone()[0] + print( + f" Sajdah ayahs: {sajdah_count}, Juzs: {juz_count}, " + f"Hizbs: {hizb_count}, Pages: {page_count}" + ) db_size = os.path.getsize(DB_FILE) / (1024 * 1024) print(f" Database size: {db_size:.1f} MB") diff --git a/docs/architecture.md b/docs/architecture.md index fe34ac2..a22bc02 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -68,9 +68,11 @@ juzs surahs hizbs ayahs 1 -------- many ayah_edition many -------- 1 editions | +-- page, juz_id, rub_id, sajda + | + +-- pages (604 descriptive ayah ranges) ``` -The enriched SQLite and PostgreSQL targets contain six tables: +The enriched SQLite and PostgreSQL targets contain seven tables: | Table | Purpose | Expected rows | | --- | --- | ---: | @@ -80,6 +82,7 @@ The enriched SQLite and PostgreSQL targets contain six tables: | `ayah_edition` | Text for each ayah and edition pairing | 835,624 | | `juzs` | Thirty juz lookup records with ayah ranges | 30 | | `hizbs` | Sixty hizb lookup records with ayah ranges | 60 | +| `pages` | Madinah Mushaf page lookup records with ayah ranges | 604 | `ayah_edition` is the high-volume junction between `ayahs` and `editions`. Indexes support common navigation by surah, juz, hizb, page, ayah number, and @@ -106,6 +109,9 @@ The converters rename it to `ayahs.rub_id` and constrain it with join that would return wrong rows. Derive the hizb with `FLOOR((rub_id - 1) / 4) + 1` when you need the 60-row lookup. `ayahs.juz_id` spans 1–30 and does correspond to the `juzs` lookup. +`ayahs.page` spans 1–604; the converters derive the `pages` table directly +from that source field. The range endpoints describe the first and last ayah +assigned to each page, rather than attempting to model a visual page layout. Range endpoints such as `start_ayah_id` and `end_ayah_id` are descriptive data. Changes to these values should be validated against the source and should not @@ -118,7 +124,7 @@ At a high level, each converter: 1. creates or recreates its target schema; 2. streams and parses supported `INSERT` statements from `quran.sql`; 3. loads `surahs`, `ayahs`, `editions`, and `ayah_edition`; -4. populates the `juzs` and `hizbs` lookup tables; +4. populates the `juzs`, `hizbs`, and `pages` lookup tables; 5. creates the convenience views; 6. prints row counts for inspection. diff --git a/docs/provenance.md b/docs/provenance.md index 86442ff..a8db4ab 100644 --- a/docs/provenance.md +++ b/docs/provenance.md @@ -28,6 +28,16 @@ apart inside this repository. We are three hops from the printed Mushaf. None of those hops is us retyping the text. +## Page mapping + +The `pages` lookup table in the SQLite and PostgreSQL exports is derived from +the `ayahs.page` field in this repository's versioned MySQL dump. That field +contains every integer from 1 through 604, the page count of the Madinah +Mushaf. The converters group the imported ayahs by that supplied value and +record each group's first and last ayah ID; they reject an incomplete or +non-contiguous mapping. This is distribution metadata derived from this dump, +not an independent verification of the printed page layout. + ## What the text actually is Tanzil **Uthmani**, exported with pause marks, sajdah signs, rub-el-hizb diff --git a/schema/postgres/schema.sql b/schema/postgres/schema.sql index 97c5916..b9f34c6 100644 --- a/schema/postgres/schema.sql +++ b/schema/postgres/schema.sql @@ -45,6 +45,14 @@ CREATE TABLE ayahs ( updated_at TIMESTAMP ); +CREATE TABLE pages ( + id INTEGER PRIMARY KEY CHECK(id BETWEEN 1 AND 604), + page_number INTEGER NOT NULL UNIQUE CHECK(page_number BETWEEN 1 AND 604), + start_ayah_id INTEGER NOT NULL REFERENCES ayahs(id), + end_ayah_id INTEGER NOT NULL REFERENCES ayahs(id), + CHECK(start_ayah_id <= end_ayah_id) +); + CREATE TABLE editions ( id INTEGER PRIMARY KEY, identifier TEXT NOT NULL UNIQUE, diff --git a/schema/sqlite/schema.sql b/schema/sqlite/schema.sql index 65851e7..40da363 100644 --- a/schema/sqlite/schema.sql +++ b/schema/sqlite/schema.sql @@ -45,6 +45,14 @@ CREATE TABLE IF NOT EXISTS ayahs ( updated_at TEXT ); +CREATE TABLE IF NOT EXISTS pages ( + id INTEGER PRIMARY KEY CHECK(id BETWEEN 1 AND 604), + page_number INTEGER NOT NULL UNIQUE CHECK(page_number BETWEEN 1 AND 604), + start_ayah_id INTEGER NOT NULL REFERENCES ayahs(id), + end_ayah_id INTEGER NOT NULL REFERENCES ayahs(id), + CHECK(start_ayah_id <= end_ayah_id) +); + CREATE TABLE IF NOT EXISTS editions ( id INTEGER PRIMARY KEY, identifier TEXT NOT NULL UNIQUE, diff --git a/tests/test_export_schema.py b/tests/test_export_schema.py index 9f285b3..546c6a3 100644 --- a/tests/test_export_schema.py +++ b/tests/test_export_schema.py @@ -7,7 +7,10 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] -EXPECTED_TABLES = {"surahs", "ayahs", "editions", "ayah_edition", "juzs", "hizbs"} +EXPECTED_TABLES = { + "surahs", "ayahs", "editions", "ayah_edition", "juzs", "hizbs", "pages" +} +MYSQL_SOURCE_TABLES = EXPECTED_TABLES - {"pages"} EXPECTED_VIEWS = {"surah_stats", "ayah_with_translation"} @@ -67,6 +70,18 @@ def test_rub_id_rejects_values_outside_the_quarter_range(self) -> None: with self.subTest(rub_id=bad), self.assertRaises(sqlite3.IntegrityError): self.db.execute(row, (bad + 100, bad)) + def test_pages_table_maps_ayah_ranges_within_the_mushaf_domain(self) -> None: + self.db.execute("INSERT INTO surahs VALUES (1,1,'a','a','a','Meccan',NULL,NULL)") + ayah = "INSERT INTO ayahs VALUES (?,1,'t',1,1,1,1,1,0,NULL,NULL)" + self.db.executemany(ayah, [(1,), (2,)]) + self.db.execute("INSERT INTO pages VALUES (1,1,1,2)") + self.assertEqual( + self.db.execute("SELECT start_ayah_id, end_ayah_id FROM pages").fetchone(), + (1, 2), + ) + with self.assertRaises(sqlite3.IntegrityError): + self.db.execute("INSERT INTO pages VALUES (605,605,1,2)") + def test_carries_no_drop_statements(self) -> None: # A reference someone might paste into a live database must not open # by dropping their tables. @@ -77,7 +92,7 @@ def test_carries_no_drop_statements(self) -> None: class SourceSchemaTests(unittest.TestCase): def test_mysql_reference_describes_the_full_source_dump(self) -> None: mysql = read("mysql/schema.sql") - for table in EXPECTED_TABLES | {"users", "password_resets", "migrations"}: + for table in MYSQL_SOURCE_TABLES | {"users", "password_resets", "migrations"}: self.assertIn(f"CREATE TABLE `{table}`", mysql) def test_mysql_reference_excludes_row_data(self) -> None: diff --git a/tests/test_pages.py b/tests/test_pages.py new file mode 100644 index 0000000..719d0a0 --- /dev/null +++ b/tests/test_pages.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import importlib.util +import sqlite3 +import types +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def load_converter() -> types.ModuleType: + spec = importlib.util.spec_from_file_location( + "convert_to_sqlite", ROOT / "convert_to_sqlite.py" + ) + if spec is None or spec.loader is None: + raise RuntimeError("unable to load convert_to_sqlite.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +converter = load_converter() + + +class PageMappingTests(unittest.TestCase): + def setUp(self) -> None: + self.db = sqlite3.connect(":memory:") + converter.create_sqlite_schema(self.db) + self.db.execute("INSERT INTO surahs VALUES (1,1,'a','a','a','Meccan',NULL,NULL)") + + def tearDown(self) -> None: + self.db.close() + + def add_full_ayah_set(self, page_for_id: int | None = None) -> None: + rows = [] + for ayah_id in range(1, 6237): + page = page_for_id or min(604, (ayah_id - 1) * 604 // 6236 + 1) + rows.append( + (ayah_id, ayah_id, "t", ayah_id, page, 1, 1, 1, 0, None, None) + ) + self.db.executemany("INSERT INTO ayahs VALUES (?,?,?,?,?,?,?,?,?,?,?)", rows) + + def test_derives_and_validates_the_complete_mapping(self) -> None: + self.add_full_ayah_set() + + converter.populate_pages(self.db) + converter.validate_page_mapping(self.db) + + self.assertEqual( + self.db.execute( + "SELECT COUNT(*), MIN(page_number), MAX(page_number) FROM pages" + ).fetchone(), + (604, 1, 604), + ) + self.assertEqual( + self.db.execute( + "SELECT start_ayah_id, end_ayah_id FROM pages WHERE id = 1" + ).fetchone(), + (1, 11), + ) + self.assertEqual( + self.db.execute( + "SELECT start_ayah_id, end_ayah_id FROM pages WHERE id = 604" + ).fetchone(), + (6227, 6236), + ) + + def test_rejects_an_incomplete_full_source_mapping(self) -> None: + self.add_full_ayah_set(page_for_id=1) + converter.populate_pages(self.db) + + with self.assertRaisesRegex(ValueError, "complete 604-page"): + converter.validate_page_mapping(self.db) + + +if __name__ == "__main__": + unittest.main()