Skip to content

fix(build): close documentation sqlite handles #39

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
imathwy merged 1 commit into from
Sep 5, 2026
Merged

fix(build): close documentation sqlite handles #39

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 29 additions & 27 deletions sdk/build/src/reasbook_build_sdk/docs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

from __future__ import annotations

from contextlib import contextmanager
from contextlib import closing, contextmanager
from dataclasses import dataclass
import fcntl
import hashlib
Expand Down Expand Up @@ -1397,20 +1397,21 @@ def _set_database_source_urls(
)
expected = {module for module, _url in module_urls}
try:
with sqlite3.connect(database) as connection:
rows = connection.execute("SELECT name FROM modules").fetchall()
actual = {str(row[0]) for row in rows}
if actual != expected:
missing = sorted(expected - actual)
unexpected = sorted(actual - expected)
raise BuildFailed(
"documentation database module mismatch"
f"; missing={missing[:10]}; unexpected={unexpected[:10]}"
with closing(sqlite3.connect(database)) as connection:
with connection:
rows = connection.execute("SELECT name FROM modules").fetchall()
actual = {str(row[0]) for row in rows}
if actual != expected:
missing = sorted(expected - actual)
unexpected = sorted(actual - expected)
raise BuildFailed(
"documentation database module mismatch"
f"; missing={missing[:10]}; unexpected={unexpected[:10]}"
)
connection.executemany(
"UPDATE modules SET source_url = ? WHERE name = ?",
((url, module) for module, url in module_urls),
)
connection.executemany(
"UPDATE modules SET source_url = ? WHERE name = ?",
((url, module) for module, url in module_urls),
)
except sqlite3.Error as exc:
raise BuildFailed(f"cannot update documentation database: {exc}") from exc

Expand Down Expand Up @@ -1820,8 +1821,8 @@ def _validate_analysis_database(
raise BuildFailed(f"documentation analysis database is unsafe: {database}")
try:
suffix = "?mode=ro" + ("&immutable=1" if immutable else "")
with sqlite3.connect(
database.resolve().as_uri() + suffix, uri=True
with closing(
sqlite3.connect(database.resolve().as_uri() + suffix, uri=True)
) as connection:
check = connection.execute("PRAGMA quick_check").fetchone()
if check != ("ok",):
Expand Down Expand Up @@ -1901,18 +1902,19 @@ def _backup_analysis_database(source: Path, destination: Path) -> None:
if source.is_symlink() or not source.is_file() or destination.exists():
raise BuildFailed(f"documentation analysis database is unsafe: {source}")
try:
with sqlite3.connect(
source.resolve().as_uri() + "?mode=ro", uri=True
with closing(
sqlite3.connect(source.resolve().as_uri() + "?mode=ro", uri=True)
) as source_connection:
with sqlite3.connect(destination) as destination_connection:
source_connection.backup(destination_connection)
journal_mode = destination_connection.execute(
"PRAGMA journal_mode=DELETE"
).fetchone()
if journal_mode != ("delete",):
raise BuildFailed(
"cannot make documentation analysis snapshot standalone"
)
with closing(sqlite3.connect(destination)) as destination_connection:
with destination_connection:
source_connection.backup(destination_connection)
journal_mode = destination_connection.execute(
"PRAGMA journal_mode=DELETE"
).fetchone()
if journal_mode != ("delete",):
raise BuildFailed(
"cannot make documentation analysis snapshot standalone"
)
except sqlite3.Error as exc:
raise BuildFailed(
f"cannot snapshot documentation analysis database: {exc}"
Expand Down
93 changes: 69 additions & 24 deletions sdk/build/tests/test_docs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from __future__ import annotations

from contextlib import closing
import json
import platform
import re
Expand All @@ -21,6 +22,16 @@
from reasbook_build_sdk import docs as docs_module


class _TrackingConnection(sqlite3.Connection):
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.was_closed = False

def close(self) -> None:
self.was_closed = True
super().close()


class _DocsRunner:
def __init__(self) -> None:
self.modules: list[str] = []
Expand All @@ -36,31 +47,35 @@ def run(self, command):
build = Path(argv[-3])
modules = tuple(control.read_text(encoding="utf-8").splitlines())
database = build / argv[-2]
with sqlite3.connect(database) as connection:
connection.execute("PRAGMA journal_mode=WAL")
connection.execute(
"CREATE TABLE IF NOT EXISTS modules "
"(name TEXT PRIMARY KEY, source_url TEXT)"
)
connection.execute(
"CREATE TABLE IF NOT EXISTS schema_meta "
"(key TEXT PRIMARY KEY, value TEXT NOT NULL)"
)
connection.executemany(
"INSERT OR REPLACE INTO schema_meta (key, value) VALUES (?, ?)",
(("ddl_hash", "fixture-ddl"), ("type_hash", "fixture-types")),
)
for table in sorted(
docs_module._ANALYZER_REQUIRED_TABLES
- {"modules", "schema_meta"}
):
with closing(sqlite3.connect(database)) as connection:
with connection:
connection.execute("PRAGMA journal_mode=WAL")
connection.execute(
f'CREATE TABLE IF NOT EXISTS "{table}" (fixture INTEGER)'
"CREATE TABLE IF NOT EXISTS modules "
"(name TEXT PRIMARY KEY, source_url TEXT)"
)
connection.execute(
"CREATE TABLE IF NOT EXISTS schema_meta "
"(key TEXT PRIMARY KEY, value TEXT NOT NULL)"
)
connection.executemany(
"INSERT OR REPLACE INTO schema_meta (key, value) VALUES (?, ?)",
(
("ddl_hash", "fixture-ddl"),
("type_hash", "fixture-types"),
),
)
for table in sorted(
docs_module._ANALYZER_REQUIRED_TABLES
- {"modules", "schema_meta"}
):
connection.execute(
f'CREATE TABLE IF NOT EXISTS "{table}" (fixture INTEGER)'
)
connection.executemany(
"INSERT INTO modules (name, source_url) VALUES (?, NULL)",
((module,) for module in modules),
)
connection.executemany(
"INSERT INTO modules (name, source_url) VALUES (?, NULL)",
((module,) for module in modules),
)
else:
build = Path(argv[-2])
modules = tuple(
Expand All @@ -72,7 +87,7 @@ def run(self, command):
build = Path(argv[argv.index("--build") + 1])
if "fromDb" in argv and not self.modules:
database = Path(argv[argv.index("--manifest") + 2])
with sqlite3.connect(database) as connection:
with closing(sqlite3.connect(database)) as connection:
self.modules.extend(
str(row[0])
for row in connection.execute(
Expand Down Expand Up @@ -641,6 +656,36 @@ def test_compiled_artifact_changes_invalidate_docs_and_analysis(self) -> None:
)
previous = current

def test_database_connections_close_before_documentation_publication(self) -> None:
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
project = self._project(root, modern=True)
output = root / "cache" / "docs"
connections: list[_TrackingConnection] = []
connect = sqlite3.connect

def tracked_connect(*args, **kwargs):
kwargs["factory"] = _TrackingConnection
connection = connect(*args, **kwargs)
connections.append(connection)
return connection

with patch.object(sqlite3, "connect", side_effect=tracked_connect):
ProjectDocumentationBuilder(runner=_DocsRunner()).build(
project, ("Books.Demo.Book",), output
)

self.assertGreater(len(connections), 0)
self.assertTrue(all(connection.was_closed for connection in connections))
self.assertEqual(
[
path
for path in output.rglob("*")
if path.name.endswith(("-wal", "-shm"))
],
[],
)

def test_unmanaged_metadata_cannot_suppress_dependency_hashing(self) -> None:
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
Expand Down
Loading