From c07522c6696ba699619f8746cb83b701219b33d8 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 5 Sep 2026 22:42:11 +0800 Subject: [PATCH] fix(release): retain project API docs on Pages --- .github/workflows/publish_release_pages.yml | 6 +- CHANGELOG.md | 5 +- config/deploy/github-pages.yml | 9 +- .../0002-target-specific-release-artifacts.md | 6 +- .../0008-project-owned-api-docs-on-pages.md | 116 ++++++++ sdk/deploy/README.md | 8 +- .../src/reasbook_deploy_sdk/release/README.md | 17 +- .../reasbook_deploy_sdk/release/artifacts.py | 279 +++++++++++++++--- .../src/reasbook_deploy_sdk/release/models.py | 8 +- sdk/deploy/tests/test_release.py | 143 ++++++++- tests/test_github_workflows.py | 3 +- 11 files changed, 522 insertions(+), 78 deletions(-) create mode 100644 docs/decisions/0008-project-owned-api-docs-on-pages.md diff --git a/.github/workflows/publish_release_pages.yml b/.github/workflows/publish_release_pages.yml index 6cb81c30a..10d729973 100644 --- a/.github/workflows/publish_release_pages.yml +++ b/.github/workflows/publish_release_pages.yml @@ -213,8 +213,10 @@ jobs: raise SystemExit("Pages artifact must use the /ReasBook/ base path") if manifest.get("artifact") != "pages": raise SystemExit("GitHub Pages requires the pages artifact") - if manifest["total_bytes"] > 850_000_000: - raise SystemExit("Pages site exceeds the 850 MB operational budget") + if manifest["total_bytes"] > 1_000_000_000: + raise SystemExit("Pages site exceeds GitHub's 1 GB hard limit") + if manifest["total_bytes"] > 920_000_000: + raise SystemExit("Pages site exceeds the 920 MB operational budget") if manifest["file_count"] > 60_000: raise SystemExit("Pages site exceeds the 60,000-file budget") artifacts = release_set.get("artifacts") diff --git a/CHANGELOG.md b/CHANGELOG.md index 77664aca1..b6b42e697 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,8 +17,9 @@ Significant user-facing and release-engineering changes are recorded here. ### Changed - GitHub Pages is a publish-only target; full Lean builds run on SiFlow. -- GitHub Pages now receives a size-bounded canonical projection with lightweight - compatibility redirects and detailed-API placeholders; the full historical +- GitHub Pages now retains every selected project's API pages while using + lightweight placeholders only for external dependencies. Its operational + site budget is 920 MB with an independent 1 GB hard gate; the full historical site remains unchanged for self-hosting. - Repository scripts are thin project adapters while reusable behavior lives in the SDK packages. diff --git a/config/deploy/github-pages.yml b/config/deploy/github-pages.yml index f748d480f..1f5b72dd4 100644 --- a/config/deploy/github-pages.yml +++ b/config/deploy/github-pages.yml @@ -25,9 +25,10 @@ policy: # A release has one build identity and two immutable delivery artifacts. # The full artifact retains bounded reachable project-module API docs and every -# project version for self-hosting. Pages keeps canonical Verso versions and -# project entry API pages; omitted detailed API links become explicit -# placeholders whose source pages must exist in the verified full artifact. +# project version for self-hosting. Pages keeps canonical Verso routes and all +# API pages owned by every selected ProjectSpec; omitted external-dependency +# API links become explicit placeholders whose source pages must exist in the +# verified full artifact. artifacts: full: history_mode: full @@ -40,7 +41,7 @@ artifacts: history_mode: canonical dependency_docs: stubs max_site_files: 60000 - max_site_bytes: 850000000 + max_site_bytes: 920000000 max_archive_members: 180000 max_bundle_bytes: 950000000 diff --git a/docs/decisions/0002-target-specific-release-artifacts.md b/docs/decisions/0002-target-specific-release-artifacts.md index 3a249a6a6..c786b0bb5 100644 --- a/docs/decisions/0002-target-specific-release-artifacts.md +++ b/docs/decisions/0002-target-specific-release-artifacts.md @@ -2,7 +2,7 @@ ## Status -Accepted +Accepted; project-owned API projection superseded in part by ADR-0008 ## Date @@ -53,6 +53,8 @@ pages, and theorem map remained available at their original URLs. exists in the full site. This preserves link closure without allowing the projection to hide a missing build output. The complete API documentation remains in the `full` artifact for self-hosting. + **Superseded by ADR-0008:** Pages now retains every API page owned by a + selected `ProjectSpec`; only external-dependency API pages use placeholders. - Every theorem map, whether generated or curated, carries a generated release context that matches its project specification. Repository source links are pinned to the full project commit; the branch name remains display metadata @@ -100,6 +102,8 @@ pages, and theorem map remained available at their original URLs. except exact `.git` and `.github` path segments, which are rejected because GitHub always omits them. These are operational margins below GitHub's hard limits, not targets to fill. + **Superseded by ADR-0008:** the extracted-content operational limit is now + 920 MB, with a separately enforced 1 GB hard gate. The other limits remain. - Self-hosted deployment accepts only the `full` artifact. It installs into a versioned directory, exposes the configured base path below `public/`, and atomically switches a `current` symlink. A failed health check restores the diff --git a/docs/decisions/0008-project-owned-api-docs-on-pages.md b/docs/decisions/0008-project-owned-api-docs-on-pages.md new file mode 100644 index 000000000..85ff773f6 --- /dev/null +++ b/docs/decisions/0008-project-owned-api-docs-on-pages.md @@ -0,0 +1,116 @@ +# ADR-0008: Retain project-owned API documentation on GitHub Pages + +## Status + +Accepted + +## Date + +2026-09-05 + +## Context + +ADR-0002 introduced a bounded GitHub Pages projection because the complete +release site is already larger than GitHub Pages' 1 GB published-site limit. +That projection retained only one API entry page per canonical project and +replaced every deeper API link with a small placeholder. + +The resulting site fit comfortably within its 850,000,000-byte operational +budget, but it did not provide usable API documentation. In release +`site-20260905T124421Z-e986b071fe51`, 5,520 of 7,183 HTML files in the Pages +artifact were projection placeholders. A reader could open a `Book`, `Paper`, +or `Main` entry and then immediately leave the real documentation when +following a chapter or declaration link. + +The same release provides enough evidence to choose a more useful boundary: + +- the existing Pages artifact is 712,116,853 bytes; +- restoring every ReleaseSpec-owned project API page projects to about + 885 MB; and +- the complete self-hosted artifact is about 2.36 GB and therefore remains + unsuitable for GitHub Pages. + +GitHub documents a [1 GB limit for a published Pages +site](https://docs.github.com/en/pages/getting-started-with-github-pages/github-pages-limits). +The temporary GitHub deployment needs a smaller fail-closed operational +budget, while the long-term self-hosted artifact must remain unchanged. + +## Decision + +- A `ProjectSpec` is the authority for API-document ownership. For every + selected project version on a branch, the Pages projector copies all + existing documentation shapes derived from its kind and project identifier, + including nested `Books` or `Papers` layouts, flat namespace directories, + and a same-name top-level HTML entry. +- Canonical selection continues to control catalog redirects and Verso routes. + It does not turn a non-canonical project version's API pages into external + dependency placeholders when a canonical project links to them. +- A project API page may still link to a non-canonical Verso route omitted by + the Pages history policy. The projector closes that link with a lightweight + redirect to the project's verified canonical Verso route, preserving the + matching deep suffix when it exists. This is a history redirect, not an API + documentation placeholder. +- The projector copies project documentation before computing link closure. + Consequently, links introduced by detailed project pages participate in the + same validation pass. +- API pages outside all ReleaseSpec-derived ownership roots become explicit + external-dependency placeholders only when the verified source already marks + them external or their namespace is an allowed dependency such as Mathlib, + Lean, Batteries, or Init. Unknown unmarked namespaces fail closed instead of + silently hiding a project page. +- The Pages operational limit is 920,000,000 bytes. A separate + 1,000,000,000-byte GitHub hard limit is enforced independently in profile + validation, local projection, bundle verification, and the publish workflow. + Raising the operational budget cannot disable the host limit. +- Capacity overflow fails during local or remote packaging before an immutable + Release is published. Selection is never changed implicitly to make an + oversized artifact fit. +- The `full` artifact, its version-qualified URLs, and the self-hosted atomic + installation process are unchanged. + +This decision supersedes ADR-0002 only where it says that Pages retains project +entry API pages while replacing detailed project API pages with placeholders. +ADR-0002 continues to govern target-specific bundles, canonical history, +ReleaseSet binding, immutable publication, and self-hosted delivery. + +## Alternatives considered + +### Keep only project API entry pages + +Rejected because the entry is not useful when its first project-module link +leads to a generic capacity notice. + +### Copy every versioned documentation directory without ownership checks + +Rejected because it erases the contract between project content and external +dependencies. ReleaseSpec-derived roots make the boundary deterministic and +cause an unknown project layout to fail instead of being silently reclassified. + +### Set the operational budget equal to GitHub's hard limit + +Rejected because metadata and normal project growth need headroom. The +920 MB budget leaves an independent 80 MB host margin and makes the next +capacity decision explicit. + +### Change or trim the self-hosted artifact + +Rejected because the full artifact is the durable deployment target and is not +subject to GitHub Pages' temporary hosting constraint. + +## Consequences + +- Detailed project API links work on GitHub Pages and remain byte-identical to + the verified branch output. +- External dependency links remain visibly bounded instead of silently + increasing the artifact with transitive documentation. +- Links from retained historical API pages to omitted historical Verso pages + converge on the explicit canonical project version without copying the + approximately 152 MB historical route trees. +- The current release family has roughly 35 MB of operational headroom. A + future release that crosses 920 MB must reduce a measured large component + (for example a generated Verso route) or move public serving to the + self-hosted deployment; it must not restore broad placeholders implicitly. +- Changing this projection and its capacity policy changes the artifact-policy + digest and therefore requires a new immutable Release. Existing Lean, + documentation, Verso, and theorem-map branch artifacts remain reusable after + identity revalidation. diff --git a/sdk/deploy/README.md b/sdk/deploy/README.md index 3e35df6a5..39adb9f48 100644 --- a/sdk/deploy/README.md +++ b/sdk/deploy/README.md @@ -269,9 +269,11 @@ for the allowed large archive is network-dominated. The Pages capacity policy is single-source: local packaging, local acceptance, and the workflow's `release verify --profile github-pages --artifact-policy -pages` use the same checked-in 850 MB, 60,000-file, and -180,000-archive-member limits. A bundle that the workflow would reject cannot -pass local packaging merely because a verifier default was looser. +pages` use the same checked-in 920 MB, 60,000-file, and +180,000-archive-member operational limits. A separate 1 GB hard gate is +enforced even if the operational policy is later raised. A bundle that the +workflow would reject cannot pass local packaging merely because a verifier +default was looser. When `--wait` is present, the command first waits up to 1,800 seconds for the workflow and then up to 300 seconds for the public Pages ReleaseSpec to diff --git a/sdk/deploy/src/reasbook_deploy_sdk/release/README.md b/sdk/deploy/src/reasbook_deploy_sdk/release/README.md index aaf7caa49..96e344a90 100644 --- a/sdk/deploy/src/reasbook_deploy_sdk/release/README.md +++ b/sdk/deploy/src/reasbook_deploy_sdk/release/README.md @@ -34,7 +34,7 @@ The two artifacts have deliberately different content contracts: | Name | Contract | Intended target | | --- | --- | --- | | `full` | Every assembled project version, reachable project-module API docs, Verso, theorem maps, and explicit dependency stubs | Self-hosted server | -| `pages` | Canonical version-qualified Verso routes, project entry API pages, theorem maps, and link-closed API placeholders | GitHub Pages | +| `pages` | Canonical version-qualified Verso routes, every selected project's API pages, theorem maps, and external-dependency API placeholders | GitHub Pages | Both retain the `/ReasBook/` public base path. `release-set.json` binds their bundle hashes, site-tree hashes, sizes, and projection-policy hash to one @@ -223,7 +223,8 @@ workflow uses GitHub's Release and per-asset verification commands, independently checks the tag target, archive/manifest/spec/ReleaseSet bindings, and recomputes the policy digest from its trusted checked-out profile. It enforces the compressed, -180,000-archive-member, 850 MB site, and 60,000-file budgets before extraction. +180,000-archive-member, 920 MB site, and 60,000-file operational budgets before +extraction, plus an independent 1 GB published-site hard gate. Local packaging and acceptance load the same artifact policy, so these limits cannot first appear as a remote-only rejection. It has no full archive and performs no source build. Publishing an existing tag is @@ -284,12 +285,12 @@ docs start at each configured entry root and include every reachable project-owned module, processed in batches of at most 128; Mathlib, Lean, and other external libraries are not rendered. The `full` artifact retains those generated module pages and every assembled project version. The Pages -projection is assembled from an allowlist: explicit canonical versions, -project entry API pages, Verso pages, theorem maps, and referenced runtime -assets. Detailed API links omitted from that bounded artifact resolve to small -explanatory placeholders only when the verified full site contains the target; -they never become silent 404s. Unversioned catalog compatibility paths are -redirects rather than copies of the large Verso trees. +projection is assembled from an allowlist: canonical Verso routes, every API +documentation shape owned by a selected `ProjectSpec`, theorem maps, and +referenced runtime assets. API links outside those project-owned roots resolve +to small external-dependency placeholders only when the verified full site +contains the target; they never become silent 404s. Unversioned catalog +compatibility paths are redirects rather than copies of the large Verso trees. ## Self-hosted installation diff --git a/sdk/deploy/src/reasbook_deploy_sdk/release/artifacts.py b/sdk/deploy/src/reasbook_deploy_sdk/release/artifacts.py index 812bc987d..7ddfb7c3e 100644 --- a/sdk/deploy/src/reasbook_deploy_sdk/release/artifacts.py +++ b/sdk/deploy/src/reasbook_deploy_sdk/release/artifacts.py @@ -18,7 +18,11 @@ from ..errors import DeployExecutionError from ..git import version_key -from .models import ReleaseArtifactPolicy, ReleaseSpec +from .models import ( + GITHUB_PAGES_HARD_SITE_BYTES, + ReleaseArtifactPolicy, + ReleaseSpec, +) from .results import ( BundleInfo, ReleaseArtifactRecord, @@ -29,6 +33,19 @@ _SHARED_DOC_DIRECTORIES = {"declarations", "find", "src"} +_EXTERNAL_DOC_NAMESPACES = { + "Aesop", + "Batteries", + "Cli", + "ImportGraph", + "Init", + "Lake", + "Lean", + "Mathlib", + "Plausible", + "Qq", + "Std", +} _PAGES_ROOT_FILES = { ".nojekyll", "-verso-docs.json", @@ -94,8 +111,10 @@ class PagesSiteProjector: """Create a bounded Pages view without changing the full release tree. Canonical project pages retain their original version-qualified URLs. - Detailed API pages omitted from Pages become explicit lightweight placeholders; - referenced styles, scripts, images, and search data remain byte-identical. + Every API page owned by a selected project remains byte-identical. External + dependency pages reached by those documents become explicit lightweight + placeholders; referenced styles, scripts, images, and search data remain + byte-identical. """ def __init__(self, *, max_site_bytes: int | None = None) -> None: @@ -146,18 +165,18 @@ def project( Path("versions") / "index.html", ) self._write_catalog_redirects(spec, staged, canonical_routes) - self._close_internal_references(spec, full_site, staged) + self._close_internal_references( + spec, + full_site, + staged, + canonical_routes, + ) self._validate_required_content(spec, staged, canonical_routes) self._validate_site(staged) - if self.max_site_bytes is not None: - total_bytes = sum( - path.stat().st_size for path in staged.rglob("*") if path.is_file() - ) - if total_bytes > self.max_site_bytes: - raise DeployExecutionError( - f"Pages projection is {total_bytes} bytes; budget is " - f"{self.max_site_bytes}" - ) + total_bytes = sum( + path.stat().st_size for path in staged.rglob("*") if path.is_file() + ) + self._validate_capacity(total_bytes) self._publish(staged, target, backup) return target finally: @@ -228,14 +247,38 @@ def _project_branch( ) target = versions / branch target.mkdir() + branch_projects = tuple( + project for project in spec.projects if project.branch == branch + ) projects = tuple( - project for project in spec.canonical_projects() if project.branch == branch + project for project in branch_projects if project.canonical ) for relative in (Path("static"), Path("-verso-docs.json")): self._copy_if_present(source, target, relative) self._copy_doc_runtime(source, target) + # Documentation ownership is recorded for every ProjectSpec, not only + # the projects selected as catalog canonicals. Copy all known project + # namespaces before link closure so cross-project API links remain real. + project_docs: dict[str, Path] = {} + for project in branch_projects: + if "docs" not in project.outputs: + continue + docs = self._copy_project_docs( + source, + target, + project.kind, + project.project_id, + project.build_target, + ) + if docs is None: + raise DeployExecutionError( + "Pages projection has no project-owned API entry: " + f"{project.key}@{branch}" + ) + project_docs[project.key] = docs + project_routes: list[tuple[str, str]] = [] canonical_routes: dict[str, tuple[Path | None, Path | None]] = {} for project in projects: @@ -256,13 +299,7 @@ def _project_branch( target, Path("theorem-maps") / project.kind / project.slug, ) - docs = self._copy_project_docs( - source, - target, - project.kind, - project.project_id, - project.build_target, - ) + docs = project_docs.get(project.key) if "docs" in project.outputs and docs is None: raise DeployExecutionError( f"Pages projection has no canonical API entry: {project.key}" @@ -374,24 +411,40 @@ def _copy_project_docs( ) ) ) - chosen: Path | None = None + entry_candidates: list[Path] = [] for relative in candidates: item = source / relative if item.is_dir(): for name in entry_names: entry = relative / f"{name}.html" - if self._copy_if_present(source, destination, entry): - if chosen is None: - chosen = entry + if (source / entry).is_file(): + entry_candidates.append(entry) break - elif item.is_file() and self._copy_if_present( - source, - destination, - relative, - ): - if chosen is None: - chosen = relative - return chosen + elif item.is_file(): + entry_candidates.append(relative) + if not entry_candidates: + return None + + # A project may have both a top-level module HTML file and a sibling + # namespace directory (for example an explicit Lake root). Once an + # entry establishes ownership, retain every ProjectSpec-derived shape. + for relative in candidates: + self._copy_if_present(source, destination, relative) + return entry_candidates[0] + + def _validate_capacity(self, total_bytes: int) -> None: + """Apply GitHub's hard limit independently of the profile safety line.""" + + if total_bytes > GITHUB_PAGES_HARD_SITE_BYTES: + raise DeployExecutionError( + f"Pages projection is {total_bytes} bytes; GitHub Pages hard " + f"limit is {GITHUB_PAGES_HARD_SITE_BYTES}" + ) + if self.max_site_bytes is not None and total_bytes > self.max_site_bytes: + raise DeployExecutionError( + f"Pages projection is {total_bytes} bytes; budget is " + f"{self.max_site_bytes}" + ) def _write_catalog_redirects( self, @@ -485,8 +538,8 @@ def _branch_index( '
', '

Canonical Pages Projection

', f'

{html.escape(branch)}

', - "

Detailed API links omitted from this bounded artifact " - "resolve to explicit placeholders.

", + "

Project API documentation is complete. External " + "dependency links resolve to explicit placeholders.

", ' ", @@ -503,6 +556,7 @@ def _close_internal_references( spec: ReleaseSpec, full_site: Path, pages_site: Path, + canonical_routes: dict[str, tuple[Path | None, Path | None]], ) -> None: origin = "https://reasbook.invalid" base_path = spec.base_path @@ -532,21 +586,47 @@ def _close_internal_references( source = full_site / relative if source.is_file(): if source.suffix.lower() in {".html", ".htm"}: - if self._is_omitted_api_doc(relative): + if self._is_external_api_doc(relative, source): self._write_api_stub(target, spec.base_path) + elif history_target := self._canonical_history_target( + spec, + relative, + pages_site, + canonical_routes, + ): + self._write_history_redirect( + target, + history_target, + spec.base_path, + ) else: errors.append( - f"{relative_document}: omitted non-dependency page {value}" + f"{relative_document}: omitted non-dependency " + f"page {value}" ) else: target.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(source, target) elif source.is_dir() and (source / "index.html").is_file(): - if self._is_omitted_api_doc(relative / "index.html"): + if self._is_external_api_doc( + relative / "index.html", + source / "index.html", + ): self._write_api_stub( target / "index.html", spec.base_path, ) + elif history_target := self._canonical_history_target( + spec, + relative / "index.html", + pages_site, + canonical_routes, + ): + self._write_history_redirect( + target / "index.html", + history_target, + spec.base_path, + ) else: errors.append( f"{relative_document}: omitted non-dependency directory {value}" @@ -619,6 +699,75 @@ def _is_omitted_api_doc(relative: Path) -> bool: parts = relative.parts return len(parts) >= 4 and parts[0] == "versions" and parts[2] == "docs" + @classmethod + def _is_external_api_doc(cls, relative: Path, source: Path) -> bool: + """Recognize dependency docs without silently reclassifying project pages.""" + + if not cls._is_omitted_api_doc(relative): + return False + if source.is_file() and 'data-reasbook-doc-stub="true"' in source.read_text( + encoding="utf-8", + errors="replace", + ): + return True + tail = relative.parts[3:] + if tail and tail[0] == "ReasBook": + tail = tail[1:] + if not tail: + return False + return Path(tail[0]).stem in _EXTERNAL_DOC_NAMESPACES + + @staticmethod + def _canonical_history_target( + spec: ReleaseSpec, + relative: Path, + pages_site: Path, + canonical_routes: dict[str, tuple[Path | None, Path | None]], + ) -> str | None: + parts = relative.parts + if len(parts) < 4 or parts[0] != "versions": + return None + branch = parts[1] + route = Path(*parts[2:]) + for project in spec.projects: + if project.branch != branch or project.canonical: + continue + for root in (Path(project.kind) / project.slug, Path(project.slug)): + if route == root or root in route.parents: + canonical = next( + ( + candidate + for candidate in spec.projects + if candidate.key == project.key and candidate.canonical + ), + None, + ) + if canonical is None: + return None + canonical_route = canonical_routes.get(canonical.key) + if canonical_route is None or canonical_route[0] is None: + return None + canonical_root = canonical_route[0] + canonical_index = pages_site / canonical_root / "index.html" + if not canonical_index.is_file(): + return None + + # Preserve a deep route when the canonical version exposes + # the same suffix. Otherwise land on its verified root. + suffix = route.relative_to(root) + candidate = pages_site / canonical_root / suffix + if candidate.is_file(): + route_target = canonical_root / suffix + if route_target.name == "index.html": + return ( + spec.base_path + + route_target.parent.as_posix().rstrip("/") + + "/" + ) + return spec.base_path + route_target.as_posix() + return spec.base_path + canonical_root.as_posix().rstrip("/") + "/" + return None + @staticmethod def _write_api_stub(path: Path, base_path: str) -> None: path.parent.mkdir(parents=True, exist_ok=True) @@ -632,17 +781,57 @@ def _write_api_stub(path: Path, base_path: str) -> None: ' ', ' ', - " Detailed API documentation", + " External dependency documentation", + f' ', + "", + '', + '
', + '

Canonical Pages Projection

', + "

External dependency documentation

", + "

This external API page is outside the project-owned " + "documentation retained by GitHub Pages. Project API pages " + "remain available in full.

", + f'

' + "Back to ReasBook

", + "
", + "", + "", + "", + ] + ), + encoding="utf-8", + ) + + @staticmethod + def _write_history_redirect( + path: Path, + target: str, + base_path: str, + ) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + home = html.escape(base_path, quote=True) + escaped_target = html.escape(target, quote=True) + path.write_text( + "\n".join( + [ + "", + '', + "", + ' ', + ' ', + f' ', + f' ', + " Canonical project version", f' ', "", - "", + '', '
', '

Canonical Pages Projection

', - "

Detailed API documentation

", - "

This API page is not included in the bounded Pages " - "artifact. The project entry page, source, and theorem map " - "remain available; the complete API is in the self-hosted " - "artifact.

", + "

Canonical project version

", + "

This historical link now resolves to the project's " + "explicit canonical version.

", + f'

Open canonical version

', f'

' "Back to ReasBook

", "
", diff --git a/sdk/deploy/src/reasbook_deploy_sdk/release/models.py b/sdk/deploy/src/reasbook_deploy_sdk/release/models.py index f18fbce07..16d856603 100644 --- a/sdk/deploy/src/reasbook_deploy_sdk/release/models.py +++ b/sdk/deploy/src/reasbook_deploy_sdk/release/models.py @@ -22,6 +22,7 @@ SHA256_RE = re.compile(r"^sha256:[0-9a-f]{64}$") RELEASE_ID_RE = re.compile(r"^site-\d{8}T\d{6}Z-(?P[0-9a-f]{12})$") ARTIFACT_NAME_RE = re.compile(r"^[a-z][a-z0-9_-]{0,31}$") +GITHUB_PAGES_HARD_SITE_BYTES = 1_000_000_000 def _text(value: object, *, field: str) -> str: @@ -218,7 +219,7 @@ def default_artifact_policies() -> tuple[ReleaseArtifactPolicy, ...]: history_mode="canonical", dependency_docs="stubs", max_site_files=60_000, - max_site_bytes=850_000_000, + max_site_bytes=920_000_000, max_archive_members=180_000, max_bundle_bytes=950_000_000, ), @@ -269,6 +270,11 @@ def __post_init__(self) -> None: "the pages artifact must retain canonical history and stub " "dependency docs" ) + if pages.max_site_bytes > GITHUB_PAGES_HARD_SITE_BYTES: + raise DeployConfigError( + "the pages artifact operational budget cannot exceed the " + f"GitHub Pages hard limit of {GITHUB_PAGES_HARD_SITE_BYTES} bytes" + ) def artifact(self, name: str) -> ReleaseArtifactPolicy: for artifact in self.artifacts: diff --git a/sdk/deploy/tests/test_release.py b/sdk/deploy/tests/test_release.py index 17ea212d1..3382de54b 100644 --- a/sdk/deploy/tests/test_release.py +++ b/sdk/deploy/tests/test_release.py @@ -40,6 +40,7 @@ from reasbook_deploy_sdk.release.models import ( CanonicalProjects, DeploymentProfile, + GITHUB_PAGES_HARD_SITE_BYTES, GitHubPublishProfile, ProjectSpec, ReleaseArtifactPolicy, @@ -304,6 +305,38 @@ def profile(root: Path, *, historical: bool = True) -> DeploymentProfile: class ReleasePlanningTests(unittest.TestCase): + def test_project_docs_copy_flat_entry_and_sibling_namespace(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + source = root / "source" + destination = root / "destination" + docs = source / "docs" / "ReasBook" + namespace = docs / "TR_LALM_theory" + namespace.mkdir(parents=True) + (docs / "TR_LALM_theory.html").write_text("entry", encoding="utf-8") + (namespace / "Current.html").write_text("detail", encoding="utf-8") + + entry = PagesSiteProjector()._copy_project_docs( + source, + destination, + "papers", + "TR_LALM_theory", + "TR_LALM_theory", + ) + + self.assertEqual(entry, Path("docs/ReasBook/TR_LALM_theory.html")) + self.assertEqual( + (destination / "docs/ReasBook/TR_LALM_theory.html").read_text(), + "entry", + ) + self.assertEqual( + ( + destination + / "docs/ReasBook/TR_LALM_theory/Current.html" + ).read_text(), + "detail", + ) + def test_pages_route_score_prefers_indexed_landing_over_larger_fragment( self, ) -> None: @@ -384,6 +417,57 @@ def test_deployment_profile_rejects_unimplemented_artifact_projection(self) -> N ), ) + def test_deployment_profile_rejects_pages_budget_above_github_hard_limit( + self, + ) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + default = profile(root) + oversized_pages = replace( + default.artifact("pages"), + max_site_bytes=GITHUB_PAGES_HARD_SITE_BYTES + 1, + ) + with self.assertRaisesRegex(DeployConfigError, "hard limit"): + replace( + default, + artifacts=(default.artifact("full"), oversized_pages), + ) + + def test_pages_projection_applies_operational_and_hard_capacity_layers( + self, + ) -> None: + projector = PagesSiteProjector(max_site_bytes=920_000_000) + with self.assertRaisesRegex(DeployExecutionError, "budget is 920000000"): + projector._validate_capacity(920_000_001) + with self.assertRaisesRegex(DeployExecutionError, "hard limit is 1000000000"): + projector._validate_capacity(GITHUB_PAGES_HARD_SITE_BYTES + 1) + + def test_external_api_classification_fails_closed_for_unknown_real_docs( + self, + ) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + source = root / "Unknown" / "Page.html" + source.parent.mkdir(parents=True) + source.write_text("", encoding="utf-8") + unknown = Path("versions/v4.30.0/docs/ReasBook/Unknown/Page.html") + mathlib = Path("versions/v4.30.0/docs/ReasBook/Mathlib/Page.html") + + self.assertFalse( + PagesSiteProjector._is_external_api_doc(unknown, source) + ) + self.assertTrue( + PagesSiteProjector._is_external_api_doc(mathlib, source) + ) + + source.write_text( + '', + encoding="utf-8", + ) + self.assertTrue( + PagesSiteProjector._is_external_api_doc(unknown, source) + ) + def test_pages_projection_rejects_encoded_parent_path(self) -> None: with tempfile.TemporaryDirectory() as temp: pages = Path(temp) / "pages" @@ -658,6 +742,10 @@ def test_policy_digest_cli_reads_the_trusted_profile(self) -> None: profile_value.artifact("pages").max_archive_members, 180_000, ) + self.assertEqual( + profile_value.artifact("pages").max_site_bytes, + 920_000_000, + ) self.assertEqual( profile_value.artifact("full").max_archive_members, 1_501_024, @@ -1167,6 +1255,15 @@ def test_release_set_packages_slim_pages_and_installable_full_site(self) -> None books={"Demo": "canonical"}, papers={}, ) + historical_doc = ( + branch_sites["v4.26.0"] + / "docs/ReasBook/Books/Demo/Book.html" + ) + historical_doc.write_text( + '' + "historical chapter", + encoding="utf-8", + ) rich_route = branch_sites["v4.30.0"] / "demo" for relative in ( Path("index.html"), @@ -1198,6 +1295,8 @@ def test_release_set_packages_slim_pages_and_installable_full_site(self) -> None ) project_chapter = project_doc.with_name("Chapter.html") project_chapter.write_text("p" * 200_000, encoding="utf-8") + project_orphan = project_doc.with_name("UnlinkedProjectModule.html") + project_orphan.write_text("project-owned", encoding="utf-8") project_doc.write_text( '' @@ -1252,7 +1351,7 @@ def test_release_set_packages_slim_pages_and_installable_full_site(self) -> None ).unlink() with self.assertRaisesRegex( DeployExecutionError, - "no canonical API entry", + "no project-owned API entry", ): PagesSiteProjector().project( spec, @@ -1276,12 +1375,12 @@ def test_release_set_packages_slim_pages_and_installable_full_site(self) -> None / "Heavy.html" ) self.assertIn( - "not included in the bounded Pages artifact", + 'data-reasbook-doc-stub="true"', stub.read_text(encoding="utf-8"), ) self.assertLess(stub.stat().st_size, dependency.stat().st_size) self.assertIn( - "not included in the bounded Pages artifact", + "External dependency documentation", stub.with_name("RefreshTarget.html").read_text(encoding="utf-8"), ) projected_chapter = ( @@ -1294,17 +1393,39 @@ def test_release_set_packages_slim_pages_and_installable_full_site(self) -> None / "Demo" / "Chapter.html" ) - self.assertIn( - "not included in the bounded Pages artifact", - projected_chapter.read_text(encoding="utf-8"), + self.assertEqual( + projected_chapter.read_bytes(), + project_chapter.read_bytes(), ) - self.assertLess( - projected_chapter.stat().st_size, - project_chapter.stat().st_size, + self.assertEqual( + projected_chapter.with_name("UnlinkedProjectModule.html").read_bytes(), + project_orphan.read_bytes(), ) - self.assertFalse( - (layout.pages_site / "versions" / "v4.26.0" / "books" / "demo").exists() + self.assertTrue( + ( + layout.pages_site + / "versions" + / "v4.26.0" + / "docs" + / "ReasBook" + / "Papers" + / "OnlyOld" + / "Paper.html" + ).is_file() + ) + history_redirect = ( + layout.pages_site + / "versions/v4.26.0/books/demo/chapter/index.html" + ).read_text(encoding="utf-8") + self.assertIn( + 'data-reasbook-history-redirect="true"', + history_redirect, + ) + self.assertIn( + 'content="0; url=/ReasBook/versions/v4.30.0/books/demo/chapter/"', + history_redirect, ) + self.assertNotIn('data-reasbook-doc-stub="true"', history_redirect) self.assertFalse((layout.pages_site / "books").exists()) self.assertFalse((layout.pages_site / "papers").exists()) self.assertFalse( diff --git a/tests/test_github_workflows.py b/tests/test_github_workflows.py index c800d5ddf..50eb4fce0 100644 --- a/tests/test_github_workflows.py +++ b/tests/test_github_workflows.py @@ -48,7 +48,8 @@ def test_release_pages_has_one_derived_input_and_scoped_permissions(self) -> Non "GitHub publication requires a clean commit-derived tooling revision", text, ) - self.assertIn("850_000_000", text) + self.assertIn("920_000_000", text) + self.assertIn("1_000_000_000", text) self.assertIn("SHA256SUMS", text) self.assertIn("cmp .release/release-manifest.json", text) self.assertGreaterEqual(text.count('gh release verify "$RELEASE_TAG"'), 2)