Prototype schema browser

docs/user-guide/cli.rst

@@ -4,3 +4,19 @@ Command Line Interface
 .. click:: felis.cli:cli
    :prog: felis
    :nested: full
+
+Browser prototype
+================
+
+The ``browser`` command generates a static HTML site for browsing schema
+structure (schemas, tables, and columns):
+
+.. code-block:: bash
+
+   felis browser --output-dir ./site tests/data/test.yml tests/data/sales.yaml
+
+You can use shell expansion to provide file lists:
+
+.. code-block:: bash
+
+   felis browser --output-dir ./site ../sdm_schemas/yml/*.yaml

pyproject.toml

@@ -27,6 +27,7 @@ dependencies = [
     "astropy",
     "click",
     "deepdiff",
+    "jinja2",
     "lsst-resources",
     "lsst-utils",
     "numpy",
@@ -43,6 +44,7 @@ Source = "https://github.com/lsst/felis"
 
 [project.optional-dependencies]
 test = [
+    "beautifulsoup4>=4.12",
     "pytest >= 3.2"
 ]
 dev = [
@@ -61,7 +63,14 @@ where = ["python"]
 zip-safe = true
 
 [tool.setuptools.package-data]
-"felis" = ["py.typed", "config/tap_schema/*.yaml", "config/tap_schema/*.csv"]
+"felis" = [
+    "py.typed",
+    "config/tap_schema/*.yaml",
+    "config/tap_schema/*.csv",
+    "browser/templates/*.j2",
+    "browser/static/*.css",
+    "browser/static/*.js",
+]
 
 [tool.setuptools.dynamic]
 version = { attr = "lsst_versions.get_lsst_version" }

python/felis/browser/__init__.py

@@ -0,0 +1,5 @@
+"""Static schema browser generator."""
+
+from .render import render_static_site
+
+__all__ = ["render_static_site"]

python/felis/browser/render.py

@@ -0,0 +1,331 @@
+from __future__ import annotations
+
+import logging
+import re
+from importlib import resources
+from pathlib import Path
+
+from jinja2 import Environment, PackageLoader, select_autoescape
+
+from felis.datamodel import Schema
+
+logger = logging.getLogger(__name__)
+_GRAPHIC_SUFFIXES = {".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp", ".ico"}
+
+
+def _slugify(value: str) -> str:
+    slug = re.sub(r"[^a-zA-Z0-9]+", "-", value.strip().lower()).strip("-")
+    return slug or "schema"
+
+
+def _schema_name(schema: Schema, source_path: str) -> str:
+    if schema.name:
+        return schema.name
+    return Path(source_path).stem
+
+
+def _column_label(column_id: str) -> str:
+    return column_id.rsplit(".", 1)[-1].lstrip("#")
+
+
+def _table_label(column_id: str) -> str:
+    return column_id.lstrip("#").split(".", 1)[0]
+
+
+def _format_field_value(value: object) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, list):
+        return ", ".join(str(item) for item in value)
+    if isinstance(value, dict):
+        return str(value)
+    return str(value)
+
+
+def _validate_site_icon(site_icon: Path) -> None:
+    if site_icon.suffix.lower() not in _GRAPHIC_SUFFIXES:
+        raise ValueError(f"Site icon must be a graphics file: {site_icon}")
+
+
+def render_static_site(
+    schemas: list[Schema],
+    source_paths: list[str],
+    output_dir: Path,
+    include_column_details: bool = True,
+    site_icon: Path | None = None,
+    icon_link_url: str | None = None,
+) -> None:
+    """Render a minimal static browser site for browsing schemas.
+
+    The generated pages include schema, table, and column navigation.
+
+    Parameters
+    ----------
+    schemas
+        List of schemas to render.
+    source_paths
+        List of source paths corresponding to the schemas, used for display
+        purposes.
+    output_dir
+        Directory where the rendered HTML files will be saved.
+    include_column_details
+        If True, include expandable per-column details in table pages.
+    site_icon
+        Optional path to a graphics file copied into assets and shown in the
+        page header.
+    icon_link_url
+        Optional URL to open when the site icon is clicked.
+    """
+    output_dir.mkdir(parents=True, exist_ok=True)
+    (output_dir / "assets").mkdir(parents=True, exist_ok=True)
+
+    site_icon_href: str | None = None
+    if site_icon is not None:
+        _validate_site_icon(site_icon)
+        site_icon_ext = site_icon.suffix.lower()
+        icon_output_path = output_dir / "assets" / f"site-icon{site_icon_ext}"
+        icon_output_path.write_bytes(site_icon.read_bytes())
+        site_icon_href = f"assets/{icon_output_path.name}"
+    elif icon_link_url:
+        logger.warning("Icon link URL provided but no site icon was configured; ignoring icon link")
+
+    env = Environment(
+        loader=PackageLoader("felis.browser", "templates"),
+        autoescape=select_autoescape(["html", "xml"]),
+        trim_blocks=True,
+        lstrip_blocks=True,
+    )
+
+    schema_pages: list[dict[str, object]] = []
+    filename_counts: dict[str, int] = {}
+    table_filename_counts: dict[str, int] = {}
+
+    for schema, source_path in zip(schemas, source_paths, strict=True):
+        name = _schema_name(schema, source_path)
+        logger.debug("Building schema: %s", name)
+        schema_slug = _slugify(name)
+        count = filename_counts.get(schema_slug, 0) + 1
+        filename_counts[schema_slug] = count
+        filename = f"{schema_slug}.html" if count == 1 else f"{schema_slug}-{count}.html"
+
+        prepared_tables: list[dict[str, object]] = []
+        for table in schema.tables:
+            logger.debug("Building table: %s.%s", name, table.name)
+            table_anchor = f"table-{_slugify(table.name)}"
+            table_base = f"{schema_slug}--{_slugify(table.name)}"
+            table_count = table_filename_counts.get(table_base, 0) + 1
+            table_filename_counts[table_base] = table_count
+            table_filename = f"{table_base}.html" if table_count == 1 else f"{table_base}-{table_count}.html"
+
+            columns: list[dict[str, object]] = []
+            for column in table.columns:
+                logger.debug("Building column: %s.%s.%s", name, table.name, column.name)
+                col_slug = _slugify(column.name)
+                details_fields: list[dict[str, str]] = []
+                if include_column_details:
+                    model = column.__class__
+                    defaults_by_alias = {
+                        (field_info.alias or field_name): field_info.default
+                        for field_name, field_info in model.model_fields.items()
+                    }
+
+                    column_data = column.model_dump(by_alias=True, exclude_none=False, exclude_defaults=False)
+                    for key, value in column_data.items():
+                        if key == "@id":
+                            continue
+                        if value is None:
+                            continue
+                        if key in defaults_by_alias and value == defaults_by_alias[key]:
+                            continue
+                        details_fields.append({"key": key, "display_value": _format_field_value(value)})
+
+                columns.append(
+                    {
+                        "id": column.id,
+                        "name": column.name,
+                        "datatype": str(column.datatype),
+                        "ucd": column.ivoa_ucd or "",
+                        "description": column.description or "",
+                        "unit": column.ivoa_unit or column.fits_tunit or "",
+                        "anchor": f"{table_anchor}-col-{col_slug}",
+                        "details_anchor": f"{table_anchor}-col-{col_slug}-details"
+                        if include_column_details
+                        else "",
+                        "details_fields": details_fields,
+                    }
+                )
+
+            prepared_tables.append(
+                {
+                    "table": table,
+                    "anchor": table_anchor,
+                    "page_filename": table_filename,
+                    "columns": columns,
+                }
+            )
+
+        schema_column_targets: dict[str, dict[str, str]] = {}
+        for prepared in prepared_tables:
+            table_name = str(getattr(prepared["table"], "name"))
+            table_page_filename = str(prepared["page_filename"])
+            for column in prepared["columns"]:
+                column_id = str(column["id"])
+                schema_column_targets[column_id] = {
+                    "id": column_id,
+                    "name": str(column["name"]),
+                    "anchor": str(column["anchor"]),
+                    "table_name": table_name,
+                    "table_page_filename": table_page_filename,
+                }
+
+        table_entries: list[dict[str, object]] = []
+        for prepared in prepared_tables:
+            table = prepared["table"]
+            table_anchor = str(prepared["anchor"])
+            table_filename = str(prepared["page_filename"])
+            columns: list[dict[str, object]] = prepared["columns"]
+            column_anchors = {str(column["id"]): str(column["anchor"]) for column in columns}
+
+            def _column_entry(column_id: str) -> dict[str, str]:
+                if column_id in schema_column_targets:
+                    return dict(schema_column_targets[column_id])
+                return {
+                    "id": column_id,
+                    "name": _column_label(column_id),
+                    "anchor": column_anchors.get(column_id, ""),
+                    "table_name": _table_label(column_id),
+                    "table_page_filename": "",
+                }
+
+            primary_key_entries: list[dict[str, str]] = []
+            if table.primary_key is not None:
+                primary_key_columns = (
+                    table.primary_key if isinstance(table.primary_key, list) else [table.primary_key]
+                )
+                for column_id in primary_key_columns:
+                    column_id_str = str(column_id)
+                    primary_key_entries.append(_column_entry(column_id_str))
+
+            index_entries: list[dict[str, object]] = []
+            index_filename_counts: dict[str, int] = {}
+            for index in table.indexes:
+                index_base = f"{table_anchor}-index-{_slugify(index.name)}"
+                index_count = index_filename_counts.get(index_base, 0) + 1
+                index_filename_counts[index_base] = index_count
+                index_anchor = f"{index_base}" if index_count == 1 else f"{index_base}-{index_count}"
+                if index.columns is not None:
+                    definition_kind = "columns"
+                    definition_values = [_column_entry(str(column_id)) for column_id in index.columns]
+                else:
+                    definition_kind = "expressions"
+                    definition_values = list(index.expressions or [])
+                index_entries.append(
+                    {
+                        "name": index.name,
+                        "description": index.description or "",
+                        "anchor": index_anchor,
+                        "definition_kind": definition_kind,
+                        "definition_values": definition_values,
+                    }
+                )
+
+            constraint_entries: list[dict[str, object]] = []
+            constraint_filename_counts: dict[str, int] = {}
+            for constraint in table.constraints:
+                constraint_base = f"{table_anchor}-constraint-{_slugify(constraint.name)}"
+                constraint_count = constraint_filename_counts.get(constraint_base, 0) + 1
+                constraint_filename_counts[constraint_base] = constraint_count
+                constraint_anchor = (
+                    f"{constraint_base}" if constraint_count == 1 else f"{constraint_base}-{constraint_count}"
+                )
+                constraint_type = str(getattr(constraint, "type", ""))
+                entry: dict[str, object] = {
+                    "name": constraint.name,
+                    "type": constraint_type,
+                    "description": constraint.description or "",
+                    "anchor": constraint_anchor,
+                    "deferrable": constraint.deferrable,
+                    "initially": constraint.initially or "",
+                }
+
+                if constraint_type == "Check":
+                    entry["expression"] = str(getattr(constraint, "expression", ""))
+                elif constraint_type == "Unique":
+                    unique_columns = [str(col) for col in getattr(constraint, "columns", [])]
+                    entry["columns"] = [_column_entry(column_id) for column_id in unique_columns]
+                elif constraint_type == "ForeignKey":
+                    fk_columns = [str(col) for col in getattr(constraint, "columns", [])]
+                    ref_columns = [str(col) for col in getattr(constraint, "referenced_columns", [])]
+                    entry["columns"] = [_column_entry(column_id) for column_id in fk_columns]
+                    entry["referenced_columns"] = [_column_entry(column_id) for column_id in ref_columns]
+                    entry["on_delete"] = str(getattr(constraint, "on_delete", "") or "")
+                    entry["on_update"] = str(getattr(constraint, "on_update", "") or "")
+
+                constraint_entries.append(entry)
+
+            table_entries.append(
+                {
+                    "name": table.name,
+                    "description": table.description or "",
+                    "anchor": table_anchor,
+                    "page_filename": table_filename,
+                    "primary_key": primary_key_entries,
+                    "include_column_details": include_column_details,
+                    "columns": columns,
+                    "indexes": index_entries,
+                    "constraints": constraint_entries,
+                }
+            )
+
+        schema_pages.append(
+            {
+                "name": name,
+                "description": schema.description or "",
+                "source_path": source_path,
+                "filename": filename,
+                "tables": table_entries,
+            }
+        )
+
+    schema_pages.sort(key=lambda page: str(page["name"]).lower())
+
+    index_template = env.get_template("index.html.j2")
+    schema_template = env.get_template("schema.html.j2")
+    table_template = env.get_template("table.html.j2")
+
+    (output_dir / "index.html").write_text(
+        index_template.render(
+            schemas=schema_pages, site_icon_href=site_icon_href, icon_link_url=icon_link_url
+        ),
+        encoding="utf-8",
+    )
+
+    for page in schema_pages:
+        (output_dir / str(page["filename"])).write_text(
+            schema_template.render(
+                current_schema=page,
+                schemas=schema_pages,
+                site_icon_href=site_icon_href,
+                icon_link_url=icon_link_url,
+            ),
+            encoding="utf-8",
+        )
+
+        for table in page["tables"]:
+            (output_dir / str(table["page_filename"])).write_text(
+                table_template.render(
+                    current_schema=page,
+                    current_table=table,
+                    schemas=schema_pages,
+                    site_icon_href=site_icon_href,
+                    icon_link_url=icon_link_url,
+                ),
+                encoding="utf-8",
+            )
+
+    css_source = resources.files("felis.browser").joinpath("static/style.css").read_text(encoding="utf-8")
+    (output_dir / "assets" / "style.css").write_text(css_source, encoding="utf-8")
+    js_source = resources.files("felis.browser").joinpath("static/sidebar.js").read_text(encoding="utf-8")
+    (output_dir / "assets" / "sidebar.js").write_text(js_source, encoding="utf-8")