From 4ed13fc00575e115711537796e7ed1d90f76f938 Mon Sep 17 00:00:00 2001 From: Lissan <150966211+DataForSolution@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:21:51 -0400 Subject: [PATCH] Add lightweight portfolio CI baseline --- .github/workflows/portfolio-ci.yml | 55 ++++++++++++++ README.md | 6 ++ scripts/validate_repository.py | 113 +++++++++++++++++++++++++++++ 3 files changed, 174 insertions(+) create mode 100644 .github/workflows/portfolio-ci.yml create mode 100644 scripts/validate_repository.py diff --git a/.github/workflows/portfolio-ci.yml b/.github/workflows/portfolio-ci.yml new file mode 100644 index 0000000..98fba69 --- /dev/null +++ b/.github/workflows/portfolio-ci.yml @@ -0,0 +1,55 @@ +name: Portfolio CI + +on: + pull_request: + branches: [Master] + push: + branches: [Master] + +permissions: + contents: read + +jobs: + structure: + name: Structural validation + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Check out repository + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Validate source, notebooks, links, and sensitive text + run: python scripts/validate_repository.py + - name: Check whitespace errors + run: git diff --check + + tests: + name: Unit tests + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Check out repository + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install tested dependency set + run: | + python -m pip install --upgrade pip + python -m pip install "setuptools>=69" wheel "pytest>=8,<9" "numpy>=1.26,<3" "pandas>=2.2,<3" "scikit-learn>=1.8,<1.10" "torch>=2.7,<3" + for project in portfolio/*; do + if [ -f "$project/pyproject.toml" ]; then + python -m pip install --no-build-isolation --no-deps --editable "$project" + fi + done + - name: Run portfolio unit tests + run: | + for project in portfolio/*; do + if [ -d "$project/tests" ]; then + python -m pytest -q "$project/tests" + fi + done diff --git a/README.md b/README.md index 34a3f71..ab7f4f2 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,12 @@ The current branch is intentionally minimal: `README.md` plus the canonical `por Each featured project maintains its own README, audit notes, source modules, tests, environment definition, and interpretation boundaries as appropriate. +## Continuous integration + +Portfolio CI uses Python 3.11 to parse all Python source and notebook code cells, check local Markdown links and sensitive/local-path patterns, detect whitespace errors, and run the 76 network-free unit tests across all ten projects. The shared test environment keeps scikit-learn below 1.10, which also preserves Pima's required `<1.11` bound until its probability-estimation methodology is deliberately migrated and revalidated. + +CI deliberately does not execute notebooks, download datasets or models, install optional research frameworks, use GPUs, retrain models, or regenerate project results. Those operations are excluded because the repository's baseline checks protect code and presentation integrity without implying that historical experiments have been reproduced. + ## Use and interpretation Several projects use healthcare or other high-stakes datasets. They are educational/research engineering projects and **are not clinical, diagnostic, financial, or security decision systems** unless a project explicitly states otherwise. diff --git a/scripts/validate_repository.py b/scripts/validate_repository.py new file mode 100644 index 0000000..404b26c --- /dev/null +++ b/scripts/validate_repository.py @@ -0,0 +1,113 @@ +"""Fast, network-free structural checks for the curated portfolio.""" + +from __future__ import annotations + +import ast +import json +import re +import subprocess +from pathlib import Path +from urllib.parse import unquote + + +ROOT = Path(__file__).resolve().parents[1] + + +def tracked_files() -> list[Path]: + output = subprocess.check_output( + ["git", "ls-files", "--cached", "--others", "--exclude-standard", "-z"], + cwd=ROOT, + text=True, + ) + return [ROOT / name for name in output.split("\0") if name] + + +def validate_python(files: list[Path]) -> int: + python_files = [path for path in files if path.suffix == ".py"] + for path in python_files: + ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + return len(python_files) + + +def validate_notebooks(files: list[Path]) -> tuple[int, int]: + notebooks = [path for path in files if path.suffix == ".ipynb"] + code_cells = 0 + for path in notebooks: + notebook = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(notebook.get("cells"), list): + raise ValueError(f"Notebook has no cells array: {path.relative_to(ROOT)}") + for index, cell in enumerate(notebook["cells"]): + if cell.get("cell_type") != "code": + continue + source = cell.get("source", []) + code = "".join(source) if isinstance(source, list) else source + ast.parse(code, filename=f"{path}#cell-{index + 1}") + code_cells += 1 + return len(notebooks), code_cells + + +def validate_markdown_links(files: list[Path]) -> int: + markdown_files = [path for path in files if path.suffix.lower() == ".md"] + link_pattern = re.compile(r"(?") + if not target or target.startswith(("#", "http://", "https://", "mailto:")): + continue + local_target = unquote(target.split("#", 1)[0]) + if not local_target: + continue + checked += 1 + if not (path.parent / local_target).resolve().exists(): + failures.append(f"{path.relative_to(ROOT)} -> {target}") + if failures: + raise ValueError("Broken local Markdown links:\n" + "\n".join(failures)) + return checked + + +def validate_sensitive_text(files: list[Path]) -> int: + slash = chr(47) + patterns = { + "Google API key": re.compile("AI" + r"za[0-9A-Za-z_-]{35}"), + "GitHub token": re.compile("gh" + r"[pousr]_[0-9A-Za-z]{20,}"), + "AWS access key": re.compile("AK" + r"IA[0-9A-Z]{16}"), + "private-key header": re.compile("BEGIN " + r"(?:RSA |EC |OPENSSH )?PRIVATE KEY"), + "Windows user path": re.compile(r"[A-Za-z]:\\Users\\[^\\\s]+", re.I), + "macOS user path": re.compile(slash + "Users" + slash + r"[^/\s]+"), + "Linux home path": re.compile(slash + "home" + slash + r"[^/\s]+"), + } + text_suffixes = {".md", ".py", ".toml", ".yml", ".yaml", ".json", ".txt"} + scanned = 0 + failures: list[str] = [] + for path in files: + if path.suffix.lower() not in text_suffixes: + continue + text = path.read_text(encoding="utf-8") + scanned += 1 + for label, pattern in patterns.items(): + if pattern.search(text): + failures.append(f"{path.relative_to(ROOT)}: {label}") + if failures: + raise ValueError("Sensitive or local-path patterns found:\n" + "\n".join(failures)) + return scanned + + +def main() -> None: + files = tracked_files() + python_count = validate_python(files) + notebook_count, code_cell_count = validate_notebooks(files) + link_count = validate_markdown_links(files) + text_count = validate_sensitive_text(files) + print( + "Structural validation passed: " + f"{python_count} Python files, {notebook_count} notebooks/" + f"{code_cell_count} code cells, {link_count} local Markdown links, " + f"{text_count} text files scanned." + ) + + +if __name__ == "__main__": + main()