docs: define agent pull request and bookkeeping workflow

.github/workflows/ai-review.yml

@@ -0,0 +1,38 @@
+name: AI Code Reviewer
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+permissions:
+  pull-requests: write
+  contents: read
+
+jobs:
+  review:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install google-generativeai==0.8.3 PyGithub==2.5.0 pathspec==0.12.1
+
+      - name: Run AI Reviewer
+        env:
+          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          REPO_NAME: ${{ github.repository }}
+          GEMINI_MODEL: ${{ vars.GEMINI_MODEL }}
+        run: python scripts/ai_pr_reviewer.py

agents.md

@@ -52,3 +52,18 @@ This project is bootstrapped to work seamlessly with VS Code / Antigravity IDE w
 2. **STRICTLY LOCAL SCOPE:** Do not write or modify any VS Code/IDE configurations outside the `data-engineering` project directory.
 3. **PANTS COMPLIANCE:** Always follow the target-based workflow for Pants commands.
 4. **SPARK COMPATIBILITY:** Run PySpark tasks using Java 17 via the `JAVA_HOME` configuration found in `.env`.
+
+---
+
+## 4. Pull Request & Bookkeeping Workflow
+
+When implementing new features or bug fixes, follow this workflow to coordinate PR creation:
+
+1. **Local Implementation & Verification:** Implement changes locally, verify they run/test successfully, and document them in `README.md` and `agents.md`.
+2. **User Review & Local Commit:** Present the changes to the user. The user will review the code locally and run `git commit` manually.
+3. **PR Creation:** Once the user commits the changes, they will instruct you to create the Pull Request (or you can offer to do so).
+4. **PR Formatting:** Use the `github` MCP server to create the PR. The PR must have a detailed description containing:
+   * **Summary:** A clear explanation of *why* the changes were made.
+   * **Key Changes:** A bulleted list of modified modules/files and what was updated.
+   * **Verification:** Documentation of successful test/run commands executed during verification.
+

scripts/ai_pr_reviewer.py

@@ -0,0 +1,170 @@
+import os
+import sys
+from typing import List
+from github import Github
+from github.PullRequest import PullRequest
+import google.generativeai as genai
+import pathspec
+
+# Max character limit for diff payload to stay within token limits
+MAX_DIFF_CHARACTERS = 150000
+
+def get_ignore_spec() -> pathspec.PathSpec:
+    """Loads .gitignore patterns and appends custom file exclusion wildcards."""
+    ignore_patterns = []
+    if os.path.exists(".gitignore"):
+        try:
+            with open(".gitignore", "r", encoding="utf-8") as f:
+                lines = [
+                    line.strip()
+                    for line in f
+                    if line.strip() and not line.strip().startswith("#")
+                ]
+                ignore_patterns.extend(lines)
+        except Exception as e:
+            print(f"Warning: Failed to parse .gitignore: {e}", file=sys.stderr)
+
+    # Custom wildcards for lockfiles and binary assets to skip
+    ignore_patterns.extend([
+        "*.lock",
+        "*.png",
+        "*.jpg",
+        "*.jpeg",
+        "*.zip",
+        "*.pdf"
+    ])
+
+    return pathspec.PathSpec.from_lines("gitwildmatch", ignore_patterns)
+
+
+def build_diff_content(pr: PullRequest, ignore_spec: pathspec.PathSpec) -> str:
+    """Retrieves and filters PR file diffs, handling size limits and early exits."""
+    diff_content: List[str] = []
+    current_size = 0
+
+    for file in pr.get_files():
+        # Optimization: Stop fetching additional diffs once we are past the limit
+        if current_size >= MAX_DIFF_CHARACTERS:
+            diff_content.append("\n\n... [TRUNCATED: MAX CHARACTER LIMIT REACHED] ...")
+            print("Max character limit reached during diff generation. Stopping file retrieval.")
+            break
+
+        # Skip files matching ignore patterns
+        if ignore_spec.match_file(file.filename):
+            print(f"Skipping {file.filename} (ignored)")
+            continue
+
+        file_header = f"=== File: {file.filename} ===\n"
+        patch_str = ""
+        if file.patch:
+            if len(file.patch) > 30000:
+                patch_str = f"{file_header}[File patch omitted: Exceeds single-file size limit]\n"
+                print(f"Skipping patch for {file.filename} (exceeds 30,000 character limit)")
+            else:
+                patch_str = f"{file_header}{file.patch}\n"
+        else:
+            patch_str = f"{file_header}[File modified, but no patch details available]\n"
+
+        diff_content.append(patch_str)
+        current_size += len(patch_str)
+
+    return "\n".join(diff_content)
+
+
+def generate_review(gemini_api_key: str, model_name: str, diff: str) -> str:
+    """Sends the PR diff to the Gemini API and returns the markdown review."""
+    genai.configure(api_key=gemini_api_key)
+
+    system_instruction = (
+        "You are an expert Data Engineer and Python Code Reviewer.\n"
+        "Your task is to conduct a professional, constructive code review.\n\n"
+        "Specific areas to analyze:\n"
+        "1. PySpark & Data Engineering Best Practices (unpartitioned writes, redundant caching, .collect() issues).\n"
+        "2. Python Code Quality (Ruff/PEP8 standards, naming, docstrings).\n"
+        "3. Bugs & Edge Cases (logical bugs, unhandled exceptions).\n\n"
+        "Format your review in Markdown with the following sections:\n"
+        "- 🤖 AI PR Review Summary\n"
+        "- 💡 Key Feedback & Recommendations (with before/after code blocks)\n"
+        "- ✅ Verdict (Approve, Comment, Request Changes)"
+    )
+
+    model = genai.GenerativeModel(
+        model_name,
+        system_instruction=system_instruction
+    )
+
+    prompt = f"Please review the following PR Diff:\n\n{diff}"
+    response = model.generate_content(prompt)
+    return response.text
+
+
+def post_review(pr: PullRequest, review_body: str) -> None:
+    """Submits the review as an official Pull Request Review on GitHub. Handles fork permissions gracefully."""
+    try:
+        pr.create_review(body=review_body, event="COMMENT")
+    except Exception as e:
+        print(f"Warning: Failed to post PR review comment: {e}", file=sys.stderr)
+        print("This is expected for Pull Requests from external forks where GITHUB_TOKEN has read-only access.")
+        print("Exiting gracefully with code 0.")
+        sys.exit(0)
+
+
+def main():
+    # Load Environment Variables
+    gemini_api_key = os.getenv("GEMINI_API_KEY")
+    github_token = os.getenv("GITHUB_TOKEN")
+    pr_number_str = os.getenv("PR_NUMBER")
+    repo_name = os.getenv("REPO_NAME")
+    gemini_model = os.getenv("GEMINI_MODEL")
+    if not gemini_model or not gemini_model.strip():
+        gemini_model = "gemini-1.5-flash"
+
+    # Handle missing API Key gracefully (e.g. for PRs from external forks)
+    if not gemini_api_key:
+        print("Warning: GEMINI_API_KEY is missing. Skipping AI Review (expected for external forks).")
+        sys.exit(0)
+
+    if not all([github_token, pr_number_str, repo_name]):
+        print("Error: Missing required environment variables (GITHUB_TOKEN, PR_NUMBER, or REPO_NAME).", file=sys.stderr)
+        sys.exit(1)
+
+    try:
+        pr_number = int(pr_number_str)
+    except ValueError:
+        print(f"Error: PR_NUMBER '{pr_number_str}' is not a valid integer.", file=sys.stderr)
+        sys.exit(1)
+
+    # Initialize client and fetch PR
+    print(f"Connecting to repo {repo_name} and fetching PR #{pr_number}...")
+    g = Github(github_token)
+    repo = g.get_repo(repo_name)
+    pr = repo.get_pull(pr_number)
+
+    # Compile ignore patterns and build diff
+    ignore_spec = get_ignore_spec()
+    diff = build_diff_content(pr, ignore_spec)
+
+    if not diff.strip():
+        print("No code changes to review.")
+        sys.exit(0)
+
+    # Generate review
+    print(f"Generating review with Gemini (model: {gemini_model})...")
+    try:
+        review_body = generate_review(gemini_api_key, gemini_model, diff)
+    except Exception as e:
+        print(f"Error generating review via Gemini API: {e}", file=sys.stderr)
+        sys.exit(1)
+
+    # Post review
+    print("Posting review back to GitHub...")
+    try:
+        post_review(pr, review_body)
+    except Exception as e:
+        print(f"Error posting review comment to GitHub: {e}", file=sys.stderr)
+        sys.exit(1)
+    print("Successfully posted PR review!")
+
+
+if __name__ == "__main__":
+    main()