From e6b116dac7b3f41c92e869bf8f711b85685d7555 Mon Sep 17 00:00:00 2001 From: Nicholas Adamou <10106289+nicholasadamou@users.noreply.github.com> Date: Wed, 2 Sep 2026 09:45:40 -0400 Subject: [PATCH] fix: resolve ruff lint failures blocking CI Narrow exception catches, drop unnecessary pass statements, and align typing/imports with current ruff rules so the Lint workflow passes. Co-authored-by: Cursor --- .gitignore | 1 + pyproject.toml | 3 +++ src/webgrab/capture/browser.py | 13 ++++++++++--- src/webgrab/capture/engine.py | 2 +- src/webgrab/capture/filters.py | 5 +---- src/webgrab/capture/processor.py | 5 +++-- src/webgrab/cli.py | 3 +-- src/webgrab/errors.py | 14 -------------- src/webgrab/models.py | 3 +-- src/webgrab/storage/saver.py | 17 +++++++++-------- 10 files changed, 30 insertions(+), 36 deletions(-) diff --git a/.gitignore b/.gitignore index 22fc702..3f2cade 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ __pycache__ .pytest_cache venv/ +.venv/ *.pyc *.pyo *.egg-info/ diff --git a/pyproject.toml b/pyproject.toml index d2139a1..19d4b35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,9 @@ Issues = "https://github.com/smeltery/webgrab/issues" [tool.hatch.build.targets.wheel] packages = ["src/webgrab"] +[tool.ruff.lint.flake8-bugbear] +extend-immutable-calls = ["typer.Argument", "typer.Option"] + [tool.pytest.ini_options] testpaths = ["tests"] python_files = ["test_*.py"] diff --git a/src/webgrab/capture/browser.py b/src/webgrab/capture/browser.py index 8c3d152..8c6227e 100644 --- a/src/webgrab/capture/browser.py +++ b/src/webgrab/capture/browser.py @@ -1,9 +1,16 @@ """Low-level Playwright browser operations.""" import asyncio -from typing import Callable +from collections.abc import Callable +from typing import Self -from playwright.async_api import Browser, BrowserContext, Page, Response, async_playwright +from playwright.async_api import ( + Browser, + BrowserContext, + Page, + Response, + async_playwright, +) from ..errors import BrowserError, NavigationError from ..models import CaptureConfig @@ -23,7 +30,7 @@ def __init__(self, config: CaptureConfig) -> None: self.context: BrowserContext | None = None self.page: Page | None = None - async def __aenter__(self) -> "BrowserManager": + async def __aenter__(self) -> Self: """Launch browser and create context.""" try: self.playwright = await async_playwright().start() diff --git a/src/webgrab/capture/engine.py b/src/webgrab/capture/engine.py index b2a9aeb..96a6cdb 100644 --- a/src/webgrab/capture/engine.py +++ b/src/webgrab/capture/engine.py @@ -2,7 +2,7 @@ import asyncio import time -from typing import Callable +from collections.abc import Callable from playwright.async_api import Response diff --git a/src/webgrab/capture/filters.py b/src/webgrab/capture/filters.py index 69af1c8..a216d30 100644 --- a/src/webgrab/capture/filters.py +++ b/src/webgrab/capture/filters.py @@ -41,10 +41,7 @@ def should_capture(self, url: str, content_type: str, status_code: int) -> bool: return False # Skip data URLs, blob URLs, etc. - if should_skip_url(url): - return False - - return True + return not should_skip_url(url) class CompositeFilter: diff --git a/src/webgrab/capture/processor.py b/src/webgrab/capture/processor.py index 2665a6c..af437b3 100644 --- a/src/webgrab/capture/processor.py +++ b/src/webgrab/capture/processor.py @@ -1,8 +1,9 @@ """Resource processing with streaming architecture.""" import asyncio -from typing import AsyncIterator, Callable +from collections.abc import AsyncIterator, Callable +from playwright.async_api import Error as PlaywrightError from playwright.async_api import Response from ..models import CaptureStats, Resource @@ -62,7 +63,7 @@ async def process_response(self, response: Response) -> Resource | None: headers=headers, status_code=status, ) - except Exception as e: + except PlaywrightError as e: self.stats.failed_captures += 1 if self.on_progress: self.on_progress(f"Failed to capture {url}: {e}") diff --git a/src/webgrab/cli.py b/src/webgrab/cli.py index 17d56e1..987e7fa 100644 --- a/src/webgrab/cli.py +++ b/src/webgrab/cli.py @@ -2,7 +2,6 @@ import asyncio from pathlib import Path -from typing import Optional import typer from rich.console import Console @@ -34,7 +33,7 @@ def capture( ..., help="URL of the webpage to capture resources from.", ), - output: Optional[Path] = typer.Option( + output: Path | None = typer.Option( None, "--output", "-o", help="Output directory for saved resources. Defaults to ./webgrab_output", diff --git a/src/webgrab/errors.py b/src/webgrab/errors.py index 5466ebf..aec0457 100644 --- a/src/webgrab/errors.py +++ b/src/webgrab/errors.py @@ -4,26 +4,18 @@ class WebGrabError(Exception): """Base exception for all webgrab errors.""" - pass - class CaptureError(WebGrabError): """Base exception for capture-related errors.""" - pass - class BrowserError(CaptureError): """Exception raised for browser-related errors.""" - pass - class NavigationError(BrowserError): """Exception raised when page navigation fails.""" - pass - class ResourceError(CaptureError): """Exception raised when capturing a resource fails.""" @@ -44,14 +36,10 @@ def __init__(self, url: str, message: str, original_error: Exception | None = No class StorageError(WebGrabError): """Base exception for storage-related errors.""" - pass - class PathResolutionError(StorageError): """Exception raised when URL to path resolution fails.""" - pass - class FileWriteError(StorageError): """Exception raised when writing a file fails.""" @@ -71,5 +59,3 @@ def __init__(self, path: str, message: str, original_error: Exception | None = N class ConfigurationError(WebGrabError): """Exception raised for configuration errors.""" - - pass diff --git a/src/webgrab/models.py b/src/webgrab/models.py index 79bf8a0..a1cad16 100644 --- a/src/webgrab/models.py +++ b/src/webgrab/models.py @@ -2,7 +2,6 @@ from dataclasses import dataclass, field from pathlib import Path -from typing import Optional @dataclass(frozen=True) @@ -28,7 +27,7 @@ class CaptureConfig: url: str wait_time: int = 0 timeout: int = 60000 - user_agent: Optional[str] = None + user_agent: str | None = None include_external: bool = False headless: bool = True bypass_csp: bool = True diff --git a/src/webgrab/storage/saver.py b/src/webgrab/storage/saver.py index 69f5978..e510b2a 100644 --- a/src/webgrab/storage/saver.py +++ b/src/webgrab/storage/saver.py @@ -2,6 +2,7 @@ from pathlib import Path +from ..errors import StorageError from ..mime.detector import infer_extension from ..models import Resource, SaveConfig, SaveResult from ..url.parser import is_same_origin @@ -30,6 +31,11 @@ def save_resource(self, resource: Resource) -> Path | None: Returns: Path where resource was saved, or None if skipped. + + Raises: + StorageError: If writing the resource fails. + OSError: If path resolution or filesystem operations fail. + ValueError: If URL or path inputs are invalid. """ # Filter external resources if not included if not self.config.include_external and not is_same_origin( @@ -48,13 +54,8 @@ def save_resource(self, resource: Resource) -> Path | None: # Deduplicate if path already used local_path = self.deduplicator.get_unique_path(local_path) - # Write content - try: - write_file(local_path, resource.body) - return local_path - except Exception: - # Return None to indicate failure (caller will track this) - return None + write_file(local_path, resource.body) + return local_path def save_resources(self, resources: list[Resource]) -> SaveResult: """Save all resources to disk. @@ -74,7 +75,7 @@ def save_resources(self, resources: list[Resource]) -> SaveResult: result.saved_paths.append(saved_path) else: result.skipped_count += 1 - except Exception as e: + except (StorageError, OSError, ValueError) as e: result.failed_saves.append((resource.url, e)) return result