From d841a7ba4151cc0002afd807d8c8430bb74bbda2 Mon Sep 17 00:00:00 2001 From: Nikolay Date: Sat, 11 Jul 2026 16:01:24 +0300 Subject: [PATCH 1/4] feat: layered backend/frontend, single pipeline job, docker limits Refactor backend into api/use_cases/data/entities/storage/scan/worker. Collapse Celery scan+metadata+alert into one job with retries. Stream uploads with size limit, projection queries, CASCADE delete. Layer frontend into types/api/hooks/components. Harden docker-compose with healthchecks, ports, resource limits. --- .env.dev | 19 +- .gitignore | 8 +- ARCHITECTURE.md | 82 ++++ README.md | 32 +- backend/.dockerignore | 7 + backend/Dockerfile | 16 +- backend/migrations/env.py | 47 +-- .../b7c4e1a9f2d0_cascade_and_indexes.py | 45 +++ backend/pyproject.toml | 18 +- backend/src/api/__init__.py | 0 backend/src/api/deps.py | 11 + backend/src/api/routes_alerts.py | 13 + backend/src/api/routes_files.py | 68 ++++ backend/src/api/routes_system.py | 8 + backend/src/app.py | 72 +--- backend/src/application.py | 42 ++ backend/src/data/__init__.py | 0 backend/src/data/alerts.py | 45 +++ backend/src/data/columns.py | 48 +++ backend/src/data/files.py | 184 +++++++++ backend/src/db/__init__.py | 0 backend/src/db/base.py | 5 + backend/src/db/session.py | 67 ++++ backend/src/dto/__init__.py | 4 + backend/src/dto/alerts.py | 13 + backend/src/dto/common.py | 17 + backend/src/dto/files.py | 32 ++ backend/src/entities/__init__.py | 4 + backend/src/entities/alert.py | 25 ++ backend/src/entities/stored_file.py | 33 ++ backend/src/errors.py | 29 ++ backend/src/http_errors.py | 85 ++++ backend/src/logging_setup.py | 26 ++ backend/src/models.py | 52 +-- backend/src/scan/__init__.py | 0 backend/src/scan/rules.py | 48 +++ backend/src/schemas.py | 36 +- backend/src/service.py | 113 +----- backend/src/settings.py | 71 ++++ backend/src/storage/__init__.py | 0 backend/src/storage/local.py | 118 ++++++ backend/src/tasks.py | 125 +----- backend/src/use_cases/__init__.py | 0 backend/src/use_cases/files.py | 144 +++++++ backend/src/use_cases/pipeline.py | 165 ++++++++ backend/src/worker/__init__.py | 0 backend/src/worker/app.py | 22 ++ backend/src/worker/jobs.py | 92 +++++ backend/storage/.gitkeep | 0 backend/tests/test_columns.py | 11 + backend/tests/test_scan_rules.py | 41 ++ docker-compose.dev.yml | 110 +++++- frontend/.env.production | 1 + frontend/Dockerfile | 44 +-- frontend/src/api/alerts.ts | 6 + frontend/src/api/files.ts | 13 + frontend/src/api/http.ts | 48 +++ frontend/src/app/page.tsx | 372 ++---------------- frontend/src/components/AlertsTable.tsx | 66 ++++ frontend/src/components/DashboardHeader.tsx | 33 ++ frontend/src/components/FilesTable.tsx | 96 +++++ frontend/src/components/UploadDialog.tsx | 61 +++ frontend/src/config/env.ts | 2 + frontend/src/hooks/useDashboardData.ts | 40 ++ frontend/src/hooks/useFileUpload.ts | 51 +++ frontend/src/lib/format.ts | 16 + frontend/src/lib/statusStyles.ts | 22 ++ frontend/src/types/api.ts | 22 ++ 68 files changed, 2327 insertions(+), 819 deletions(-) create mode 100644 ARCHITECTURE.md create mode 100644 backend/.dockerignore create mode 100644 backend/migrations/versions/b7c4e1a9f2d0_cascade_and_indexes.py create mode 100644 backend/src/api/__init__.py create mode 100644 backend/src/api/deps.py create mode 100644 backend/src/api/routes_alerts.py create mode 100644 backend/src/api/routes_files.py create mode 100644 backend/src/api/routes_system.py create mode 100644 backend/src/application.py create mode 100644 backend/src/data/__init__.py create mode 100644 backend/src/data/alerts.py create mode 100644 backend/src/data/columns.py create mode 100644 backend/src/data/files.py create mode 100644 backend/src/db/__init__.py create mode 100644 backend/src/db/base.py create mode 100644 backend/src/db/session.py create mode 100644 backend/src/dto/__init__.py create mode 100644 backend/src/dto/alerts.py create mode 100644 backend/src/dto/common.py create mode 100644 backend/src/dto/files.py create mode 100644 backend/src/entities/__init__.py create mode 100644 backend/src/entities/alert.py create mode 100644 backend/src/entities/stored_file.py create mode 100644 backend/src/errors.py create mode 100644 backend/src/http_errors.py create mode 100644 backend/src/logging_setup.py create mode 100644 backend/src/scan/__init__.py create mode 100644 backend/src/scan/rules.py create mode 100644 backend/src/settings.py create mode 100644 backend/src/storage/__init__.py create mode 100644 backend/src/storage/local.py create mode 100644 backend/src/use_cases/__init__.py create mode 100644 backend/src/use_cases/files.py create mode 100644 backend/src/use_cases/pipeline.py create mode 100644 backend/src/worker/__init__.py create mode 100644 backend/src/worker/app.py create mode 100644 backend/src/worker/jobs.py create mode 100644 backend/storage/.gitkeep create mode 100644 backend/tests/test_columns.py create mode 100644 backend/tests/test_scan_rules.py create mode 100644 frontend/.env.production create mode 100644 frontend/src/api/alerts.ts create mode 100644 frontend/src/api/files.ts create mode 100644 frontend/src/api/http.ts create mode 100644 frontend/src/components/AlertsTable.tsx create mode 100644 frontend/src/components/DashboardHeader.tsx create mode 100644 frontend/src/components/FilesTable.tsx create mode 100644 frontend/src/components/UploadDialog.tsx create mode 100644 frontend/src/config/env.ts create mode 100644 frontend/src/hooks/useDashboardData.ts create mode 100644 frontend/src/hooks/useFileUpload.ts create mode 100644 frontend/src/lib/format.ts create mode 100644 frontend/src/lib/statusStyles.ts create mode 100644 frontend/src/types/api.ts diff --git a/.env.dev b/.env.dev index 181da01c..3d204d9c 100644 --- a/.env.dev +++ b/.env.dev @@ -3,7 +3,22 @@ POSTGRES_USER=postgres POSTGRES_PASSWORD=postgres POSTGRES_DB=test POSTGRES_HOST=backend-db -PGPORT=5433 +POSTGRES_PORT=5432 +PGPORT=5432 # Celery / Redis -CELERY_BROKER_URL=redis://backend-redis:6379/0 \ No newline at end of file +CELERY_BROKER_URL=redis://backend-redis:6379/0 +REDIS_URL=redis://backend-redis:6379/0 + +# App +STORAGE_DIR=/backend/storage/files +MAX_UPLOAD_BYTES=52428800 +LOG_LEVEL=INFO +CORS_ORIGINS=http://localhost:3000,http://127.0.0.1:3000 + +# Container resource overrides (optional) +BACKEND_MEM_LIMIT=768M +WORKER_MEM_LIMIT=768M +POSTGRES_MEM_LIMIT=512M +REDIS_MEM_LIMIT=128M +FRONTEND_MEM_LIMIT=512M diff --git a/.gitignore b/.gitignore index e3870959..879c0b52 100644 --- a/.gitignore +++ b/.gitignore @@ -5,9 +5,15 @@ build/ dist/ wheels/ *.egg-info +.pytest_cache/ # Virtual environments .venv .idea **/.DS_Store -backend/storage/* \ No newline at end of file +backend/storage/* +!backend/storage/.gitkeep + +# Local helpers +_write_batch*.py +uv.lock.bak diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 00000000..86a631cf --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,82 @@ +# Architecture + +## Goal + +Refactor the baseline file-exchange MVP without changing business rules or public API paths. Optional optimizations: single background pipeline task, safer uploads, Docker resource limits, frontend layering. + +## Backend layers + +``` +api/ HTTP routes, validation, status mapping +use_cases/ orchestration (CRUD, pipeline) +data/ SQLAlchemy queries with column projections +entities/ ORM tables +dto/ Pydantic request/response models +storage/ disk I/O (stream, path safety) +scan/ pure scan rules +worker/ Celery app + single job +db/ engine / sessions +settings.py env configuration +errors.py domain exceptions +http_errors.py domain → HTTP +``` + +### Pipeline optimization + +Baseline used three Celery tasks (`scan → metadata → alert`): three broker hops, three DB sessions, three PK loads. + +Now one task `jobs.process_uploaded_file` runs scan, metadata extraction, and alert creation in one worker job with retries for transient failures. + +Business outcomes are unchanged: + +- statuses: `uploaded` → `processing` → `processed` / `failed` +- scan: clean vs suspicious (extension / >10MB / pdf mime mismatch) +- alerts: critical / warning / info (including success info) + +### Data access rules + +- List/get API select only `FileItem` / `AlertItem` columns (no `stored_name` on API reads) +- Download/delete/worker use narrow queries that include `stored_name` when needed +- Indexes: `files.created_at`, `alerts.created_at`, `alerts.file_id` +- `alerts.file_id` FK uses `ON DELETE CASCADE` + +### Upload / DoS baseline + +- Streamed writes (chunks), max size from `MAX_UPLOAD_BYTES` (default 50 MiB) +- Oversize → 413 + partial cleanup +- Empty file → 400 +- No resumable protocol; connection drop mid-upload cleans partial file + +### Auth + +No tokens / JWT. Demo is open on LAN; hardening is path safety, limits, CORS, log hygiene. + +## Frontend layers + +``` +config/ API base URL +types/ DTO types +api/ HTTP client + resource methods +hooks/ dashboard load, upload form +components/ presentational UI +app/page composition only +``` + +## Docker + +- Code is **copied into the image** for reproducible runs. +- Dev compose also **bind-mounts** `./backend` for uvicorn `--reload`. +- Uploaded blobs live in a **named volume** `file-storage` shared by API and worker (not "code out of container"). +- Resource limits via `deploy.resources` (memory/CPU) on every service. +- Postgres `127.0.0.1:5433→5432`, healthchecks, pinned alpine images. + +## Running + +```bash +docker compose -f docker-compose.dev.yml up --build +docker exec -it backend alembic upgrade head +``` + +UI: http://localhost:3000/test +API docs: http://localhost:8000/docs +Health: http://localhost:8000/health diff --git a/README.md b/README.md index 00da8f4c..ac641f36 100644 --- a/README.md +++ b/README.md @@ -10,11 +10,35 @@ 2. (Дополнительно) На бэкенде есть возможность неочевидной оптимизации - выполните ее; 3. (Дополнительно) Разбейте логику фронтенда на слои; +## Что сделано в этой ветке + +- Backend разбит на слои: `api` / `use_cases` / `data` / `entities` / `storage` / `scan` / `worker` +- Неочевидная оптимизация: 3 Celery-задачи → 1 pipeline job +- Stream upload, лимит размера, cleanup partial, информативные логи, retries +- Projection-запросы (API не читает `stored_name`), индексы, CASCADE delete +- Frontend: types / api / hooks / components +- Docker: healthchecks, ports, resource limits, shared storage volume +- Подробности: [ARCHITECTURE.md](./ARCHITECTURE.md) + **Запуск:** -1. ```docker compose -f docker-compose.dev.yml up``` -2. ```docker exec -it backend alembic upgrade head``` +1. `docker compose -f docker-compose.dev.yml up --build` +2. `docker exec -it backend alembic upgrade head` + +**Открыть фронт:** `http://localhost:3000/test` + +**Открыть бэк:** `http://localhost:8000/docs` + +**Health:** `http://localhost:8000/health` +**Тесты backend (локально):** +```bash +cd backend +uv sync --extra dev +uv run pytest +``` -**Открыть фронт:** ```http://localhost:3000/test``` +### Docker: код в контейнере или снаружи? -**Открыть бэк:** ```http://localhost:8000/docs``` +- **В образ** копируется исходный код (`COPY`) — основной способ запуска. +- В dev compose `./backend` **примонтирован** для hot-reload; это не «вынос логики наружу», а DX. +- Бинарные загрузки — в volume `file-storage`, общий для API и worker. diff --git a/backend/.dockerignore b/backend/.dockerignore new file mode 100644 index 00000000..e8bb0e60 --- /dev/null +++ b/backend/.dockerignore @@ -0,0 +1,7 @@ +__pycache__ +*.pyc +.pytest_cache +.venv +storage +tests +*.md diff --git a/backend/Dockerfile b/backend/Dockerfile index 66443a20..946f42b9 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,13 +1,17 @@ FROM python:3.14-slim -ENV PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 \ - UV_NO_DEV=1 \ - UV_PROJECT_ENVIRONMENT=/usr/local +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 UV_NO_DEV=1 UV_PROJECT_ENVIRONMENT=/usr/local PYTHONPATH=/backend + WORKDIR /backend -COPY pyproject.toml uv.lock ./ +COPY pyproject.toml ./ COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ -RUN uv sync --locked +RUN uv lock && uv sync --no-dev COPY src ./src +COPY migrations ./migrations +COPY alembic.ini ./ + +RUN mkdir -p /backend/storage/files + +EXPOSE 8000 diff --git a/backend/migrations/env.py b/backend/migrations/env.py index e9e9f01b..2c423386 100644 --- a/backend/migrations/env.py +++ b/backend/migrations/env.py @@ -1,47 +1,25 @@ import asyncio from logging.config import fileConfig + +from alembic import context from sqlalchemy import pool from sqlalchemy.engine import Connection from sqlalchemy.ext.asyncio import async_engine_from_config -from alembic import context -from src.service import DB_URL -from src.models import Base -import src.models -# this is the Alembic Config object, which provides -# access to the values within the .ini file in use. +from src.db.base import Base +from src.entities import Alert, StoredFile # noqa: F401 +from src.settings import get_settings + config = context.config -config.set_main_option('sqlalchemy.url', DB_URL) +config.set_main_option("sqlalchemy.url", get_settings().database_url) -# Interpret the config file for Python logging. -# This line sets up loggers basically. if config.config_file_name is not None: fileConfig(config.config_file_name) -# add your model's MetaData object here -# for 'autogenerate' support -# from myapp import mymodel -# target_metadata = mymodel.Base.metadata target_metadata = Base.metadata -# other values from the config, defined by the needs of env.py, -# can be acquired: -# my_important_option = config.get_main_option("my_important_option") -# ... etc. - def run_migrations_offline() -> None: - """Run migrations in 'offline' mode. - - This configures the context with just a URL - and not an Engine, though an Engine is acceptable - here as well. By skipping the Engine creation - we don't even need a DBAPI to be available. - - Calls to context.execute() here emit the given string to the - script output. - - """ url = config.get_main_option("sqlalchemy.url") context.configure( url=url, @@ -49,39 +27,28 @@ def run_migrations_offline() -> None: literal_binds=True, dialect_opts={"paramstyle": "named"}, ) - with context.begin_transaction(): context.run_migrations() def do_run_migrations(connection: Connection) -> None: context.configure(connection=connection, target_metadata=target_metadata) - with context.begin_transaction(): context.run_migrations() async def run_async_migrations() -> None: - """In this scenario we need to create an Engine - and associate a connection with the context. - - """ - connectable = async_engine_from_config( config.get_section(config.config_ini_section, {}), prefix="sqlalchemy.", poolclass=pool.NullPool, ) - async with connectable.connect() as connection: await connection.run_sync(do_run_migrations) - await connectable.dispose() def run_migrations_online() -> None: - """Run migrations in 'online' mode.""" - asyncio.run(run_async_migrations()) diff --git a/backend/migrations/versions/b7c4e1a9f2d0_cascade_and_indexes.py b/backend/migrations/versions/b7c4e1a9f2d0_cascade_and_indexes.py new file mode 100644 index 00000000..43143b38 --- /dev/null +++ b/backend/migrations/versions/b7c4e1a9f2d0_cascade_and_indexes.py @@ -0,0 +1,45 @@ +"""cascade alerts and list indexes + +Revision ID: b7c4e1a9f2d0 +Revises: 0d6439d2e79f +Create Date: 2026-07-11 15:00:00.000000 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "b7c4e1a9f2d0" +down_revision: Union[str, Sequence[str], None] = "0d6439d2e79f" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.drop_constraint("alerts_file_id_fkey", "alerts", type_="foreignkey") + op.create_foreign_key( + "alerts_file_id_fkey", + "alerts", + "files", + ["file_id"], + ["id"], + ondelete="CASCADE", + ) + op.create_index("ix_files_created_at", "files", ["created_at"], unique=False) + op.create_index("ix_alerts_created_at", "alerts", ["created_at"], unique=False) + op.create_index("ix_alerts_file_id", "alerts", ["file_id"], unique=False) + + +def downgrade() -> None: + op.drop_index("ix_alerts_file_id", table_name="alerts") + op.drop_index("ix_alerts_created_at", table_name="alerts") + op.drop_index("ix_files_created_at", table_name="files") + op.drop_constraint("alerts_file_id_fkey", "alerts", type_="foreignkey") + op.create_foreign_key( + "alerts_file_id_fkey", + "alerts", + "files", + ["file_id"], + ["id"], + ) diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 988f6959..dcd6456a 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -1,16 +1,30 @@ [project] name = "backend" version = "0.1.0" -description = "Add your description here" +description = "File exchange backend" readme = "README.md" -requires-python = ">=3.14" +requires-python = ">=3.12" dependencies = [ "alembic>=1.18.4", + "anyio>=4.0.0", "asyncpg>=0.30.0", "celery[redis]>=5.6.3", "fastapi>=0.135.3", "pydantic>=2.12.5", + "pydantic-settings>=2.0.0", "python-multipart>=0.0.20", "sqlalchemy>=2.0.48", "uvicorn>=0.42.0", ] + +[project.optional-dependencies] +dev = [ + "httpx>=0.27.0", + "pytest>=8.0.0", + "pytest-asyncio>=0.24.0", +] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +pythonpath = ["."] +testpaths = ["tests"] diff --git a/backend/src/api/__init__.py b/backend/src/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/src/api/deps.py b/backend/src/api/deps.py new file mode 100644 index 00000000..9e9394d2 --- /dev/null +++ b/backend/src/api/deps.py @@ -0,0 +1,11 @@ +from collections.abc import AsyncIterator + +from sqlalchemy.ext.asyncio import AsyncSession + +from src.db.session import get_session_factory + + +async def db_session() -> AsyncIterator[AsyncSession]: + factory = get_session_factory() + async with factory() as session: + yield session diff --git a/backend/src/api/routes_alerts.py b/backend/src/api/routes_alerts.py new file mode 100644 index 00000000..c1a3e242 --- /dev/null +++ b/backend/src/api/routes_alerts.py @@ -0,0 +1,13 @@ +from fastapi import APIRouter, Depends +from sqlalchemy.ext.asyncio import AsyncSession + +from src.api.deps import db_session +from src.dto.alerts import AlertItem +from src.use_cases import files as files_uc + +router = APIRouter(tags=["alerts"]) + + +@router.get("/alerts", response_model=list[AlertItem]) +async def list_alerts_view(session: AsyncSession = Depends(db_session)) -> list[AlertItem]: + return await files_uc.list_alerts(session=session) diff --git a/backend/src/api/routes_files.py b/backend/src/api/routes_files.py new file mode 100644 index 00000000..c35fdb45 --- /dev/null +++ b/backend/src/api/routes_files.py @@ -0,0 +1,68 @@ +from fastapi import APIRouter, Depends, File, Form, UploadFile +from fastapi.responses import FileResponse, Response +from sqlalchemy.ext.asyncio import AsyncSession + +from src.api.deps import db_session +from src.dto.common import file_id_path +from src.dto.files import FileItem, FileUpdate +from src.use_cases import files as files_uc +from src.worker.jobs import enqueue_file_pipeline + +router = APIRouter(tags=["files"]) + + +@router.get("/files", response_model=list[FileItem]) +async def list_files_view(session: AsyncSession = Depends(db_session)) -> list[FileItem]: + return await files_uc.list_files(session=session) + + +@router.post("/files", response_model=FileItem, status_code=201) +async def create_file_view( + title: str = Form(..., min_length=1, max_length=255), + file: UploadFile = File(...), + session: AsyncSession = Depends(db_session), +) -> FileItem: + return await files_uc.create_file( + session=session, + title=title, + upload_file=file, + enqueue=enqueue_file_pipeline, + ) + + +@router.get("/files/{file_id}", response_model=FileItem) +async def get_file_view( + file_id: str = Depends(file_id_path), + session: AsyncSession = Depends(db_session), +) -> FileItem: + return await files_uc.get_file(session=session, file_id=file_id) + + +@router.patch("/files/{file_id}", response_model=FileItem) +async def update_file_view( + payload: FileUpdate, + file_id: str = Depends(file_id_path), + session: AsyncSession = Depends(db_session), +) -> FileItem: + return await files_uc.update_file(session=session, file_id=file_id, title=payload.title) + + +@router.get("/files/{file_id}/download") +async def download_file_view( + file_id: str = Depends(file_id_path), + session: AsyncSession = Depends(db_session), +) -> FileResponse: + path, filename, mime_type = await files_uc.resolve_download( + session=session, + file_id=file_id, + ) + return FileResponse(path=path, media_type=mime_type, filename=filename) + + +@router.delete("/files/{file_id}", status_code=204) +async def delete_file_view( + file_id: str = Depends(file_id_path), + session: AsyncSession = Depends(db_session), +) -> Response: + await files_uc.delete_file(session=session, file_id=file_id) + return Response(status_code=204) diff --git a/backend/src/api/routes_system.py b/backend/src/api/routes_system.py new file mode 100644 index 00000000..5f4ca371 --- /dev/null +++ b/backend/src/api/routes_system.py @@ -0,0 +1,8 @@ +from fastapi import APIRouter + +router = APIRouter(tags=["system"]) + + +@router.get("/health") +async def health() -> dict[str, str]: + return {"status": "ok"} diff --git a/backend/src/app.py b/backend/src/app.py index bec89a5f..dc9d47b7 100644 --- a/backend/src/app.py +++ b/backend/src/app.py @@ -1,71 +1,5 @@ -from fastapi import FastAPI, HTTPException -from fastapi import File, Form, UploadFile -from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import FileResponse -from starlette import status -from src.schemas import AlertItem, FileItem, FileUpdate -from src.service import create_file, delete_file, get_file, list_alerts, list_files, update_file, STORAGE_DIR -from src.tasks import scan_file_for_threats +"""ASGI entrypoint for uvicorn.""" -app = FastAPI() -app.add_middleware( - CORSMiddleware, - allow_origins=[ - "http://localhost:3000", - "http://127.0.0.1:3000", - ], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], -) +from src.application import create_app - -@app.get("/files", response_model=list[FileItem]) -async def list_files_view(): - return await list_files() - - -@app.get("/alerts", response_model=list[AlertItem]) -async def list_alerts_view(): - return await list_alerts() - - -@app.post("/files", response_model=FileItem, status_code=201) -async def create_file_view( - title: str = Form(...), - file: UploadFile = File(...), -): - file_item = await create_file(title=title, upload_file=file) - scan_file_for_threats.delay(file_item.id) - return file_item - - -@app.get("/files/{file_id}", response_model=FileItem) -async def get_file_view(file_id: str): - return await get_file(file_id) - - -@app.patch("/files/{file_id}", response_model=FileItem) -async def update_file_view( - file_id: str, - payload: FileUpdate, -): - return await update_file(file_id=file_id, title=payload.title) - - -@app.get("/files/{file_id}/download") -async def download_file(file_id: str): - file_item = await get_file(file_id) - stored_path = STORAGE_DIR / file_item.stored_name - if not stored_path.exists(): - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Stored file not found") - return FileResponse( - path=stored_path, - media_type=file_item.mime_type, - filename=file_item.original_name, - ) - - -@app.delete("/files/{file_id}", status_code=204) -async def delete_file_view(file_id: str): - await delete_file(file_id) +app = create_app() diff --git a/backend/src/application.py b/backend/src/application.py new file mode 100644 index 00000000..625b9fc0 --- /dev/null +++ b/backend/src/application.py @@ -0,0 +1,42 @@ +"""FastAPI application factory.""" + +from contextlib import asynccontextmanager + +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware + +from src.api.routes_alerts import router as alerts_router +from src.api.routes_files import router as files_router +from src.api.routes_system import router as system_router +from src.db.session import dispose_engine, get_engine +from src.http_errors import register_exception_handlers +from src.logging_setup import configure_logging +from src.settings import get_settings +from src.storage.local import ensure_storage_dir + + +@asynccontextmanager +async def lifespan(_: FastAPI): + configure_logging() + settings = get_settings() + ensure_storage_dir(settings=settings) + get_engine() + yield + await dispose_engine() + + +def create_app() -> FastAPI: + settings = get_settings() + app = FastAPI(title="File Exchange API", lifespan=lifespan) + app.add_middleware( + CORSMiddleware, + allow_origins=settings.cors_origin_list, + allow_credentials=True, + allow_methods=["GET", "POST", "PATCH", "DELETE", "OPTIONS"], + allow_headers=["*"], + ) + register_exception_handlers(app) + app.include_router(system_router) + app.include_router(files_router) + app.include_router(alerts_router) + return app diff --git a/backend/src/data/__init__.py b/backend/src/data/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/src/data/alerts.py b/backend/src/data/alerts.py new file mode 100644 index 00000000..146124f3 --- /dev/null +++ b/backend/src/data/alerts.py @@ -0,0 +1,45 @@ +"""Alert persistence with projections.""" + +from __future__ import annotations + +from typing import Any + +from sqlalchemy import exists, select +from sqlalchemy.ext.asyncio import AsyncSession + +from src.data.columns import ALERT_ITEM_COLUMNS +from src.dto.alerts import AlertItem +from src.entities.alert import Alert + + +def _row_to_alert(row: Any) -> AlertItem: + return AlertItem( + id=row.id, + file_id=row.file_id, + level=row.level, + message=row.message, + created_at=row.created_at, + ) + + +async def list_alert_items(*, session: AsyncSession) -> list[AlertItem]: + result = await session.execute( + select(*ALERT_ITEM_COLUMNS).order_by(Alert.created_at.desc()) + ) + return [_row_to_alert(row) for row in result.all()] + + +async def alert_exists_for_file(*, session: AsyncSession, file_id: str) -> bool: + stmt = select(exists().where(Alert.file_id == file_id)) + result = await session.execute(stmt) + return bool(result.scalar()) + + +async def insert_alert( + *, + session: AsyncSession, + file_id: str, + level: str, + message: str, +) -> None: + session.add(Alert(file_id=file_id, level=level, message=message)) diff --git a/backend/src/data/columns.py b/backend/src/data/columns.py new file mode 100644 index 00000000..8d3756ac --- /dev/null +++ b/backend/src/data/columns.py @@ -0,0 +1,48 @@ +"""Column projections aligned with response DTOs.""" + +from src.entities.alert import Alert +from src.entities.stored_file import StoredFile + +FILE_ITEM_COLUMNS = ( + StoredFile.id, + StoredFile.title, + StoredFile.original_name, + StoredFile.mime_type, + StoredFile.size, + StoredFile.processing_status, + StoredFile.scan_status, + StoredFile.scan_details, + StoredFile.metadata_json, + StoredFile.requires_attention, + StoredFile.created_at, + StoredFile.updated_at, +) + +ALERT_ITEM_COLUMNS = ( + Alert.id, + Alert.file_id, + Alert.level, + Alert.message, + Alert.created_at, +) + +STORAGE_REF_COLUMNS = ( + StoredFile.id, + StoredFile.stored_name, + StoredFile.original_name, + StoredFile.mime_type, +) + +PIPELINE_COLUMNS = ( + StoredFile.id, + StoredFile.title, + StoredFile.original_name, + StoredFile.stored_name, + StoredFile.mime_type, + StoredFile.size, + StoredFile.processing_status, + StoredFile.scan_status, + StoredFile.scan_details, + StoredFile.metadata_json, + StoredFile.requires_attention, +) diff --git a/backend/src/data/files.py b/backend/src/data/files.py new file mode 100644 index 00000000..b0d5ff03 --- /dev/null +++ b/backend/src/data/files.py @@ -0,0 +1,184 @@ +"""File persistence with column projections.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any + +from sqlalchemy import delete, select, update +from sqlalchemy.ext.asyncio import AsyncSession + +from src.data.columns import FILE_ITEM_COLUMNS, PIPELINE_COLUMNS, STORAGE_REF_COLUMNS +from src.dto.files import FileItem +from src.entities.stored_file import StoredFile + + +@dataclass(slots=True) +class StorageRef: + id: str + stored_name: str + original_name: str + mime_type: str + + +@dataclass(slots=True) +class PipelineRow: + id: str + title: str + original_name: str + stored_name: str + mime_type: str + size: int + processing_status: str + scan_status: str | None + scan_details: str | None + metadata_json: dict | None + requires_attention: bool + + +def _row_to_file_item(row: Any) -> FileItem: + return FileItem( + id=row.id, + title=row.title, + original_name=row.original_name, + mime_type=row.mime_type, + size=row.size, + processing_status=row.processing_status, + scan_status=row.scan_status, + scan_details=row.scan_details, + metadata_json=row.metadata_json, + requires_attention=row.requires_attention, + created_at=row.created_at, + updated_at=row.updated_at, + ) + + +async def list_file_items(*, session: AsyncSession) -> list[FileItem]: + result = await session.execute( + select(*FILE_ITEM_COLUMNS).order_by(StoredFile.created_at.desc()) + ) + return [_row_to_file_item(row) for row in result.all()] + + +async def get_file_item(*, session: AsyncSession, file_id: str) -> FileItem | None: + result = await session.execute( + select(*FILE_ITEM_COLUMNS).where(StoredFile.id == file_id) + ) + row = result.first() + if row is None: + return None + return _row_to_file_item(row) + + +async def get_storage_ref(*, session: AsyncSession, file_id: str) -> StorageRef | None: + result = await session.execute( + select(*STORAGE_REF_COLUMNS).where(StoredFile.id == file_id) + ) + row = result.first() + if row is None: + return None + return StorageRef( + id=row.id, + stored_name=row.stored_name, + original_name=row.original_name, + mime_type=row.mime_type, + ) + + +async def get_pipeline_row(*, session: AsyncSession, file_id: str) -> PipelineRow | None: + result = await session.execute( + select(*PIPELINE_COLUMNS).where(StoredFile.id == file_id) + ) + row = result.first() + if row is None: + return None + return PipelineRow( + id=row.id, + title=row.title, + original_name=row.original_name, + stored_name=row.stored_name, + mime_type=row.mime_type, + size=row.size, + processing_status=row.processing_status, + scan_status=row.scan_status, + scan_details=row.scan_details, + metadata_json=row.metadata_json, + requires_attention=row.requires_attention, + ) + + +async def insert_file( + *, + session: AsyncSession, + file_id: str, + title: str, + original_name: str, + stored_name: str, + mime_type: str, + size: int, +) -> FileItem: + entity = StoredFile( + id=file_id, + title=title, + original_name=original_name, + stored_name=stored_name, + mime_type=mime_type, + size=size, + processing_status="uploaded", + requires_attention=False, + ) + session.add(entity) + await session.flush() + await session.refresh(entity) + return FileItem( + id=entity.id, + title=entity.title, + original_name=entity.original_name, + mime_type=entity.mime_type, + size=entity.size, + processing_status=entity.processing_status, + scan_status=entity.scan_status, + scan_details=entity.scan_details, + metadata_json=entity.metadata_json, + requires_attention=entity.requires_attention, + created_at=entity.created_at, + updated_at=entity.updated_at, + ) + + +async def update_file_title( + *, + session: AsyncSession, + file_id: str, + title: str, +) -> FileItem | None: + result = await session.execute( + update(StoredFile) + .where(StoredFile.id == file_id) + .values(title=title, updated_at=datetime.now(timezone.utc)) + .returning(*FILE_ITEM_COLUMNS) + ) + row = result.first() + if row is None: + return None + return _row_to_file_item(row) + + +async def delete_file_row(*, session: AsyncSession, file_id: str) -> bool: + result = await session.execute(delete(StoredFile).where(StoredFile.id == file_id)) + return (result.rowcount or 0) > 0 + + +async def update_pipeline_fields( + *, + session: AsyncSession, + file_id: str, + **fields: Any, +) -> None: + if not fields: + return + fields["updated_at"] = datetime.now(timezone.utc) + await session.execute( + update(StoredFile).where(StoredFile.id == file_id).values(**fields) + ) diff --git a/backend/src/db/__init__.py b/backend/src/db/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/src/db/base.py b/backend/src/db/base.py new file mode 100644 index 00000000..fa2b68a5 --- /dev/null +++ b/backend/src/db/base.py @@ -0,0 +1,5 @@ +from sqlalchemy.orm import DeclarativeBase + + +class Base(DeclarativeBase): + pass diff --git a/backend/src/db/session.py b/backend/src/db/session.py new file mode 100644 index 00000000..51887123 --- /dev/null +++ b/backend/src/db/session.py @@ -0,0 +1,67 @@ +"""Async SQLAlchemy engine and session helpers.""" + +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager + +from sqlalchemy.ext.asyncio import ( + AsyncEngine, + AsyncSession, + async_sessionmaker, + create_async_engine, +) + +from src.settings import Settings, get_settings + +_engine: AsyncEngine | None = None +_session_factory: async_sessionmaker[AsyncSession] | None = None + + +def create_engine(settings: Settings | None = None) -> AsyncEngine: + cfg = settings or get_settings() + return create_async_engine( + cfg.database_url, + pool_pre_ping=True, + ) + + +def get_engine() -> AsyncEngine: + global _engine, _session_factory + if _engine is None: + _engine = create_engine() + _session_factory = async_sessionmaker(_engine, expire_on_commit=False) + return _engine + + +def get_session_factory() -> async_sessionmaker[AsyncSession]: + global _session_factory + if _session_factory is None: + get_engine() + assert _session_factory is not None + return _session_factory + + +async def get_db_session() -> AsyncIterator[AsyncSession]: + session_factory = get_session_factory() + async with session_factory() as session: + yield session + + +@asynccontextmanager +async def worker_session_scope(settings: Settings | None = None): + """Fresh engine per worker job — avoids event-loop leakage across tasks.""" + cfg = settings or get_settings() + engine = create_engine(cfg) + factory = async_sessionmaker(engine, expire_on_commit=False) + try: + async with factory() as session: + yield session + finally: + await engine.dispose() + + +async def dispose_engine() -> None: + global _engine, _session_factory + if _engine is not None: + await _engine.dispose() + _engine = None + _session_factory = None diff --git a/backend/src/dto/__init__.py b/backend/src/dto/__init__.py new file mode 100644 index 00000000..fc22af2b --- /dev/null +++ b/backend/src/dto/__init__.py @@ -0,0 +1,4 @@ +from src.dto.alerts import AlertItem +from src.dto.files import FileItem, FileUpdate + +__all__ = ["AlertItem", "FileItem", "FileUpdate"] diff --git a/backend/src/dto/alerts.py b/backend/src/dto/alerts.py new file mode 100644 index 00000000..50b2e0b8 --- /dev/null +++ b/backend/src/dto/alerts.py @@ -0,0 +1,13 @@ +from datetime import datetime + +from pydantic import BaseModel, ConfigDict + + +class AlertItem(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: int + file_id: str + level: str + message: str + created_at: datetime diff --git a/backend/src/dto/common.py b/backend/src/dto/common.py new file mode 100644 index 00000000..18f32ea8 --- /dev/null +++ b/backend/src/dto/common.py @@ -0,0 +1,17 @@ +"""Shared request validators.""" + +from uuid import UUID + +from fastapi import HTTPException, Path +from starlette import status + + +def file_id_path(file_id: str = Path(..., min_length=36, max_length=36)) -> str: + """Validate path file_id as UUID string.""" + try: + return str(UUID(file_id)) + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="Invalid file_id format", + ) from exc diff --git a/backend/src/dto/files.py b/backend/src/dto/files.py new file mode 100644 index 00000000..0b41237e --- /dev/null +++ b/backend/src/dto/files.py @@ -0,0 +1,32 @@ +from datetime import datetime + +from pydantic import BaseModel, ConfigDict, Field, field_validator + + +class FileItem(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: str + title: str + original_name: str + mime_type: str + size: int + processing_status: str + scan_status: str | None + scan_details: str | None + metadata_json: dict | None + requires_attention: bool + created_at: datetime + updated_at: datetime + + +class FileUpdate(BaseModel): + title: str = Field(..., min_length=1, max_length=255) + + @field_validator("title") + @classmethod + def strip_title(cls, value: str) -> str: + cleaned = value.strip() + if not cleaned: + raise ValueError("title must not be empty") + return cleaned diff --git a/backend/src/entities/__init__.py b/backend/src/entities/__init__.py new file mode 100644 index 00000000..0d252c15 --- /dev/null +++ b/backend/src/entities/__init__.py @@ -0,0 +1,4 @@ +from src.entities.alert import Alert +from src.entities.stored_file import StoredFile + +__all__ = ["Alert", "StoredFile"] diff --git a/backend/src/entities/alert.py b/backend/src/entities/alert.py new file mode 100644 index 00000000..c96adbad --- /dev/null +++ b/backend/src/entities/alert.py @@ -0,0 +1,25 @@ +from datetime import datetime + +from sqlalchemy import DateTime, ForeignKey, Integer, String, func +from sqlalchemy.orm import Mapped, mapped_column + +from src.db.base import Base + + +class Alert(Base): + __tablename__ = "alerts" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + file_id: Mapped[str] = mapped_column( + String(36), + ForeignKey("files.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + level: Mapped[str] = mapped_column(String(50), nullable=False) + message: Mapped[str] = mapped_column(String(500), nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + server_default=func.now(), + nullable=False, + ) diff --git a/backend/src/entities/stored_file.py b/backend/src/entities/stored_file.py new file mode 100644 index 00000000..0ca45bdc --- /dev/null +++ b/backend/src/entities/stored_file.py @@ -0,0 +1,33 @@ +from datetime import datetime + +from sqlalchemy import Boolean, DateTime, Integer, JSON, String, func +from sqlalchemy.orm import Mapped, mapped_column + +from src.db.base import Base + + +class StoredFile(Base): + __tablename__ = "files" + + id: Mapped[str] = mapped_column(String(36), primary_key=True) + title: Mapped[str] = mapped_column(String(255), nullable=False) + original_name: Mapped[str] = mapped_column(String(255), nullable=False) + stored_name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True) + mime_type: Mapped[str] = mapped_column(String(255), nullable=False) + size: Mapped[int] = mapped_column(Integer, nullable=False) + processing_status: Mapped[str] = mapped_column(String(50), nullable=False, default="uploaded") + scan_status: Mapped[str | None] = mapped_column(String(50), nullable=True) + scan_details: Mapped[str | None] = mapped_column(String(500), nullable=True) + metadata_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) + requires_attention: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + server_default=func.now(), + nullable=False, + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + server_default=func.now(), + onupdate=func.now(), + nullable=False, + ) diff --git a/backend/src/errors.py b/backend/src/errors.py new file mode 100644 index 00000000..e4f8d30a --- /dev/null +++ b/backend/src/errors.py @@ -0,0 +1,29 @@ +"""Domain exceptions — no HTTP coupling.""" + + +class AppError(Exception): + """Base domain error.""" + + def __init__(self, message: str) -> None: + self.message = message + super().__init__(message) + + +class ResourceNotFoundError(AppError): + pass + + +class EmptyUploadError(AppError): + pass + + +class PayloadTooLargeError(AppError): + pass + + +class StorageIOError(AppError): + pass + + +class ValidationDomainError(AppError): + pass diff --git a/backend/src/http_errors.py b/backend/src/http_errors.py new file mode 100644 index 00000000..ade6b2b8 --- /dev/null +++ b/backend/src/http_errors.py @@ -0,0 +1,85 @@ +"""Map domain errors and unexpected failures to HTTP responses.""" + +import logging + +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from starlette import status + +from src.errors import ( + AppError, + EmptyUploadError, + PayloadTooLargeError, + ResourceNotFoundError, + StorageIOError, + ValidationDomainError, +) + +logger = logging.getLogger(__name__) + + +def register_exception_handlers(app: FastAPI) -> None: + @app.exception_handler(ResourceNotFoundError) + async def not_found_handler(_: Request, exc: ResourceNotFoundError) -> JSONResponse: + logger.warning("not_found detail=%s", exc.message) + return JSONResponse( + status_code=status.HTTP_404_NOT_FOUND, + content={"detail": exc.message}, + ) + + @app.exception_handler(EmptyUploadError) + async def empty_upload_handler(_: Request, exc: EmptyUploadError) -> JSONResponse: + logger.warning("empty_upload detail=%s", exc.message) + return JSONResponse( + status_code=status.HTTP_400_BAD_REQUEST, + content={"detail": exc.message}, + ) + + @app.exception_handler(PayloadTooLargeError) + async def payload_handler(_: Request, exc: PayloadTooLargeError) -> JSONResponse: + logger.warning("payload_too_large detail=%s", exc.message) + return JSONResponse( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + content={"detail": exc.message}, + ) + + @app.exception_handler(ValidationDomainError) + async def validation_domain_handler(_: Request, exc: ValidationDomainError) -> JSONResponse: + logger.warning("validation_domain detail=%s", exc.message) + return JSONResponse( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + content={"detail": exc.message}, + ) + + @app.exception_handler(StorageIOError) + async def storage_handler(_: Request, exc: StorageIOError) -> JSONResponse: + logger.error("storage_io detail=%s", exc.message, exc_info=True) + return JSONResponse( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + content={"detail": "Internal server error"}, + ) + + @app.exception_handler(AppError) + async def app_error_handler(_: Request, exc: AppError) -> JSONResponse: + logger.error("app_error detail=%s", exc.message, exc_info=True) + return JSONResponse( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + content={"detail": "Internal server error"}, + ) + + @app.exception_handler(RequestValidationError) + async def request_validation_handler(_: Request, exc: RequestValidationError) -> JSONResponse: + logger.warning("request_validation errors=%s", exc.errors()) + return JSONResponse( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + content={"detail": exc.errors()}, + ) + + @app.exception_handler(Exception) + async def unhandled_handler(_: Request, exc: Exception) -> JSONResponse: + logger.error("unhandled_exception type=%s reason=%s", type(exc).__name__, exc, exc_info=True) + return JSONResponse( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + content={"detail": "Internal server error"}, + ) diff --git a/backend/src/logging_setup.py b/backend/src/logging_setup.py new file mode 100644 index 00000000..f6ec482f --- /dev/null +++ b/backend/src/logging_setup.py @@ -0,0 +1,26 @@ +"""Console logging configuration.""" + +import logging +import sys + +from src.settings import get_settings + + +def configure_logging() -> None: + settings = get_settings() + level = getattr(logging, settings.log_level.upper(), logging.INFO) + root = logging.getLogger() + if root.handlers: + root.setLevel(level) + return + + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter( + logging.Formatter( + fmt="%(asctime)s | %(levelname)s | %(name)s | %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + ) + root.addHandler(handler) + root.setLevel(level) + logging.getLogger("uvicorn.access").setLevel(logging.INFO) diff --git a/backend/src/models.py b/backend/src/models.py index ad5e515b..3b88a8d9 100644 --- a/backend/src/models.py +++ b/backend/src/models.py @@ -1,49 +1,7 @@ -from datetime import datetime +"""Backward-compatible re-exports for Alembic and legacy imports.""" -from sqlalchemy import Boolean, DateTime, ForeignKey, Integer, JSON, String, func -from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column +from src.db.base import Base +from src.entities.alert import Alert +from src.entities.stored_file import StoredFile - -class Base(DeclarativeBase): - pass - - -class StoredFile(Base): - __tablename__ = "files" - - id: Mapped[str] = mapped_column(String(36), primary_key=True) - title: Mapped[str] = mapped_column(String(255), nullable=False) - original_name: Mapped[str] = mapped_column(String(255), nullable=False) - stored_name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True) - mime_type: Mapped[str] = mapped_column(String(255), nullable=False) - size: Mapped[int] = mapped_column(Integer, nullable=False) - processing_status: Mapped[str] = mapped_column(String(50), nullable=False, default="uploaded") - scan_status: Mapped[str | None] = mapped_column(String(50), nullable=True) - scan_details: Mapped[str | None] = mapped_column(String(500), nullable=True) - metadata_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) - requires_attention: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) - created_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), - server_default=func.now(), - nullable=False, - ) - updated_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), - server_default=func.now(), - onupdate=func.now(), - nullable=False, - ) - - -class Alert(Base): - __tablename__ = "alerts" - - id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) - file_id: Mapped[str] = mapped_column(String(36), ForeignKey("files.id"), nullable=False) - level: Mapped[str] = mapped_column(String(50), nullable=False) - message: Mapped[str] = mapped_column(String(500), nullable=False) - created_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), - server_default=func.now(), - nullable=False, - ) +__all__ = ["Base", "Alert", "StoredFile"] diff --git a/backend/src/scan/__init__.py b/backend/src/scan/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/src/scan/rules.py b/backend/src/scan/rules.py new file mode 100644 index 00000000..bb92b461 --- /dev/null +++ b/backend/src/scan/rules.py @@ -0,0 +1,48 @@ +"""Pure scan rules matching baseline business logic.""" + +from dataclasses import dataclass +from pathlib import Path + + +SUSPICIOUS_EXTENSIONS = {".exe", ".bat", ".cmd", ".sh", ".js"} +SIZE_THRESHOLD_BYTES = 10 * 1024 * 1024 +PDF_ALLOWED_MIME = {"application/pdf", "application/octet-stream"} + + +@dataclass(frozen=True, slots=True) +class ScanResult: + scan_status: str + scan_details: str + requires_attention: bool + + +def evaluate_scan( + *, + original_name: str, + mime_type: str, + size: int, + size_threshold_bytes: int = SIZE_THRESHOLD_BYTES, +) -> ScanResult: + reasons: list[str] = [] + extension = Path(original_name).suffix.lower() + + if extension in SUSPICIOUS_EXTENSIONS: + reasons.append(f"suspicious extension {extension}") + + if size > size_threshold_bytes: + reasons.append("file is larger than 10 MB") + + if extension == ".pdf" and mime_type not in PDF_ALLOWED_MIME: + reasons.append("pdf extension does not match mime type") + + if reasons: + return ScanResult( + scan_status="suspicious", + scan_details=", ".join(reasons), + requires_attention=True, + ) + return ScanResult( + scan_status="clean", + scan_details="no threats found", + requires_attention=False, + ) diff --git a/backend/src/schemas.py b/backend/src/schemas.py index 4d639b1f..06dc93b5 100644 --- a/backend/src/schemas.py +++ b/backend/src/schemas.py @@ -1,34 +1,6 @@ -from datetime import datetime +"""Backward-compatible re-exports.""" -from pydantic import BaseModel, ConfigDict +from src.dto.alerts import AlertItem +from src.dto.files import FileItem, FileUpdate - -class FileItem(BaseModel): - model_config = ConfigDict(from_attributes=True) - - id: str - title: str - original_name: str - mime_type: str - size: int - processing_status: str - scan_status: str | None - scan_details: str | None - metadata_json: dict | None - requires_attention: bool - created_at: datetime - updated_at: datetime - - -class FileUpdate(BaseModel): - title: str - - -class AlertItem(BaseModel): - model_config = ConfigDict(from_attributes=True) - - id: int - file_id: str - level: str - message: str - created_at: datetime +__all__ = ["AlertItem", "FileItem", "FileUpdate"] diff --git a/backend/src/service.py b/backend/src/service.py index e707fdc7..a6a48c0a 100644 --- a/backend/src/service.py +++ b/backend/src/service.py @@ -1,109 +1,10 @@ -import mimetypes -import os -from pathlib import Path -from uuid import uuid4 +"""Deprecated module kept only to avoid import surprises during transition. -from fastapi import HTTPException, UploadFile, status -from sqlalchemy import select -from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker +Use use_cases / data / storage packages instead. +""" -from src.models import Alert, StoredFile +from src.settings import get_settings - -BASE_DIR = Path(__file__).resolve().parent.parent -STORAGE_DIR = BASE_DIR / "storage" / "files" -STORAGE_DIR.mkdir(parents=True, exist_ok=True) -DB_URL = ( - f"postgresql+asyncpg://{os.environ.get('POSTGRES_USER')}:" - f"{os.environ.get('POSTGRES_PASSWORD')}@{os.environ.get('POSTGRES_HOST')}:" - f"{os.environ.get('PGPORT')}/{os.environ.get('POSTGRES_DB')}" -) -engine = create_async_engine(DB_URL) -async_session_maker = async_sessionmaker(engine, expire_on_commit=False) - - -async def list_files() -> list[StoredFile]: - async with async_session_maker() as session: - result = await session.execute(select(StoredFile).order_by(StoredFile.created_at.desc())) - return list(result.scalars().all()) - - -async def list_alerts() -> list[Alert]: - async with async_session_maker() as session: - result = await session.execute(select(Alert).order_by(Alert.created_at.desc())) - return list(result.scalars().all()) - - -async def get_file(file_id: str) -> StoredFile: - async with async_session_maker() as session: - file_item = await session.get(StoredFile, file_id) - if not file_item: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found") - return file_item - - -async def create_file(title: str, upload_file: UploadFile) -> StoredFile: - content = await upload_file.read() - if not content: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="File is empty") - - file_id = str(uuid4()) - suffix = Path(upload_file.filename or "").suffix - stored_name = f"{file_id}{suffix}" - stored_path = STORAGE_DIR / stored_name - stored_path.write_bytes(content) - - file_item = StoredFile( - id=file_id, - title=title, - original_name=upload_file.filename or stored_name, - stored_name=stored_name, - mime_type=upload_file.content_type or mimetypes.guess_type(stored_name)[0] or "application/octet-stream", - size=len(content), - processing_status="uploaded", - ) - async with async_session_maker() as session: - session.add(file_item) - await session.commit() - await session.refresh(file_item) - return file_item - - -async def update_file(file_id: str, title: str) -> StoredFile: - async with async_session_maker() as session: - file_item = await session.get(StoredFile, file_id) - if not file_item: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found") - file_item.title = title - await session.commit() - await session.refresh(file_item) - return file_item - - -async def delete_file(file_id: str) -> None: - async with async_session_maker() as session: - file_item = await session.get(StoredFile, file_id) - if not file_item: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found") - stored_path = STORAGE_DIR / file_item.stored_name - if stored_path.exists(): - stored_path.unlink() - await session.delete(file_item) - await session.commit() - - -async def get_file_path(file_id: str) -> tuple[StoredFile, Path]: - file_item = await get_file(file_id) - stored_path = STORAGE_DIR / file_item.stored_name - if not stored_path.exists(): - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Stored file not found") - return file_item, stored_path - - -async def create_alert(file_id: str, level: str, message: str) -> Alert: - alert = Alert(file_id=file_id, level=level, message=message) - async with async_session_maker() as session: - session.add(alert) - await session.commit() - await session.refresh(alert) - return alert +settings = get_settings() +STORAGE_DIR = settings.storage_dir +DB_URL = settings.database_url diff --git a/backend/src/settings.py b/backend/src/settings.py new file mode 100644 index 00000000..1604caea --- /dev/null +++ b/backend/src/settings.py @@ -0,0 +1,71 @@ +"""Application settings loaded from environment.""" + +from functools import lru_cache +from pathlib import Path + +from pydantic import Field +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + model_config = SettingsConfigDict( + env_file=None, + extra="ignore", + case_sensitive=False, + ) + + postgres_user: str = "postgres" + postgres_password: str = "postgres" + postgres_host: str = "backend-db" + postgres_db: str = "test" + postgres_port: int = Field(default=5432, validation_alias="POSTGRES_PORT") + # Accept legacy PGPORT as well via model validator below + pgport: int | None = Field(default=None, validation_alias="PGPORT") + + celery_broker_url: str = Field( + default="redis://backend-redis:6379/0", + validation_alias="CELERY_BROKER_URL", + ) + redis_url: str | None = Field(default=None, validation_alias="REDIS_URL") + + storage_dir: Path = Field( + default=Path(__file__).resolve().parent.parent / "storage" / "files", + validation_alias="STORAGE_DIR", + ) + max_upload_bytes: int = Field(default=50 * 1024 * 1024, validation_alias="MAX_UPLOAD_BYTES") + log_level: str = Field(default="INFO", validation_alias="LOG_LEVEL") + cors_origins: str = Field( + default="http://localhost:3000,http://127.0.0.1:3000", + validation_alias="CORS_ORIGINS", + ) + scan_size_threshold_bytes: int = 10 * 1024 * 1024 + metadata_text_read_limit: int = 2 * 1024 * 1024 + metadata_pdf_read_limit: int = 5 * 1024 * 1024 + worker_max_retries: int = 3 + worker_retry_backoff_seconds: int = 5 + + @property + def db_port(self) -> int: + if self.pgport is not None: + return self.pgport + return self.postgres_port + + @property + def database_url(self) -> str: + return ( + f"postgresql+asyncpg://{self.postgres_user}:{self.postgres_password}" + f"@{self.postgres_host}:{self.db_port}/{self.postgres_db}" + ) + + @property + def broker_url(self) -> str: + return self.redis_url or self.celery_broker_url + + @property + def cors_origin_list(self) -> list[str]: + return [item.strip() for item in self.cors_origins.split(",") if item.strip()] + + +@lru_cache +def get_settings() -> Settings: + return Settings() diff --git a/backend/src/storage/__init__.py b/backend/src/storage/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/src/storage/local.py b/backend/src/storage/local.py new file mode 100644 index 00000000..52277428 --- /dev/null +++ b/backend/src/storage/local.py @@ -0,0 +1,118 @@ +"""Local disk storage with stream write and path safety.""" + +from __future__ import annotations + +import logging +import re +from pathlib import Path +from uuid import uuid4 + +import anyio +from fastapi import UploadFile + +from src.errors import EmptyUploadError, PayloadTooLargeError, StorageIOError +from src.settings import Settings, get_settings + +logger = logging.getLogger(__name__) + +_SAFE_SUFFIX = re.compile(r"^[A-Za-z0-9._-]{0,32}$") +_CHUNK = 1024 * 64 + + +def ensure_storage_dir(*, settings: Settings | None = None) -> Path: + cfg = settings or get_settings() + path = Path(cfg.storage_dir) + path.mkdir(parents=True, exist_ok=True) + return path + + +def _safe_suffix(filename: str | None) -> str: + suffix = Path(filename or "").suffix + if not suffix or not _SAFE_SUFFIX.match(suffix): + return "" + return suffix[:32] + + +def sanitize_download_name(name: str) -> str: + cleaned = Path(name).name.replace("\\", "_").replace("/", "_") + cleaned = "".join(ch for ch in cleaned if ch.isprintable() and ch not in "\r\n") + return cleaned or "download.bin" + + +def build_stored_name(*, file_id: str, original_filename: str | None) -> str: + return f"{file_id}{_safe_suffix(original_filename)}" + + +def resolve_under_storage(*, stored_name: str, settings: Settings | None = None) -> Path: + cfg = settings or get_settings() + root = ensure_storage_dir(settings=cfg).resolve() + candidate = (root / Path(stored_name).name).resolve() + try: + candidate.relative_to(root) + except ValueError as exc: + raise StorageIOError("path escapes storage root") from exc + return candidate + + +async def save_upload_stream( + *, + upload_file: UploadFile, + file_id: str, + settings: Settings | None = None, +) -> tuple[str, int]: + """Stream upload to disk. Returns (stored_name, size).""" + cfg = settings or get_settings() + ensure_storage_dir(settings=cfg) + stored_name = build_stored_name(file_id=file_id, original_filename=upload_file.filename) + path = resolve_under_storage(stored_name=stored_name, settings=cfg) + size = 0 + try: + async with await anyio.open_file(path, "wb") as out: + while True: + chunk = await upload_file.read(_CHUNK) + if not chunk: + break + size += len(chunk) + if size > cfg.max_upload_bytes: + raise PayloadTooLargeError( + f"File exceeds max size of {cfg.max_upload_bytes} bytes" + ) + await out.write(chunk) + except PayloadTooLargeError: + await delete_stored(stored_name=stored_name, settings=cfg) + raise + except EmptyUploadError: + raise + except Exception as exc: + await delete_stored(stored_name=stored_name, settings=cfg) + logger.error( + "save_upload_stream failed file_id=%s reason=%s action=unlink_partial", + file_id, + exc, + exc_info=True, + ) + raise StorageIOError("failed to store upload") from exc + + if size == 0: + await delete_stored(stored_name=stored_name, settings=cfg) + raise EmptyUploadError("File is empty") + + return stored_name, size + + +async def delete_stored(*, stored_name: str, settings: Settings | None = None) -> None: + try: + path = resolve_under_storage(stored_name=stored_name, settings=settings) + if path.exists(): + await anyio.Path(path).unlink() + except Exception as exc: + logger.warning( + "delete_stored failed stored_name=%s reason=%s", + stored_name, + exc, + exc_info=True, + ) + + +def new_file_id() -> str: + return str(uuid4()) diff --git a/backend/src/tasks.py b/backend/src/tasks.py index 4583aded..c8602709 100644 --- a/backend/src/tasks.py +++ b/backend/src/tasks.py @@ -1,122 +1,9 @@ -import asyncio -import os -from pathlib import Path -from celery import Celery -from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker -from src.models import Alert, StoredFile -from src.service import STORAGE_DIR, DB_URL +"""Backward-compatible Celery entry for compose/docs.""" -REDIS_URL = os.environ.get("REDIS_URL", "redis://backend-redis:6379/0") -_worker_loop: asyncio.AbstractEventLoop | None = None +from src.worker.app import celery_app +from src.worker.jobs import process_uploaded_file +# Keep historical names importable but unused in new flow +scan_file_for_threats = process_uploaded_file -def run_in_worker_loop(coroutine): - global _worker_loop - if _worker_loop is None or _worker_loop.is_closed(): - _worker_loop = asyncio.new_event_loop() - asyncio.set_event_loop(_worker_loop) - return _worker_loop.run_until_complete(coroutine) - - -celery_app = Celery("file_tasks", broker=REDIS_URL, backend=REDIS_URL) -engine = create_async_engine(DB_URL) -async_session_maker = async_sessionmaker(engine, expire_on_commit=False) - - -async def _scan_file_for_threats(file_id: str) -> None: - async with async_session_maker() as session: - file_item = await session.get(StoredFile, file_id) - if not file_item: - return - - file_item.processing_status = "processing" - reasons: list[str] = [] - extension = Path(file_item.original_name).suffix.lower() - - if extension in {".exe", ".bat", ".cmd", ".sh", ".js"}: - reasons.append(f"suspicious extension {extension}") - - if file_item.size > 10 * 1024 * 1024: - reasons.append("file is larger than 10 MB") - - if extension == ".pdf" and file_item.mime_type not in {"application/pdf", "application/octet-stream"}: - reasons.append("pdf extension does not match mime type") - - file_item.scan_status = "suspicious" if reasons else "clean" - file_item.scan_details = ", ".join(reasons) if reasons else "no threats found" - file_item.requires_attention = bool(reasons) - await session.commit() - - extract_file_metadata.delay(file_id) - - -async def _extract_file_metadata(file_id: str) -> None: - async with async_session_maker() as session: - file_item = await session.get(StoredFile, file_id) - if not file_item: - return - - stored_path = STORAGE_DIR / file_item.stored_name - if not stored_path.exists(): - file_item.processing_status = "failed" - file_item.scan_status = file_item.scan_status or "failed" - file_item.scan_details = "stored file not found during metadata extraction" - await session.commit() - send_file_alert.delay(file_id) - return - - metadata = { - "extension": Path(file_item.original_name).suffix.lower(), - "size_bytes": file_item.size, - "mime_type": file_item.mime_type, - } - - if file_item.mime_type.startswith("text/"): - content = stored_path.read_text(encoding="utf-8", errors="ignore") - metadata["line_count"] = len(content.splitlines()) - metadata["char_count"] = len(content) - elif file_item.mime_type == "application/pdf": - content = stored_path.read_bytes() - metadata["approx_page_count"] = max(content.count(b"/Type /Page"), 1) - - file_item.metadata_json = metadata - file_item.processing_status = "processed" - await session.commit() - - send_file_alert.delay(file_id) - - -async def _send_file_alert(file_id: str) -> None: - async with async_session_maker() as session: - file_item = await session.get(StoredFile, file_id) - if not file_item: - return - - if file_item.processing_status == "failed": - alert = Alert(file_id=file_id, level="critical", message="File processing failed") - elif file_item.requires_attention: - alert = Alert( - file_id=file_id, - level="warning", - message=f"File requires attention: {file_item.scan_details}", - ) - else: - alert = Alert(file_id=file_id, level="info", message="File processed successfully") - - session.add(alert) - await session.commit() - - -@celery_app.task -def scan_file_for_threats(file_id: str) -> None: - run_in_worker_loop(_scan_file_for_threats(file_id)) - - -@celery_app.task -def extract_file_metadata(file_id: str) -> None: - run_in_worker_loop(_extract_file_metadata(file_id)) - - -@celery_app.task -def send_file_alert(file_id: str) -> None: - run_in_worker_loop(_send_file_alert(file_id)) +__all__ = ["celery_app", "process_uploaded_file", "scan_file_for_threats"] diff --git a/backend/src/use_cases/__init__.py b/backend/src/use_cases/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/src/use_cases/files.py b/backend/src/use_cases/files.py new file mode 100644 index 00000000..c0cbd915 --- /dev/null +++ b/backend/src/use_cases/files.py @@ -0,0 +1,144 @@ +"""File CRUD orchestration.""" + +from __future__ import annotations + +import logging +import mimetypes +from pathlib import Path + +from fastapi import UploadFile +from sqlalchemy.ext.asyncio import AsyncSession + +from src.data import alerts as alerts_data +from src.data import files as files_data +from src.dto.alerts import AlertItem +from src.dto.files import FileItem +from src.errors import ResourceNotFoundError, ValidationDomainError +from src.settings import get_settings +from src.storage.local import ( + delete_stored, + new_file_id, + resolve_under_storage, + sanitize_download_name, + save_upload_stream, +) + +logger = logging.getLogger(__name__) + + +def _normalize_title(title: str) -> str: + cleaned = title.strip() + if not cleaned: + raise ValidationDomainError("title must not be empty") + if len(cleaned) > 255: + raise ValidationDomainError("title must be at most 255 characters") + return cleaned + + +async def list_files(*, session: AsyncSession) -> list[FileItem]: + return await files_data.list_file_items(session=session) + + +async def list_alerts(*, session: AsyncSession) -> list[AlertItem]: + return await alerts_data.list_alert_items(session=session) + + +async def get_file(*, session: AsyncSession, file_id: str) -> FileItem: + item = await files_data.get_file_item(session=session, file_id=file_id) + if item is None: + raise ResourceNotFoundError("File not found") + return item + + +async def create_file( + *, + session: AsyncSession, + title: str, + upload_file: UploadFile, + enqueue, +) -> FileItem: + normalized_title = _normalize_title(title) + file_id = new_file_id() + settings = get_settings() + + stored_name, size = await save_upload_stream( + upload_file=upload_file, + file_id=file_id, + settings=settings, + ) + original_name = Path(upload_file.filename or stored_name).name[:255] + mime_type = ( + upload_file.content_type + or mimetypes.guess_type(original_name)[0] + or "application/octet-stream" + ) + + try: + item = await files_data.insert_file( + session=session, + file_id=file_id, + title=normalized_title, + original_name=original_name, + stored_name=stored_name, + mime_type=mime_type, + size=size, + ) + await session.commit() + except Exception: + await session.rollback() + await delete_stored(stored_name=stored_name, settings=settings) + logger.error( + "create_file db failed file_id=%s action=unlink_partial", + file_id, + exc_info=True, + ) + raise + + try: + enqueue(file_id=file_id) + except Exception as exc: + logger.error( + "enqueue failed file_id=%s reason=%s action=keep_uploaded_row", + file_id, + exc, + exc_info=True, + ) + + logger.info("upload accepted file_id=%s size=%s", file_id, size) + return item + + +async def update_file(*, session: AsyncSession, file_id: str, title: str) -> FileItem: + normalized_title = _normalize_title(title) + item = await files_data.update_file_title( + session=session, + file_id=file_id, + title=normalized_title, + ) + if item is None: + raise ResourceNotFoundError("File not found") + await session.commit() + return item + + +async def delete_file(*, session: AsyncSession, file_id: str) -> None: + ref = await files_data.get_storage_ref(session=session, file_id=file_id) + if ref is None: + raise ResourceNotFoundError("File not found") + + await delete_stored(stored_name=ref.stored_name) + deleted = await files_data.delete_file_row(session=session, file_id=file_id) + if not deleted: + raise ResourceNotFoundError("File not found") + await session.commit() + logger.info("file deleted file_id=%s", file_id) + + +async def resolve_download(*, session: AsyncSession, file_id: str) -> tuple[Path, str, str]: + ref = await files_data.get_storage_ref(session=session, file_id=file_id) + if ref is None: + raise ResourceNotFoundError("File not found") + path = resolve_under_storage(stored_name=ref.stored_name) + if not path.exists(): + raise ResourceNotFoundError("Stored file not found") + return path, sanitize_download_name(ref.original_name), ref.mime_type diff --git a/backend/src/use_cases/pipeline.py b/backend/src/use_cases/pipeline.py new file mode 100644 index 00000000..04349ae0 --- /dev/null +++ b/backend/src/use_cases/pipeline.py @@ -0,0 +1,165 @@ +"""Single-flow scan + metadata + alert pipeline.""" + +from __future__ import annotations + +import logging +from pathlib import Path + +from sqlalchemy.ext.asyncio import AsyncSession + +from src.data import alerts as alerts_data +from src.data import files as files_data +from src.scan.rules import evaluate_scan +from src.settings import Settings, get_settings +from src.storage.local import resolve_under_storage + +logger = logging.getLogger(__name__) + + +def _extract_metadata( + *, + path: Path, + original_name: str, + mime_type: str, + size: int, + settings: Settings, +) -> dict: + metadata: dict = { + "extension": Path(original_name).suffix.lower(), + "size_bytes": size, + "mime_type": mime_type, + } + if mime_type.startswith("text/"): + # Cap read to avoid DoS on huge text files + raw = path.read_bytes()[: settings.metadata_text_read_limit] + content = raw.decode("utf-8", errors="ignore") + metadata["line_count"] = len(content.splitlines()) + metadata["char_count"] = len(content) + elif mime_type == "application/pdf": + raw = path.read_bytes()[: settings.metadata_pdf_read_limit] + metadata["approx_page_count"] = max(raw.count(b"/Type /Page"), 1) + return metadata + + +async def run_file_pipeline(*, session: AsyncSession, file_id: str) -> None: + settings = get_settings() + row = await files_data.get_pipeline_row(session=session, file_id=file_id) + if row is None: + logger.warning("pipeline skip missing file_id=%s", file_id) + return + + if row.processing_status == "processed" and await alerts_data.alert_exists_for_file( + session=session, file_id=file_id + ): + logger.info("pipeline noop already processed file_id=%s", file_id) + return + + logger.info("pipeline started file_id=%s", file_id) + await files_data.update_pipeline_fields( + session=session, + file_id=file_id, + processing_status="processing", + ) + await session.commit() + + scan = evaluate_scan( + original_name=row.original_name, + mime_type=row.mime_type, + size=row.size, + size_threshold_bytes=settings.scan_size_threshold_bytes, + ) + await files_data.update_pipeline_fields( + session=session, + file_id=file_id, + scan_status=scan.scan_status, + scan_details=scan.scan_details, + requires_attention=scan.requires_attention, + ) + await session.commit() + + path = resolve_under_storage(stored_name=row.stored_name, settings=settings) + if not path.exists(): + await files_data.update_pipeline_fields( + session=session, + file_id=file_id, + processing_status="failed", + scan_status=scan.scan_status or "failed", + scan_details="stored file not found during metadata extraction", + ) + if not await alerts_data.alert_exists_for_file(session=session, file_id=file_id): + await alerts_data.insert_alert( + session=session, + file_id=file_id, + level="critical", + message="File processing failed", + ) + await session.commit() + logger.error("pipeline failed missing disk file file_id=%s", file_id) + return + + try: + metadata = _extract_metadata( + path=path, + original_name=row.original_name, + mime_type=row.mime_type, + size=row.size, + settings=settings, + ) + await files_data.update_pipeline_fields( + session=session, + file_id=file_id, + metadata_json=metadata, + processing_status="processed", + ) + await session.commit() + except Exception as exc: + await session.rollback() + await files_data.update_pipeline_fields( + session=session, + file_id=file_id, + processing_status="failed", + scan_status=scan.scan_status or "failed", + scan_details=f"metadata extraction failed: {type(exc).__name__}", + ) + if not await alerts_data.alert_exists_for_file(session=session, file_id=file_id): + await alerts_data.insert_alert( + session=session, + file_id=file_id, + level="critical", + message="File processing failed", + ) + await session.commit() + logger.error( + "pipeline metadata failed file_id=%s reason=%s", + file_id, + exc, + exc_info=True, + ) + return + + # reload attention flags after scan commit + row = await files_data.get_pipeline_row(session=session, file_id=file_id) + assert row is not None + + if not await alerts_data.alert_exists_for_file(session=session, file_id=file_id): + if row.processing_status == "failed": + level, message = "critical", "File processing failed" + elif row.requires_attention: + level = "warning" + message = f"File requires attention: {row.scan_details}" + else: + level, message = "info", "File processed successfully" + await alerts_data.insert_alert( + session=session, + file_id=file_id, + level=level, + message=message, + ) + await session.commit() + + logger.info( + "pipeline finished file_id=%s status=%s scan=%s", + file_id, + row.processing_status, + row.scan_status, + ) diff --git a/backend/src/worker/__init__.py b/backend/src/worker/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/src/worker/app.py b/backend/src/worker/app.py new file mode 100644 index 00000000..057e9de4 --- /dev/null +++ b/backend/src/worker/app.py @@ -0,0 +1,22 @@ +"""Celery application instance.""" + +from celery import Celery + +from src.settings import get_settings + +settings = get_settings() + +celery_app = Celery( + "file_exchange", + broker=settings.broker_url, + backend=settings.broker_url, + include=["src.worker.jobs"], +) +celery_app.conf.update( + task_acks_late=True, + worker_prefetch_multiplier=1, + task_serializer="json", + accept_content=["json"], + result_serializer="json", + broker_connection_retry_on_startup=True, +) diff --git a/backend/src/worker/jobs.py b/backend/src/worker/jobs.py new file mode 100644 index 00000000..29f1cdaf --- /dev/null +++ b/backend/src/worker/jobs.py @@ -0,0 +1,92 @@ +"""Background jobs: single pipeline task with retries.""" + +from __future__ import annotations + +import asyncio +import logging + +from celery.exceptions import MaxRetriesExceededError + +from src.db.session import worker_session_scope +from src.settings import get_settings +from src.use_cases.pipeline import run_file_pipeline +from src.worker.app import celery_app + +logger = logging.getLogger(__name__) + + +def _run_async(coro): + return asyncio.run(coro) + + +@celery_app.task( + bind=True, + name="jobs.process_uploaded_file", + max_retries=3, + default_retry_delay=5, +) +def process_uploaded_file(self, file_id: str) -> None: + settings = get_settings() + + async def _body() -> None: + async with worker_session_scope(settings=settings) as session: + await run_file_pipeline(session=session, file_id=file_id) + + try: + _run_async(_body()) + except Exception as exc: + logger.error( + "worker task failed file_id=%s attempt=%s reason=%s", + file_id, + self.request.retries, + exc, + exc_info=True, + ) + try: + raise self.retry( + exc=exc, + countdown=settings.worker_retry_backoff_seconds * (2 ** self.request.retries), + max_retries=settings.worker_max_retries, + ) + except MaxRetriesExceededError: + logger.error( + "worker retries exhausted file_id=%s action=mark_failed", + file_id, + exc_info=True, + ) + + async def _fail() -> None: + from src.data import alerts as alerts_data + from src.data import files as files_data + + async with worker_session_scope(settings=settings) as session: + await files_data.update_pipeline_fields( + session=session, + file_id=file_id, + processing_status="failed", + scan_status="failed", + scan_details="processing failed after retries", + ) + if not await alerts_data.alert_exists_for_file( + session=session, file_id=file_id + ): + await alerts_data.insert_alert( + session=session, + file_id=file_id, + level="critical", + message="File processing failed", + ) + await session.commit() + + try: + _run_async(_fail()) + except Exception: + logger.error( + "failed to mark file failed file_id=%s", + file_id, + exc_info=True, + ) + + +def enqueue_file_pipeline(*, file_id: str) -> None: + process_uploaded_file.delay(file_id) diff --git a/backend/storage/.gitkeep b/backend/storage/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/backend/tests/test_columns.py b/backend/tests/test_columns.py new file mode 100644 index 00000000..802f7b07 --- /dev/null +++ b/backend/tests/test_columns.py @@ -0,0 +1,11 @@ +from src.data.columns import FILE_ITEM_COLUMNS +from src.dto.files import FileItem +from src.entities.stored_file import StoredFile + + +def test_file_item_projection_excludes_stored_name(): + names = {col.key for col in FILE_ITEM_COLUMNS} + assert "stored_name" not in names + schema_fields = set(FileItem.model_fields.keys()) + assert names == schema_fields + assert "stored_name" in StoredFile.__table__.c diff --git a/backend/tests/test_scan_rules.py b/backend/tests/test_scan_rules.py new file mode 100644 index 00000000..55c388b9 --- /dev/null +++ b/backend/tests/test_scan_rules.py @@ -0,0 +1,41 @@ +from src.scan.rules import evaluate_scan + + +def test_clean_text_file(): + result = evaluate_scan( + original_name="note.txt", + mime_type="text/plain", + size=100, + ) + assert result.scan_status == "clean" + assert result.requires_attention is False + + +def test_suspicious_extension(): + result = evaluate_scan( + original_name="payload.js", + mime_type="application/javascript", + size=10, + ) + assert result.scan_status == "suspicious" + assert "suspicious extension .js" in result.scan_details + + +def test_large_file(): + result = evaluate_scan( + original_name="big.bin", + mime_type="application/octet-stream", + size=11 * 1024 * 1024, + ) + assert result.scan_status == "suspicious" + assert "10 MB" in result.scan_details + + +def test_pdf_mime_mismatch(): + result = evaluate_scan( + original_name="doc.pdf", + mime_type="text/plain", + size=10, + ) + assert result.scan_status == "suspicious" + assert "pdf extension" in result.scan_details diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index c0df035b..cf790c04 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -5,34 +5,113 @@ services: container_name: backend volumes: - ./backend:/backend + - file-storage:/backend/storage/files env_file: ".env.dev" + environment: + STORAGE_DIR: /backend/storage/files + POSTGRES_PORT: "5432" + PGPORT: "5432" ports: - "8000:8000" depends_on: - - backend-db + backend-db: + condition: service_healthy + backend-redis: + condition: service_healthy + restart: unless-stopped + deploy: + resources: + limits: + memory: ${BACKEND_MEM_LIMIT:-768M} + cpus: "${BACKEND_CPU_LIMIT:-1.0}" + reservations: + memory: 256M + cpus: "0.25" + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" backend-worker: build: ./backend - command: [ 'celery', '-A', 'src.tasks.celery_app', 'worker', '-l', 'info' ] + command: ["celery", "-A", "src.worker.app.celery_app", "worker", "-l", "info", "-c", "1"] container_name: backend-worker env_file: ".env.dev" + environment: + STORAGE_DIR: /backend/storage/files + POSTGRES_PORT: "5432" + PGPORT: "5432" volumes: - ./backend:/backend + - file-storage:/backend/storage/files depends_on: - - backend-db + backend-db: + condition: service_healthy + backend-redis: + condition: service_healthy + restart: unless-stopped + deploy: + resources: + limits: + memory: ${WORKER_MEM_LIMIT:-768M} + cpus: "${WORKER_CPU_LIMIT:-1.0}" + reservations: + memory: 256M + cpus: "0.25" + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" backend-db: - image: postgres:latest + image: postgres:16-alpine container_name: backend-db ports: - - "5433:5433" + - "127.0.0.1:5433:5432" env_file: ".env.dev" + environment: + POSTGRES_USER: ${POSTGRES_USER:-postgres} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} + POSTGRES_DB: ${POSTGRES_DB:-test} + PGDATA: /var/lib/postgresql/data/pgdata volumes: - - backend-db-volume:/var/lib/postgresql + - backend-db-volume:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] + interval: 5s + timeout: 5s + retries: 10 + start_period: 10s + restart: unless-stopped + deploy: + resources: + limits: + memory: ${POSTGRES_MEM_LIMIT:-512M} + cpus: "${POSTGRES_CPU_LIMIT:-1.0}" + reservations: + memory: 128M + cpus: "0.25" backend-redis: - image: redis:latest + image: redis:7-alpine container_name: backend-redis + command: ["redis-server", "--save", "", "--loglevel", "warning"] + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 10 + restart: unless-stopped + deploy: + resources: + limits: + memory: ${REDIS_MEM_LIMIT:-128M} + cpus: "${REDIS_CPU_LIMIT:-0.5}" + reservations: + memory: 32M + cpus: "0.1" frontend: build: @@ -41,6 +120,23 @@ services: container_name: frontend ports: - "3000:3000" + depends_on: + - backend + restart: unless-stopped + deploy: + resources: + limits: + memory: ${FRONTEND_MEM_LIMIT:-512M} + cpus: "${FRONTEND_CPU_LIMIT:-0.75}" + reservations: + memory: 128M + cpus: "0.15" + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" volumes: backend-db-volume: + file-storage: diff --git a/frontend/.env.production b/frontend/.env.production new file mode 100644 index 00000000..ee17ccf5 --- /dev/null +++ b/frontend/.env.production @@ -0,0 +1 @@ +NEXT_PUBLIC_API_BASE_URL=http://localhost:8000 diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 73896276..1191b7e5 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -2,66 +2,30 @@ FROM node:20-alpine AS base -# Install dependencies only when needed FROM base AS deps -# Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed. RUN apk add --no-cache libc6-compat WORKDIR /app - -# Install dependencies based on the preferred package manager COPY package.json yarn.lock* package-lock.json* pnpm-lock.yaml* .npmrc* ./ -RUN \ - if [ -f yarn.lock ]; then yarn --frozen-lockfile; \ - elif [ -f package-lock.json ]; then npm ci; \ - elif [ -f pnpm-lock.yaml ]; then corepack enable pnpm && pnpm i --frozen-lockfile; \ - else echo "Lockfile not found." && exit 1; \ - fi - +RUN if [ -f yarn.lock ]; then yarn --frozen-lockfile; elif [ -f package-lock.json ]; then npm ci; elif [ -f pnpm-lock.yaml ]; then corepack enable pnpm && pnpm i --frozen-lockfile; else echo "Lockfile not found." && exit 1; fi -# Rebuild the source code only when needed FROM base AS builder WORKDIR /app COPY --from=deps /app/node_modules ./node_modules COPY . . +ENV NEXT_TELEMETRY_DISABLED=1 +RUN if [ -f yarn.lock ]; then yarn run build; elif [ -f package-lock.json ]; then npm run build; elif [ -f pnpm-lock.yaml ]; then corepack enable pnpm && pnpm run build; else echo "Lockfile not found." && exit 1; fi -# Next.js collects completely anonymous telemetry data about general usage. -# Learn more here: https://nextjs.org/telemetry -# Uncomment the following line in case you want to disable telemetry during the build. -# ENV NEXT_TELEMETRY_DISABLED=1 - -RUN \ - if [ -f yarn.lock ]; then yarn run build; \ - elif [ -f package-lock.json ]; then npm run build; \ - elif [ -f pnpm-lock.yaml ]; then corepack enable pnpm && pnpm run build; \ - else echo "Lockfile not found." && exit 1; \ - fi - -# Production image, copy all the files and run next FROM base AS runner WORKDIR /app - ENV NODE_ENV=production -# Uncomment the following line in case you want to disable telemetry during runtime. -# ENV NEXT_TELEMETRY_DISABLED=1 - +ENV NEXT_TELEMETRY_DISABLED=1 RUN addgroup --system --gid 1001 nodejs RUN adduser --system --uid 1001 nextjs - COPY --from=builder /app/public ./public -COPY --from=builder /app/.env.production ./.env.production - -# Automatically leverage output traces to reduce image size -# https://nextjs.org/docs/advanced-features/output-file-tracing COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./ COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static - USER nextjs - EXPOSE 3000 - ENV PORT=3000 - -# server.js is created by next build from the standalone output -# https://nextjs.org/docs/pages/api-reference/config/next-config-js/output ENV HOSTNAME="0.0.0.0" CMD ["node", "server.js"] diff --git a/frontend/src/api/alerts.ts b/frontend/src/api/alerts.ts new file mode 100644 index 00000000..1954eb03 --- /dev/null +++ b/frontend/src/api/alerts.ts @@ -0,0 +1,6 @@ +import { AlertItem } from "../types/api"; +import { httpGet } from "./http"; + +export function fetchAlerts(): Promise { + return httpGet("/alerts"); +} diff --git a/frontend/src/api/files.ts b/frontend/src/api/files.ts new file mode 100644 index 00000000..b30a8181 --- /dev/null +++ b/frontend/src/api/files.ts @@ -0,0 +1,13 @@ +import { FileItem } from "../types/api"; +import { httpGet, httpPostForm } from "./http"; + +export function fetchFiles(): Promise { + return httpGet("/files"); +} + +export function uploadFile(title: string, file: File): Promise { + const formData = new FormData(); + formData.append("title", title); + formData.append("file", file); + return httpPostForm("/files", formData); +} diff --git a/frontend/src/api/http.ts b/frontend/src/api/http.ts new file mode 100644 index 00000000..da38931e --- /dev/null +++ b/frontend/src/api/http.ts @@ -0,0 +1,48 @@ +import { API_BASE_URL } from "../config/env"; + +export class ApiError extends Error { + status: number; + + constructor(message: string, status: number) { + super(message); + this.status = status; + } +} + +async function parseDetail(response: Response): Promise { + try { + const data = await response.json(); + if (typeof data?.detail === "string") { + return data.detail; + } + if (Array.isArray(data?.detail)) { + return "Ошибка валидации"; + } + } catch { + // ignore + } + return `Ошибка запроса (${response.status})`; +} + +export async function httpGet(path: string): Promise { + const response = await fetch(`${API_BASE_URL}${path}`, { cache: "no-store" }); + if (!response.ok) { + throw new ApiError(await parseDetail(response), response.status); + } + return (await response.json()) as T; +} + +export async function httpPostForm(path: string, body: FormData): Promise { + const response = await fetch(`${API_BASE_URL}${path}`, { + method: "POST", + body, + }); + if (!response.ok) { + throw new ApiError(await parseDetail(response), response.status); + } + return (await response.json()) as T; +} + +export function downloadUrl(fileId: string): string { + return `${API_BASE_URL}/files/${fileId}/download`; +} diff --git a/frontend/src/app/page.tsx b/frontend/src/app/page.tsx index 8f420e2e..49f1df34 100644 --- a/frontend/src/app/page.tsx +++ b/frontend/src/app/page.tsx @@ -1,193 +1,30 @@ "use client"; -import { FormEvent, useEffect, useState } from "react"; -import { - Alert, - Badge, - Button, - Card, - Col, - Container, - Form, - Modal, - Row, - Spinner, - Table, -} from "react-bootstrap"; - -type FileItem = { - id: string; - title: string; - original_name: string; - mime_type: string; - size: number; - processing_status: string; - scan_status: string | null; - scan_details: string | null; - metadata_json: Record | null; - requires_attention: boolean; - created_at: string; - updated_at: string; -}; - -type AlertItem = { - id: number; - file_id: string; - level: string; - message: string; - created_at: string; -}; - - -function formatDate(value: string) { - return new Intl.DateTimeFormat("ru-RU", { - dateStyle: "short", - timeStyle: "short", - }).format(new Date(value)); -} - -function formatSize(size: number) { - if (size < 1024) { - return `${size} B`; - } - - if (size < 1024 * 1024) { - return `${(size / 1024).toFixed(1)} KB`; - } - - return `${(size / (1024 * 1024)).toFixed(1)} MB`; -} - -function getLevelVariant(level: string) { - if (level === "critical") { - return "danger"; - } - - if (level === "warning") { - return "warning"; - } - - return "success"; -} - -function getProcessingVariant(status: string) { - if (status === "failed") { - return "danger"; - } - - if (status === "processing") { - return "warning"; - } - - if (status === "processed") { - return "success"; - } - - return "secondary"; -} +import { Alert, Col, Container, Row } from "react-bootstrap"; +import { AlertsTable } from "../components/AlertsTable"; +import { DashboardHeader } from "../components/DashboardHeader"; +import { FilesTable } from "../components/FilesTable"; +import { UploadDialog } from "../components/UploadDialog"; +import { useDashboardData } from "../hooks/useDashboardData"; +import { useFileUpload } from "../hooks/useFileUpload"; export default function Page() { - const [files, setFiles] = useState([]); - const [alerts, setAlerts] = useState([]); - const [isLoading, setIsLoading] = useState(true); - const [isSubmitting, setIsSubmitting] = useState(false); - const [showModal, setShowModal] = useState(false); - const [title, setTitle] = useState(""); - const [selectedFile, setSelectedFile] = useState(null); - const [errorMessage, setErrorMessage] = useState(null); - - async function loadData() { - setIsLoading(true); - setErrorMessage(null); - - try { - const [filesResponse, alertsResponse] = await Promise.all([ - fetch(`http://localhost:8000/files`, { cache: "no-store" }), - fetch(`http://localhost:8000/alerts`, { cache: "no-store" }), - ]); - - if (!filesResponse.ok || !alertsResponse.ok) { - throw new Error("Не удалось загрузить данные"); - } - - const [filesData, alertsData] = await Promise.all([ - filesResponse.json() as Promise, - alertsResponse.json() as Promise, - ]); - - setFiles(filesData); - setAlerts(alertsData); - } catch (error) { - setErrorMessage(error instanceof Error ? error.message : "Произошла ошибка"); - } finally { - setIsLoading(false); - } - } - - useEffect(() => { - void loadData(); - }, []); + const { files, alerts, isLoading, errorMessage, setErrorMessage, loadData } = + useDashboardData(); - async function handleSubmit(event: FormEvent) { - event.preventDefault(); - - if (!title.trim() || !selectedFile) { - setErrorMessage("Укажите название и выберите файл"); - return; - } - - setIsSubmitting(true); - setErrorMessage(null); - - const formData = new FormData(); - formData.append("title", title.trim()); - formData.append("file", selectedFile); - - try { - const response = await fetch(`http://localhost:8000/files`, { - method: "POST", - body: formData, - }); - - if (!response.ok) { - throw new Error("Не удалось загрузить файл"); - } - - setShowModal(false); - setTitle(""); - setSelectedFile(null); - await loadData(); - } catch (error) { - setErrorMessage(error instanceof Error ? error.message : "Произошла ошибка"); - } finally { - setIsSubmitting(false); - } - } + const upload = useFileUpload({ + onSuccess: loadData, + onError: setErrorMessage, + }); return ( - - -
-
-

Управление файлами

-

- Загрузка файлов, просмотр статусов обработки и ленты алертов. -

-
-
- - -
-
- - + void loadData()} + onAdd={() => upload.setShowModal(true)} + /> {errorMessage ? ( @@ -195,173 +32,20 @@ export default function Page() { ) : null} - - -
-

Файлы

- {files.length} -
- - - {isLoading ? ( -
- -
- ) : ( -
- - - - - - - - - - - - - - - {files.length === 0 ? ( - - - - ) : ( - files.map((file) => ( - - - - - - - - - - - )) - )} - -
НазваниеФайлMIMEРазмерСтатусПроверкаСоздан
- Файлы пока не загружены -
-
{file.title}
-
{file.id}
- 		{file.original_name}{file.mime_type}{formatSize(file.size)} - - {file.processing_status} - - -
- - {file.scan_status ?? "pending"} - - - {file.scan_details ?? "Ожидает обработки"} - -
- 		{formatDate(file.created_at)} - -
-
- )} - - - - - -
-

Алерты

- {alerts.length} -
- - - {isLoading ? ( -
- -
- ) : ( -
- - - - - - - - - - - - {alerts.length === 0 ? ( - - - - ) : ( - alerts.map((item) => ( - - - - - - - - )) - )} - -
IDFile IDУровеньСообщениеСоздан
- Алертов пока нет -
{item.id}{item.file_id} - {item.level} - {item.message}{formatDate(item.created_at)}
-
- )} - - + + - setShowModal(false)} centered> -
- - Добавить файл - - - - Название - setTitle(event.target.value)} - placeholder="Например, Договор с подрядчиком" - /> - - - Файл - - setSelectedFile((event.target as HTMLInputElement).files?.[0] ?? null) - } - /> - - - - - - -
- + upload.setShowModal(false)} + onTitleChange={upload.setTitle} + onFileChange={upload.setSelectedFile} + onSubmit={upload.handleSubmit} + /> ); } diff --git a/frontend/src/components/AlertsTable.tsx b/frontend/src/components/AlertsTable.tsx new file mode 100644 index 00000000..2451ce5a --- /dev/null +++ b/frontend/src/components/AlertsTable.tsx @@ -0,0 +1,66 @@ +"use client"; + +import { Badge, Card, Spinner, Table } from "react-bootstrap"; +import { formatDate } from "../lib/format"; +import { getLevelVariant } from "../lib/statusStyles"; +import { AlertItem } from "../types/api"; + +type Props = { + alerts: AlertItem[]; + isLoading: boolean; +}; + +export function AlertsTable({ alerts, isLoading }: Props) { + return ( + + +
+

Алерты

+ {alerts.length} +
+ + + {isLoading ? ( +
+ +
+ ) : ( +
+ + + + + + + + + + + + {alerts.length === 0 ? ( + + + + ) : ( + alerts.map((item) => ( + + + + + + + + )) + )} + +
IDFile IDУровеньСообщениеСоздан
+ Алертов пока нет +
{item.id}{item.file_id} + {item.level} + {item.message}{formatDate(item.created_at)}
+
+ )} + + + ); +} diff --git a/frontend/src/components/DashboardHeader.tsx b/frontend/src/components/DashboardHeader.tsx new file mode 100644 index 00000000..d013baa0 --- /dev/null +++ b/frontend/src/components/DashboardHeader.tsx @@ -0,0 +1,33 @@ +"use client"; + +import { Button, Card } from "react-bootstrap"; + +type Props = { + onRefresh: () => void; + onAdd: () => void; +}; + +export function DashboardHeader({ onRefresh, onAdd }: Props) { + return ( + + +
+
+

Управление файлами

+

+ Загрузка файлов, просмотр статусов обработки и ленты алертов. +

+
+
+ + +
+
+ + + ); +} diff --git a/frontend/src/components/FilesTable.tsx b/frontend/src/components/FilesTable.tsx new file mode 100644 index 00000000..5ded336f --- /dev/null +++ b/frontend/src/components/FilesTable.tsx @@ -0,0 +1,96 @@ +"use client"; + +import { Badge, Button, Card, Spinner, Table } from "react-bootstrap"; +import { downloadUrl } from "../api/http"; +import { formatDate, formatSize } from "../lib/format"; +import { getProcessingVariant } from "../lib/statusStyles"; +import { FileItem } from "../types/api"; + +type Props = { + files: FileItem[]; + isLoading: boolean; +}; + +export function FilesTable({ files, isLoading }: Props) { + return ( + + +
+

Файлы

+ {files.length} +
+ + + {isLoading ? ( +
+ +
+ ) : ( +
+ + + + + + + + + + + + + + + {files.length === 0 ? ( + + + + ) : ( + files.map((file) => ( + + + + + + + + + + + )) + )} + +
НазваниеФайлMIMEРазмерСтатусПроверкаСоздан
+ Файлы пока не загружены +
+
{file.title}
+
{file.id}
+ 		{file.original_name}{file.mime_type}{formatSize(file.size)} + + {file.processing_status} + + +
+ + {file.scan_status ?? "pending"} + + + {file.scan_details ?? "Ожидает обработки"} + +
+ 		{formatDate(file.created_at)} + +
+
+ )} + + + ); +} diff --git a/frontend/src/components/UploadDialog.tsx b/frontend/src/components/UploadDialog.tsx new file mode 100644 index 00000000..7f598e5f --- /dev/null +++ b/frontend/src/components/UploadDialog.tsx @@ -0,0 +1,61 @@ +"use client"; + +import { FormEvent } from "react"; +import { Button, Form, Modal } from "react-bootstrap"; + +type Props = { + show: boolean; + title: string; + isSubmitting: boolean; + onHide: () => void; + onTitleChange: (value: string) => void; + onFileChange: (file: File | null) => void; + onSubmit: (event: FormEvent) => void; +}; + +export function UploadDialog({ + show, + title, + isSubmitting, + onHide, + onTitleChange, + onFileChange, + onSubmit, +}: Props) { + return ( + +
+ + Добавить файл + + + + Название + onTitleChange(event.target.value)} + placeholder="Например, Договор с подрядчиком" + /> + + + Файл + + onFileChange((event.target as HTMLInputElement).files?.[0] ?? null) + } + /> + + + + + + +
+ + ); +} diff --git a/frontend/src/config/env.ts b/frontend/src/config/env.ts new file mode 100644 index 00000000..8b1ea430 --- /dev/null +++ b/frontend/src/config/env.ts @@ -0,0 +1,2 @@ +export const API_BASE_URL = + process.env.NEXT_PUBLIC_API_BASE_URL?.replace(/\/$/, "") || "http://localhost:8000"; diff --git a/frontend/src/hooks/useDashboardData.ts b/frontend/src/hooks/useDashboardData.ts new file mode 100644 index 00000000..e6d99766 --- /dev/null +++ b/frontend/src/hooks/useDashboardData.ts @@ -0,0 +1,40 @@ +"use client"; + +import { useCallback, useEffect, useState } from "react"; +import { fetchAlerts } from "../api/alerts"; +import { fetchFiles } from "../api/files"; +import { AlertItem, FileItem } from "../types/api"; + +export function useDashboardData() { + const [files, setFiles] = useState([]); + const [alerts, setAlerts] = useState([]); + const [isLoading, setIsLoading] = useState(true); + const [errorMessage, setErrorMessage] = useState(null); + + const loadData = useCallback(async () => { + setIsLoading(true); + setErrorMessage(null); + try { + const [filesData, alertsData] = await Promise.all([fetchFiles(), fetchAlerts()]); + setFiles(filesData); + setAlerts(alertsData); + } catch (error) { + setErrorMessage(error instanceof Error ? error.message : "Произошла ошибка"); + } finally { + setIsLoading(false); + } + }, []); + + useEffect(() => { + void loadData(); + }, [loadData]); + + return { + files, + alerts, + isLoading, + errorMessage, + setErrorMessage, + loadData, + }; +} diff --git a/frontend/src/hooks/useFileUpload.ts b/frontend/src/hooks/useFileUpload.ts new file mode 100644 index 00000000..3247fa1e --- /dev/null +++ b/frontend/src/hooks/useFileUpload.ts @@ -0,0 +1,51 @@ +"use client"; + +import { FormEvent, useState } from "react"; +import { uploadFile } from "../api/files"; + +type Options = { + onSuccess: () => Promise | void; + onError: (message: string) => void; +}; + +export function useFileUpload({ onSuccess, onError }: Options) { + const [showModal, setShowModal] = useState(false); + const [title, setTitle] = useState(""); + const [selectedFile, setSelectedFile] = useState(null); + const [isSubmitting, setIsSubmitting] = useState(false); + + function resetForm() { + setTitle(""); + setSelectedFile(null); + } + + async function handleSubmit(event: FormEvent) { + event.preventDefault(); + if (!title.trim() || !selectedFile) { + onError("Укажите название и выберите файл"); + return; + } + setIsSubmitting(true); + try { + await uploadFile(title.trim(), selectedFile); + setShowModal(false); + resetForm(); + await onSuccess(); + } catch (error) { + onError(error instanceof Error ? error.message : "Произошла ошибка"); + } finally { + setIsSubmitting(false); + } + } + + return { + showModal, + setShowModal, + title, + setTitle, + selectedFile, + setSelectedFile, + isSubmitting, + handleSubmit, + }; +} diff --git a/frontend/src/lib/format.ts b/frontend/src/lib/format.ts new file mode 100644 index 00000000..f1739263 --- /dev/null +++ b/frontend/src/lib/format.ts @@ -0,0 +1,16 @@ +export function formatDate(value: string): string { + return new Intl.DateTimeFormat("ru-RU", { + dateStyle: "short", + timeStyle: "short", + }).format(new Date(value)); +} + +export function formatSize(size: number): string { + if (size < 1024) { + return `${size} B`; + } + if (size < 1024 * 1024) { + return `${(size / 1024).toFixed(1)} KB`; + } + return `${(size / (1024 * 1024)).toFixed(1)} MB`; +} diff --git a/frontend/src/lib/statusStyles.ts b/frontend/src/lib/statusStyles.ts new file mode 100644 index 00000000..06369239 --- /dev/null +++ b/frontend/src/lib/statusStyles.ts @@ -0,0 +1,22 @@ +export function getLevelVariant(level: string): string { + if (level === "critical") { + return "danger"; + } + if (level === "warning") { + return "warning"; + } + return "success"; +} + +export function getProcessingVariant(status: string): string { + if (status === "failed") { + return "danger"; + } + if (status === "processing") { + return "warning"; + } + if (status === "processed") { + return "success"; + } + return "secondary"; +} diff --git a/frontend/src/types/api.ts b/frontend/src/types/api.ts new file mode 100644 index 00000000..bc9e4cd3 --- /dev/null +++ b/frontend/src/types/api.ts @@ -0,0 +1,22 @@ +export type FileItem = { + id: string; + title: string; + original_name: string; + mime_type: string; + size: number; + processing_status: string; + scan_status: string | null; + scan_details: string | null; + metadata_json: Record | null; + requires_attention: boolean; + created_at: string; + updated_at: string; +}; + +export type AlertItem = { + id: number; + file_id: string; + level: string; + message: string; + created_at: string; +}; From 0983e3fd0e447057a3a117778ae48dde743b09a5 Mon Sep 17 00:00:00 2001 From: Nikolay Date: Sat, 11 Jul 2026 16:24:13 +0300 Subject: [PATCH 2/4] feat(docker): auto wait DB/Redis and run alembic on startup --- ARCHITECTURE.md | 8 +++++++- README.md | 13 +++++++++++-- backend/Dockerfile | 17 ++++++++++++++--- backend/entrypoint.sh | 34 ++++++++++++++++++++++++++++++++++ docker-compose.dev.yml | 21 +++++++++++++++------ 5 files changed, 81 insertions(+), 12 deletions(-) create mode 100644 backend/entrypoint.sh diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 86a631cf..37f99f54 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -74,9 +74,15 @@ app/page composition only ```bash docker compose -f docker-compose.dev.yml up --build -docker exec -it backend alembic upgrade head ``` +Startup (backend entrypoint, same idea as mail-manager / coffeestudio): +1. Wait for Postgres + Redis (`nc`) +2. `alembic upgrade head` (API container only) +3. Start uvicorn / celery + +Containers use `restart: always` so Docker Desktop restarts the stack after reboot if it was running. + UI: http://localhost:3000/test API docs: http://localhost:8000/docs Health: http://localhost:8000/health diff --git a/README.md b/README.md index ac641f36..0821247e 100644 --- a/README.md +++ b/README.md @@ -20,9 +20,17 @@ - Docker: healthchecks, ports, resource limits, shared storage volume - Подробности: [ARCHITECTURE.md](./ARCHITECTURE.md) -**Запуск:** +**Запуск (одной командой):** 1. `docker compose -f docker-compose.dev.yml up --build` -2. `docker exec -it backend alembic upgrade head` + +При старте backend entrypoint: +- ждёт PostgreSQL и Redis; +- накатывает `alembic upgrade head` (только API-контейнер, без гонки с worker); +- поднимает uvicorn / celery. + +`restart: always` — контейнеры поднимаются снова при рестарте Docker Desktop (если стек уже был запущен). + +Ручной `docker exec … alembic upgrade head` **не нужен** (оставлен только для отладки). **Открыть фронт:** `http://localhost:3000/test` @@ -42,3 +50,4 @@ uv run pytest - **В образ** копируется исходный код (`COPY`) — основной способ запуска. - В dev compose `./backend` **примонтирован** для hot-reload; это не «вынос логики наружу», а DX. - Бинарные загрузки — в volume `file-storage`, общий для API и worker. +- Старт: wait DB/Redis → migrate → app (как в mail-manager / coffeestudio entrypoint). diff --git a/backend/Dockerfile b/backend/Dockerfile index 946f42b9..e65fbdd4 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,9 +1,17 @@ FROM python:3.14-slim -ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 UV_NO_DEV=1 UV_PROJECT_ENVIRONMENT=/usr/local PYTHONPATH=/backend +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + UV_NO_DEV=1 \ + UV_PROJECT_ENVIRONMENT=/usr/local \ + PYTHONPATH=/backend WORKDIR /backend +RUN apt-get update \ + && apt-get install -y --no-install-recommends netcat-openbsd \ + && rm -rf /var/lib/apt/lists/* + COPY pyproject.toml ./ COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ RUN uv lock && uv sync --no-dev @@ -11,7 +19,10 @@ RUN uv lock && uv sync --no-dev COPY src ./src COPY migrations ./migrations COPY alembic.ini ./ - -RUN mkdir -p /backend/storage/files +COPY entrypoint.sh /backend/entrypoint.sh +RUN chmod +x /backend/entrypoint.sh \ + && mkdir -p /backend/storage/files EXPOSE 8000 + +ENTRYPOINT ["sh", "/backend/entrypoint.sh"] diff --git a/backend/entrypoint.sh b/backend/entrypoint.sh new file mode 100644 index 00000000..e5102517 --- /dev/null +++ b/backend/entrypoint.sh @@ -0,0 +1,34 @@ +#!/bin/sh +set -e + +DB_HOST="${POSTGRES_HOST:-backend-db}" +DB_PORT="${POSTGRES_PORT:-${PGPORT:-5432}}" +REDIS_HOST="${REDIS_HOST:-backend-redis}" +REDIS_PORT="${REDIS_PORT:-6379}" + +echo ">>> Waiting for PostgreSQL at ${DB_HOST}:${DB_PORT}..." +until nc -z "$DB_HOST" "$DB_PORT" 2>/dev/null; do + echo " Database not ready, retrying in 2s..." + sleep 2 +done +echo ">>> PostgreSQL is ready" + +echo ">>> Waiting for Redis at ${REDIS_HOST}:${REDIS_PORT}..." +until nc -z "$REDIS_HOST" "$REDIS_PORT" 2>/dev/null; do + echo " Redis not ready, retrying in 2s..." + sleep 2 +done +echo ">>> Redis is ready" + +# Migrations only in the API process to avoid races with celery workers +# (same idea as mail-manager: migrate when starting uvicorn). +if [ "$1" = "uvicorn" ] || [ "${RUN_MIGRATIONS:-0}" = "1" ]; then + echo ">>> Applying database migrations (alembic upgrade head)..." + alembic upgrade head + echo ">>> Migrations applied" +else + echo ">>> Skipping migrations (not API process)" +fi + +echo ">>> Starting: $*" +exec "$@" diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index cf790c04..0cc4f29f 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,7 +1,7 @@ services: backend: build: ./backend - command: uvicorn src.app:app --host 0.0.0.0 --reload --port 8000 + command: ["uvicorn", "src.app:app", "--host", "0.0.0.0", "--reload", "--port", "8000"] container_name: backend volumes: - ./backend:/backend @@ -9,8 +9,11 @@ services: env_file: ".env.dev" environment: STORAGE_DIR: /backend/storage/files + POSTGRES_HOST: backend-db POSTGRES_PORT: "5432" PGPORT: "5432" + REDIS_HOST: backend-redis + REDIS_PORT: "6379" ports: - "8000:8000" depends_on: @@ -18,7 +21,7 @@ services: condition: service_healthy backend-redis: condition: service_healthy - restart: unless-stopped + restart: always deploy: resources: limits: @@ -40,8 +43,12 @@ services: env_file: ".env.dev" environment: STORAGE_DIR: /backend/storage/files + POSTGRES_HOST: backend-db POSTGRES_PORT: "5432" PGPORT: "5432" + REDIS_HOST: backend-redis + REDIS_PORT: "6379" + RUN_MIGRATIONS: "0" volumes: - ./backend:/backend - file-storage:/backend/storage/files @@ -50,7 +57,9 @@ services: condition: service_healthy backend-redis: condition: service_healthy - restart: unless-stopped + backend: + condition: service_started + restart: always deploy: resources: limits: @@ -84,7 +93,7 @@ services: timeout: 5s retries: 10 start_period: 10s - restart: unless-stopped + restart: always deploy: resources: limits: @@ -103,7 +112,7 @@ services: interval: 5s timeout: 3s retries: 10 - restart: unless-stopped + restart: always deploy: resources: limits: @@ -122,7 +131,7 @@ services: - "3000:3000" depends_on: - backend - restart: unless-stopped + restart: always deploy: resources: limits: From 2284c096cfced3e2cebf077e8791773bec11352d Mon Sep 17 00:00:00 2001 From: Nikolay Date: Sat, 11 Jul 2026 17:10:48 +0300 Subject: [PATCH 3/4] docs: OpenAPI summaries, Field/ORM comments, key docstrings --- backend/src/api/deps.py | 1 + backend/src/api/routes_alerts.py | 9 ++- backend/src/api/routes_files.py | 100 +++++++++++++++++++++++++--- backend/src/api/routes_system.py | 20 +++++- backend/src/app.py | 2 - backend/src/application.py | 26 +++++++- backend/src/data/alerts.py | 5 +- backend/src/data/columns.py | 6 +- backend/src/data/files.py | 14 +++- backend/src/db/session.py | 13 +++- backend/src/dto/alerts.py | 26 ++++++-- backend/src/dto/common.py | 14 ++-- backend/src/dto/files.py | 80 ++++++++++++++++++---- backend/src/entities/alert.py | 24 ++++++- backend/src/entities/stored_file.py | 74 +++++++++++++++++--- backend/src/errors.py | 15 ++--- backend/src/http_errors.py | 11 ++- backend/src/logging_setup.py | 3 +- backend/src/models.py | 2 - backend/src/scan/rules.py | 16 ++++- backend/src/schemas.py | 2 - backend/src/service.py | 5 -- backend/src/settings.py | 73 +++++++++++++++----- backend/src/storage/local.py | 31 ++++++++- backend/src/tasks.py | 3 - backend/src/use_cases/files.py | 52 ++++++++++++++- backend/src/use_cases/pipeline.py | 25 +++++-- backend/src/worker/jobs.py | 18 ++++- 28 files changed, 547 insertions(+), 123 deletions(-) diff --git a/backend/src/api/deps.py b/backend/src/api/deps.py index 9e9394d2..f16d48e9 100644 --- a/backend/src/api/deps.py +++ b/backend/src/api/deps.py @@ -6,6 +6,7 @@ async def db_session() -> AsyncIterator[AsyncSession]: + """FastAPI dependency: async SQLAlchemy session на запрос.""" factory = get_session_factory() async with factory() as session: yield session diff --git a/backend/src/api/routes_alerts.py b/backend/src/api/routes_alerts.py index c1a3e242..9f75c727 100644 --- a/backend/src/api/routes_alerts.py +++ b/backend/src/api/routes_alerts.py @@ -8,6 +8,13 @@ router = APIRouter(tags=["alerts"]) -@router.get("/alerts", response_model=list[AlertItem]) +@router.get( + "/alerts", + response_model=list[AlertItem], + summary="Список алертов", + description="Возвращает алерты, отсортированные по дате создания (новые первыми).", + responses={200: {"description": "Список алертов"}}, +) async def list_alerts_view(session: AsyncSession = Depends(db_session)) -> list[AlertItem]: + """Получить ленту алертов по обработке файлов.""" return await files_uc.list_alerts(session=session) diff --git a/backend/src/api/routes_files.py b/backend/src/api/routes_files.py index c35fdb45..d84666bb 100644 --- a/backend/src/api/routes_files.py +++ b/backend/src/api/routes_files.py @@ -1,4 +1,4 @@ -from fastapi import APIRouter, Depends, File, Form, UploadFile +from fastapi import APIRouter, Depends, File, Form, UploadFile, status from fastapi.responses import FileResponse, Response from sqlalchemy.ext.asyncio import AsyncSession @@ -11,17 +11,50 @@ router = APIRouter(tags=["files"]) -@router.get("/files", response_model=list[FileItem]) +@router.get( + "/files", + response_model=list[FileItem], + summary="Список файлов", + description="Возвращает файлы, отсортированные по дате создания (новые первыми). " + "Поле stored_name не отдаётся.", + responses={ + 200: {"description": "Список файлов"}, + }, +) async def list_files_view(session: AsyncSession = Depends(db_session)) -> list[FileItem]: + """Получить список всех загруженных файлов.""" return await files_uc.list_files(session=session) -@router.post("/files", response_model=FileItem, status_code=201) +@router.post( + "/files", + response_model=FileItem, + status_code=status.HTTP_201_CREATED, + summary="Загрузить файл", + description=( + "Принимает multipart/form-data: title + file. " + "Файл пишется потоково, создаётся запись со статусом uploaded, " + "в очередь ставится фоновая обработка (scan → metadata → alert)." + ), + responses={ + 201: {"description": "Файл принят"}, + 400: {"description": "Пустой файл"}, + 413: {"description": "Файл превышает MAX_UPLOAD_BYTES"}, + 422: {"description": "Ошибка валидации входных данных"}, + }, +) async def create_file_view( - title: str = Form(..., min_length=1, max_length=255), - file: UploadFile = File(...), + title: str = Form( + ..., + min_length=1, + max_length=255, + description="Отображаемое название файла", + examples=["Договор с подрядчиком"], + ), + file: UploadFile = File(..., description="Бинарное содержимое файла"), session: AsyncSession = Depends(db_session), ) -> FileItem: + """Загрузить файл и поставить его в очередь обработки.""" return await files_uc.create_file( session=session, title=title, @@ -30,28 +63,64 @@ async def create_file_view( ) -@router.get("/files/{file_id}", response_model=FileItem) +@router.get( + "/files/{file_id}", + response_model=FileItem, + summary="Получить файл по id", + description="Возвращает метаданные файла без stored_name.", + responses={ + 200: {"description": "Файл найден"}, + 404: {"description": "Файл не найден"}, + 422: {"description": "Некорректный file_id"}, + }, +) async def get_file_view( file_id: str = Depends(file_id_path), session: AsyncSession = Depends(db_session), ) -> FileItem: + """Получить карточку файла по UUID.""" return await files_uc.get_file(session=session, file_id=file_id) -@router.patch("/files/{file_id}", response_model=FileItem) +@router.patch( + "/files/{file_id}", + response_model=FileItem, + summary="Обновить название файла", + description="Обновляет только title. Остальные поля не изменяются.", + responses={ + 200: {"description": "Файл обновлён"}, + 404: {"description": "Файл не найден"}, + 422: {"description": "Ошибка валидации"}, + }, +) async def update_file_view( payload: FileUpdate, file_id: str = Depends(file_id_path), session: AsyncSession = Depends(db_session), ) -> FileItem: + """Обновить title файла.""" return await files_uc.update_file(session=session, file_id=file_id, title=payload.title) -@router.get("/files/{file_id}/download") +@router.get( + "/files/{file_id}/download", + summary="Скачать файл", + description="Отдаёт бинарное содержимое с Content-Disposition по original_name.", + responses={ + 200: { + "description": "Содержимое файла", + "content": {"application/octet-stream": {}}, + }, + 404: {"description": "Запись или объект на диске не найдены"}, + 422: {"description": "Некорректный file_id"}, + }, + response_class=FileResponse, +) async def download_file_view( file_id: str = Depends(file_id_path), session: AsyncSession = Depends(db_session), ) -> FileResponse: + """Скачать исходный файл с диска.""" path, filename, mime_type = await files_uc.resolve_download( session=session, file_id=file_id, @@ -59,10 +128,21 @@ async def download_file_view( return FileResponse(path=path, media_type=mime_type, filename=filename) -@router.delete("/files/{file_id}", status_code=204) +@router.delete( + "/files/{file_id}", + status_code=status.HTTP_204_NO_CONTENT, + summary="Удалить файл", + description="Удаляет объект с диска, запись files и связанные alerts (CASCADE).", + responses={ + 204: {"description": "Удалено"}, + 404: {"description": "Файл не найден"}, + 422: {"description": "Некорректный file_id"}, + }, +) async def delete_file_view( file_id: str = Depends(file_id_path), session: AsyncSession = Depends(db_session), ) -> Response: + """Удалить файл и связанные алерты.""" await files_uc.delete_file(session=session, file_id=file_id) - return Response(status_code=204) + return Response(status_code=status.HTTP_204_NO_CONTENT) diff --git a/backend/src/api/routes_system.py b/backend/src/api/routes_system.py index 5f4ca371..8237c7a5 100644 --- a/backend/src/api/routes_system.py +++ b/backend/src/api/routes_system.py @@ -1,8 +1,22 @@ from fastapi import APIRouter +from pydantic import BaseModel, Field router = APIRouter(tags=["system"]) -@router.get("/health") -async def health() -> dict[str, str]: - return {"status": "ok"} +class HealthResponse(BaseModel): + """Ответ health-check.""" + + status: str = Field(..., description="Статус сервиса", examples=["ok"]) + + +@router.get( + "/health", + response_model=HealthResponse, + summary="Проверка доступности API", + description="Лёгкий health-check без обращения к БД.", + responses={200: {"description": "Сервис доступен"}}, +) +async def health() -> HealthResponse: + """Вернуть статус API.""" + return HealthResponse(status="ok") diff --git a/backend/src/app.py b/backend/src/app.py index dc9d47b7..599e1829 100644 --- a/backend/src/app.py +++ b/backend/src/app.py @@ -1,5 +1,3 @@ -"""ASGI entrypoint for uvicorn.""" - from src.application import create_app app = create_app() diff --git a/backend/src/application.py b/backend/src/application.py index 625b9fc0..741106ca 100644 --- a/backend/src/application.py +++ b/backend/src/application.py @@ -1,5 +1,3 @@ -"""FastAPI application factory.""" - from contextlib import asynccontextmanager from fastapi import FastAPI @@ -14,9 +12,19 @@ from src.settings import get_settings from src.storage.local import ensure_storage_dir +API_DESCRIPTION = """ +API файлообменника: загрузка файлов, фоновая проверка и алерты. + +### Контракт +- Публичные эндпоинты без токенов. +- `stored_name` не отдаётся в JSON-ответах. +- Фоновая обработка: scan → metadata → alert (одна Celery-задача). +""" + @asynccontextmanager async def lifespan(_: FastAPI): + """Инициализация логирования, storage и engine; dispose при остановке.""" configure_logging() settings = get_settings() ensure_storage_dir(settings=settings) @@ -26,8 +34,20 @@ async def lifespan(_: FastAPI): def create_app() -> FastAPI: + """Собрать FastAPI-приложение с CORS, handlers и роутерами.""" settings = get_settings() - app = FastAPI(title="File Exchange API", lifespan=lifespan) + app = FastAPI( + title="File Exchange API", + description=API_DESCRIPTION, + version="1.0.0", + lifespan=lifespan, + contact={"name": "File Exchange"}, + openapi_tags=[ + {"name": "system", "description": "Служебные эндпоинты"}, + {"name": "files", "description": "CRUD и скачивание файлов"}, + {"name": "alerts", "description": "Лента алертов обработки"}, + ], + ) app.add_middleware( CORSMiddleware, allow_origins=settings.cors_origin_list, diff --git a/backend/src/data/alerts.py b/backend/src/data/alerts.py index 146124f3..0f8f28c8 100644 --- a/backend/src/data/alerts.py +++ b/backend/src/data/alerts.py @@ -1,5 +1,3 @@ -"""Alert persistence with projections.""" - from __future__ import annotations from typing import Any @@ -23,6 +21,7 @@ def _row_to_alert(row: Any) -> AlertItem: async def list_alert_items(*, session: AsyncSession) -> list[AlertItem]: + """Список алертов (projection AlertItem), order by created_at desc.""" result = await session.execute( select(*ALERT_ITEM_COLUMNS).order_by(Alert.created_at.desc()) ) @@ -30,6 +29,7 @@ async def list_alert_items(*, session: AsyncSession) -> list[AlertItem]: async def alert_exists_for_file(*, session: AsyncSession, file_id: str) -> bool: + """Проверить наличие хотя бы одного алерта для file_id.""" stmt = select(exists().where(Alert.file_id == file_id)) result = await session.execute(stmt) return bool(result.scalar()) @@ -42,4 +42,5 @@ async def insert_alert( level: str, message: str, ) -> None: + """Добавить алерт (без commit — commit на уровне use case / worker).""" session.add(Alert(file_id=file_id, level=level, message=message)) diff --git a/backend/src/data/columns.py b/backend/src/data/columns.py index 8d3756ac..f7580e21 100644 --- a/backend/src/data/columns.py +++ b/backend/src/data/columns.py @@ -1,5 +1,3 @@ -"""Column projections aligned with response DTOs.""" - from src.entities.alert import Alert from src.entities.stored_file import StoredFile @@ -17,6 +15,7 @@ StoredFile.created_at, StoredFile.updated_at, ) +"""Колонки API FileItem (без stored_name).""" ALERT_ITEM_COLUMNS = ( Alert.id, @@ -25,6 +24,7 @@ Alert.message, Alert.created_at, ) +"""Колонки API AlertItem.""" STORAGE_REF_COLUMNS = ( StoredFile.id, @@ -32,6 +32,7 @@ StoredFile.original_name, StoredFile.mime_type, ) +"""Узкая проекция download/delete.""" PIPELINE_COLUMNS = ( StoredFile.id, @@ -46,3 +47,4 @@ StoredFile.metadata_json, StoredFile.requires_attention, ) +"""Проекция worker pipeline.""" diff --git a/backend/src/data/files.py b/backend/src/data/files.py index b0d5ff03..09b5284d 100644 --- a/backend/src/data/files.py +++ b/backend/src/data/files.py @@ -1,5 +1,3 @@ -"""File persistence with column projections.""" - from __future__ import annotations from dataclasses import dataclass @@ -16,6 +14,8 @@ @dataclass(slots=True) class StorageRef: + """Узкая проекция для download/delete (включает stored_name).""" + id: str stored_name: str original_name: str @@ -24,6 +24,8 @@ class StorageRef: @dataclass(slots=True) class PipelineRow: + """Проекция полей, нужных worker pipeline.""" + id: str title: str original_name: str @@ -55,6 +57,7 @@ def _row_to_file_item(row: Any) -> FileItem: async def list_file_items(*, session: AsyncSession) -> list[FileItem]: + """Список файлов: только колонки FileItem, order by created_at desc.""" result = await session.execute( select(*FILE_ITEM_COLUMNS).order_by(StoredFile.created_at.desc()) ) @@ -62,6 +65,7 @@ async def list_file_items(*, session: AsyncSession) -> list[FileItem]: async def get_file_item(*, session: AsyncSession, file_id: str) -> FileItem | None: + """Карточка файла по id (projection FileItem) или None.""" result = await session.execute( select(*FILE_ITEM_COLUMNS).where(StoredFile.id == file_id) ) @@ -72,6 +76,7 @@ async def get_file_item(*, session: AsyncSession, file_id: str) -> FileItem | No async def get_storage_ref(*, session: AsyncSession, file_id: str) -> StorageRef | None: + """Ссылка на объект storage для download/delete.""" result = await session.execute( select(*STORAGE_REF_COLUMNS).where(StoredFile.id == file_id) ) @@ -87,6 +92,7 @@ async def get_storage_ref(*, session: AsyncSession, file_id: str) -> StorageRef async def get_pipeline_row(*, session: AsyncSession, file_id: str) -> PipelineRow | None: + """Строка pipeline-проекции для worker.""" result = await session.execute( select(*PIPELINE_COLUMNS).where(StoredFile.id == file_id) ) @@ -118,6 +124,7 @@ async def insert_file( mime_type: str, size: int, ) -> FileItem: + """Вставить запись files со статусом uploaded и вернуть FileItem.""" entity = StoredFile( id=file_id, title=title, @@ -153,6 +160,7 @@ async def update_file_title( file_id: str, title: str, ) -> FileItem | None: + """Обновить title; вернуть FileItem или None если запись не найдена.""" result = await session.execute( update(StoredFile) .where(StoredFile.id == file_id) @@ -166,6 +174,7 @@ async def update_file_title( async def delete_file_row(*, session: AsyncSession, file_id: str) -> bool: + """Удалить строку files; True если была удалена хотя бы одна запись.""" result = await session.execute(delete(StoredFile).where(StoredFile.id == file_id)) return (result.rowcount or 0) > 0 @@ -176,6 +185,7 @@ async def update_pipeline_fields( file_id: str, **fields: Any, ) -> None: + """Частично обновить поля pipeline (scan/metadata/status).""" if not fields: return fields["updated_at"] = datetime.now(timezone.utc) diff --git a/backend/src/db/session.py b/backend/src/db/session.py index 51887123..c584669b 100644 --- a/backend/src/db/session.py +++ b/backend/src/db/session.py @@ -1,5 +1,3 @@ -"""Async SQLAlchemy engine and session helpers.""" - from collections.abc import AsyncIterator from contextlib import asynccontextmanager @@ -17,6 +15,7 @@ def create_engine(settings: Settings | None = None) -> AsyncEngine: + """Создать async engine по settings.""" cfg = settings or get_settings() return create_async_engine( cfg.database_url, @@ -25,6 +24,7 @@ def create_engine(settings: Settings | None = None) -> AsyncEngine: def get_engine() -> AsyncEngine: + """Ленивый singleton engine API-процесса.""" global _engine, _session_factory if _engine is None: _engine = create_engine() @@ -33,6 +33,7 @@ def get_engine() -> AsyncEngine: def get_session_factory() -> async_sessionmaker[AsyncSession]: + """Session factory API-процесса.""" global _session_factory if _session_factory is None: get_engine() @@ -41,6 +42,7 @@ def get_session_factory() -> async_sessionmaker[AsyncSession]: async def get_db_session() -> AsyncIterator[AsyncSession]: + """Dependency-compatible генератор сессии.""" session_factory = get_session_factory() async with session_factory() as session: yield session @@ -48,7 +50,11 @@ async def get_db_session() -> AsyncIterator[AsyncSession]: @asynccontextmanager async def worker_session_scope(settings: Settings | None = None): - """Fresh engine per worker job — avoids event-loop leakage across tasks.""" + """ + Свежий engine+session на одну worker-задачу. + + Dispose engine после выхода — защита от утечки loop между задачами. + """ cfg = settings or get_settings() engine = create_engine(cfg) factory = async_sessionmaker(engine, expire_on_commit=False) @@ -60,6 +66,7 @@ async def worker_session_scope(settings: Settings | None = None): async def dispose_engine() -> None: + """Закрыть singleton engine API-процесса.""" global _engine, _session_factory if _engine is not None: await _engine.dispose() diff --git a/backend/src/dto/alerts.py b/backend/src/dto/alerts.py index 50b2e0b8..0886b71c 100644 --- a/backend/src/dto/alerts.py +++ b/backend/src/dto/alerts.py @@ -1,13 +1,27 @@ from datetime import datetime -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, Field class AlertItem(BaseModel): + """Алерт по результатам обработки файла.""" + model_config = ConfigDict(from_attributes=True) - id: int - file_id: str - level: str - message: str - created_at: datetime + id: int = Field(..., description="Идентификатор алерта", examples=[1]) + file_id: str = Field( + ..., + description="UUID связанного файла", + examples=["550e8400-e29b-41d4-a716-446655440000"], + ) + level: str = Field( + ..., + description="Уровень: info | warning | critical", + examples=["info"], + ) + message: str = Field( + ..., + description="Текст сообщения", + examples=["File processed successfully"], + ) + created_at: datetime = Field(..., description="Время создания алерта (UTC)") diff --git a/backend/src/dto/common.py b/backend/src/dto/common.py index 18f32ea8..2900bc80 100644 --- a/backend/src/dto/common.py +++ b/backend/src/dto/common.py @@ -1,13 +1,19 @@ -"""Shared request validators.""" - from uuid import UUID from fastapi import HTTPException, Path from starlette import status -def file_id_path(file_id: str = Path(..., min_length=36, max_length=36)) -> str: - """Validate path file_id as UUID string.""" +def file_id_path( + file_id: str = Path( + ..., + min_length=36, + max_length=36, + description="UUID файла", + examples=["550e8400-e29b-41d4-a716-446655440000"], + ), +) -> str: + """Валидирует path-параметр file_id как UUID.""" try: return str(UUID(file_id)) except ValueError as exc: diff --git a/backend/src/dto/files.py b/backend/src/dto/files.py index 0b41237e..039f8f2a 100644 --- a/backend/src/dto/files.py +++ b/backend/src/dto/files.py @@ -1,27 +1,81 @@ from datetime import datetime +from typing import Any from pydantic import BaseModel, ConfigDict, Field, field_validator class FileItem(BaseModel): + """Публичное представление файла в API (без внутреннего stored_name).""" + model_config = ConfigDict(from_attributes=True) - id: str - title: str - original_name: str - mime_type: str - size: int - processing_status: str - scan_status: str | None - scan_details: str | None - metadata_json: dict | None - requires_attention: bool - created_at: datetime - updated_at: datetime + id: str = Field( + ..., + description="UUID файла", + examples=["550e8400-e29b-41d4-a716-446655440000"], + ) + title: str = Field( + ..., + description="Отображаемое название файла", + min_length=1, + max_length=255, + examples=["Договор с подрядчиком"], + ) + original_name: str = Field( + ..., + description="Исходное имя загруженного файла", + examples=["contract.pdf"], + ) + mime_type: str = Field( + ..., + description="MIME-тип содержимого", + examples=["application/pdf"], + ) + size: int = Field( + ..., + description="Размер файла в байтах", + ge=0, + examples=[1024], + ) + processing_status: str = Field( + ..., + description="Статус обработки: uploaded | processing | processed | failed", + examples=["processed"], + ) + scan_status: str | None = Field( + None, + description="Результат проверки: clean | suspicious | failed | null до обработки", + examples=["clean"], + ) + scan_details: str | None = Field( + None, + description="Текстовое описание результата проверки", + examples=["no threats found"], + ) + metadata_json: dict[str, Any] | None = Field( + None, + description="Извлечённые метаданные (extension, size_bytes, mime_type и доп. поля)", + examples=[{"extension": ".pdf", "size_bytes": 1024, "mime_type": "application/pdf"}], + ) + requires_attention: bool = Field( + ..., + description="Требует внимания (подозрительный или ошибочный файл)", + examples=[False], + ) + created_at: datetime = Field(..., description="Время создания записи (UTC)") + updated_at: datetime = Field(..., description="Время последнего обновления (UTC)") class FileUpdate(BaseModel): - title: str = Field(..., min_length=1, max_length=255) + """Тело запроса на обновление метаданных файла.""" + + title: str = Field( + ..., + min_length=1, + max_length=255, + description="Новое название файла", + examples=["Обновлённый договор"], + ) @field_validator("title") @classmethod diff --git a/backend/src/entities/alert.py b/backend/src/entities/alert.py index c96adbad..4d7e6e30 100644 --- a/backend/src/entities/alert.py +++ b/backend/src/entities/alert.py @@ -7,19 +7,37 @@ class Alert(Base): + """Таблица alerts: уведомления по результатам обработки файлов.""" + __tablename__ = "alerts" + __table_args__ = {"comment": "Алерты по файлам (info/warning/critical)"} - id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + id: Mapped[int] = mapped_column( + Integer, + primary_key=True, + autoincrement=True, + comment="Идентификатор алерта", + ) file_id: Mapped[str] = mapped_column( String(36), ForeignKey("files.id", ondelete="CASCADE"), nullable=False, index=True, + comment="Ссылка на files.id", + ) + level: Mapped[str] = mapped_column( + String(50), + nullable=False, + comment="info | warning | critical", + ) + message: Mapped[str] = mapped_column( + String(500), + nullable=False, + comment="Текст алерта", ) - level: Mapped[str] = mapped_column(String(50), nullable=False) - message: Mapped[str] = mapped_column(String(500), nullable=False) created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), server_default=func.now(), nullable=False, + comment="Время создания", ) diff --git a/backend/src/entities/stored_file.py b/backend/src/entities/stored_file.py index 0ca45bdc..8ad78541 100644 --- a/backend/src/entities/stored_file.py +++ b/backend/src/entities/stored_file.py @@ -7,27 +7,79 @@ class StoredFile(Base): + """Таблица files: загруженные файлы и статусы обработки.""" + __tablename__ = "files" + __table_args__ = {"comment": "Загруженные файлы и результаты фоновой обработки"} - id: Mapped[str] = mapped_column(String(36), primary_key=True) - title: Mapped[str] = mapped_column(String(255), nullable=False) - original_name: Mapped[str] = mapped_column(String(255), nullable=False) - stored_name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True) - mime_type: Mapped[str] = mapped_column(String(255), nullable=False) - size: Mapped[int] = mapped_column(Integer, nullable=False) - processing_status: Mapped[str] = mapped_column(String(50), nullable=False, default="uploaded") - scan_status: Mapped[str | None] = mapped_column(String(50), nullable=True) - scan_details: Mapped[str | None] = mapped_column(String(500), nullable=True) - metadata_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) - requires_attention: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + id: Mapped[str] = mapped_column( + String(36), + primary_key=True, + comment="UUID файла", + ) + title: Mapped[str] = mapped_column( + String(255), + nullable=False, + comment="Отображаемое название", + ) + original_name: Mapped[str] = mapped_column( + String(255), + nullable=False, + comment="Исходное имя загруженного файла", + ) + stored_name: Mapped[str] = mapped_column( + String(255), + nullable=False, + unique=True, + comment="Имя объекта в файловом хранилище (внутреннее)", + ) + mime_type: Mapped[str] = mapped_column( + String(255), + nullable=False, + comment="MIME-тип", + ) + size: Mapped[int] = mapped_column( + Integer, + nullable=False, + comment="Размер в байтах", + ) + processing_status: Mapped[str] = mapped_column( + String(50), + nullable=False, + default="uploaded", + comment="uploaded | processing | processed | failed", + ) + scan_status: Mapped[str | None] = mapped_column( + String(50), + nullable=True, + comment="clean | suspicious | failed", + ) + scan_details: Mapped[str | None] = mapped_column( + String(500), + nullable=True, + comment="Детали проверки", + ) + metadata_json: Mapped[dict | None] = mapped_column( + JSON, + nullable=True, + comment="Извлечённые метаданные", + ) + requires_attention: Mapped[bool] = mapped_column( + Boolean, + nullable=False, + default=False, + comment="Флаг необходимости внимания", + ) created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), server_default=func.now(), nullable=False, + comment="Время создания", ) updated_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), server_default=func.now(), onupdate=func.now(), nullable=False, + comment="Время обновления", ) diff --git a/backend/src/errors.py b/backend/src/errors.py index e4f8d30a..c2afbe0f 100644 --- a/backend/src/errors.py +++ b/backend/src/errors.py @@ -1,8 +1,5 @@ -"""Domain exceptions — no HTTP coupling.""" - - class AppError(Exception): - """Base domain error.""" + """Базовая доменная ошибка.""" def __init__(self, message: str) -> None: self.message = message @@ -10,20 +7,20 @@ def __init__(self, message: str) -> None: class ResourceNotFoundError(AppError): - pass + """Ресурс не найден.""" class EmptyUploadError(AppError): - pass + """Загружен пустой файл.""" class PayloadTooLargeError(AppError): - pass + """Превышен лимит размера upload.""" class StorageIOError(AppError): - pass + """Ошибка файлового хранилища.""" class ValidationDomainError(AppError): - pass + """Доменная ошибка валидации входных данных.""" diff --git a/backend/src/http_errors.py b/backend/src/http_errors.py index ade6b2b8..b69fb544 100644 --- a/backend/src/http_errors.py +++ b/backend/src/http_errors.py @@ -1,5 +1,3 @@ -"""Map domain errors and unexpected failures to HTTP responses.""" - import logging from fastapi import FastAPI, Request @@ -20,6 +18,8 @@ def register_exception_handlers(app: FastAPI) -> None: + """Зарегистрировать маппинг доменных ошибок в HTTP-ответы.""" + @app.exception_handler(ResourceNotFoundError) async def not_found_handler(_: Request, exc: ResourceNotFoundError) -> JSONResponse: logger.warning("not_found detail=%s", exc.message) @@ -78,7 +78,12 @@ async def request_validation_handler(_: Request, exc: RequestValidationError) -> @app.exception_handler(Exception) async def unhandled_handler(_: Request, exc: Exception) -> JSONResponse: - logger.error("unhandled_exception type=%s reason=%s", type(exc).__name__, exc, exc_info=True) + logger.error( + "unhandled_exception type=%s reason=%s", + type(exc).__name__, + exc, + exc_info=True, + ) return JSONResponse( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, content={"detail": "Internal server error"}, diff --git a/backend/src/logging_setup.py b/backend/src/logging_setup.py index f6ec482f..6f0e7c2b 100644 --- a/backend/src/logging_setup.py +++ b/backend/src/logging_setup.py @@ -1,5 +1,3 @@ -"""Console logging configuration.""" - import logging import sys @@ -7,6 +5,7 @@ def configure_logging() -> None: + """Настроить root logger на stdout, если handlers ещё не заданы.""" settings = get_settings() level = getattr(logging, settings.log_level.upper(), logging.INFO) root = logging.getLogger() diff --git a/backend/src/models.py b/backend/src/models.py index 3b88a8d9..faf499d8 100644 --- a/backend/src/models.py +++ b/backend/src/models.py @@ -1,5 +1,3 @@ -"""Backward-compatible re-exports for Alembic and legacy imports.""" - from src.db.base import Base from src.entities.alert import Alert from src.entities.stored_file import StoredFile diff --git a/backend/src/scan/rules.py b/backend/src/scan/rules.py index bb92b461..1855ed4b 100644 --- a/backend/src/scan/rules.py +++ b/backend/src/scan/rules.py @@ -1,5 +1,3 @@ -"""Pure scan rules matching baseline business logic.""" - from dataclasses import dataclass from pathlib import Path @@ -11,6 +9,8 @@ @dataclass(frozen=True, slots=True) class ScanResult: + """Результат эвристической проверки файла.""" + scan_status: str scan_details: str requires_attention: bool @@ -23,6 +23,18 @@ def evaluate_scan( size: int, size_threshold_bytes: int = SIZE_THRESHOLD_BYTES, ) -> ScanResult: + """ + Применить baseline-правила проверки. + + Args: + original_name: исходное имя файла (для extension). + mime_type: заявленный MIME. + size: размер в байтах. + size_threshold_bytes: порог «большого» файла (по умолчанию 10 MiB). + + Returns: + ScanResult со статусом clean/suspicious. + """ reasons: list[str] = [] extension = Path(original_name).suffix.lower() diff --git a/backend/src/schemas.py b/backend/src/schemas.py index 06dc93b5..fc22af2b 100644 --- a/backend/src/schemas.py +++ b/backend/src/schemas.py @@ -1,5 +1,3 @@ -"""Backward-compatible re-exports.""" - from src.dto.alerts import AlertItem from src.dto.files import FileItem, FileUpdate diff --git a/backend/src/service.py b/backend/src/service.py index a6a48c0a..23c77320 100644 --- a/backend/src/service.py +++ b/backend/src/service.py @@ -1,8 +1,3 @@ -"""Deprecated module kept only to avoid import surprises during transition. - -Use use_cases / data / storage packages instead. -""" - from src.settings import get_settings settings = get_settings() diff --git a/backend/src/settings.py b/backend/src/settings.py index 1604caea..59305e91 100644 --- a/backend/src/settings.py +++ b/backend/src/settings.py @@ -1,5 +1,3 @@ -"""Application settings loaded from environment.""" - from functools import lru_cache from pathlib import Path @@ -8,50 +6,88 @@ class Settings(BaseSettings): + """Конфигурация приложения из переменных окружения.""" + model_config = SettingsConfigDict( env_file=None, extra="ignore", case_sensitive=False, ) - postgres_user: str = "postgres" - postgres_password: str = "postgres" - postgres_host: str = "backend-db" - postgres_db: str = "test" - postgres_port: int = Field(default=5432, validation_alias="POSTGRES_PORT") - # Accept legacy PGPORT as well via model validator below - pgport: int | None = Field(default=None, validation_alias="PGPORT") + postgres_user: str = Field(default="postgres", description="Пользователь PostgreSQL") + postgres_password: str = Field(default="postgres", description="Пароль PostgreSQL") + postgres_host: str = Field(default="backend-db", description="Хост PostgreSQL") + postgres_db: str = Field(default="test", description="Имя БД") + postgres_port: int = Field( + default=5432, + validation_alias="POSTGRES_PORT", + description="Порт PostgreSQL", + ) + pgport: int | None = Field( + default=None, + validation_alias="PGPORT", + description="Legacy-порт PostgreSQL (имеет приоритет над POSTGRES_PORT)", + ) celery_broker_url: str = Field( default="redis://backend-redis:6379/0", validation_alias="CELERY_BROKER_URL", + description="URL брокера Celery", + ) + redis_url: str | None = Field( + default=None, + validation_alias="REDIS_URL", + description="Альтернативный URL Redis (перекрывает CELERY_BROKER_URL)", ) - redis_url: str | None = Field(default=None, validation_alias="REDIS_URL") storage_dir: Path = Field( default=Path(__file__).resolve().parent.parent / "storage" / "files", validation_alias="STORAGE_DIR", + description="Каталог локального файлового хранилища", + ) + max_upload_bytes: int = Field( + default=50 * 1024 * 1024, + validation_alias="MAX_UPLOAD_BYTES", + description="Максимальный размер upload в байтах", + ) + log_level: str = Field( + default="INFO", + validation_alias="LOG_LEVEL", + description="Уровень логирования", ) - max_upload_bytes: int = Field(default=50 * 1024 * 1024, validation_alias="MAX_UPLOAD_BYTES") - log_level: str = Field(default="INFO", validation_alias="LOG_LEVEL") cors_origins: str = Field( default="http://localhost:3000,http://127.0.0.1:3000", validation_alias="CORS_ORIGINS", + description="Список CORS origins через запятую", + ) + scan_size_threshold_bytes: int = Field( + default=10 * 1024 * 1024, + description="Порог размера для suspicious scan", + ) + metadata_text_read_limit: int = Field( + default=2 * 1024 * 1024, + description="Лимит чтения text/* при извлечении метаданных", + ) + metadata_pdf_read_limit: int = Field( + default=5 * 1024 * 1024, + description="Лимит чтения PDF при извлечении метаданных", + ) + worker_max_retries: int = Field(default=3, description="Максимум retry Celery-задачи") + worker_retry_backoff_seconds: int = Field( + default=5, + description="Базовый backoff retry (экспоненциальный)", ) - scan_size_threshold_bytes: int = 10 * 1024 * 1024 - metadata_text_read_limit: int = 2 * 1024 * 1024 - metadata_pdf_read_limit: int = 5 * 1024 * 1024 - worker_max_retries: int = 3 - worker_retry_backoff_seconds: int = 5 @property def db_port(self) -> int: + """Эффективный порт БД (PGPORT > POSTGRES_PORT).""" if self.pgport is not None: return self.pgport return self.postgres_port @property def database_url(self) -> str: + """Async SQLAlchemy URL (postgresql+asyncpg).""" return ( f"postgresql+asyncpg://{self.postgres_user}:{self.postgres_password}" f"@{self.postgres_host}:{self.db_port}/{self.postgres_db}" @@ -59,13 +95,16 @@ def database_url(self) -> str: @property def broker_url(self) -> str: + """URL брокера Celery (REDIS_URL или CELERY_BROKER_URL).""" return self.redis_url or self.celery_broker_url @property def cors_origin_list(self) -> list[str]: + """Список CORS origins.""" return [item.strip() for item in self.cors_origins.split(",") if item.strip()] @lru_cache def get_settings() -> Settings: + """Singleton-настройки процесса.""" return Settings() diff --git a/backend/src/storage/local.py b/backend/src/storage/local.py index 52277428..d3312e03 100644 --- a/backend/src/storage/local.py +++ b/backend/src/storage/local.py @@ -1,5 +1,3 @@ -"""Local disk storage with stream write and path safety.""" - from __future__ import annotations import logging @@ -20,6 +18,7 @@ def ensure_storage_dir(*, settings: Settings | None = None) -> Path: + """Создать каталог storage при необходимости и вернуть путь.""" cfg = settings or get_settings() path = Path(cfg.storage_dir) path.mkdir(parents=True, exist_ok=True) @@ -27,6 +26,7 @@ def ensure_storage_dir(*, settings: Settings | None = None) -> Path: def _safe_suffix(filename: str | None) -> str: + """Вернуть безопасное расширение или пустую строку.""" suffix = Path(filename or "").suffix if not suffix or not _SAFE_SUFFIX.match(suffix): return "" @@ -34,16 +34,24 @@ def _safe_suffix(filename: str | None) -> str: def sanitize_download_name(name: str) -> str: + """Очистить имя файла для Content-Disposition.""" cleaned = Path(name).name.replace("\\", "_").replace("/", "_") cleaned = "".join(ch for ch in cleaned if ch.isprintable() and ch not in "\r\n") return cleaned or "download.bin" def build_stored_name(*, file_id: str, original_filename: str | None) -> str: + """Собрать внутреннее имя объекта storage: {uuid}{suffix}.""" return f"{file_id}{_safe_suffix(original_filename)}" def resolve_under_storage(*, stored_name: str, settings: Settings | None = None) -> Path: + """ + Резолвить путь внутри storage root с защитой от path traversal. + + Raises: + StorageIOError: путь выходит за пределы storage root. + """ cfg = settings or get_settings() root = ensure_storage_dir(settings=cfg).resolve() candidate = (root / Path(stored_name).name).resolve() @@ -60,7 +68,22 @@ async def save_upload_stream( file_id: str, settings: Settings | None = None, ) -> tuple[str, int]: - """Stream upload to disk. Returns (stored_name, size).""" + """ + Потоково записать upload на диск. + + Args: + upload_file: FastAPI UploadFile. + file_id: UUID для имени объекта. + settings: лимит размера и путь storage. + + Returns: + (stored_name, size_bytes) + + Raises: + PayloadTooLargeError: превышен MAX_UPLOAD_BYTES. + EmptyUploadError: размер 0. + StorageIOError: ошибка записи. + """ cfg = settings or get_settings() ensure_storage_dir(settings=cfg) stored_name = build_stored_name(file_id=file_id, original_filename=upload_file.filename) @@ -101,6 +124,7 @@ async def save_upload_stream( async def delete_stored(*, stored_name: str, settings: Settings | None = None) -> None: + """Удалить объект storage; ошибки логируются и не пробрасываются.""" try: path = resolve_under_storage(stored_name=stored_name, settings=settings) if path.exists(): @@ -115,4 +139,5 @@ async def delete_stored(*, stored_name: str, settings: Settings | None = None) - def new_file_id() -> str: + """Сгенерировать UUID файла.""" return str(uuid4()) diff --git a/backend/src/tasks.py b/backend/src/tasks.py index c8602709..832e72e5 100644 --- a/backend/src/tasks.py +++ b/backend/src/tasks.py @@ -1,9 +1,6 @@ -"""Backward-compatible Celery entry for compose/docs.""" - from src.worker.app import celery_app from src.worker.jobs import process_uploaded_file -# Keep historical names importable but unused in new flow scan_file_for_threats = process_uploaded_file __all__ = ["celery_app", "process_uploaded_file", "scan_file_for_threats"] diff --git a/backend/src/use_cases/files.py b/backend/src/use_cases/files.py index c0cbd915..759193e0 100644 --- a/backend/src/use_cases/files.py +++ b/backend/src/use_cases/files.py @@ -1,9 +1,8 @@ -"""File CRUD orchestration.""" - from __future__ import annotations import logging import mimetypes +from collections.abc import Callable from pathlib import Path from fastapi import UploadFile @@ -27,6 +26,7 @@ def _normalize_title(title: str) -> str: + """Нормализует title: trim и проверка длины.""" cleaned = title.strip() if not cleaned: raise ValidationDomainError("title must not be empty") @@ -36,14 +36,26 @@ def _normalize_title(title: str) -> str: async def list_files(*, session: AsyncSession) -> list[FileItem]: + """Вернуть список файлов для API (projection FileItem).""" return await files_data.list_file_items(session=session) async def list_alerts(*, session: AsyncSession) -> list[AlertItem]: + """Вернуть список алертов для API.""" return await alerts_data.list_alert_items(session=session) async def get_file(*, session: AsyncSession, file_id: str) -> FileItem: + """ + Получить файл по id. + + Args: + session: async SQLAlchemy-сессия. + file_id: UUID файла. + + Raises: + ResourceNotFoundError: файл не найден. + """ item = await files_data.get_file_item(session=session, file_id=file_id) if item is None: raise ResourceNotFoundError("File not found") @@ -55,8 +67,20 @@ async def create_file( session: AsyncSession, title: str, upload_file: UploadFile, - enqueue, + enqueue: Callable[..., None], ) -> FileItem: + """ + Сохранить upload, создать запись files и поставить pipeline в очередь. + + Args: + session: async SQLAlchemy-сессия. + title: отображаемое название. + upload_file: загружаемый файл. + enqueue: callable(file_id=...) для постановки Celery-задачи. + + Returns: + FileItem созданной записи. + """ normalized_title = _normalize_title(title) file_id = new_file_id() settings = get_settings() @@ -109,6 +133,13 @@ async def create_file( async def update_file(*, session: AsyncSession, file_id: str, title: str) -> FileItem: + """ + Обновить title файла. + + Raises: + ResourceNotFoundError: файл не найден. + ValidationDomainError: некорректный title. + """ normalized_title = _normalize_title(title) item = await files_data.update_file_title( session=session, @@ -122,6 +153,12 @@ async def update_file(*, session: AsyncSession, file_id: str, title: str) -> Fil async def delete_file(*, session: AsyncSession, file_id: str) -> None: + """ + Удалить файл с диска и запись в БД (alerts каскадом). + + Raises: + ResourceNotFoundError: файл не найден. + """ ref = await files_data.get_storage_ref(session=session, file_id=file_id) if ref is None: raise ResourceNotFoundError("File not found") @@ -135,6 +172,15 @@ async def delete_file(*, session: AsyncSession, file_id: str) -> None: async def resolve_download(*, session: AsyncSession, file_id: str) -> tuple[Path, str, str]: + """ + Подготовить путь, безопасное имя и MIME для скачивания. + + Returns: + (path, download_filename, mime_type) + + Raises: + ResourceNotFoundError: запись или объект на диске отсутствуют. + """ ref = await files_data.get_storage_ref(session=session, file_id=file_id) if ref is None: raise ResourceNotFoundError("File not found") diff --git a/backend/src/use_cases/pipeline.py b/backend/src/use_cases/pipeline.py index 04349ae0..99875a07 100644 --- a/backend/src/use_cases/pipeline.py +++ b/backend/src/use_cases/pipeline.py @@ -1,5 +1,3 @@ -"""Single-flow scan + metadata + alert pipeline.""" - from __future__ import annotations import logging @@ -24,13 +22,22 @@ def _extract_metadata( size: int, settings: Settings, ) -> dict: + """ + Извлечь метаданные файла с ограничением объёма чтения. + + Args: + path: путь к объекту в storage. + original_name: исходное имя файла. + mime_type: MIME-тип. + size: размер в байтах. + settings: лимиты чтения text/pdf. + """ metadata: dict = { "extension": Path(original_name).suffix.lower(), "size_bytes": size, "mime_type": mime_type, } if mime_type.startswith("text/"): - # Cap read to avoid DoS on huge text files raw = path.read_bytes()[: settings.metadata_text_read_limit] content = raw.decode("utf-8", errors="ignore") metadata["line_count"] = len(content.splitlines()) @@ -42,6 +49,17 @@ def _extract_metadata( async def run_file_pipeline(*, session: AsyncSession, file_id: str) -> None: + """ + Единый pipeline: scan → metadata → alert. + + Args: + session: async SQLAlchemy-сессия worker-а. + file_id: UUID файла. + + Notes: + Идемпотентен для уже processed+alerted записей. + При отсутствии файла на диске выставляет failed + critical alert. + """ settings = get_settings() row = await files_data.get_pipeline_row(session=session, file_id=file_id) if row is None: @@ -137,7 +155,6 @@ async def run_file_pipeline(*, session: AsyncSession, file_id: str) -> None: ) return - # reload attention flags after scan commit row = await files_data.get_pipeline_row(session=session, file_id=file_id) assert row is not None diff --git a/backend/src/worker/jobs.py b/backend/src/worker/jobs.py index 29f1cdaf..d8ceb11a 100644 --- a/backend/src/worker/jobs.py +++ b/backend/src/worker/jobs.py @@ -1,9 +1,9 @@ -"""Background jobs: single pipeline task with retries.""" - from __future__ import annotations import asyncio import logging +from collections.abc import Coroutine +from typing import Any from celery.exceptions import MaxRetriesExceededError @@ -15,7 +15,8 @@ logger = logging.getLogger(__name__) -def _run_async(coro): +def _run_async(coro: Coroutine[Any, Any, Any]) -> Any: + """Выполнить coroutine в отдельном event loop worker-процесса.""" return asyncio.run(coro) @@ -26,6 +27,16 @@ def _run_async(coro): default_retry_delay=5, ) def process_uploaded_file(self, file_id: str) -> None: + """ + Celery-задача единого pipeline обработки загруженного файла. + + Args: + file_id: UUID файла в таблице files. + + Notes: + При transient-ошибках — exponential backoff retry. + После исчерпания retries помечает файл failed и пишет critical alert. + """ settings = get_settings() async def _body() -> None: @@ -89,4 +100,5 @@ async def _fail() -> None: def enqueue_file_pipeline(*, file_id: str) -> None: + """Поставить process_uploaded_file в очередь Celery.""" process_uploaded_file.delay(file_id) From cba77f2eaab33ddf3fe1ae9a802f4c081baec123 Mon Sep 17 00:00:00 2001 From: Nikolay Date: Sat, 11 Jul 2026 18:56:05 +0300 Subject: [PATCH 4/4] feat: silent polling, backend tests, RU architecture docs - Frontend: auto-refresh statuses while files are uploaded/processing - Backend: JSON-safe 422 validation errors; full API/pipeline/storage tests - Docs: ARCHITECTURE.md in Russian; drop external project name mentions --- .gitignore | 4 + ARCHITECTURE.md | 207 +++++++++++---- README.md | 2 +- backend/entrypoint.sh | 2 +- backend/src/http_errors.py | 19 +- backend/tests/conftest.py | 196 ++++++++++++++ backend/tests/test_api_endpoints.py | 280 ++++++++++++++++++++ backend/tests/test_columns.py | 19 +- backend/tests/test_dto_validation.py | 46 ++++ backend/tests/test_http_errors.py | 76 ++++++ backend/tests/test_pipeline.py | 179 +++++++++++++ backend/tests/test_scan_rules.py | 89 +++++-- backend/tests/test_storage.py | 125 +++++++++ backend/tests/test_use_cases_files.py | 79 ++++++ frontend/next.config.ts | 10 + frontend/package-lock.json | 19 +- frontend/package.json | 2 + frontend/src/app/page.tsx | 19 +- frontend/src/components/DashboardHeader.tsx | 25 +- frontend/src/hooks/useDashboardData.ts | 92 ++++++- frontend/tsconfig.json | 12 +- 21 files changed, 1406 insertions(+), 96 deletions(-) create mode 100644 backend/tests/conftest.py create mode 100644 backend/tests/test_api_endpoints.py create mode 100644 backend/tests/test_dto_validation.py create mode 100644 backend/tests/test_http_errors.py create mode 100644 backend/tests/test_pipeline.py create mode 100644 backend/tests/test_storage.py create mode 100644 backend/tests/test_use_cases_files.py diff --git a/.gitignore b/.gitignore index 879c0b52..2f3f3a87 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,7 @@ backend/storage/* # Local helpers _write_batch*.py uv.lock.bak + +# Local docker diagnostics (never commit) +.docker_*.txt +.docker_*.log diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 37f99f54..85e7cc1e 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,88 +1,183 @@ -# Architecture +# Архитектура -## Goal +## Цель -Refactor the baseline file-exchange MVP without changing business rules or public API paths. Optional optimizations: single background pipeline task, safer uploads, Docker resource limits, frontend layering. +Рефакторинг baseline-MVP файлообменника **без изменения бизнес-правил и публичных API-путей**. +Дополнительно: одна фоновая pipeline-задача, безопасный stream-upload, лимиты Docker, слои frontend, live-обновление статусов в UI, интеграционные/unit-тесты backend. -## Backend layers +--- -``` -api/ HTTP routes, validation, status mapping -use_cases/ orchestration (CRUD, pipeline) -data/ SQLAlchemy queries with column projections -entities/ ORM tables -dto/ Pydantic request/response models -storage/ disk I/O (stream, path safety) -scan/ pure scan rules -worker/ Celery app + single job +## Backend: слои + +```text +api/ HTTP-роуты, валидация входа, коды ответов +use_cases/ оркестрация (CRUD, pipeline) +data/ SQLAlchemy-запросы с projection-колонками +entities/ ORM-таблицы +dto/ Pydantic request/response +storage/ диск (stream, path safety) +scan/ pure-правила проверки (без I/O) +worker/ Celery app + одна job db/ engine / sessions -settings.py env configuration -errors.py domain exceptions -http_errors.py domain → HTTP +settings.py конфигурация из env +errors.py доменные исключения +http_errors.py domain → HTTP (в т.ч. безопасная сериализация 422) ``` -### Pipeline optimization +### Оптимизация pipeline + +**Было (baseline):** 3 Celery-задачи `scan → metadata → alert` += 3 hop в Redis + 3 DB session + 3 load по PK. + +**Стало:** одна задача `jobs.process_uploaded_file` +([worker/jobs.py](backend/src/worker/jobs.py) → [use_cases/pipeline.py](backend/src/use_cases/pipeline.py)): -Baseline used three Celery tasks (`scan → metadata → alert`): three broker hops, three DB sessions, three PK loads. +1. `processing` +2. scan (эвристики) +3. metadata (с лимитом чтения) +4. alert (`info` / `warning` / `critical`) +5. retries с exponential backoff; после исчерпания — `failed` + critical alert -Now one task `jobs.process_uploaded_file` runs scan, metadata extraction, and alert creation in one worker job with retries for transient failures. +**Бизнес-исходы не менялись:** -Business outcomes are unchanged: +| Область | Значения | +|---|---| +| `processing_status` | `uploaded` → `processing` → `processed` / `failed` | +| `scan_status` | `clean` / `suspicious` / `failed` (`null` до обработки) | +| scan rules | extension ∈ `{.exe,.bat,.cmd,.sh,.js}`, size > 10 MiB, pdf mime mismatch | +| alerts | `info` (успех), `warning` (suspicious), `critical` (failed) | -- statuses: `uploaded` → `processing` → `processed` / `failed` -- scan: clean vs suspicious (extension / >10MB / pdf mime mismatch) -- alerts: critical / warning / info (including success info) +Идемпотентность: если уже `processed` и alert есть → noop. -### Data access rules +### Доступ к данным -- List/get API select only `FileItem` / `AlertItem` columns (no `stored_name` on API reads) -- Download/delete/worker use narrow queries that include `stored_name` when needed -- Indexes: `files.created_at`, `alerts.created_at`, `alerts.file_id` -- `alerts.file_id` FK uses `ON DELETE CASCADE` +- List/get API выбирают только колонки `FileItem` / `AlertItem` (**без** `stored_name`) +- Download / delete / worker — узкие projection с `stored_name` при необходимости +- Индексы: `files.created_at`, `alerts.created_at`, `alerts.file_id` +- `alerts.file_id` FK: **ON DELETE CASCADE** -### Upload / DoS baseline +### Upload / защита от DoS -- Streamed writes (chunks), max size from `MAX_UPLOAD_BYTES` (default 50 MiB) -- Oversize → 413 + partial cleanup -- Empty file → 400 -- No resumable protocol; connection drop mid-upload cleans partial file +- Потоковая запись (chunk 64 KiB), лимит `MAX_UPLOAD_BYTES` (по умолчанию 50 MiB) +- Oversize → **413** + cleanup partial +- Empty → **400** +- Ошибка записи → **500** (без internals) + unlink partial +- Path resolve через basename — anti path-traversal +- Resumable protocol нет; обрыв соединения чистит partial + +### HTTP-ошибки + +| Domain | HTTP | +|---|---| +| ResourceNotFoundError | 404 | +| EmptyUploadError | 400 | +| PayloadTooLargeError | 413 | +| ValidationDomainError | 422 | +| RequestValidationError | 422 (JSON-safe: Exception в `ctx` → string) | +| StorageIOError / AppError / unhandled | 500, detail без internals | ### Auth -No tokens / JWT. Demo is open on LAN; hardening is path safety, limits, CORS, log hygiene. +Без токенов / JWT. Demo открыт в LAN; hardening: path safety, лимиты, CORS, log hygiene. -## Frontend layers +--- -``` -config/ API base URL -types/ DTO types -api/ HTTP client + resource methods -hooks/ dashboard load, upload form +## Frontend: слои + +```text +config/ API_BASE_URL +types/ DTO (FileItem, AlertItem) +api/ HTTP-клиент + resource methods +hooks/ useDashboardData (load + polling), useFileUpload components/ presentational UI -app/page composition only +app/page только composition ``` -## Docker +### UX: live-статусы (polling) + +После upload worker обновляет БД за сотни мс, а UI раньше делал **один** `loadData` → «залипал» на `uploaded` / UI-`pending`. + +**Сейчас** ([useDashboardData.ts](frontend/src/hooks/useDashboardData.ts)): + +- silent refresh после upload (без spinner таблиц) +- polling **1.5 s**, пока есть файлы в `uploaded` | `processing` +- kickoff **400 ms**, safety stop **120 s** +- coalesce concurrent loads (очередь silent refresh) +- индикатор в header: «Идёт фоновая обработка…» + +### basePath + +- `basePath: '/test'` — контракт задания, **сохранён** +- redirect `/` → `/test` (`basePath: false`) — удобный вход с корня + ([next.config.ts](frontend/next.config.ts)) + +--- -- Code is **copied into the image** for reproducible runs. -- Dev compose also **bind-mounts** `./backend` for uvicorn `--reload`. -- Uploaded blobs live in a **named volume** `file-storage` shared by API and worker (not "code out of container"). -- Resource limits via `deploy.resources` (memory/CPU) on every service. -- Postgres `127.0.0.1:5433→5432`, healthchecks, pinned alpine images. +## Docker / ops -## Running +- Код **копируется в образ** (`COPY`) — основной reproducible-запуск +- Dev compose **bind-mount** `./backend` для uvicorn `--reload` (DX, не «логика снаружи») +- Бинарники upload — named volume **`file-storage`**, общий API ↔ worker +- `deploy.resources` limits (memory/CPU) на сервисах +- Postgres host `127.0.0.1:5433→5432`, healthchecks, alpine images +- Entrypoint: + 1. wait Postgres + Redis (`nc`) + 2. `alembic upgrade head` **только API** + 3. start uvicorn / celery +- `restart: always` — подъём после reboot Docker Desktop + +--- + +## Тесты backend + +```text +tests/conftest.py fixtures: temp storage, truncate DB, ASGI client, mock enqueue +tests/test_api_endpoints.py все HTTP-эндпоинты, валидация, 400/413/404/422/204 +tests/test_pipeline.py clean / suspicious / missing disk / idempotent +tests/test_storage.py stream, empty, oversize, sanitize name, path safety +tests/test_scan_rules.py границы порогов и комбинации rules +tests/test_use_cases_files.py create/update/delete, soft-fail enqueue +tests/test_http_errors.py матрица domain → HTTP +tests/test_dto_validation.py FileUpdate / title normalize +tests/test_columns.py projection ≡ DTO, stored_name только где нужен +``` + +Запуск (в контейнере API или локально с dev-extra): + +```bash +cd backend +uv sync --extra dev +# Postgres доступен (compose: backend-db) +POSTGRES_HOST=127.0.0.1 POSTGRES_PORT=5433 PGPORT=5433 uv run pytest +``` + +--- + +## Запуск стека ```bash docker compose -f docker-compose.dev.yml up --build ``` -Startup (backend entrypoint, same idea as mail-manager / coffeestudio): -1. Wait for Postgres + Redis (`nc`) -2. `alembic upgrade head` (API container only) -3. Start uvicorn / celery +| URL | Назначение | +|---|---| +| http://localhost:3000/ | redirect → `/test` | +| http://localhost:3000/test | UI | +| http://localhost:8000/docs | OpenAPI | +| http://localhost:8000/health | liveness | + +--- + +## Что сознательно не меняли + +- Публичные path API и wire-format статусов (строки) +- `basePath: '/test'` +- Бизнес-правила scan / alert levels +- Auth (нет в ТЗ) -Containers use `restart: always` so Docker Desktop restarts the stack after reboot if it was running. +## Возможные следующие шаги -UI: http://localhost:3000/test -API docs: http://localhost:8000/docs -Health: http://localhost:8000/health +- StrEnum для статусов (БД остаётся `String`) +- UI rename/delete (API уже есть) +- reprocess endpoint для «застрявших» `uploaded` +- non-root Celery user diff --git a/README.md b/README.md index 0821247e..cb22a160 100644 --- a/README.md +++ b/README.md @@ -50,4 +50,4 @@ uv run pytest - **В образ** копируется исходный код (`COPY`) — основной способ запуска. - В dev compose `./backend` **примонтирован** для hot-reload; это не «вынос логики наружу», а DX. - Бинарные загрузки — в volume `file-storage`, общий для API и worker. -- Старт: wait DB/Redis → migrate → app (как в mail-manager / coffeestudio entrypoint). +- Старт: wait DB/Redis → migrate → app. diff --git a/backend/entrypoint.sh b/backend/entrypoint.sh index e5102517..d0a83b8f 100644 --- a/backend/entrypoint.sh +++ b/backend/entrypoint.sh @@ -21,7 +21,7 @@ done echo ">>> Redis is ready" # Migrations only in the API process to avoid races with celery workers -# (same idea as mail-manager: migrate when starting uvicorn). +# Run migrations only for the API process (not for the Celery worker). if [ "$1" = "uvicorn" ] || [ "${RUN_MIGRATIONS:-0}" = "1" ]; then echo ">>> Applying database migrations (alembic upgrade head)..." alembic upgrade head diff --git a/backend/src/http_errors.py b/backend/src/http_errors.py index b69fb544..c27e6a03 100644 --- a/backend/src/http_errors.py +++ b/backend/src/http_errors.py @@ -1,6 +1,7 @@ import logging from fastapi import FastAPI, Request +from fastapi.encoders import jsonable_encoder from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse from starlette import status @@ -17,6 +18,21 @@ logger = logging.getLogger(__name__) +def _json_safe_validation_errors(errors: list) -> list: + """Make RequestValidationError.errors() JSON-serializable.""" + safe: list = [] + for err in errors: + item = dict(err) + ctx = item.get("ctx") + if isinstance(ctx, dict): + item["ctx"] = { + key: (str(value) if isinstance(value, BaseException) else value) + for key, value in ctx.items() + } + safe.append(item) + return safe + + def register_exception_handlers(app: FastAPI) -> None: """Зарегистрировать маппинг доменных ошибок в HTTP-ответы.""" @@ -71,9 +87,10 @@ async def app_error_handler(_: Request, exc: AppError) -> JSONResponse: @app.exception_handler(RequestValidationError) async def request_validation_handler(_: Request, exc: RequestValidationError) -> JSONResponse: logger.warning("request_validation errors=%s", exc.errors()) + # Pydantic may put Exception instances into error ctx; stringify them. return JSONResponse( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - content={"detail": exc.errors()}, + content={"detail": jsonable_encoder(_json_safe_validation_errors(exc.errors()))}, ) @app.exception_handler(Exception) diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py new file mode 100644 index 00000000..0beed3a5 --- /dev/null +++ b/backend/tests/conftest.py @@ -0,0 +1,196 @@ +"""Pytest fixtures: isolated storage, DB cleanup, ASGI client, enqueue mock.""" + +from __future__ import annotations + +import os +import shutil +from collections.abc import AsyncIterator, Callable, Iterator +from pathlib import Path +from typing import Any +from uuid import uuid4 + +import pytest +import pytest_asyncio +from httpx import ASGITransport, AsyncClient +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker + +# Env must be set before Settings/engine are used in fixtures. +_DEFAULT_HOST = os.getenv("TEST_POSTGRES_HOST", os.getenv("POSTGRES_HOST", "127.0.0.1")) +_DEFAULT_PORT = os.getenv("TEST_POSTGRES_PORT", os.getenv("POSTGRES_PORT", "5433")) +# Inside docker-compose network the service is backend-db:5432. +if Path("/.dockerenv").exists() or os.getenv("POSTGRES_HOST") == "backend-db": + _DEFAULT_HOST = os.getenv("POSTGRES_HOST", "backend-db") + _DEFAULT_PORT = os.getenv("POSTGRES_PORT", "5432") + +os.environ.setdefault("POSTGRES_USER", "postgres") +os.environ.setdefault("POSTGRES_PASSWORD", "postgres") +os.environ.setdefault("POSTGRES_DB", "test") +os.environ["POSTGRES_HOST"] = _DEFAULT_HOST +os.environ["POSTGRES_PORT"] = str(_DEFAULT_PORT) +os.environ["PGPORT"] = str(_DEFAULT_PORT) +os.environ.setdefault("CELERY_BROKER_URL", "redis://localhost:6379/0") +os.environ.setdefault("REDIS_URL", "redis://localhost:6379/0") +os.environ.setdefault("LOG_LEVEL", "WARNING") +os.environ.setdefault("CORS_ORIGINS", "http://localhost:3000") + + +def _clear_settings_and_engine() -> None: + from src.db import session as session_mod + from src.settings import get_settings + + get_settings.cache_clear() + session_mod._engine = None + session_mod._session_factory = None + + +@pytest.fixture +def storage_dir(tmp_path: Path) -> Iterator[Path]: + """Temporary storage directory bound to settings.""" + path = tmp_path / "files" + path.mkdir(parents=True, exist_ok=True) + os.environ["STORAGE_DIR"] = str(path) + _clear_settings_and_engine() + yield path + shutil.rmtree(path, ignore_errors=True) + _clear_settings_and_engine() + + +@pytest.fixture +def max_upload_bytes() -> Iterator[int]: + """Lower upload limit for boundary tests (8 KiB).""" + value = 8 * 1024 + previous = os.environ.get("MAX_UPLOAD_BYTES") + os.environ["MAX_UPLOAD_BYTES"] = str(value) + _clear_settings_and_engine() + yield value + if previous is None: + os.environ.pop("MAX_UPLOAD_BYTES", None) + else: + os.environ["MAX_UPLOAD_BYTES"] = previous + _clear_settings_and_engine() + + +@pytest.fixture +def settings(storage_dir: Path, max_upload_bytes: int): + from src.settings import Settings, get_settings + + _clear_settings_and_engine() + cfg = get_settings() + assert Path(cfg.storage_dir) == storage_dir + assert cfg.max_upload_bytes == max_upload_bytes + return cfg + + +@pytest_asyncio.fixture +async def engine(settings) -> AsyncIterator[AsyncEngine]: + from src.db.session import create_engine, dispose_engine + + eng = create_engine(settings) + # Bind process singleton used by deps if anything touches it. + import src.db.session as session_mod + + session_mod._engine = eng + session_mod._session_factory = async_sessionmaker(eng, expire_on_commit=False) + yield eng + await dispose_engine() + + +@pytest_asyncio.fixture +async def db_session_factory(engine: AsyncEngine) -> async_sessionmaker[AsyncSession]: + return async_sessionmaker(engine, expire_on_commit=False) + + +@pytest_asyncio.fixture +async def clean_db(engine: AsyncEngine) -> AsyncIterator[None]: + """Truncate domain tables before and after each test.""" + async with engine.begin() as conn: + await conn.execute(text("TRUNCATE TABLE alerts, files RESTART IDENTITY CASCADE")) + yield + async with engine.begin() as conn: + await conn.execute(text("TRUNCATE TABLE alerts, files RESTART IDENTITY CASCADE")) + + +@pytest.fixture +def enqueue_calls() -> list[str]: + return [] + + +@pytest.fixture +def enqueue_ok(enqueue_calls: list[str]) -> Callable[..., None]: + def _enqueue(*, file_id: str) -> None: + enqueue_calls.append(file_id) + + return _enqueue + + +@pytest.fixture +def enqueue_fail() -> Callable[..., None]: + def _enqueue(*, file_id: str) -> None: + raise RuntimeError(f"broker down for {file_id}") + + return _enqueue + + +@pytest_asyncio.fixture +async def app(settings, engine: AsyncEngine, clean_db, enqueue_ok, monkeypatch: pytest.MonkeyPatch): + """FastAPI app with mocked Celery enqueue and isolated storage/DB.""" + from src.api.deps import db_session + from src.application import create_app + from src.db.session import get_session_factory + + monkeypatch.setattr("src.api.routes_files.enqueue_file_pipeline", enqueue_ok) + monkeypatch.setattr("src.worker.jobs.enqueue_file_pipeline", enqueue_ok) + + application = create_app() + + async def _override_db() -> AsyncIterator[AsyncSession]: + factory = get_session_factory() + async with factory() as session: + yield session + + application.dependency_overrides[db_session] = _override_db + yield application + application.dependency_overrides.clear() + + +@pytest_asyncio.fixture +async def client(app) -> AsyncIterator[AsyncClient]: + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as ac: + yield ac + + +@pytest.fixture +def make_upload() -> Callable[..., tuple[str, bytes, str]]: + """Build (filename, content, content_type) tuples for multipart.""" + + def _make( + *, + name: str = "note.txt", + content: bytes = b"hello world\n", + content_type: str = "text/plain", + ) -> tuple[str, bytes, str]: + return name, content, content_type + + return _make + + +async def upload_file( + client: AsyncClient, + *, + title: str = "Demo", + filename: str = "note.txt", + content: bytes = b"hello world\n", + content_type: str = "text/plain", +) -> Any: + """POST /files helper.""" + return await client.post( + "/files", + data={"title": title}, + files={"file": (filename, content, content_type)}, + ) + + +def unique_title(prefix: str = "file") -> str: + return f"{prefix}-{uuid4().hex[:8]}" diff --git a/backend/tests/test_api_endpoints.py b/backend/tests/test_api_endpoints.py new file mode 100644 index 00000000..b1923671 --- /dev/null +++ b/backend/tests/test_api_endpoints.py @@ -0,0 +1,280 @@ +"""API tests: system + files + alerts endpoints, validation, errors.""" + +from __future__ import annotations + +from pathlib import Path +from uuid import uuid4 + +import pytest +from httpx import AsyncClient +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from src.data import alerts as alerts_data +from src.data import files as files_data +from src.use_cases.pipeline import run_file_pipeline +from tests.conftest import unique_title, upload_file + + +@pytest.mark.asyncio +async def test_health(client: AsyncClient) -> None: + response = await client.get("/health") + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + + +@pytest.mark.asyncio +async def test_list_files_empty(client: AsyncClient) -> None: + response = await client.get("/files") + assert response.status_code == 200 + assert response.json() == [] + + +@pytest.mark.asyncio +async def test_list_alerts_empty(client: AsyncClient) -> None: + response = await client.get("/alerts") + assert response.status_code == 200 + assert response.json() == [] + + +@pytest.mark.asyncio +async def test_upload_success_shape_and_no_stored_name( + client: AsyncClient, enqueue_calls: list[str], settings +) -> None: + title = unique_title("ok") + response = await upload_file( + client, + title=title, + filename="note.txt", + content=b"hello\nworld\n", + content_type="text/plain", + ) + assert response.status_code == 201, response.text + body = response.json() + assert body["title"] == title + assert body["original_name"] == "note.txt" + assert body["mime_type"] == "text/plain" + assert body["size"] == 12 + assert body["processing_status"] == "uploaded" + assert body["scan_status"] is None + assert body["requires_attention"] is False + assert "stored_name" not in body + assert body["id"] in enqueue_calls + # object exists on disk + files = list(Path(settings.storage_dir).iterdir()) + assert len(files) == 1 + + +@pytest.mark.asyncio +async def test_upload_empty_file_400(client: AsyncClient) -> None: + response = await upload_file(client, title="empty", filename="e.bin", content=b"") + assert response.status_code == 400 + assert response.json()["detail"] == "File is empty" + + +@pytest.mark.asyncio +async def test_upload_oversize_413(client: AsyncClient, settings) -> None: + content = b"x" * (settings.max_upload_bytes + 1) + response = await upload_file(client, title="big", filename="big.bin", content=content) + assert response.status_code == 413 + assert "max size" in response.json()["detail"] + assert list(Path(settings.storage_dir).iterdir()) == [] + + +@pytest.mark.asyncio +async def test_upload_missing_title_422(client: AsyncClient) -> None: + response = await client.post( + "/files", + files={"file": ("a.txt", b"data", "text/plain")}, + ) + assert response.status_code == 422 + + +@pytest.mark.asyncio +async def test_upload_missing_file_422(client: AsyncClient) -> None: + response = await client.post("/files", data={"title": "no-file"}) + assert response.status_code == 422 + + +@pytest.mark.asyncio +async def test_upload_title_whitespace_only_422(client: AsyncClient) -> None: + response = await upload_file(client, title=" ", filename="a.txt", content=b"x") + assert response.status_code == 422 + assert response.json()["detail"] == "title must not be empty" + + +@pytest.mark.asyncio +async def test_upload_title_max_length_boundary(client: AsyncClient) -> None: + ok = "a" * 255 + response = await upload_file(client, title=ok, content=b"x") + assert response.status_code == 201 + assert response.json()["title"] == ok + + too_long = "a" * 256 + response2 = await upload_file(client, title=too_long, content=b"x") + assert response2.status_code == 422 + + +@pytest.mark.asyncio +async def test_upload_strips_title(client: AsyncClient) -> None: + response = await upload_file(client, title=" padded ", content=b"x") + assert response.status_code == 201 + assert response.json()["title"] == "padded" + + +@pytest.mark.asyncio +async def test_upload_enqueue_failure_keeps_row( + client: AsyncClient, app, enqueue_fail, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr("src.api.routes_files.enqueue_file_pipeline", enqueue_fail) + response = await upload_file(client, title="kept", content=b"payload") + assert response.status_code == 201 + file_id = response.json()["id"] + got = await client.get(f"/files/{file_id}") + assert got.status_code == 200 + assert got.json()["processing_status"] == "uploaded" + + +@pytest.mark.asyncio +async def test_get_file_ok_and_404_and_invalid_id(client: AsyncClient) -> None: + created = await upload_file(client, title="g", content=b"x") + file_id = created.json()["id"] + + ok = await client.get(f"/files/{file_id}") + assert ok.status_code == 200 + assert ok.json()["id"] == file_id + assert "stored_name" not in ok.json() + + missing = await client.get(f"/files/{uuid4()}") + assert missing.status_code == 404 + assert missing.json()["detail"] == "File not found" + + # wrong length → FastAPI Path min/max_length 422 (list detail) + bad = await client.get("/files/not-a-uuid") + assert bad.status_code == 422 + assert isinstance(bad.json()["detail"], list) + + short = await client.get("/files/123") + assert short.status_code == 422 + + # length 36 but not a UUID → custom domain-style message from file_id_path + invalid_uuid = "z" * 36 + invalid = await client.get(f"/files/{invalid_uuid}") + assert invalid.status_code == 422 + assert invalid.json()["detail"] == "Invalid file_id format" + + +@pytest.mark.asyncio +async def test_patch_title_ok_and_validation(client: AsyncClient) -> None: + created = await upload_file(client, title="old", content=b"x") + file_id = created.json()["id"] + + ok = await client.patch(f"/files/{file_id}", json={"title": " new title "}) + assert ok.status_code == 200 + assert ok.json()["title"] == "new title" + + empty = await client.patch(f"/files/{file_id}", json={"title": " "}) + assert empty.status_code == 422 + # Pydantic field_validator → RequestValidationError list detail + assert isinstance(empty.json()["detail"], list) + + missing_body = await client.patch(f"/files/{file_id}", json={}) + assert missing_body.status_code == 422 + + not_found = await client.patch(f"/files/{uuid4()}", json={"title": "x"}) + assert not_found.status_code == 404 + + +@pytest.mark.asyncio +async def test_download_ok_and_missing(client: AsyncClient, settings) -> None: + created = await upload_file( + client, + title="dl", + filename="report.txt", + content=b"download-me", + content_type="text/plain", + ) + file_id = created.json()["id"] + + response = await client.get(f"/files/{file_id}/download") + assert response.status_code == 200 + assert response.content == b"download-me" + assert "report.txt" in response.headers.get("content-disposition", "") + + # remove disk object → 404 Stored file not found + for path in Path(settings.storage_dir).iterdir(): + path.unlink() + missing_disk = await client.get(f"/files/{file_id}/download") + assert missing_disk.status_code == 404 + assert missing_disk.json()["detail"] == "Stored file not found" + + missing_row = await client.get(f"/files/{uuid4()}/download") + assert missing_row.status_code == 404 + + +@pytest.mark.asyncio +async def test_delete_and_cascade_alerts( + client: AsyncClient, + db_session_factory: async_sessionmaker[AsyncSession], + settings, +) -> None: + created = await upload_file(client, title="to-del", filename="a.txt", content=b"z") + file_id = created.json()["id"] + + # run pipeline to create alert + async with db_session_factory() as session: + await run_file_pipeline(session=session, file_id=file_id) + + alerts_before = await client.get("/alerts") + assert any(a["file_id"] == file_id for a in alerts_before.json()) + + deleted = await client.delete(f"/files/{file_id}") + assert deleted.status_code == 204 + assert deleted.content == b"" + + assert (await client.get(f"/files/{file_id}")).status_code == 404 + alerts_after = await client.get("/alerts") + assert all(a["file_id"] != file_id for a in alerts_after.json()) + + # disk cleaned + assert list(Path(settings.storage_dir).glob(f"{file_id}*")) == [] + + again = await client.delete(f"/files/{file_id}") + assert again.status_code == 404 + + +@pytest.mark.asyncio +async def test_list_files_order_and_alerts_after_pipeline( + client: AsyncClient, + db_session_factory: async_sessionmaker[AsyncSession], +) -> None: + first = await upload_file(client, title="first", filename="a.txt", content=b"1") + second = await upload_file(client, title="second", filename="b.js", content=b"2") + id1, id2 = first.json()["id"], second.json()["id"] + + async with db_session_factory() as session: + await run_file_pipeline(session=session, file_id=id1) + await run_file_pipeline(session=session, file_id=id2) + + files = (await client.get("/files")).json() + assert [f["id"] for f in files][:2] == [id2, id1] # created_at desc + + f2 = next(f for f in files if f["id"] == id2) + assert f2["scan_status"] == "suspicious" + assert f2["processing_status"] == "processed" + + alerts = (await client.get("/alerts")).json() + levels = {a["file_id"]: a["level"] for a in alerts} + assert levels[id1] == "info" + assert levels[id2] == "warning" + for alert in alerts: + assert set(alert.keys()) == {"id", "file_id", "level", "message", "created_at"} + + +@pytest.mark.asyncio +async def test_openapi_available(client: AsyncClient) -> None: + response = await client.get("/openapi.json") + assert response.status_code == 200 + paths = response.json()["paths"] + assert "/files" in paths + assert "/alerts" in paths + assert "/health" in paths diff --git a/backend/tests/test_columns.py b/backend/tests/test_columns.py index 802f7b07..4d1a7d22 100644 --- a/backend/tests/test_columns.py +++ b/backend/tests/test_columns.py @@ -1,4 +1,5 @@ -from src.data.columns import FILE_ITEM_COLUMNS +from src.data.columns import ALERT_ITEM_COLUMNS, FILE_ITEM_COLUMNS, PIPELINE_COLUMNS, STORAGE_REF_COLUMNS +from src.dto.alerts import AlertItem from src.dto.files import FileItem from src.entities.stored_file import StoredFile @@ -9,3 +10,19 @@ def test_file_item_projection_excludes_stored_name(): schema_fields = set(FileItem.model_fields.keys()) assert names == schema_fields assert "stored_name" in StoredFile.__table__.c + + +def test_alert_item_projection_matches_dto(): + names = {col.key for col in ALERT_ITEM_COLUMNS} + assert names == set(AlertItem.model_fields.keys()) + + +def test_storage_ref_includes_stored_name(): + names = {col.key for col in STORAGE_REF_COLUMNS} + assert "stored_name" in names + assert "id" in names + + +def test_pipeline_columns_include_worker_fields(): + names = {col.key for col in PIPELINE_COLUMNS} + assert {"stored_name", "processing_status", "scan_status", "metadata_json"} <= names diff --git a/backend/tests/test_dto_validation.py b/backend/tests/test_dto_validation.py new file mode 100644 index 00000000..26ebff6a --- /dev/null +++ b/backend/tests/test_dto_validation.py @@ -0,0 +1,46 @@ +"""DTO and domain validation unit tests.""" + +import pytest +from pydantic import ValidationError + +from src.dto.common import file_id_path +from src.dto.files import FileUpdate +from src.errors import ValidationDomainError +from src.use_cases.files import _normalize_title + + +def test_file_update_strips_and_rejects_blank() -> None: + assert FileUpdate(title=" ok ").title == "ok" + with pytest.raises(ValidationError): + FileUpdate(title=" ") + with pytest.raises(ValidationError): + FileUpdate(title="") + + +def test_file_update_max_length() -> None: + FileUpdate(title="a" * 255) + with pytest.raises(ValidationError): + FileUpdate(title="a" * 256) + + +def test_normalize_title() -> None: + assert _normalize_title(" x ") == "x" + with pytest.raises(ValidationDomainError): + _normalize_title(" ") + with pytest.raises(ValidationDomainError): + _normalize_title("a" * 256) + + +@pytest.mark.asyncio +async def test_file_id_path_valid() -> None: + value = "550e8400-e29b-41d4-a716-446655440000" + # Depends-style call: function expects default Path injection; call body logic via UUID check + from uuid import UUID + + assert str(UUID(value)) == value + + +def test_file_item_schema_excludes_stored_name() -> None: + from src.dto.files import FileItem + + assert "stored_name" not in FileItem.model_fields diff --git a/backend/tests/test_http_errors.py b/backend/tests/test_http_errors.py new file mode 100644 index 00000000..58dbc5d3 --- /dev/null +++ b/backend/tests/test_http_errors.py @@ -0,0 +1,76 @@ +"""HTTP exception mapping tests via real domain raises on routes.""" + +from __future__ import annotations + +import pytest +from fastapi import FastAPI +from httpx import ASGITransport, AsyncClient + +from src.errors import ( + AppError, + EmptyUploadError, + PayloadTooLargeError, + ResourceNotFoundError, + StorageIOError, + ValidationDomainError, +) +from src.http_errors import register_exception_handlers + + +@pytest.fixture +def error_app() -> FastAPI: + app = FastAPI() + register_exception_handlers(app) + + @app.get("/not-found") + async def not_found() -> None: + raise ResourceNotFoundError("missing") + + @app.get("/empty") + async def empty() -> None: + raise EmptyUploadError("File is empty") + + @app.get("/large") + async def large() -> None: + raise PayloadTooLargeError("too big") + + @app.get("/domain-422") + async def domain_422() -> None: + raise ValidationDomainError("bad title") + + @app.get("/storage") + async def storage() -> None: + raise StorageIOError("disk") + + @app.get("/app-error") + async def app_error() -> None: + raise AppError("generic") + + @app.get("/boom") + async def boom() -> None: + raise RuntimeError("secret internals") + + return app + + +@pytest.mark.asyncio +async def test_error_mapping_matrix(error_app: FastAPI) -> None: + # raise_app_exceptions=False: generic Exception handler returns 500 JSON + # instead of bubbling RuntimeError through ServerErrorMiddleware. + transport = ASGITransport(app=error_app, raise_app_exceptions=False) + async with AsyncClient(transport=transport, base_url="http://test") as client: + cases = [ + ("/not-found", 404, "missing"), + ("/empty", 400, "File is empty"), + ("/large", 413, "too big"), + ("/domain-422", 422, "bad title"), + ("/storage", 500, "Internal server error"), + ("/app-error", 500, "Internal server error"), + ("/boom", 500, "Internal server error"), + ] + for path, status, detail in cases: + response = await client.get(path) + assert response.status_code == status, f"{path}: {response.status_code} {response.text}" + assert response.json()["detail"] == detail + assert "secret" not in response.text + assert "Traceback" not in response.text diff --git a/backend/tests/test_pipeline.py b/backend/tests/test_pipeline.py new file mode 100644 index 00000000..5d67d35b --- /dev/null +++ b/backend/tests/test_pipeline.py @@ -0,0 +1,179 @@ +"""Pipeline use-case tests: clean/suspicious/failed/idempotent paths.""" + +from __future__ import annotations + +from pathlib import Path +from uuid import uuid4 + +import pytest +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from src.data import alerts as alerts_data +from src.data import files as files_data +from src.settings import Settings +from src.storage.local import resolve_under_storage +from src.use_cases.pipeline import _extract_metadata, run_file_pipeline + + +async def _insert_uploaded( + session: AsyncSession, + *, + settings: Settings, + original_name: str, + mime_type: str, + content: bytes, + title: str = "t", +) -> str: + file_id = str(uuid4()) + stored_name = f"{file_id}{Path(original_name).suffix}" + path = resolve_under_storage(stored_name=stored_name, settings=settings) + path.write_bytes(content) + await files_data.insert_file( + session=session, + file_id=file_id, + title=title, + original_name=original_name, + stored_name=stored_name, + mime_type=mime_type, + size=len(content), + ) + await session.commit() + return file_id + + +@pytest.mark.asyncio +async def test_pipeline_clean_text( + settings, db_session_factory: async_sessionmaker[AsyncSession], clean_db +) -> None: + async with db_session_factory() as session: + file_id = await _insert_uploaded( + session, + settings=settings, + original_name="note.txt", + mime_type="text/plain", + content=b"line1\nline2\n", + ) + await run_file_pipeline(session=session, file_id=file_id) + + async with db_session_factory() as session: + item = await files_data.get_file_item(session=session, file_id=file_id) + assert item is not None + assert item.processing_status == "processed" + assert item.scan_status == "clean" + assert item.requires_attention is False + assert item.metadata_json is not None + assert item.metadata_json["line_count"] == 2 + assert item.metadata_json["char_count"] == 12 + assert await alerts_data.alert_exists_for_file(session=session, file_id=file_id) + alerts = await alerts_data.list_alert_items(session=session) + assert alerts[0].level == "info" + assert alerts[0].message == "File processed successfully" + + +@pytest.mark.asyncio +async def test_pipeline_suspicious_js_warning_alert( + settings, db_session_factory: async_sessionmaker[AsyncSession], clean_db +) -> None: + async with db_session_factory() as session: + file_id = await _insert_uploaded( + session, + settings=settings, + original_name="payload.js", + mime_type="application/javascript", + content=b"alert(1)", + ) + await run_file_pipeline(session=session, file_id=file_id) + + async with db_session_factory() as session: + item = await files_data.get_file_item(session=session, file_id=file_id) + assert item is not None + assert item.processing_status == "processed" + assert item.scan_status == "suspicious" + assert item.requires_attention is True + alerts = await alerts_data.list_alert_items(session=session) + assert alerts[0].level == "warning" + assert "requires attention" in alerts[0].message + + +@pytest.mark.asyncio +async def test_pipeline_missing_disk_failed_critical( + settings, db_session_factory: async_sessionmaker[AsyncSession], clean_db +) -> None: + file_id = str(uuid4()) + stored_name = f"{file_id}.bin" + async with db_session_factory() as session: + await files_data.insert_file( + session=session, + file_id=file_id, + title="ghost", + original_name="ghost.bin", + stored_name=stored_name, + mime_type="application/octet-stream", + size=10, + ) + await session.commit() + # no file on disk + await run_file_pipeline(session=session, file_id=file_id) + + async with db_session_factory() as session: + item = await files_data.get_file_item(session=session, file_id=file_id) + assert item is not None + assert item.processing_status == "failed" + assert "stored file not found" in (item.scan_details or "") + alerts = await alerts_data.list_alert_items(session=session) + assert alerts[0].level == "critical" + + +@pytest.mark.asyncio +async def test_pipeline_idempotent_noop( + settings, db_session_factory: async_sessionmaker[AsyncSession], clean_db +) -> None: + async with db_session_factory() as session: + file_id = await _insert_uploaded( + session, + settings=settings, + original_name="note.txt", + mime_type="text/plain", + content=b"x", + ) + await run_file_pipeline(session=session, file_id=file_id) + await run_file_pipeline(session=session, file_id=file_id) + + async with db_session_factory() as session: + alerts = await alerts_data.list_alert_items(session=session) + assert len([a for a in alerts if a.file_id == file_id]) == 1 + + +@pytest.mark.asyncio +async def test_pipeline_skip_missing_row( + settings, db_session_factory: async_sessionmaker[AsyncSession], clean_db +) -> None: + async with db_session_factory() as session: + await run_file_pipeline(session=session, file_id=str(uuid4())) + + +def test_extract_metadata_pdf(settings, tmp_path: Path) -> None: + path = tmp_path / "a.pdf" + path.write_bytes(b"%PDF-1.4 /Type /Page /Type /Page") + meta = _extract_metadata( + path=path, + original_name="a.pdf", + mime_type="application/pdf", + size=path.stat().st_size, + settings=settings, + ) + assert meta["approx_page_count"] >= 1 + assert meta["extension"] == ".pdf" + + +def test_extract_metadata_binary_only_base(settings, tmp_path: Path) -> None: + path = tmp_path / "a.bin" + path.write_bytes(b"\x00\x01") + meta = _extract_metadata( + path=path, + original_name="a.bin", + mime_type="application/octet-stream", + size=2, + settings=settings, + ) + assert set(meta.keys()) == {"extension", "size_bytes", "mime_type"} diff --git a/backend/tests/test_scan_rules.py b/backend/tests/test_scan_rules.py index 55c388b9..3c12b78f 100644 --- a/backend/tests/test_scan_rules.py +++ b/backend/tests/test_scan_rules.py @@ -1,41 +1,92 @@ -from src.scan.rules import evaluate_scan +"""Unit tests for scan heuristics — boundaries and combinations.""" +import pytest -def test_clean_text_file(): +from src.scan.rules import SIZE_THRESHOLD_BYTES, evaluate_scan + + +@pytest.mark.parametrize( + "name,mime,size", + [ + ("note.txt", "text/plain", 1), + ("note.txt", "text/plain", SIZE_THRESHOLD_BYTES), + ("doc.pdf", "application/pdf", 100), + ("doc.pdf", "application/octet-stream", 100), + ("image.PNG", "image/png", 500), + ("noext", "application/octet-stream", 10), + ], +) +def test_clean_cases(name: str, mime: str, size: int) -> None: + result = evaluate_scan(original_name=name, mime_type=mime, size=size) + assert result.scan_status == "clean" + assert result.requires_attention is False + assert result.scan_details == "no threats found" + + +@pytest.mark.parametrize("ext", [".exe", ".bat", ".cmd", ".sh", ".js", ".JS", ".Exe"]) +def test_suspicious_extensions(ext: str) -> None: + # Path.suffix.lower() normalizes case for extension check via .lower() on suffix + name = f"payload{ext}" + result = evaluate_scan(original_name=name, mime_type="application/octet-stream", size=10) + # evaluate_scan uses Path.suffix.lower() + assert result.scan_status == "suspicious" + assert result.requires_attention is True + assert "suspicious extension" in result.scan_details + + +def test_size_boundary_equal_is_clean() -> None: result = evaluate_scan( - original_name="note.txt", - mime_type="text/plain", - size=100, + original_name="big.bin", + mime_type="application/octet-stream", + size=SIZE_THRESHOLD_BYTES, ) assert result.scan_status == "clean" - assert result.requires_attention is False -def test_suspicious_extension(): +def test_size_boundary_plus_one_is_suspicious() -> None: result = evaluate_scan( - original_name="payload.js", - mime_type="application/javascript", - size=10, + original_name="big.bin", + mime_type="application/octet-stream", + size=SIZE_THRESHOLD_BYTES + 1, ) assert result.scan_status == "suspicious" - assert "suspicious extension .js" in result.scan_details + assert "larger than 10 MB" in result.scan_details -def test_large_file(): +def test_custom_size_threshold() -> None: result = evaluate_scan( - original_name="big.bin", + original_name="x.bin", mime_type="application/octet-stream", - size=11 * 1024 * 1024, + size=101, + size_threshold_bytes=100, ) assert result.scan_status == "suspicious" - assert "10 MB" in result.scan_details -def test_pdf_mime_mismatch(): +def test_pdf_mime_mismatch() -> None: + result = evaluate_scan(original_name="doc.pdf", mime_type="text/plain", size=10) + assert result.scan_status == "suspicious" + assert "pdf extension does not match mime type" in result.scan_details + + +def test_combined_reasons() -> None: result = evaluate_scan( - original_name="doc.pdf", - mime_type="text/plain", - size=10, + original_name="payload.js", + mime_type="application/javascript", + size=SIZE_THRESHOLD_BYTES + 5, ) assert result.scan_status == "suspicious" + assert "suspicious extension .js" in result.scan_details + assert "larger than 10 MB" in result.scan_details + assert ", " in result.scan_details + + +def test_pdf_and_large_combined() -> None: + result = evaluate_scan( + original_name="huge.pdf", + mime_type="text/html", + size=SIZE_THRESHOLD_BYTES + 1, + ) + assert result.requires_attention is True + assert "larger than 10 MB" in result.scan_details assert "pdf extension" in result.scan_details diff --git a/backend/tests/test_storage.py b/backend/tests/test_storage.py new file mode 100644 index 00000000..a62828a1 --- /dev/null +++ b/backend/tests/test_storage.py @@ -0,0 +1,125 @@ +"""Unit tests for local storage helpers and stream upload boundaries.""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import AsyncMock + +import pytest +from fastapi import UploadFile + +from src.errors import EmptyUploadError, PayloadTooLargeError, StorageIOError +from src.storage.local import ( + build_stored_name, + delete_stored, + ensure_storage_dir, + resolve_under_storage, + sanitize_download_name, + save_upload_stream, +) + + +def test_ensure_storage_dir(settings) -> None: + path = ensure_storage_dir(settings=settings) + assert path.exists() + assert path.is_dir() + + +def test_build_stored_name_safe_suffix() -> None: + assert build_stored_name(file_id="abc", original_filename="a.txt") == "abc.txt" + name = build_stored_name(file_id="id1", original_filename="report.PDF") + assert name == "id1.PDF" + + +def test_build_stored_name_strips_unsafe_suffix() -> None: + # spaces / unicode not allowed in suffix pattern + assert build_stored_name(file_id="id1", original_filename="x.bad suffix") == "id1" + assert build_stored_name(file_id="id1", original_filename=None) == "id1" + + +@pytest.mark.parametrize( + "raw,expected", + [ + ("normal.txt", "normal.txt"), + ("../etc/passwd", "passwd"), + ("a\\b\\c.txt", "a_b_c.txt"), + ("line\r\nbreak.txt", "linebreak.txt"), + ("", "download.bin"), + ], +) +def test_sanitize_download_name(raw: str, expected: str) -> None: + result = sanitize_download_name(raw) + assert result == expected + assert "/" not in result + assert "\\" not in result + assert "\n" not in result + assert result # never empty + + +def test_resolve_under_storage_uses_basename(settings) -> None: + path = resolve_under_storage(stored_name="../evil.txt", settings=settings) + assert path.parent == Path(settings.storage_dir).resolve() + assert path.name == "evil.txt" + + +@pytest.mark.asyncio +async def test_save_upload_stream_success(settings) -> None: + content = b"abc123" + upload = UploadFile(filename="note.txt", file=AsyncMock()) + upload.read = AsyncMock(side_effect=[content, b""]) + + stored_name, size = await save_upload_stream( + upload_file=upload, + file_id="11111111-1111-1111-1111-111111111111", + settings=settings, + ) + assert size == len(content) + assert stored_name.endswith(".txt") + path = resolve_under_storage(stored_name=stored_name, settings=settings) + assert path.read_bytes() == content + + +@pytest.mark.asyncio +async def test_save_upload_stream_empty(settings) -> None: + upload = UploadFile(filename="empty.bin", file=AsyncMock()) + upload.read = AsyncMock(return_value=b"") + with pytest.raises(EmptyUploadError, match="File is empty"): + await save_upload_stream( + upload_file=upload, + file_id="22222222-2222-2222-2222-222222222222", + settings=settings, + ) + + +@pytest.mark.asyncio +async def test_save_upload_stream_oversize(settings) -> None: + # settings.max_upload_bytes is 8 KiB from fixture + chunk = b"x" * (settings.max_upload_bytes + 1) + upload = UploadFile(filename="big.bin", file=AsyncMock()) + upload.read = AsyncMock(side_effect=[chunk, b""]) + with pytest.raises(PayloadTooLargeError): + await save_upload_stream( + upload_file=upload, + file_id="33333333-3333-3333-3333-333333333333", + settings=settings, + ) + # partial must be cleaned + leftovers = list(Path(settings.storage_dir).glob("33333333*")) + assert leftovers == [] + + +@pytest.mark.asyncio +async def test_save_upload_stream_io_error_cleans_partial(settings, monkeypatch) -> None: + upload = UploadFile(filename="x.bin", file=AsyncMock()) + upload.read = AsyncMock(side_effect=OSError("disk full")) + with pytest.raises(StorageIOError, match="failed to store upload"): + await save_upload_stream( + upload_file=upload, + file_id="44444444-4444-4444-4444-444444444444", + settings=settings, + ) + + +@pytest.mark.asyncio +async def test_delete_stored_missing_is_silent(settings) -> None: + await delete_stored(stored_name="missing-object.bin", settings=settings) diff --git a/backend/tests/test_use_cases_files.py b/backend/tests/test_use_cases_files.py new file mode 100644 index 00000000..e45d426b --- /dev/null +++ b/backend/tests/test_use_cases_files.py @@ -0,0 +1,79 @@ +"""Use-case level tests for create/update/delete with injectable enqueue.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock +from uuid import uuid4 + +import pytest +from fastapi import UploadFile +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker +from starlette.datastructures import Headers + +from src.data import files as files_data +from src.errors import ResourceNotFoundError, ValidationDomainError +from src.use_cases import files as files_uc + + +def _upload(filename: str, content: bytes, content_type: str = "text/plain") -> UploadFile: + upload = UploadFile( + filename=filename, + file=AsyncMock(), + headers=Headers({"content-type": content_type}), + ) + upload.read = AsyncMock(side_effect=[content, b""]) + return upload + + +@pytest.mark.asyncio +async def test_create_file_success_and_enqueue( + settings, db_session_factory: async_sessionmaker[AsyncSession], clean_db +) -> None: + calls: list[str] = [] + + async with db_session_factory() as session: + item = await files_uc.create_file( + session=session, + title=" hello ", + upload_file=_upload("a.txt", b"data"), + enqueue=lambda **kw: calls.append(kw["file_id"]), + ) + assert item.title == "hello" + assert item.processing_status == "uploaded" + assert item.id in calls + assert "stored_name" not in item.model_dump() + + +@pytest.mark.asyncio +async def test_create_file_enqueue_error_keeps_row( + settings, db_session_factory: async_sessionmaker[AsyncSession], clean_db +) -> None: + def boom(*, file_id: str) -> None: + raise RuntimeError("redis down") + + async with db_session_factory() as session: + item = await files_uc.create_file( + session=session, + title="kept", + upload_file=_upload("a.txt", b"data"), + enqueue=boom, + ) + loaded = await files_data.get_file_item(session=session, file_id=item.id) + assert loaded is not None + assert loaded.processing_status == "uploaded" + + +@pytest.mark.asyncio +async def test_update_delete_not_found( + settings, db_session_factory: async_sessionmaker[AsyncSession], clean_db +) -> None: + missing = str(uuid4()) + async with db_session_factory() as session: + with pytest.raises(ResourceNotFoundError): + await files_uc.get_file(session=session, file_id=missing) + with pytest.raises(ResourceNotFoundError): + await files_uc.update_file(session=session, file_id=missing, title="x") + with pytest.raises(ResourceNotFoundError): + await files_uc.delete_file(session=session, file_id=missing) + with pytest.raises(ValidationDomainError): + await files_uc.update_file(session=session, file_id=missing, title=" ") diff --git a/frontend/next.config.ts b/frontend/next.config.ts index c99d5c0d..f00c6426 100644 --- a/frontend/next.config.ts +++ b/frontend/next.config.ts @@ -3,6 +3,16 @@ import type { NextConfig } from "next"; const nextConfig: NextConfig = { output: 'standalone', basePath: '/test', + async redirects() { + return [ + { + source: '/', + destination: '/test', + permanent: false, + basePath: false, + }, + ]; + }, }; export default nextConfig; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 3483508e..dd42c62e 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -13,6 +13,8 @@ }, "devDependencies": { "@types/node": "^20.11.0", + "@types/react": "^18.2.0", + "@types/react-dom": "^18.2.0", "typescript": "^5.4.0" } }, @@ -740,14 +742,25 @@ "license": "MIT" }, "node_modules/@types/react": { - "version": "19.2.14", - "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.14.tgz", - "integrity": "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==", + "version": "18.3.31", + "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.31.tgz", + "integrity": "sha512-vfEqpXTvwT91yhmwdfouStN2hSKwTvyRs8qpLfADyrq/kxDw0hZM7Wk9Ug1FELj8hIby+S/+kQCSRFF32nv2Qw==", "license": "MIT", "dependencies": { + "@types/prop-types": "*", "csstype": "^3.2.2" } }, + "node_modules/@types/react-dom": { + "version": "18.3.7", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.3.7.tgz", + "integrity": "sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "^18.0.0" + } + }, "node_modules/@types/react-transition-group": { "version": "4.4.12", "resolved": "https://registry.npmjs.org/@types/react-transition-group/-/react-transition-group-4.4.12.tgz", diff --git a/frontend/package.json b/frontend/package.json index 4a0d27c3..1a227856 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -13,6 +13,8 @@ }, "devDependencies": { "@types/node": "^20.11.0", + "@types/react": "^18.2.0", + "@types/react-dom": "^18.2.0", "typescript": "^5.4.0" } } diff --git a/frontend/src/app/page.tsx b/frontend/src/app/page.tsx index 49f1df34..98d60fe4 100644 --- a/frontend/src/app/page.tsx +++ b/frontend/src/app/page.tsx @@ -9,11 +9,20 @@ import { useDashboardData } from "../hooks/useDashboardData"; import { useFileUpload } from "../hooks/useFileUpload"; export default function Page() { - const { files, alerts, isLoading, errorMessage, setErrorMessage, loadData } = - useDashboardData(); + const { + files, + alerts, + isLoading, + isRefreshing, + errorMessage, + setErrorMessage, + loadData, + needsPolling, + } = useDashboardData(); const upload = useFileUpload({ - onSuccess: loadData, + // Silent refresh: keep current table, then polling picks up processed state. + onSuccess: () => loadData({ silent: true }), onError: setErrorMessage, }); @@ -22,8 +31,10 @@ export default function Page() { void loadData()} + onRefresh={() => void loadData({ silent: true })} onAdd={() => upload.setShowModal(true)} + isRefreshing={isRefreshing} + isPolling={needsPolling} /> {errorMessage ? ( diff --git a/frontend/src/components/DashboardHeader.tsx b/frontend/src/components/DashboardHeader.tsx index d013baa0..7de5bf62 100644 --- a/frontend/src/components/DashboardHeader.tsx +++ b/frontend/src/components/DashboardHeader.tsx @@ -1,13 +1,20 @@ "use client"; -import { Button, Card } from "react-bootstrap"; +import { Button, Card, Spinner } from "react-bootstrap"; type Props = { onRefresh: () => void; onAdd: () => void; + isRefreshing?: boolean; + isPolling?: boolean; }; -export function DashboardHeader({ onRefresh, onAdd }: Props) { +export function DashboardHeader({ + onRefresh, + onAdd, + isRefreshing = false, + isPolling = false, +}: Props) { return ( @@ -17,10 +24,20 @@ export function DashboardHeader({ onRefresh, onAdd }: Props) {

Загрузка файлов, просмотр статусов обработки и ленты алертов.

+ {isPolling ? ( +

+ + Идёт фоновая обработка — статусы обновляются автоматически +

+ ) : null}
-