From 3700bf8fcb69c5e71cf88dd241adfc3aa1ecd19e Mon Sep 17 00:00:00 2001 From: Aish-kul16 Date: Tue, 26 May 2026 23:24:02 +0530 Subject: [PATCH 1/8] feat(backend): add request ID propagation and structured logging --- backend/secuscan/database.py | 27 ++++++++++++++++--- backend/secuscan/logging_utils.py | 8 ++++++ backend/secuscan/main.py | 16 ++++++++--- backend/secuscan/request_context.py | 15 +++++++++++ backend/secuscan/request_middleware.py | 21 +++++++++++++++ backend/secuscan/workflows.py | 21 ++++++++++++--- .../integration/test_report_audit_log.py | 20 ++++++++++++++ 7 files changed, 117 insertions(+), 11 deletions(-) create mode 100644 backend/secuscan/logging_utils.py create mode 100644 backend/secuscan/request_context.py create mode 100644 backend/secuscan/request_middleware.py diff --git a/backend/secuscan/database.py b/backend/secuscan/database.py index 58a18c46..d1325ed5 100644 --- a/backend/secuscan/database.py +++ b/backend/secuscan/database.py @@ -269,18 +269,34 @@ async def log_audit( context: Optional[dict] = None, task_id: Optional[str] = None, plugin_id: Optional[str] = None, + request_id: Optional[str] = None, ): """Log an audit event.""" + + from .request_context import get_request_id + + request_id = request_id or get_request_id() + + context = context or {} + context["request_id"] = request_id + await self.execute( """ - INSERT INTO audit_log (event_type, severity, message, context_json, task_id, plugin_id) + INSERT INTO audit_log ( + event_type, + severity, + message, + context_json, + task_id, + plugin_id + ) VALUES (?, ?, ?, ?, ?, ?) """, ( event_type, severity, message, - json.dumps(context) if context else None, + json.dumps(context), task_id, plugin_id, ), @@ -293,10 +309,12 @@ async def log_audit( async def init_db(db_path: Optional[str] = None) -> Database: """Initialize the global database connection.""" global db - # Fallback to config path if not provided + path = db_path or f"{settings.data_dir}/secuscan.db" + db_instance = Database(path) await db_instance.connect() + db = db_instance return db_instance @@ -305,4 +323,5 @@ async def get_db() -> Database: """Get the global database instance.""" if db is None: raise RuntimeError("Database not initialized") - return db + + return db \ No newline at end of file diff --git a/backend/secuscan/logging_utils.py b/backend/secuscan/logging_utils.py new file mode 100644 index 00000000..46bec548 --- /dev/null +++ b/backend/secuscan/logging_utils.py @@ -0,0 +1,8 @@ +import logging +from .request_context import get_request_id + + +class RequestIDFilter(logging.Filter): + def filter(self, record): + record.request_id = get_request_id() or "no-request-id" + return True \ No newline at end of file diff --git a/backend/secuscan/main.py b/backend/secuscan/main.py index 08eb02c2..2d007eed 100644 --- a/backend/secuscan/main.py +++ b/backend/secuscan/main.py @@ -7,6 +7,7 @@ import shutil from pathlib import Path from contextlib import asynccontextmanager +from .request_middleware import RequestIDMiddleware from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware @@ -20,16 +21,23 @@ from .workflows import scheduler -# Configure logging +from .logging_utils import RequestIDFilter + logging.basicConfig( level=getattr(logging, settings.log_level), - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + format="%(asctime)s %(levelname)s [request_id=%(request_id)s] %(name)s - %(message)s", handlers=[ logging.StreamHandler(sys.stdout), - logging.FileHandler(settings.log_file) if Path(settings.log_file).parent.exists() else logging.NullHandler() + logging.FileHandler(settings.log_file) + if Path(settings.log_file).parent.exists() + else logging.NullHandler() ] ) +for handler in logging.getLogger().handlers: + handler.addFilter(RequestIDFilter()) + + logger = logging.getLogger(__name__) @@ -113,11 +121,11 @@ async def redirect_api_openapi(): allow_methods=settings.cors_allowed_methods, allow_headers=settings.cors_allowed_headers, ) +app.add_middleware(RequestIDMiddleware) # Include API routes app.include_router(router) - # Health check endpoint @app.get("/api/v1/health") async def health_check(): diff --git a/backend/secuscan/request_context.py b/backend/secuscan/request_context.py new file mode 100644 index 00000000..e4fa71d8 --- /dev/null +++ b/backend/secuscan/request_context.py @@ -0,0 +1,15 @@ +from contextvars import ContextVar +from uuid import uuid4 + +request_id_context: ContextVar[str] = ContextVar( + "request_id", + default="" +) + +def get_request_id() -> str: + return request_id_context.get() + +def set_request_id(request_id: str = None) -> str: + request_id = request_id or str(uuid4()) + request_id_context.set(request_id) + return request_id \ No newline at end of file diff --git a/backend/secuscan/request_middleware.py b/backend/secuscan/request_middleware.py new file mode 100644 index 00000000..985b4979 --- /dev/null +++ b/backend/secuscan/request_middleware.py @@ -0,0 +1,21 @@ +from starlette.middleware.base import BaseHTTPMiddleware +from fastapi import Request +from .request_context import set_request_id + + +class RequestIDMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): + + # Accept existing X-Request-ID or generate one + request_id = request.headers.get("X-Request-ID") + request_id = set_request_id(request_id) + + # Make available during request lifecycle + request.state.request_id = request_id + + response = await call_next(request) + + # Return ID in response headers + response.headers["X-Request-ID"] = request_id + + return response \ No newline at end of file diff --git a/backend/secuscan/workflows.py b/backend/secuscan/workflows.py index 52e7b638..14646f34 100644 --- a/backend/secuscan/workflows.py +++ b/backend/secuscan/workflows.py @@ -1,6 +1,7 @@ """Workflow automation and scheduling.""" from __future__ import annotations +from .request_context import get_request_id, set_request_id import asyncio import json @@ -74,13 +75,27 @@ def _should_run(self, now: datetime, last_run_at: str | None, schedule_seconds: async def _run_workflow(self, workflow_id: str, steps: List[Dict[str, Any]]): logger.info("Running workflow %s with %d step(s)", workflow_id, len(steps)) + for step in steps: plugin_id = step.get("plugin_id") inputs = step.get("inputs") or {} + if not plugin_id: continue - task_id = await executor.create_task(plugin_id, inputs, preset=step.get("preset"), consent_granted=True) - asyncio.create_task(executor.execute_task(task_id)) + request_id = get_request_id() + + task_id = await executor.create_task( + plugin_id, + inputs, + preset=step.get("preset"), + consent_granted=True + ) + + async def run_task(): + set_request_id(request_id) + await executor.execute_task(task_id) -scheduler = WorkflowScheduler() + asyncio.create_task(run_task()) + +scheduler = WorkflowScheduler() \ No newline at end of file diff --git a/testing/backend/integration/test_report_audit_log.py b/testing/backend/integration/test_report_audit_log.py index 34f62f58..29a1f7b5 100644 --- a/testing/backend/integration/test_report_audit_log.py +++ b/testing/backend/integration/test_report_audit_log.py @@ -119,3 +119,23 @@ def test_generation_failure_does_not_create_audit_entry(test_client): assert response.status_code == 500 entries = asyncio.run(get_audit_entries(task_id)) assert len(entries) == 0 + +def test_request_id_is_saved_in_audit_context(test_client): + task_id = "audit-request-id" + asyncio.run(insert_completed_task(task_id)) + + request_id = "test-request-123" + + response = test_client.get( + f"/api/v1/task/{task_id}/report/csv", + headers={"X-Request-ID": request_id}, + ) + + assert response.status_code == 200 + + entries = asyncio.run(get_audit_entries(task_id)) + assert len(entries) == 1 + + ctx = json.loads(entries[0]["context_json"]) + + assert ctx["request_id"] == request_id From 6cfe8e21dc6a1d713bef25d41e6c1e5b1a657cfd Mon Sep 17 00:00:00 2001 From: Aish-kul16 Date: Tue, 26 May 2026 23:34:44 +0530 Subject: [PATCH 2/8] feat(backend): add structured JSON logging --- backend/secuscan/logging_utils.py | 20 +++++++++++++++++++- backend/secuscan/main.py | 8 ++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/backend/secuscan/logging_utils.py b/backend/secuscan/logging_utils.py index 46bec548..b7547759 100644 --- a/backend/secuscan/logging_utils.py +++ b/backend/secuscan/logging_utils.py @@ -1,8 +1,26 @@ +import json import logging +from datetime import datetime, timezone from .request_context import get_request_id class RequestIDFilter(logging.Filter): def filter(self, record): record.request_id = get_request_id() or "no-request-id" - return True \ No newline at end of file + return True + + +class JSONFormatter(logging.Formatter): + def format(self, record): + log_entry = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "level": record.levelname, + "request_id": getattr(record, "request_id", "no-request-id"), + "logger": record.name, + "message": record.getMessage(), + } + + if record.exc_info: + log_entry["exception"] = self.formatException(record.exc_info) + + return json.dumps(log_entry) \ No newline at end of file diff --git a/backend/secuscan/main.py b/backend/secuscan/main.py index 2d007eed..ebf57f60 100644 --- a/backend/secuscan/main.py +++ b/backend/secuscan/main.py @@ -21,11 +21,8 @@ from .workflows import scheduler -from .logging_utils import RequestIDFilter - logging.basicConfig( level=getattr(logging, settings.log_level), - format="%(asctime)s %(levelname)s [request_id=%(request_id)s] %(name)s - %(message)s", handlers=[ logging.StreamHandler(sys.stdout), logging.FileHandler(settings.log_file) @@ -34,10 +31,13 @@ ] ) +from .logging_utils import RequestIDFilter, JSONFormatter + for handler in logging.getLogger().handlers: handler.addFilter(RequestIDFilter()) + handler.setFormatter(JSONFormatter()) + - logger = logging.getLogger(__name__) From c634c217ff3bd3c0b72a6cfe7b32ce015cdf13a6 Mon Sep 17 00:00:00 2001 From: Aish-kul16 Date: Tue, 26 May 2026 23:39:41 +0530 Subject: [PATCH 3/8] chore: remove trailing whitespace --- backend/secuscan/workflows.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/secuscan/workflows.py b/backend/secuscan/workflows.py index 14646f34..2563ef5e 100644 --- a/backend/secuscan/workflows.py +++ b/backend/secuscan/workflows.py @@ -96,6 +96,5 @@ async def run_task(): set_request_id(request_id) await executor.execute_task(task_id) - asyncio.create_task(run_task()) - + asyncio.create_task(run_task()) scheduler = WorkflowScheduler() \ No newline at end of file From 7e944f39216ba5666d8926b6826b95f0aee0b9b3 Mon Sep 17 00:00:00 2001 From: Aish-kul16 Date: Tue, 26 May 2026 23:44:18 +0530 Subject: [PATCH 4/8] chore: remove trailing whitespace --- backend/secuscan/workflows.py | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/backend/secuscan/workflows.py b/backend/secuscan/workflows.py index 2563ef5e..b1da1ab0 100644 --- a/backend/secuscan/workflows.py +++ b/backend/secuscan/workflows.py @@ -1,20 +1,14 @@ """Workflow automation and scheduling.""" - from __future__ import annotations from .request_context import get_request_id, set_request_id - import asyncio import json import logging from datetime import datetime, timezone from typing import Any, Dict, List - from .database import get_db from .executor import executor - logger = logging.getLogger(__name__) - - class WorkflowScheduler: def __init__(self): self._task: asyncio.Task | None = None @@ -26,7 +20,6 @@ async def start(self): self._running = True self._task = asyncio.create_task(self._run_loop()) logger.info("Workflow scheduler started") - async def stop(self): self._running = False if self._task: @@ -37,7 +30,6 @@ async def stop(self): pass self._task = None logger.info("Workflow scheduler stopped") - async def _run_loop(self): while self._running: try: @@ -45,7 +37,6 @@ async def _run_loop(self): except Exception as exc: logger.error("Workflow scheduler tick failed: %s", exc) await asyncio.sleep(5) - async def tick(self): db = await get_db() rows = await db.fetchall( @@ -55,7 +46,6 @@ async def tick(self): WHERE enabled = 1 AND schedule_seconds IS NOT NULL AND schedule_seconds > 0 """ ) - now = datetime.now(timezone.utc) for row in rows: if not self._should_run(now, row.get("last_run_at"), int(row["schedule_seconds"])): @@ -65,36 +55,28 @@ async def tick(self): "UPDATE workflows SET last_run_at = datetime('now') WHERE id = ?", (row["id"],), ) - def _should_run(self, now: datetime, last_run_at: str | None, schedule_seconds: int) -> bool: if not last_run_at: return True last = datetime.fromisoformat(last_run_at.replace("Z", "+00:00")) elapsed = (now - last).total_seconds() return elapsed >= schedule_seconds - async def _run_workflow(self, workflow_id: str, steps: List[Dict[str, Any]]): logger.info("Running workflow %s with %d step(s)", workflow_id, len(steps)) - for step in steps: plugin_id = step.get("plugin_id") inputs = step.get("inputs") or {} - if not plugin_id: continue - request_id = get_request_id() - task_id = await executor.create_task( plugin_id, inputs, preset=step.get("preset"), consent_granted=True ) - async def run_task(): set_request_id(request_id) await executor.execute_task(task_id) - - asyncio.create_task(run_task()) + asyncio.create_task(run_task()) scheduler = WorkflowScheduler() \ No newline at end of file From a2a6f1e581c138026fae91f3389adc8b3f68d8c2 Mon Sep 17 00:00:00 2001 From: Aish-kul16 Date: Tue, 26 May 2026 23:48:08 +0530 Subject: [PATCH 5/8] chore: remove trailing whitespace --- backend/secuscan/workflows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/secuscan/workflows.py b/backend/secuscan/workflows.py index b1da1ab0..c0f90209 100644 --- a/backend/secuscan/workflows.py +++ b/backend/secuscan/workflows.py @@ -78,5 +78,5 @@ async def _run_workflow(self, workflow_id: str, steps: List[Dict[str, Any]]): async def run_task(): set_request_id(request_id) await executor.execute_task(task_id) - asyncio.create_task(run_task()) + asyncio.create_task(run_task()) scheduler = WorkflowScheduler() \ No newline at end of file From edf0e3e35cb8fe843004b5be23bdf5d813d252c8 Mon Sep 17 00:00:00 2001 From: Aish-kul16 Date: Sat, 6 Jun 2026 13:09:38 +0530 Subject: [PATCH 6/8] fix: restore workflow execution after merge --- backend/secuscan/database.py | 3 ++- backend/secuscan/workflows.py | 31 ++++++++++++++++++++++++++----- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/backend/secuscan/database.py b/backend/secuscan/database.py index 7108edc8..79cd75d0 100644 --- a/backend/secuscan/database.py +++ b/backend/secuscan/database.py @@ -838,4 +838,5 @@ async def init_db(db_path: Optional[str] = None) -> Database: async def get_db() -> Database: """Get the global database instance.""" if db is None: - raise RuntimeError("Database not initialized") \ No newline at end of file + raise RuntimeError("Database not initialized") + return db \ No newline at end of file diff --git a/backend/secuscan/workflows.py b/backend/secuscan/workflows.py index 619686f3..1d250631 100644 --- a/backend/secuscan/workflows.py +++ b/backend/secuscan/workflows.py @@ -73,6 +73,7 @@ def _should_run(self, now: datetime, last_run_at: str | None, schedule_seconds: elapsed = (now - last).total_seconds() return elapsed >= schedule_seconds async def _run_workflow(self, workflow_id: str, steps: List[Dict[str, Any]]): + db = await get_db() logger.info("Running workflow %s with %d step(s)", workflow_id, len(steps)) for step in steps: plugin_id = step.get("plugin_id") @@ -80,13 +81,33 @@ async def _run_workflow(self, workflow_id: str, steps: List[Dict[str, Any]]): if not plugin_id: continue request_id = get_request_id() + execution_context = normalize_execution_context( + step.get("execution_context") or {} + ) + target_policy = await get_target_policy( + db, + "default", + execution_context.get("target_policy_id") + ) + safe_mode = bool( + settings.safe_mode_default + and not ( + target_policy + and target_policy.get("allow_public_targets") + ) + ) + effective_inputs = dict(inputs) + effective_inputs.pop("safe_mode", None) + effective_inputs["safe_mode"] = safe_mode task_id = await executor.create_task( - plugin_id, - inputs, - preset=step.get("preset"), - consent_granted=True - ) + plugin_id, + effective_inputs, + safe_mode=safe_mode, + preset=step.get("preset"), + execution_context=execution_context, + consent_granted=True, + ) async def run_task(): set_request_id(request_id) await executor.execute_task(task_id) From df09720ac31f3688ef043c2c9a00cd1e01f38df6 Mon Sep 17 00:00:00 2001 From: Aish-kul16 Date: Sat, 6 Jun 2026 14:11:11 +0530 Subject: [PATCH 7/8] test: update frontend tests for execution context --- .../unit/pages/ToolConfigDynamic.test.tsx | 5 ++++ .../testing/unit/pages/Workflows.test.tsx | 26 ++++++++++++++++--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/frontend/testing/unit/pages/ToolConfigDynamic.test.tsx b/frontend/testing/unit/pages/ToolConfigDynamic.test.tsx index 6cd5a1da..feff6ebc 100644 --- a/frontend/testing/unit/pages/ToolConfigDynamic.test.tsx +++ b/frontend/testing/unit/pages/ToolConfigDynamic.test.tsx @@ -108,6 +108,11 @@ describe('ToolConfig dynamic schema flow', () => { }), true, 'quick', + expect.objectContaining({ + evidence_level: 'standard', + scan_profile: 'standard', + validation_mode: 'proof', + }), ) }) }) diff --git a/frontend/testing/unit/pages/Workflows.test.tsx b/frontend/testing/unit/pages/Workflows.test.tsx index 7c304da8..b2cc0934 100644 --- a/frontend/testing/unit/pages/Workflows.test.tsx +++ b/frontend/testing/unit/pages/Workflows.test.tsx @@ -17,7 +17,17 @@ const mockWorkflow = { name: 'Nightly Scan', schedule_seconds: 3600, enabled: true, - steps: [{ plugin_id: 'nmap', inputs: {} }], + steps: [ + { + plugin_id: 'nmap', + inputs: {}, + execution_context: { + evidence_level: 'standard'as const, + scan_profile: 'standard', + validation_mode: 'proof' as const, + }, + }, + ], last_run_at: null, queued_task_ids: [], } @@ -130,8 +140,18 @@ describe('Workflows — create action', () => { name: 'Nightly Scan', schedule_seconds: 7200, enabled: true, - steps: [{ plugin_id: '', inputs: {} }], - }) + steps: [ + { + plugin_id: '', + inputs: {}, + execution_context: { + evidence_level: 'standard', + scan_profile: 'standard', + validation_mode: 'proof' as const, + }, + }, + ], + }) }) }) From 34454aa57adc296491aa377112ff45305270dbc8 Mon Sep 17 00:00:00 2001 From: Aish-kul16 Date: Sat, 6 Jun 2026 14:22:35 +0530 Subject: [PATCH 8/8] test: update timeout expectation --- frontend/testing/unit/pages/ToolConfigTimeout.test.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/testing/unit/pages/ToolConfigTimeout.test.tsx b/frontend/testing/unit/pages/ToolConfigTimeout.test.tsx index 4c0b2213..513ca033 100644 --- a/frontend/testing/unit/pages/ToolConfigTimeout.test.tsx +++ b/frontend/testing/unit/pages/ToolConfigTimeout.test.tsx @@ -75,6 +75,6 @@ describe('ToolConfig timeout control', () => { // min from field.validation expect(input).toHaveAttribute('min', '30') // max is min(field.validation.max, server default_timeout) - expect(input).toHaveAttribute('max', '600') + expect(input).toHaveAttribute('max', '7200') }) })