diff --git a/backend/ai-service/Dockerfile b/backend/ai-service/Dockerfile index f327f2ee..55b81091 100644 --- a/backend/ai-service/Dockerfile +++ b/backend/ai-service/Dockerfile @@ -32,6 +32,7 @@ RUN uv export \ && pip install --no-cache-dir -r /tmp/requirements-ai.txt \ && pip install --no-cache-dir --force-reinstall opencv-contrib-python-headless==4.10.0.84 +COPY backend/shared /app/backend/shared COPY backend/ai-service /app/backend/ai-service WORKDIR /app/backend/ai-service diff --git a/backend/ai-service/app/core/safe_logging.py b/backend/ai-service/app/core/safe_logging.py new file mode 100644 index 00000000..3fa281c7 --- /dev/null +++ b/backend/ai-service/app/core/safe_logging.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +_BACKEND_ROOT = Path(__file__).resolve().parents[3] +if str(_BACKEND_ROOT) not in sys.path: + sys.path.insert(0, str(_BACKEND_ROOT)) + +from shared.logging_redaction import ( # noqa: E402 + LOG_FORMAT_ERROR, + REDACTED, + REDACTED_BINARY, + install_safe_logging, + is_sensitive_key, + redact_text, + redact_value, + sanitize_log_record, +) + +__all__ = [ + "LOG_FORMAT_ERROR", + "REDACTED", + "REDACTED_BINARY", + "install_safe_logging", + "is_sensitive_key", + "redact_text", + "redact_value", + "sanitize_log_record", +] diff --git a/backend/ai-service/app/main.py b/backend/ai-service/app/main.py index ad3b9286..c30ca18b 100644 --- a/backend/ai-service/app/main.py +++ b/backend/ai-service/app/main.py @@ -1,8 +1,14 @@ -from fastapi import FastAPI +from app.core.safe_logging import install_safe_logging -from app.core.auth import enforce_internal_token as _enforce_internal_token -from app.routers.health import healthcheck, readiness, router as health_router -from app.routers.processing import ( +# Install before importing processing modules so initialization failures and +# exception paths cannot emit raw evidence or credentials. +install_safe_logging() + +from fastapi import FastAPI # noqa: E402 + +from app.core.auth import enforce_internal_token as _enforce_internal_token # noqa: E402 +from app.routers.health import healthcheck, readiness, router as health_router # noqa: E402 +from app.routers.processing import ( # noqa: E402 document_classify, document_ocr, document_quality, @@ -10,8 +16,8 @@ liveness_check, router as processing_router, ) -from app.runtime import configure_runtime_environment -from app.schemas.processing import ( +from app.runtime import configure_runtime_environment # noqa: E402 +from app.schemas.processing import ( # noqa: E402 AIResultResponse, DocumentClassificationRequest, DocumentOCRRequest, @@ -21,7 +27,7 @@ LivenessCheckRequest, ReadinessResponse, ) -from app.settings import get_settings +from app.settings import get_settings # noqa: E402 settings = get_settings() diff --git a/backend/ai-service/tests/test_safe_logging.py b/backend/ai-service/tests/test_safe_logging.py new file mode 100644 index 00000000..0942bea3 --- /dev/null +++ b/backend/ai-service/tests/test_safe_logging.py @@ -0,0 +1,74 @@ +import io +import logging + +from app.core.safe_logging import REDACTED, install_safe_logging, redact_value + + +def _capture(logger: logging.Logger, formatter: str = "%(message)s"): + stream = io.StringIO() + handler = logging.StreamHandler(stream) + handler.setFormatter(logging.Formatter(formatter)) + logger.handlers = [handler] + logger.propagate = False + logger.setLevel(logging.INFO) + return stream + + +def test_managed_ai_uses_shared_nested_redaction_corpus(): + install_safe_logging() + payload = { + "authorization": "Bearer ai-secret", + "request": { + "email": "subject@example.test", + "document_number": "P1234567", + "safe_operation": "face_compare", + }, + "biometrics": { + "face_embedding": [0.12, 0.34], + "selfie_image": "base64-selfie", + }, + } + + redacted = redact_value(payload) + + assert redacted["authorization"] == REDACTED + assert redacted["request"]["email"] == REDACTED + assert redacted["request"]["document_number"] == REDACTED + assert redacted["request"]["safe_operation"] == "face_compare" + # A container explicitly named "biometrics" is sensitive as a whole. The + # redactor intentionally fails closed instead of retaining its structure. + assert redacted["biometrics"] == REDACTED + + +def test_managed_ai_logger_redacts_structured_context_and_exception_text(): + install_safe_logging() + logger = logging.getLogger("identitycore.ai.redaction-test") + stream = _capture(logger, "%(message)s context=%(context)s") + + try: + raise ValueError( + "api_key=provider-secret email=subject@example.test " + "image_base64=raw-biometric" + ) + except ValueError: + logger.exception( + "AI processing failed Authorization: Bearer %s", + "internal-shared-token", + extra={ + "context": { + "document_storage_key": "tenant/evidence/document.jpg", + "ocr_text": "raw OCR contents", + "operation": "document_ocr", + } + }, + ) + + output = stream.getvalue() + assert "provider-secret" not in output + assert "subject@example.test" not in output + assert "raw-biometric" not in output + assert "internal-shared-token" not in output + assert "tenant/evidence/document.jpg" not in output + assert "raw OCR contents" not in output + assert "document_ocr" in output + assert "Traceback" in output diff --git a/backend/django/Dockerfile b/backend/django/Dockerfile index 33111902..6a7b8056 100644 --- a/backend/django/Dockerfile +++ b/backend/django/Dockerfile @@ -22,6 +22,7 @@ RUN uv export \ --output-file /tmp/requirements-django.txt \ && pip install --no-cache-dir -r /tmp/requirements-django.txt +COPY backend/shared /app/backend/shared COPY backend/django /app/backend/django COPY docs/openapi/identitycore-public-api.yaml /app/docs/openapi/identitycore-public-api.yaml diff --git a/backend/django/apps/core/apps.py b/backend/django/apps/core/apps.py index 68868e8b..665f028d 100644 --- a/backend/django/apps/core/apps.py +++ b/backend/django/apps/core/apps.py @@ -5,3 +5,10 @@ class CoreConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" name = "apps.core" label = "core" + + def ready(self) -> None: + # Install the redaction boundary after Django configures logging but before + # request/worker code can emit application records. + from common.safe_logging import install_safe_logging + + install_safe_logging() diff --git a/backend/django/common/safe_logging.py b/backend/django/common/safe_logging.py new file mode 100644 index 00000000..5c0ef2dc --- /dev/null +++ b/backend/django/common/safe_logging.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +_BACKEND_ROOT = Path(__file__).resolve().parents[2] +if str(_BACKEND_ROOT) not in sys.path: + sys.path.insert(0, str(_BACKEND_ROOT)) + +from shared.logging_redaction import ( # noqa: E402 + LOG_FORMAT_ERROR, + REDACTED, + REDACTED_BINARY, + install_safe_logging, + is_sensitive_key, + redact_text, + redact_value, + sanitize_log_record, +) + +__all__ = [ + "LOG_FORMAT_ERROR", + "REDACTED", + "REDACTED_BINARY", + "install_safe_logging", + "is_sensitive_key", + "redact_text", + "redact_value", + "sanitize_log_record", +] diff --git a/backend/django/common/test_safe_logging.py b/backend/django/common/test_safe_logging.py new file mode 100644 index 00000000..3c8fecd8 --- /dev/null +++ b/backend/django/common/test_safe_logging.py @@ -0,0 +1,122 @@ +import io +import logging + +from celery.utils.log import get_task_logger +from django.test import SimpleTestCase + +from common.safe_logging import REDACTED, REDACTED_BINARY, install_safe_logging, redact_value + + +class SafeLoggingTests(SimpleTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + install_safe_logging() + + def _capture(self, logger: logging.Logger, formatter: str = "%(levelname)s %(message)s"): + stream = io.StringIO() + handler = logging.StreamHandler(stream) + handler.setFormatter(logging.Formatter(formatter)) + logger.handlers = [handler] + logger.propagate = False + logger.setLevel(logging.INFO) + self.addCleanup(logger.handlers.clear) + return stream + + def test_recursive_redaction_covers_secret_pii_and_biometric_fields(self): + payload = { + "authorization": "Bearer top-secret", + "profile": { + "email": "ada@example.test", + "phone_number": "+233241234567", + "document_number": "GHA-123456789", + "safe_status": "pending_review", + }, + "evidence": { + "selfie_image": "base64-sensitive-selfie", + "face_embedding": [0.1, 0.2, 0.3], + "document_storage_key": "tenant/evidence/front.jpg", + }, + "binary": b"document-bytes", + } + + redacted = redact_value(payload) + + self.assertEqual(redacted["authorization"], REDACTED) + self.assertEqual(redacted["profile"]["email"], REDACTED) + self.assertEqual(redacted["profile"]["phone_number"], REDACTED) + self.assertEqual(redacted["profile"]["document_number"], REDACTED) + self.assertEqual(redacted["profile"]["safe_status"], "pending_review") + self.assertEqual(redacted["evidence"]["selfie_image"], REDACTED) + self.assertEqual(redacted["evidence"]["face_embedding"], REDACTED) + self.assertEqual(redacted["evidence"]["document_storage_key"], REDACTED) + self.assertEqual(redacted["binary"], REDACTED_BINARY) + + def test_django_logger_redacts_message_arguments_and_structured_extra(self): + logger = logging.getLogger("django.identitycore.redaction-test") + stream = self._capture(logger, "%(message)s payload=%(payload)s") + + logger.info( + "request rejected email=%s Authorization: Bearer %s", + "ada@example.test", + "secret-access-token", + extra={ + "payload": { + "password": "never-log-this", + "selfie_image": "raw-biometric-data", + "safe_reason": "credentials_missing", + } + }, + ) + + output = stream.getvalue() + self.assertNotIn("ada@example.test", output) + self.assertNotIn("secret-access-token", output) + self.assertNotIn("never-log-this", output) + self.assertNotIn("raw-biometric-data", output) + self.assertIn("credentials_missing", output) + self.assertIn(REDACTED, output) + + def test_celery_task_logger_uses_the_same_redaction_boundary(self): + logger = get_task_logger("identitycore.redaction-test") + stream = self._capture(logger, "%(message)s context=%(context)s") + + logger.warning( + "worker retry token=%s", + "celery-secret-token", + extra={ + "context": { + "api_key": "provider-key", + "ocr_text": "raw document text", + "verification_id": "ver_safe_public_id", + } + }, + ) + + output = stream.getvalue() + self.assertNotIn("celery-secret-token", output) + self.assertNotIn("provider-key", output) + self.assertNotIn("raw document text", output) + self.assertIn("ver_safe_public_id", output) + + def test_exception_traceback_is_preserved_without_sensitive_values(self): + logger = logging.getLogger("identitycore.exception-redaction-test") + stream = self._capture(logger) + + try: + raise RuntimeError( + "token=runtime-secret email=ada@example.test " + "phone=+233241234567 document_number=GHA-123456789" + ) + except RuntimeError: + logger.exception("provider failed first_name=Ada") + + output = stream.getvalue() + self.assertIn("Traceback", output) + self.assertIn("RuntimeError", output) + self.assertNotIn("runtime-secret", output) + self.assertNotIn("ada@example.test", output) + self.assertNotIn("+233241234567", output) + self.assertNotIn("GHA-123456789", output) + self.assertNotIn("first_name=Ada", output) + self.assertIn("first_name=[REDACTED]", output) diff --git a/backend/django/common/test_safe_logging_boundaries.py b/backend/django/common/test_safe_logging_boundaries.py new file mode 100644 index 00000000..94faf4d3 --- /dev/null +++ b/backend/django/common/test_safe_logging_boundaries.py @@ -0,0 +1,92 @@ +import io +import logging + +from django.test import SimpleTestCase + +from common.safe_logging import LOG_FORMAT_ERROR, install_safe_logging + + +class SafeLoggingBoundaryTests(SimpleTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + install_safe_logging() + + def _capture(self, logger_name: str): + stream = io.StringIO() + handler = logging.StreamHandler(stream) + handler.setFormatter(logging.Formatter("%(message)s context=%(context)s")) + logger = logging.getLogger(logger_name) + logger.handlers = [handler] + logger.propagate = False + logger.setLevel(logging.INFO) + self.addCleanup(logger.handlers.clear) + return logger, stream + + def test_storage_and_provider_loggers_share_the_global_boundary(self): + for logger_name in ("common.storage", "apps.providers.services"): + with self.subTest(logger=logger_name): + logger, stream = self._capture(logger_name) + logger.info( + "operation completed", + extra={ + "context": { + "storage_key": "tenant/evidence/private.jpg", + "client_secret": "provider-secret", + "external_reference": "customer-4482", + "device_fingerprint": "device-secret", + "user_agent": "browser-fingerprint", + "verification_subject_id": "vs_sensitive", + "provider_code": "safe-provider-code", + } + }, + ) + output = stream.getvalue() + self.assertNotIn("tenant/evidence/private.jpg", output) + self.assertNotIn("provider-secret", output) + self.assertNotIn("customer-4482", output) + self.assertNotIn("device-secret", output) + self.assertNotIn("browser-fingerprint", output) + self.assertNotIn("vs_sensitive", output) + self.assertIn("safe-provider-code", output) + + def test_exception_objects_and_sensitive_mapping_keys_are_sanitized(self): + logger, stream = self._capture("identitycore.argument-redaction-test") + error = RuntimeError("token=exception-secret; email=subject@example.test") + + logger.error( + "provider error: %s", + error, + extra={"context": {"subject@example.test": "lookup", "status": "failed"}}, + ) + + output = stream.getvalue() + self.assertNotIn("exception-secret", output) + self.assertNotIn("subject@example.test", output) + self.assertIn("RuntimeError", output) + self.assertIn("failed", output) + + def test_multiword_pii_in_free_text_is_fully_removed(self): + logger, stream = self._capture("identitycore.multiword-redaction-test") + logger.info( + "review full_name=Ada Lovelace; external_reference=customer-4482; status=pending", + extra={"context": {"status": "pending"}}, + ) + + output = stream.getvalue() + self.assertNotIn("Ada Lovelace", output) + self.assertNotIn("customer-4482", output) + self.assertIn("status=pending", output) + + def test_malformed_format_string_does_not_raise_or_render_arguments(self): + logger, stream = self._capture("identitycore.format-error-test") + logger.info( + "provider failed without placeholder", + "secret-that-must-not-render", + extra={"context": {"status": "failed"}}, + ) + + output = stream.getvalue() + self.assertNotIn("secret-that-must-not-render", output) + self.assertIn(LOG_FORMAT_ERROR, output) + self.assertIn("failed", output) diff --git a/backend/django/config/celery.py b/backend/django/config/celery.py index 1d803399..2610c9d7 100644 --- a/backend/django/config/celery.py +++ b/backend/django/config/celery.py @@ -2,6 +2,12 @@ from celery import Celery +from common.safe_logging import install_safe_logging + + +# Celery can initialize its own logging before Django app-ready hooks run. Install +# the redaction boundary here as well so broker/startup and task logs are protected. +install_safe_logging() os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.development") diff --git a/backend/shared/__init__.py b/backend/shared/__init__.py new file mode 100644 index 00000000..71cd720a --- /dev/null +++ b/backend/shared/__init__.py @@ -0,0 +1 @@ +"""Shared backend utilities used by both Django and managed AI services.""" diff --git a/backend/shared/logging_redaction.py b/backend/shared/logging_redaction.py new file mode 100644 index 00000000..b13527db --- /dev/null +++ b/backend/shared/logging_redaction.py @@ -0,0 +1,280 @@ +from __future__ import annotations + +import logging +import re +import traceback +from collections.abc import Mapping +from threading import Lock +from typing import Any + +REDACTED = "[REDACTED]" +REDACTED_BINARY = "[REDACTED_BINARY]" +MAX_REDACTION_DEPTH = 12 +LOG_FORMAT_ERROR = "[LOG_FORMAT_ERROR]" + +_SENSITIVE_KEYS = frozenset( + { + # Credentials and session material. + "access_key", + "access_token", + "api_key", + "authorization", + "client_secret", + "cookie", + "credentials", + "csrf_token", + "csrfmiddlewaretoken", + "id_token", + "password", + "private_key", + "refresh_token", + "secret", + "secret_access_key", + "session_token", + "set_cookie", + "token", + # Direct identifiers / PII / correlation values that may identify a subject. + "address", + "birth_date", + "client_ip", + "date_of_birth", + "device_fingerprint", + "dob", + "document_number", + "email", + "external_reference", + "first_name", + "full_name", + "ghana_card_number", + "ip", + "ip_address", + "last_name", + "middle_name", + "national_id", + "passport_number", + "phone", + "phone_number", + "postal_address", + "remote_addr", + "subject_id", + "tax_identification_number", + "tin", + "user_agent", + "verification_subject_id", + # Evidence locations and raw document / biometric material. + "biometric_payload", + "biometric_template", + "document_bytes", + "document_image", + "document_storage_key", + "face_embedding", + "face_image", + "image", + "image_base64", + "image_bytes", + "liveness_video", + "mrz", + "ocr_text", + "raw_document", + "raw_image", + "raw_ocr", + "selfie", + "selfie_image", + "selfie_storage_key", + "storage_key", + } +) + +_SENSITIVE_SUFFIXES = ( + "_access_token", + "_api_key", + "_authorization", + "_client_secret", + "_credential", + "_credentials", + "_fingerprint", + "_password", + "_private_key", + "_refresh_token", + "_secret", + "_session_token", + "_storage_key", + "_subject_id", + "_token", + "_user_agent", +) + +_SENSITIVE_FRAGMENTS = ( + "biometric", + "face_embedding", + "image_base64", + "image_bytes", + "liveness_video", + "selfie_image", +) + +_BEARER_RE = re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._~+/=-]+") +_JWT_RE = re.compile( + r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b" +) +_EMAIL_RE = re.compile( + r"(? str: + normalized = re.sub(r"[^a-z0-9]+", "_", str(key).strip().lower()) + return normalized.strip("_") + + +def is_sensitive_key(key: object) -> bool: + normalized = _normalize_key(key) + if not normalized: + return False + if normalized in _SENSITIVE_KEYS: + return True + if normalized.endswith(_SENSITIVE_SUFFIXES): + return True + return any(fragment in normalized for fragment in _SENSITIVE_FRAGMENTS) + + +def redact_text(value: str) -> str: + """Redact common secret and PII shapes from unstructured log text.""" + redacted = _BEARER_RE.sub("Bearer [REDACTED]", value) + redacted = _JWT_RE.sub(REDACTED, redacted) + redacted = _AWS_ACCESS_KEY_RE.sub(REDACTED, redacted) + redacted = _CREDENTIAL_ASSIGNMENT_RE.sub( + lambda match: f"{match.group(1)}={REDACTED}", redacted + ) + redacted = _EMAIL_RE.sub(REDACTED, redacted) + redacted = _IPV4_RE.sub(REDACTED, redacted) + redacted = _PHONE_RE.sub(REDACTED, redacted) + return redacted + + +def redact_value(value: Any, *, key: object | None = None, _depth: int = 0) -> Any: + """Return a logging-safe copy of a nested value. + + Redaction is key-aware for structured payloads and shape-aware for free text. + Bytes are never logged because they can contain document or biometric evidence. + """ + if key is not None and is_sensitive_key(key): + return REDACTED + if _depth >= MAX_REDACTION_DEPTH: + return "[REDACTED_DEPTH_LIMIT]" + if value is None or isinstance(value, (bool, int, float)): + return value + if isinstance(value, str): + return redact_text(value) + if isinstance(value, BaseException): + return f"{value.__class__.__name__}: {redact_text(str(value))}" + if isinstance(value, (bytes, bytearray, memoryview)): + return REDACTED_BINARY + if isinstance(value, Mapping): + redacted_mapping = {} + for item_key, item_value in value.items(): + safe_key = redact_text(str(item_key)) + redacted_mapping[safe_key] = redact_value( + item_value, + key=item_key, + _depth=_depth + 1, + ) + return redacted_mapping + if isinstance(value, tuple): + return tuple(redact_value(item, _depth=_depth + 1) for item in value) + if isinstance(value, list): + return [redact_value(item, _depth=_depth + 1) for item in value] + if isinstance(value, (set, frozenset)): + return [redact_value(item, _depth=_depth + 1) for item in value] + return redact_text(str(value)) + + +def _redact_exception( + exc_info: tuple[type[BaseException], BaseException, Any], +) -> str: + rendered = "".join(traceback.format_exception(*exc_info)) + return redact_text(rendered) + + +def sanitize_log_record(record: logging.LogRecord) -> logging.LogRecord: + """Sanitize rendered messages, structured extras, stack text, and exceptions.""" + if record.args: + # Render once using Python logging's normal interpolation rules, then redact + # the complete result. If the caller supplied a malformed format string, + # discard the args instead of letting logging break request/worker execution. + try: + rendered_message = record.getMessage() + except (TypeError, ValueError): + rendered_message = f"{redact_value(record.msg)} {LOG_FORMAT_ERROR}" + record.msg = redact_text(str(rendered_message)) + record.args = () + else: + record.msg = redact_value(record.msg) + + for field, value in list(record.__dict__.items()): + if field in _STANDARD_LOG_RECORD_ATTRS or field.startswith("_"): + continue + record.__dict__[field] = redact_value(value, key=field) + + if record.stack_info: + record.stack_info = redact_text(record.stack_info) + if record.exc_info: + record.exc_text = _redact_exception(record.exc_info) + record.exc_info = None + elif record.exc_text: + record.exc_text = redact_text(record.exc_text) + return record + + +def _safe_make_record(self, *args, **kwargs): + record = _ORIGINAL_MAKE_RECORD(self, *args, **kwargs) + return sanitize_log_record(record) + + +def install_safe_logging() -> None: + """Install the process-wide logging redaction boundary exactly once.""" + global _INSTALLED + if _INSTALLED: + return + with _INSTALL_LOCK: + if _INSTALLED: + return + logging.Logger.makeRecord = _safe_make_record + _INSTALLED = True + + +__all__ = [ + "LOG_FORMAT_ERROR", + "REDACTED", + "REDACTED_BINARY", + "install_safe_logging", + "is_sensitive_key", + "redact_text", + "redact_value", + "sanitize_log_record", +] diff --git a/docs/operations/logging-redaction.md b/docs/operations/logging-redaction.md new file mode 100644 index 00000000..c6210cbf --- /dev/null +++ b/docs/operations/logging-redaction.md @@ -0,0 +1,41 @@ +# Logging redaction and safe telemetry + +IdentityCore treats logs as an operational data stream, not as a place to store verification evidence or user data. Application logs must never contain credentials, session material, direct subject PII, raw document/OCR content, storage locations for evidence, or biometric payloads. + +## Backend boundary + +Django, Celery workers, storage/provider call paths, and the managed AI service use the shared redaction implementation in `backend/shared/logging_redaction.py`. + +The boundary is installed when Django starts, before Celery initializes its application logger, and before the AI service imports processing routes. It sanitizes every Python `LogRecord` after `extra` fields have been attached and before handlers format the record. This covers: + +- nested dictionaries/lists passed as structured logging context; +- positional and mapping logging arguments; +- bearer/JWT-like credentials and common credential assignments embedded in free text; +- common email, phone, and IP shapes in free text; +- exception messages and tracebacks; +- bytes and byte-like values, which are always replaced rather than rendered. + +Safe operational dimensions such as public request/verification IDs, operation names, status/reason codes, provider codes, durations, retry counts, and queue names may be logged when they do not themselves contain subject data. + +When a new secret, PII field, document field, or biometric representation is introduced, add its normalized key to the shared redaction corpus and add an adversarial test before using it in telemetry. + +## Frontend boundary + +Frontend code must use `safeLog` exported by `@identitycore/api-client`. The logger applies the same categories of key-aware and free-text redaction before calling the browser console. + +Direct `console.log`, `console.info`, `console.warn`, `console.error`, `console.debug`, or `console.trace` calls are rejected by the frontend lint gate for production source. This prevents a future component from bypassing the redaction helper with a raw API error, token, form state, capture payload, or server response. + +## What not to log + +Do not log: + +- authorization headers, API keys, passwords, cookies, refresh/session tokens, provider credentials, or private keys; +- names, email addresses, phone numbers, addresses, dates of birth, national/passport/document numbers, or tax identifiers; +- OCR/MRZ text, raw document images/bytes, evidence storage keys, selfies, face embeddings, liveness media, or biometric templates; +- complete request/response bodies from identity, provider, storage, or webhook operations. + +Prefer stable reason/error codes and public correlation identifiers over raw exception/request payloads. + +## Verification + +The CI suite includes adversarial tests for Django, Celery, managed AI, and frontend logging. Tests intentionally place sensitive values in nested structured fields, positional arguments, binary values, and exception text and assert that those values never reach formatted log output. Frontend lint also fails when production source introduces a direct console logging call. diff --git a/frontend/packages/api-client/package.json b/frontend/packages/api-client/package.json index db440b1f..9efdef4f 100644 --- a/frontend/packages/api-client/package.json +++ b/frontend/packages/api-client/package.json @@ -7,6 +7,9 @@ "exports": { ".": "./src/index.ts" }, + "scripts": { + "lint": "tsc -p tsconfig.json && tsc -p tsconfig.safe-logging.json && node --test test/safe-logging.test.mjs && node ../../scripts/check-safe-logging.mjs" + }, "devDependencies": { "typescript": "^5" } diff --git a/frontend/packages/api-client/src/index.ts b/frontend/packages/api-client/src/index.ts index da2ff732..1d777429 100644 --- a/frontend/packages/api-client/src/index.ts +++ b/frontend/packages/api-client/src/index.ts @@ -1,3 +1,5 @@ +export * from "./safe-logging"; + export interface ApiSuccess { success: true; data: T; diff --git a/frontend/packages/api-client/src/safe-logging.ts b/frontend/packages/api-client/src/safe-logging.ts new file mode 100644 index 00000000..3d482ed4 --- /dev/null +++ b/frontend/packages/api-client/src/safe-logging.ts @@ -0,0 +1,184 @@ +export const REDACTED = "[REDACTED]"; +export const REDACTED_BINARY = "[REDACTED_BINARY]"; + +const MAX_REDACTION_DEPTH = 12; + +const SENSITIVE_KEYS = new Set([ + "access_key", + "access_token", + "address", + "api_key", + "authorization", + "biometric_payload", + "biometric_template", + "birth_date", + "client_ip", + "client_secret", + "cookie", + "credentials", + "csrf_token", + "date_of_birth", + "device_fingerprint", + "dob", + "document_bytes", + "document_image", + "document_number", + "document_storage_key", + "email", + "external_reference", + "face_embedding", + "face_image", + "first_name", + "full_name", + "ghana_card_number", + "id_token", + "image", + "image_base64", + "image_bytes", + "ip", + "ip_address", + "last_name", + "liveness_video", + "middle_name", + "mrz", + "national_id", + "ocr_text", + "passport_number", + "password", + "phone", + "phone_number", + "postal_address", + "private_key", + "raw_document", + "raw_image", + "raw_ocr", + "refresh_token", + "remote_addr", + "secret", + "secret_access_key", + "selfie", + "selfie_image", + "selfie_storage_key", + "session_token", + "set_cookie", + "storage_key", + "subject_id", + "tax_identification_number", + "tin", + "token", + "user_agent", + "verification_subject_id", +]); + +const SENSITIVE_SUFFIXES = [ + "_access_token", + "_api_key", + "_authorization", + "_client_secret", + "_credential", + "_credentials", + "_fingerprint", + "_password", + "_private_key", + "_refresh_token", + "_secret", + "_session_token", + "_storage_key", + "_subject_id", + "_token", + "_user_agent", +]; + +const SENSITIVE_FRAGMENTS = [ + "biometric", + "face_embedding", + "image_base64", + "image_bytes", + "liveness_video", + "selfie_image", +]; + +function normalizeKey(key: PropertyKey): string { + return String(key) + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/g, "_") + .replace(/^_+|_+$/g, ""); +} + +export function isSensitiveLogKey(key: PropertyKey): boolean { + const normalized = normalizeKey(key); + if (!normalized) return false; + return ( + SENSITIVE_KEYS.has(normalized) || + SENSITIVE_SUFFIXES.some((suffix) => normalized.endsWith(suffix)) || + SENSITIVE_FRAGMENTS.some((fragment) => normalized.includes(fragment)) + ); +} + +export function redactLogText(value: string): string { + return value + .replace(/\bBearer\s+[A-Za-z0-9._~+/=-]+/gi, "Bearer [REDACTED]") + .replace( + /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g, + REDACTED, + ) + .replace(/\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g, REDACTED) + .replace( + /\b(authorization|password|passcode|secret|token|api[_-]?key|client[_-]?secret|access[_-]?key|refresh[_-]?token|session[_-]?token|cookie|email|phone(?:_number)?|first[_-]?name|last[_-]?name|full[_-]?name|address|document[_-]?number|passport[_-]?number|national[_-]?id|date[_-]?of[_-]?birth|dob|external[_-]?reference|device[_-]?fingerprint|subject[_-]?id|verification[_-]?subject[_-]?id|client[_-]?ip|ip[_-]?address|remote[_-]?addr|user[_-]?agent|selfie(?:_image)?|image[_-]?base64|ocr[_-]?text|mrz|biometric[_-]?(?:payload|template))\b\s*[:=]\s*(["']?)([^,;\n\r"'}]+)\2/gi, + (_match, label: string) => `${label}=${REDACTED}`, + ) + .replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, REDACTED) + .replace(/\b(?:\d{1,3}\.){3}\d{1,3}\b/g, REDACTED) + .replace(/(?:\+?\d[\d ().-]{7,}\d)/g, REDACTED); +} + +export function redactLogValue( + value: unknown, + key?: PropertyKey, + depth = 0, +): unknown { + if (key !== undefined && isSensitiveLogKey(key)) return REDACTED; + if (depth >= MAX_REDACTION_DEPTH) return "[REDACTED_DEPTH_LIMIT]"; + if (value === null || value === undefined) return value; + if (typeof value === "string") return redactLogText(value); + if (typeof value === "number" || typeof value === "boolean") return value; + if (value instanceof Uint8Array || value instanceof ArrayBuffer) { + return REDACTED_BINARY; + } + if (Array.isArray(value)) { + return value.map((item) => redactLogValue(item, undefined, depth + 1)); + } + if (typeof value === "object") { + const output: Record = {}; + for (const [entryKey, entryValue] of Object.entries(value)) { + output[redactLogText(entryKey)] = redactLogValue( + entryValue, + entryKey, + depth + 1, + ); + } + return output; + } + return redactLogText(String(value)); +} + +export type SafeLogLevel = "debug" | "info" | "warn" | "error"; + +const LOG_METHODS: Record void> = { + debug: console.debug.bind(console), + info: console.info.bind(console), + warn: console.warn.bind(console), + error: console.error.bind(console), +}; + +export function safeLog( + level: SafeLogLevel, + event: string, + context: Record = {}, +): void { + LOG_METHODS[level]({ + event: redactLogText(event), + context: redactLogValue(context), + }); +} diff --git a/frontend/packages/api-client/test/safe-logging.test.mjs b/frontend/packages/api-client/test/safe-logging.test.mjs new file mode 100644 index 00000000..2258c5ab --- /dev/null +++ b/frontend/packages/api-client/test/safe-logging.test.mjs @@ -0,0 +1,67 @@ +import assert from "node:assert/strict"; +import { rmSync } from "node:fs"; +import { createRequire } from "node:module"; +import { after, test } from "node:test"; + +const require = createRequire(import.meta.url); +const { + REDACTED, + REDACTED_BINARY, + redactLogText, + redactLogValue, +} = require("../.safe-logging-test/safe-logging.js"); + +after(() => { + rmSync(new URL("../.safe-logging-test", import.meta.url), { + force: true, + recursive: true, + }); +}); + +test("redacts nested credentials, PII, evidence, and binary values", () => { + const redacted = redactLogValue({ + authorization: "Bearer secret-token", + profile: { + email: "ada@example.test", + phone_number: "+233241234567", + document_number: "GHA-123456789", + external_reference: "customer-4482", + device_fingerprint: "device-secret", + user_agent: "browser-fingerprint", + verification_subject_id: "vs_sensitive", + safe_status: "pending_review", + }, + evidence: { + selfie_image: "base64-selfie", + face_embedding: [0.1, 0.2], + document_storage_key: "tenant/evidence/front.jpg", + }, + binary: new Uint8Array([1, 2, 3]), + }); + + assert.equal(redacted.authorization, REDACTED); + assert.equal(redacted.profile.email, REDACTED); + assert.equal(redacted.profile.phone_number, REDACTED); + assert.equal(redacted.profile.document_number, REDACTED); + assert.equal(redacted.profile.external_reference, REDACTED); + assert.equal(redacted.profile.device_fingerprint, REDACTED); + assert.equal(redacted.profile.user_agent, REDACTED); + assert.equal(redacted.profile.verification_subject_id, REDACTED); + assert.equal(redacted.profile.safe_status, "pending_review"); + assert.equal(redacted.evidence.selfie_image, REDACTED); + assert.equal(redacted.evidence.face_embedding, REDACTED); + assert.equal(redacted.evidence.document_storage_key, REDACTED); + assert.equal(redacted.binary, REDACTED_BINARY); +}); + +test("redacts secrets and multiword identifiers embedded in free text", () => { + const output = redactLogText( + "full_name=Ada Lovelace; token=top-secret; email=ada@example.test; phone=+233241234567; external_reference=customer-4482; document_number=GHA-123; Authorization: Bearer bearer-secret", + ); + + assert.doesNotMatch( + output, + /Ada Lovelace|top-secret|ada@example\.test|233241234567|customer-4482|GHA-123|bearer-secret/, + ); + assert.match(output, /\[REDACTED\]/); +}); diff --git a/frontend/packages/api-client/tsconfig.safe-logging.json b/frontend/packages/api-client/tsconfig.safe-logging.json new file mode 100644 index 00000000..c33b4fd3 --- /dev/null +++ b/frontend/packages/api-client/tsconfig.safe-logging.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "CommonJS", + "moduleResolution": "Node", + "strict": true, + "noEmit": false, + "declaration": false, + "outDir": ".safe-logging-test", + "lib": ["ES2022", "DOM"] + }, + "include": ["src/safe-logging.ts"] +} diff --git a/frontend/scripts/check-safe-logging.mjs b/frontend/scripts/check-safe-logging.mjs new file mode 100644 index 00000000..65fe4912 --- /dev/null +++ b/frontend/scripts/check-safe-logging.mjs @@ -0,0 +1,105 @@ +import { readFileSync, readdirSync } from "node:fs"; +import { createRequire } from "node:module"; +import { dirname, extname, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const frontendRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const require = createRequire(import.meta.url); +const ts = require( + resolve( + frontendRoot, + "packages/api-client/node_modules/typescript/lib/typescript.js", + ), +); +const allowedConsoleFile = resolve( + frontendRoot, + "packages/api-client/src/safe-logging.ts", +); +const sourceRoots = [ + "dashboard", + "developer-portal", + "identitycore", + "platform-admin", + "verification-portal", + "packages", +].map((path) => resolve(frontendRoot, path)); +const extensions = new Set([".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx"]); +const consoleMethods = new Set(["debug", "info", "log", "warn", "error", "trace"]); +const skipDirectories = new Set([ + ".next", + ".safe-logging-test", + "coverage", + "dist", + "e2e", + "node_modules", + "test", + "tests", +]); + +function scriptKind(path) { + if (path.endsWith(".tsx")) return ts.ScriptKind.TSX; + if (path.endsWith(".jsx")) return ts.ScriptKind.JSX; + if (path.endsWith(".ts")) return ts.ScriptKind.TS; + return ts.ScriptKind.JS; +} + +function containsDirectConsoleCall(path, source) { + const sourceFile = ts.createSourceFile( + path, + source, + ts.ScriptTarget.Latest, + true, + scriptKind(path), + ); + let found = false; + + function visit(node) { + if (found) return; + if ( + ts.isCallExpression(node) && + ts.isPropertyAccessExpression(node.expression) && + ts.isIdentifier(node.expression.expression) && + node.expression.expression.text === "console" && + consoleMethods.has(node.expression.name.text) + ) { + found = true; + return; + } + ts.forEachChild(node, visit); + } + + visit(sourceFile); + return found; +} + +function walk(path, findings) { + for (const entry of readdirSync(path, { withFileTypes: true })) { + if (entry.isDirectory() && skipDirectories.has(entry.name)) continue; + const absolute = join(path, entry.name); + if (entry.isDirectory()) { + walk(absolute, findings); + continue; + } + if (!extensions.has(extname(entry.name))) continue; + if (/\.(?:spec|test)\.[cm]?[jt]sx?$/.test(entry.name)) continue; + if (absolute === allowedConsoleFile) continue; + const source = readFileSync(absolute, "utf8"); + if (containsDirectConsoleCall(absolute, source)) { + findings.push(relative(frontendRoot, absolute)); + } + } +} + +const findings = []; +for (const root of sourceRoots) walk(root, findings); + +if (findings.length) { + console.error( + [ + "Unsafe direct console logging is not allowed in frontend production source.", + "Use safeLog from @identitycore/api-client so sensitive context is redacted.", + ...findings.map((path) => ` - ${path}`), + ].join("\n"), + ); + process.exitCode = 1; +} diff --git a/frontend/verification-portal/src/app/error.tsx b/frontend/verification-portal/src/app/error.tsx index e1b57b3a..008f7216 100644 --- a/frontend/verification-portal/src/app/error.tsx +++ b/frontend/verification-portal/src/app/error.tsx @@ -2,6 +2,7 @@ import { useEffect } from "react"; import { AlertTriangle } from "lucide-react"; +import { safeLog } from "@identitycore/api-client"; import { Button } from "@identitycore/ui"; import { VerificationShell } from "@/components/layout/verification-shell"; @@ -13,10 +14,11 @@ export default function GlobalError({ reset: () => void; }) { useEffect(() => { - // Errors here are boundary-level render failures, not the flow's own - // handled error state. Avoid logging identity evidence or session - // tokens; only the error/digest are safe to surface. - console.error(error); + safeLog("error", "verification_portal_render_error", { + error_name: error.name, + error_message: error.message, + digest: error.digest ?? "", + }); }, [error]); return (