From f07e1a0e65202d93e82f65ae849d733b0f4e9a78 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 10:58:56 +0000 Subject: [PATCH 01/33] feat(logging): add shared backend logging package --- backend/shared/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 backend/shared/__init__.py diff --git a/backend/shared/__init__.py b/backend/shared/__init__.py new file mode 100644 index 00000000..71cd720a --- /dev/null +++ b/backend/shared/__init__.py @@ -0,0 +1 @@ +"""Shared backend utilities used by both Django and managed AI services.""" From f19b3f7d6bc4d8081bd7b32f763be580b66848b6 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 10:59:16 +0000 Subject: [PATCH 02/33] feat(logging): centralize sensitive log redaction --- backend/shared/logging_redaction.py | 248 ++++++++++++++++++++++++++++ 1 file changed, 248 insertions(+) create mode 100644 backend/shared/logging_redaction.py diff --git a/backend/shared/logging_redaction.py b/backend/shared/logging_redaction.py new file mode 100644 index 00000000..5322e1e5 --- /dev/null +++ b/backend/shared/logging_redaction.py @@ -0,0 +1,248 @@ +from __future__ import annotations + +import logging +import re +import traceback +from collections.abc import Mapping +from threading import Lock +from typing import Any + +REDACTED = "[REDACTED]" +REDACTED_BINARY = "[REDACTED_BINARY]" +MAX_REDACTION_DEPTH = 12 + +_SENSITIVE_KEYS = frozenset( + { + # Credentials and session material. + "access_key", + "access_token", + "api_key", + "authorization", + "client_secret", + "cookie", + "credentials", + "csrf_token", + "csrfmiddlewaretoken", + "id_token", + "password", + "private_key", + "refresh_token", + "secret", + "secret_access_key", + "session_token", + "set_cookie", + "token", + # Direct identifiers / PII. + "address", + "birth_date", + "date_of_birth", + "dob", + "document_number", + "email", + "first_name", + "full_name", + "ghana_card_number", + "last_name", + "middle_name", + "national_id", + "passport_number", + "phone", + "phone_number", + "postal_address", + "tax_identification_number", + "tin", + # Evidence locations and raw document / biometric material. + "biometric_payload", + "biometric_template", + "document_bytes", + "document_image", + "document_storage_key", + "face_embedding", + "face_image", + "image", + "image_base64", + "image_bytes", + "liveness_video", + "mrz", + "ocr_text", + "raw_document", + "raw_image", + "raw_ocr", + "selfie", + "selfie_image", + "selfie_storage_key", + "storage_key", + } +) + +_SENSITIVE_SUFFIXES = ( + "_access_token", + "_api_key", + "_authorization", + "_client_secret", + "_credential", + "_credentials", + "_password", + "_private_key", + "_refresh_token", + "_secret", + "_session_token", + "_storage_key", + "_token", +) + +_SENSITIVE_FRAGMENTS = ( + "biometric", + "face_embedding", + "image_base64", + "image_bytes", + "liveness_video", + "selfie_image", +) + +_BEARER_RE = re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._~+/=-]+") +_JWT_RE = re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b") +_EMAIL_RE = re.compile( + r"(? str: + normalized = re.sub(r"[^a-z0-9]+", "_", str(key).strip().lower()) + return normalized.strip("_") + + +def is_sensitive_key(key: object) -> bool: + normalized = _normalize_key(key) + if not normalized: + return False + if normalized in _SENSITIVE_KEYS: + return True + if normalized.endswith(_SENSITIVE_SUFFIXES): + return True + return any(fragment in normalized for fragment in _SENSITIVE_FRAGMENTS) + + +def redact_text(value: str) -> str: + """Redact common secret and PII shapes from unstructured log text.""" + redacted = _BEARER_RE.sub("Bearer [REDACTED]", value) + redacted = _JWT_RE.sub(REDACTED, redacted) + redacted = _AWS_ACCESS_KEY_RE.sub(REDACTED, redacted) + redacted = _CREDENTIAL_ASSIGNMENT_RE.sub( + lambda match: f"{match.group(1)}={REDACTED}", redacted + ) + redacted = _EMAIL_RE.sub(REDACTED, redacted) + redacted = _IPV4_RE.sub(REDACTED, redacted) + redacted = _PHONE_RE.sub(REDACTED, redacted) + return redacted + + +def redact_value(value: Any, *, key: object | None = None, _depth: int = 0) -> Any: + """Return a logging-safe copy of a nested value. + + Redaction is key-aware for structured payloads and shape-aware for free text. + Bytes are never logged because they can contain document or biometric evidence. + """ + if key is not None and is_sensitive_key(key): + return REDACTED + if _depth >= MAX_REDACTION_DEPTH: + return "[REDACTED_DEPTH_LIMIT]" + if value is None or isinstance(value, (bool, int, float)): + return value + if isinstance(value, str): + return redact_text(value) + if isinstance(value, (bytes, bytearray, memoryview)): + return REDACTED_BINARY + if isinstance(value, Mapping): + return { + str(item_key): redact_value( + item_value, + key=item_key, + _depth=_depth + 1, + ) + for item_key, item_value in value.items() + } + if isinstance(value, tuple): + return tuple(redact_value(item, _depth=_depth + 1) for item in value) + if isinstance(value, list): + return [redact_value(item, _depth=_depth + 1) for item in value] + if isinstance(value, (set, frozenset)): + return [redact_value(item, _depth=_depth + 1) for item in value] + return value + + +def _redact_exception( + exc_info: tuple[type[BaseException], BaseException, Any], +) -> str: + rendered = "".join(traceback.format_exception(*exc_info)) + return redact_text(rendered) + + +def sanitize_log_record(record: logging.LogRecord) -> logging.LogRecord: + """Sanitize message args, structured extras, stack text, and exception text in place.""" + record.msg = redact_value(record.msg) + if record.args: + if isinstance(record.args, Mapping): + record.args = redact_value(record.args) + else: + record.args = tuple(redact_value(item) for item in record.args) + + standard = set(logging.LogRecord(None, 0, "", 0, "", (), None).__dict__) + standard.update({"message", "asctime"}) + for field, value in list(record.__dict__.items()): + if field in standard or field.startswith("_"): + continue + record.__dict__[field] = redact_value(value, key=field) + + if record.stack_info: + record.stack_info = redact_text(record.stack_info) + if record.exc_info: + record.exc_text = _redact_exception(record.exc_info) + record.exc_info = None + elif record.exc_text: + record.exc_text = redact_text(record.exc_text) + return record + + +def _safe_make_record(self, *args, **kwargs): + record = _ORIGINAL_MAKE_RECORD(self, *args, **kwargs) + return sanitize_log_record(record) + + +def install_safe_logging() -> None: + """Install the process-wide logging redaction boundary exactly once.""" + global _INSTALLED + if _INSTALLED: + return + with _INSTALL_LOCK: + if _INSTALLED: + return + logging.Logger.makeRecord = _safe_make_record + _INSTALLED = True + + +__all__ = [ + "REDACTED", + "REDACTED_BINARY", + "install_safe_logging", + "is_sensitive_key", + "redact_text", + "redact_value", + "sanitize_log_record", +] From 720eb3a73c94aba07b3eca9e1b3fab6f0d9c1f66 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 10:59:25 +0000 Subject: [PATCH 03/33] feat(logging): expose safe logging to Django and workers --- backend/django/common/safe_logging.py | 28 +++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 backend/django/common/safe_logging.py diff --git a/backend/django/common/safe_logging.py b/backend/django/common/safe_logging.py new file mode 100644 index 00000000..bf9971bd --- /dev/null +++ b/backend/django/common/safe_logging.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +_BACKEND_ROOT = Path(__file__).resolve().parents[2] +if str(_BACKEND_ROOT) not in sys.path: + sys.path.insert(0, str(_BACKEND_ROOT)) + +from shared.logging_redaction import ( # noqa: E402 + REDACTED, + REDACTED_BINARY, + install_safe_logging, + is_sensitive_key, + redact_text, + redact_value, + sanitize_log_record, +) + +__all__ = [ + "REDACTED", + "REDACTED_BINARY", + "install_safe_logging", + "is_sensitive_key", + "redact_text", + "redact_value", + "sanitize_log_record", +] From 1f8e8b9471d7e43aa68b54238e43bf5fad79616a Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 10:59:32 +0000 Subject: [PATCH 04/33] feat(logging): expose safe logging to managed AI --- backend/ai-service/app/core/safe_logging.py | 28 +++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 backend/ai-service/app/core/safe_logging.py diff --git a/backend/ai-service/app/core/safe_logging.py b/backend/ai-service/app/core/safe_logging.py new file mode 100644 index 00000000..67d26ee5 --- /dev/null +++ b/backend/ai-service/app/core/safe_logging.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +_BACKEND_ROOT = Path(__file__).resolve().parents[3] +if str(_BACKEND_ROOT) not in sys.path: + sys.path.insert(0, str(_BACKEND_ROOT)) + +from shared.logging_redaction import ( # noqa: E402 + REDACTED, + REDACTED_BINARY, + install_safe_logging, + is_sensitive_key, + redact_text, + redact_value, + sanitize_log_record, +) + +__all__ = [ + "REDACTED", + "REDACTED_BINARY", + "install_safe_logging", + "is_sensitive_key", + "redact_text", + "redact_value", + "sanitize_log_record", +] From 1b2fa8559898d91882e1f45af6f52b38c0033429 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 10:59:40 +0000 Subject: [PATCH 05/33] feat(logging): install redaction at Django startup --- backend/django/apps/core/apps.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/backend/django/apps/core/apps.py b/backend/django/apps/core/apps.py index 68868e8b..665f028d 100644 --- a/backend/django/apps/core/apps.py +++ b/backend/django/apps/core/apps.py @@ -5,3 +5,10 @@ class CoreConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" name = "apps.core" label = "core" + + def ready(self) -> None: + # Install the redaction boundary after Django configures logging but before + # request/worker code can emit application records. + from common.safe_logging import install_safe_logging + + install_safe_logging() From 4cb315f9ce64e2e25cedcfc7598fca9a125f451f Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 10:59:50 +0000 Subject: [PATCH 06/33] feat(logging): protect Celery startup logs --- backend/django/config/celery.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/backend/django/config/celery.py b/backend/django/config/celery.py index 1d803399..2610c9d7 100644 --- a/backend/django/config/celery.py +++ b/backend/django/config/celery.py @@ -2,6 +2,12 @@ from celery import Celery +from common.safe_logging import install_safe_logging + + +# Celery can initialize its own logging before Django app-ready hooks run. Install +# the redaction boundary here as well so broker/startup and task logs are protected. +install_safe_logging() os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.development") From 0a0359f4eaf7d0d7e0fd6ba73136ffd624e376e8 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:00:00 +0000 Subject: [PATCH 07/33] feat(logging): install redaction before AI imports --- backend/ai-service/app/main.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/backend/ai-service/app/main.py b/backend/ai-service/app/main.py index ad3b9286..c30ca18b 100644 --- a/backend/ai-service/app/main.py +++ b/backend/ai-service/app/main.py @@ -1,8 +1,14 @@ -from fastapi import FastAPI +from app.core.safe_logging import install_safe_logging -from app.core.auth import enforce_internal_token as _enforce_internal_token -from app.routers.health import healthcheck, readiness, router as health_router -from app.routers.processing import ( +# Install before importing processing modules so initialization failures and +# exception paths cannot emit raw evidence or credentials. +install_safe_logging() + +from fastapi import FastAPI # noqa: E402 + +from app.core.auth import enforce_internal_token as _enforce_internal_token # noqa: E402 +from app.routers.health import healthcheck, readiness, router as health_router # noqa: E402 +from app.routers.processing import ( # noqa: E402 document_classify, document_ocr, document_quality, @@ -10,8 +16,8 @@ liveness_check, router as processing_router, ) -from app.runtime import configure_runtime_environment -from app.schemas.processing import ( +from app.runtime import configure_runtime_environment # noqa: E402 +from app.schemas.processing import ( # noqa: E402 AIResultResponse, DocumentClassificationRequest, DocumentOCRRequest, @@ -21,7 +27,7 @@ LivenessCheckRequest, ReadinessResponse, ) -from app.settings import get_settings +from app.settings import get_settings # noqa: E402 settings = get_settings() From 3e3e62de4c56462e8b1409fe81a6313b83f16dc3 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:00:08 +0000 Subject: [PATCH 08/33] build(logging): include shared redaction package in Django image --- backend/django/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/django/Dockerfile b/backend/django/Dockerfile index 33111902..2ae3c89b 100644 --- a/backend/django/Dockerfile +++ b/backend/django/Dockerfile @@ -10,7 +10,7 @@ WORKDIR /app RUN apt-get update \ && apt-get upgrade -y \ && apt-get install -y --no-install-recommends build-essential libpq-dev \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/lists/* COPY backend/pyproject.toml backend/uv.lock /app/backend/ RUN uv export \ @@ -22,6 +22,7 @@ RUN uv export \ --output-file /tmp/requirements-django.txt \ && pip install --no-cache-dir -r /tmp/requirements-django.txt +COPY backend/shared /app/backend/shared COPY backend/django /app/backend/django COPY docs/openapi/identitycore-public-api.yaml /app/docs/openapi/identitycore-public-api.yaml From f2e80eb6039cea356856c3578f7e72ef09946190 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:00:15 +0000 Subject: [PATCH 09/33] fix(build): preserve apt cleanup path --- backend/django/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/django/Dockerfile b/backend/django/Dockerfile index 2ae3c89b..6a7b8056 100644 --- a/backend/django/Dockerfile +++ b/backend/django/Dockerfile @@ -10,7 +10,7 @@ WORKDIR /app RUN apt-get update \ && apt-get upgrade -y \ && apt-get install -y --no-install-recommends build-essential libpq-dev \ - && rm -rf /var/lib/lists/* + && rm -rf /var/lib/apt/lists/* COPY backend/pyproject.toml backend/uv.lock /app/backend/ RUN uv export \ From fc7c45716579ad1b99c202e69281756f5e1bcc76 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:00:23 +0000 Subject: [PATCH 10/33] build(logging): include shared redaction package in AI image --- backend/ai-service/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/ai-service/Dockerfile b/backend/ai-service/Dockerfile index f327f2ee..55b81091 100644 --- a/backend/ai-service/Dockerfile +++ b/backend/ai-service/Dockerfile @@ -32,6 +32,7 @@ RUN uv export \ && pip install --no-cache-dir -r /tmp/requirements-ai.txt \ && pip install --no-cache-dir --force-reinstall opencv-contrib-python-headless==4.10.0.84 +COPY backend/shared /app/backend/shared COPY backend/ai-service /app/backend/ai-service WORKDIR /app/backend/ai-service From bae85b3aad0b36e897e575e9a2899515c0d7361a Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:00:46 +0000 Subject: [PATCH 11/33] test(logging): cover Django Celery and exception redaction --- backend/django/common/test_safe_logging.py | 122 +++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 backend/django/common/test_safe_logging.py diff --git a/backend/django/common/test_safe_logging.py b/backend/django/common/test_safe_logging.py new file mode 100644 index 00000000..3c8fecd8 --- /dev/null +++ b/backend/django/common/test_safe_logging.py @@ -0,0 +1,122 @@ +import io +import logging + +from celery.utils.log import get_task_logger +from django.test import SimpleTestCase + +from common.safe_logging import REDACTED, REDACTED_BINARY, install_safe_logging, redact_value + + +class SafeLoggingTests(SimpleTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + install_safe_logging() + + def _capture(self, logger: logging.Logger, formatter: str = "%(levelname)s %(message)s"): + stream = io.StringIO() + handler = logging.StreamHandler(stream) + handler.setFormatter(logging.Formatter(formatter)) + logger.handlers = [handler] + logger.propagate = False + logger.setLevel(logging.INFO) + self.addCleanup(logger.handlers.clear) + return stream + + def test_recursive_redaction_covers_secret_pii_and_biometric_fields(self): + payload = { + "authorization": "Bearer top-secret", + "profile": { + "email": "ada@example.test", + "phone_number": "+233241234567", + "document_number": "GHA-123456789", + "safe_status": "pending_review", + }, + "evidence": { + "selfie_image": "base64-sensitive-selfie", + "face_embedding": [0.1, 0.2, 0.3], + "document_storage_key": "tenant/evidence/front.jpg", + }, + "binary": b"document-bytes", + } + + redacted = redact_value(payload) + + self.assertEqual(redacted["authorization"], REDACTED) + self.assertEqual(redacted["profile"]["email"], REDACTED) + self.assertEqual(redacted["profile"]["phone_number"], REDACTED) + self.assertEqual(redacted["profile"]["document_number"], REDACTED) + self.assertEqual(redacted["profile"]["safe_status"], "pending_review") + self.assertEqual(redacted["evidence"]["selfie_image"], REDACTED) + self.assertEqual(redacted["evidence"]["face_embedding"], REDACTED) + self.assertEqual(redacted["evidence"]["document_storage_key"], REDACTED) + self.assertEqual(redacted["binary"], REDACTED_BINARY) + + def test_django_logger_redacts_message_arguments_and_structured_extra(self): + logger = logging.getLogger("django.identitycore.redaction-test") + stream = self._capture(logger, "%(message)s payload=%(payload)s") + + logger.info( + "request rejected email=%s Authorization: Bearer %s", + "ada@example.test", + "secret-access-token", + extra={ + "payload": { + "password": "never-log-this", + "selfie_image": "raw-biometric-data", + "safe_reason": "credentials_missing", + } + }, + ) + + output = stream.getvalue() + self.assertNotIn("ada@example.test", output) + self.assertNotIn("secret-access-token", output) + self.assertNotIn("never-log-this", output) + self.assertNotIn("raw-biometric-data", output) + self.assertIn("credentials_missing", output) + self.assertIn(REDACTED, output) + + def test_celery_task_logger_uses_the_same_redaction_boundary(self): + logger = get_task_logger("identitycore.redaction-test") + stream = self._capture(logger, "%(message)s context=%(context)s") + + logger.warning( + "worker retry token=%s", + "celery-secret-token", + extra={ + "context": { + "api_key": "provider-key", + "ocr_text": "raw document text", + "verification_id": "ver_safe_public_id", + } + }, + ) + + output = stream.getvalue() + self.assertNotIn("celery-secret-token", output) + self.assertNotIn("provider-key", output) + self.assertNotIn("raw document text", output) + self.assertIn("ver_safe_public_id", output) + + def test_exception_traceback_is_preserved_without_sensitive_values(self): + logger = logging.getLogger("identitycore.exception-redaction-test") + stream = self._capture(logger) + + try: + raise RuntimeError( + "token=runtime-secret email=ada@example.test " + "phone=+233241234567 document_number=GHA-123456789" + ) + except RuntimeError: + logger.exception("provider failed first_name=Ada") + + output = stream.getvalue() + self.assertIn("Traceback", output) + self.assertIn("RuntimeError", output) + self.assertNotIn("runtime-secret", output) + self.assertNotIn("ada@example.test", output) + self.assertNotIn("+233241234567", output) + self.assertNotIn("GHA-123456789", output) + self.assertNotIn("first_name=Ada", output) + self.assertIn("first_name=[REDACTED]", output) From f6d79a485b8e7b968837f54ac26b2e57829959ed Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:01:12 +0000 Subject: [PATCH 12/33] test(logging): cover managed AI redaction failures --- backend/ai-service/tests/test_safe_logging.py | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 backend/ai-service/tests/test_safe_logging.py diff --git a/backend/ai-service/tests/test_safe_logging.py b/backend/ai-service/tests/test_safe_logging.py new file mode 100644 index 00000000..702e0704 --- /dev/null +++ b/backend/ai-service/tests/test_safe_logging.py @@ -0,0 +1,73 @@ +import io +import logging + +from app.core.safe_logging import REDACTED, install_safe_logging, redact_value + + +def _capture(logger: logging.Logger, formatter: str = "%(message)s"): + stream = io.StringIO() + handler = logging.StreamHandler(stream) + handler.setFormatter(logging.Formatter(formatter)) + logger.handlers = [handler] + logger.propagate = False + logger.setLevel(logging.INFO) + return stream + + +def test_managed_ai_uses_shared_nested_redaction_corpus(): + install_safe_logging() + payload = { + "authorization": "Bearer ai-secret", + "request": { + "email": "subject@example.test", + "document_number": "P1234567", + "safe_operation": "face_compare", + }, + "biometrics": { + "face_embedding": [0.12, 0.34], + "selfie_image": "base64-selfie", + }, + } + + redacted = redact_value(payload) + + assert redacted["authorization"] == REDACTED + assert redacted["request"]["email"] == REDACTED + assert redacted["request"]["document_number"] == REDACTED + assert redacted["request"]["safe_operation"] == "face_compare" + assert redacted["biometrics"]["face_embedding"] == REDACTED + assert redacted["biometrics"]["selfie_image"] == REDACTED + + +def test_managed_ai_logger_redacts_structured_context_and_exception_text(): + install_safe_logging() + logger = logging.getLogger("identitycore.ai.redaction-test") + stream = _capture(logger, "%(message)s context=%(context)s") + + try: + raise ValueError( + "api_key=provider-secret email=subject@example.test " + "image_base64=raw-biometric" + ) + except ValueError: + logger.exception( + "AI processing failed Authorization: Bearer %s", + "internal-shared-token", + extra={ + "context": { + "document_storage_key": "tenant/evidence/document.jpg", + "ocr_text": "raw OCR contents", + "operation": "document_ocr", + } + }, + ) + + output = stream.getvalue() + assert "provider-secret" not in output + assert "subject@example.test" not in output + assert "raw-biometric" not in output + assert "internal-shared-token" not in output + assert "tenant/evidence/document.jpg" not in output + assert "raw OCR contents" not in output + assert "document_ocr" in output + assert "Traceback" in output From c8f851cb3679e8aad6acf5e4215fc33e09059258 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:02:09 +0000 Subject: [PATCH 13/33] feat(logging): add frontend safe logging boundary --- .../packages/api-client/src/safe-logging.ts | 168 ++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 frontend/packages/api-client/src/safe-logging.ts diff --git a/frontend/packages/api-client/src/safe-logging.ts b/frontend/packages/api-client/src/safe-logging.ts new file mode 100644 index 00000000..40997145 --- /dev/null +++ b/frontend/packages/api-client/src/safe-logging.ts @@ -0,0 +1,168 @@ +export const REDACTED = "[REDACTED]"; +export const REDACTED_BINARY = "[REDACTED_BINARY]"; + +const MAX_REDACTION_DEPTH = 12; + +const SENSITIVE_KEYS = new Set([ + "access_key", + "access_token", + "address", + "api_key", + "authorization", + "biometric_payload", + "biometric_template", + "birth_date", + "client_secret", + "cookie", + "credentials", + "csrf_token", + "date_of_birth", + "dob", + "document_bytes", + "document_image", + "document_number", + "document_storage_key", + "email", + "face_embedding", + "face_image", + "first_name", + "full_name", + "ghana_card_number", + "id_token", + "image", + "image_base64", + "image_bytes", + "last_name", + "liveness_video", + "middle_name", + "mrz", + "national_id", + "ocr_text", + "passport_number", + "password", + "phone", + "phone_number", + "postal_address", + "private_key", + "raw_document", + "raw_image", + "raw_ocr", + "refresh_token", + "secret", + "secret_access_key", + "selfie", + "selfie_image", + "selfie_storage_key", + "session_token", + "set_cookie", + "storage_key", + "tax_identification_number", + "tin", + "token", +]); + +const SENSITIVE_SUFFIXES = [ + "_access_token", + "_api_key", + "_authorization", + "_client_secret", + "_credential", + "_credentials", + "_password", + "_private_key", + "_refresh_token", + "_secret", + "_session_token", + "_storage_key", + "_token", +]; + +const SENSITIVE_FRAGMENTS = [ + "biometric", + "face_embedding", + "image_base64", + "image_bytes", + "liveness_video", + "selfie_image", +]; + +function normalizeKey(key: PropertyKey): string { + return String(key) + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/g, "_") + .replace(/^_+|_+$/g, ""); +} + +export function isSensitiveLogKey(key: PropertyKey): boolean { + const normalized = normalizeKey(key); + if (!normalized) return false; + return ( + SENSITIVE_KEYS.has(normalized) || + SENSITIVE_SUFFIXES.some((suffix) => normalized.endsWith(suffix)) || + SENSITIVE_FRAGMENTS.some((fragment) => normalized.includes(fragment)) + ); +} + +export function redactLogText(value: string): string { + return value + .replace(/\bBearer\s+[A-Za-z0-9._~+/=-]+/gi, "Bearer [REDACTED]") + .replace( + /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g, + REDACTED, + ) + .replace(/\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g, REDACTED) + .replace( + /\b(authorization|password|passcode|secret|token|api[_-]?key|client[_-]?secret|access[_-]?key|refresh[_-]?token|session[_-]?token|cookie|email|phone(?:_number)?|first[_-]?name|last[_-]?name|full[_-]?name|address|document[_-]?number|passport[_-]?number|national[_-]?id|date[_-]?of[_-]?birth|dob|selfie(?:_image)?|image[_-]?base64|ocr[_-]?text|mrz|biometric[_-]?(?:payload|template))\b\s*[:=]\s*(["']?)([^\s,;"'}]+)\2/gi, + (_match, label: string) => `${label}=${REDACTED}`, + ) + .replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, REDACTED) + .replace(/\b(?:\d{1,3}\.){3}\d{1,3}\b/g, REDACTED) + .replace(/(?:\+?\d[\d ().-]{7,}\d)/g, REDACTED); +} + +export function redactLogValue( + value: unknown, + key?: PropertyKey, + depth = 0, +): unknown { + if (key !== undefined && isSensitiveLogKey(key)) return REDACTED; + if (depth >= MAX_REDACTION_DEPTH) return "[REDACTED_DEPTH_LIMIT]"; + if (value === null || value === undefined) return value; + if (typeof value === "string") return redactLogText(value); + if (typeof value === "number" || typeof value === "boolean") return value; + if (value instanceof Uint8Array || value instanceof ArrayBuffer) { + return REDACTED_BINARY; + } + if (Array.isArray(value)) { + return value.map((item) => redactLogValue(item, undefined, depth + 1)); + } + if (typeof value === "object") { + const output: Record = {}; + for (const [entryKey, entryValue] of Object.entries(value)) { + output[entryKey] = redactLogValue(entryValue, entryKey, depth + 1); + } + return output; + } + return redactLogText(String(value)); +} + +export type SafeLogLevel = "debug" | "info" | "warn" | "error"; + +const LOG_METHODS: Record void> = { + debug: console.debug.bind(console), + info: console.info.bind(console), + warn: console.warn.bind(console), + error: console.error.bind(console), +}; + +export function safeLog( + level: SafeLogLevel, + event: string, + context: Record = {}, +): void { + LOG_METHODS[level]({ + event: redactLogText(event), + context: redactLogValue(context), + }); +} From be3f15cb11f3d52f183569e8d4c1565b0e27f094 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:02:16 +0000 Subject: [PATCH 14/33] test(logging): add frontend redaction test build config --- .../packages/api-client/tsconfig.safe-logging.json | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 frontend/packages/api-client/tsconfig.safe-logging.json diff --git a/frontend/packages/api-client/tsconfig.safe-logging.json b/frontend/packages/api-client/tsconfig.safe-logging.json new file mode 100644 index 00000000..c33b4fd3 --- /dev/null +++ b/frontend/packages/api-client/tsconfig.safe-logging.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "CommonJS", + "moduleResolution": "Node", + "strict": true, + "noEmit": false, + "declaration": false, + "outDir": ".safe-logging-test", + "lib": ["ES2022", "DOM"] + }, + "include": ["src/safe-logging.ts"] +} From 15c2edcfc694b7521a401625afdac5eee62cb120 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:02:27 +0000 Subject: [PATCH 15/33] test(logging): add frontend adversarial redaction tests --- .../api-client/test/safe-logging.test.mjs | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 frontend/packages/api-client/test/safe-logging.test.mjs diff --git a/frontend/packages/api-client/test/safe-logging.test.mjs b/frontend/packages/api-client/test/safe-logging.test.mjs new file mode 100644 index 00000000..2d94251f --- /dev/null +++ b/frontend/packages/api-client/test/safe-logging.test.mjs @@ -0,0 +1,59 @@ +import assert from "node:assert/strict"; +import { rmSync } from "node:fs"; +import { createRequire } from "node:module"; +import { after, test } from "node:test"; + +const require = createRequire(import.meta.url); +const { + REDACTED, + REDACTED_BINARY, + redactLogText, + redactLogValue, +} = require("../.safe-logging-test/safe-logging.js"); + +after(() => { + rmSync(new URL("../.safe-logging-test", import.meta.url), { + force: true, + recursive: true, + }); +}); + +test("redacts nested credentials, PII, evidence, and binary values", () => { + const redacted = redactLogValue({ + authorization: "Bearer secret-token", + profile: { + email: "ada@example.test", + phone_number: "+233241234567", + document_number: "GHA-123456789", + safe_status: "pending_review", + }, + evidence: { + selfie_image: "base64-selfie", + face_embedding: [0.1, 0.2], + document_storage_key: "tenant/evidence/front.jpg", + }, + binary: new Uint8Array([1, 2, 3]), + }); + + assert.equal(redacted.authorization, REDACTED); + assert.equal(redacted.profile.email, REDACTED); + assert.equal(redacted.profile.phone_number, REDACTED); + assert.equal(redacted.profile.document_number, REDACTED); + assert.equal(redacted.profile.safe_status, "pending_review"); + assert.equal(redacted.evidence.selfie_image, REDACTED); + assert.equal(redacted.evidence.face_embedding, REDACTED); + assert.equal(redacted.evidence.document_storage_key, REDACTED); + assert.equal(redacted.binary, REDACTED_BINARY); +}); + +test("redacts secrets and identifiers embedded in free text", () => { + const output = redactLogText( + "token=top-secret email=ada@example.test phone=+233241234567 document_number=GHA-123 Authorization: Bearer bearer-secret", + ); + + assert.doesNotMatch( + output, + /top-secret|ada@example\.test|233241234567|GHA-123|bearer-secret/, + ); + assert.match(output, /\[REDACTED\]/); +}); From e6bbe3355eb5495c186063b2cc2daa2ff88126c9 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:02:38 +0000 Subject: [PATCH 16/33] test(logging): fail CI on unsafe frontend console logging --- frontend/scripts/check-safe-logging.mjs | 59 +++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 frontend/scripts/check-safe-logging.mjs diff --git a/frontend/scripts/check-safe-logging.mjs b/frontend/scripts/check-safe-logging.mjs new file mode 100644 index 00000000..4e1e3013 --- /dev/null +++ b/frontend/scripts/check-safe-logging.mjs @@ -0,0 +1,59 @@ +import { readFileSync, readdirSync } from "node:fs"; +import { dirname, extname, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const frontendRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const allowedConsoleFile = resolve( + frontendRoot, + "packages/api-client/src/safe-logging.ts", +); +const sourceRoots = [ + "dashboard", + "developer-portal", + "identitycore", + "platform-admin", + "verification-portal", + "packages", +].map((path) => resolve(frontendRoot, path)); +const extensions = new Set([".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx"]); +const directConsole = /\bconsole\.(?:debug|info|log|warn|error|trace)\s*\(/; +const skipDirectories = new Set([ + ".next", + ".safe-logging-test", + "coverage", + "dist", + "e2e", + "node_modules", + "test", + "tests", +]); + +function walk(path, findings) { + for (const entry of readdirSync(path, { withFileTypes: true })) { + if (entry.isDirectory() && skipDirectories.has(entry.name)) continue; + const absolute = join(path, entry.name); + if (entry.isDirectory()) { + walk(absolute, findings); + continue; + } + if (!extensions.has(extname(entry.name))) continue; + if (/\.(?:spec|test)\.[cm]?[jt]sx?$/.test(entry.name)) continue; + if (absolute === allowedConsoleFile) continue; + const source = readFileSync(absolute, "utf8"); + if (directConsole.test(source)) findings.push(relative(frontendRoot, absolute)); + } +} + +const findings = []; +for (const root of sourceRoots) walk(root, findings); + +if (findings.length) { + console.error( + [ + "Unsafe direct console logging is not allowed in frontend production source.", + "Use safeLog from @identitycore/api-client so sensitive context is redacted.", + ...findings.map((path) => ` - ${path}`), + ].join("\n"), + ); + process.exitCode = 1; +} From 273be6831f57a6f32749bf490ddb2027b1fa5adc Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:02:56 +0000 Subject: [PATCH 17/33] feat(logging): export frontend safe logger --- frontend/packages/api-client/src/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/packages/api-client/src/index.ts b/frontend/packages/api-client/src/index.ts index da2ff732..1d777429 100644 --- a/frontend/packages/api-client/src/index.ts +++ b/frontend/packages/api-client/src/index.ts @@ -1,3 +1,5 @@ +export * from "./safe-logging"; + export interface ApiSuccess { success: true; data: T; From 6b1dbbf35d9a850d49e4ffc6531c351fc6b55dca Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:03:02 +0000 Subject: [PATCH 18/33] test(logging): enforce safe frontend logging in lint --- frontend/packages/api-client/package.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frontend/packages/api-client/package.json b/frontend/packages/api-client/package.json index db440b1f..9efdef4f 100644 --- a/frontend/packages/api-client/package.json +++ b/frontend/packages/api-client/package.json @@ -7,6 +7,9 @@ "exports": { ".": "./src/index.ts" }, + "scripts": { + "lint": "tsc -p tsconfig.json && tsc -p tsconfig.safe-logging.json && node --test test/safe-logging.test.mjs && node ../../scripts/check-safe-logging.mjs" + }, "devDependencies": { "typescript": "^5" } From 878335c30bb52d35bb2b8ab46c5f490d0621449a Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:03:29 +0000 Subject: [PATCH 19/33] docs(logging): document safe telemetry and redaction rules --- docs/operations/logging-redaction.md | 41 ++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 docs/operations/logging-redaction.md diff --git a/docs/operations/logging-redaction.md b/docs/operations/logging-redaction.md new file mode 100644 index 00000000..c6210cbf --- /dev/null +++ b/docs/operations/logging-redaction.md @@ -0,0 +1,41 @@ +# Logging redaction and safe telemetry + +IdentityCore treats logs as an operational data stream, not as a place to store verification evidence or user data. Application logs must never contain credentials, session material, direct subject PII, raw document/OCR content, storage locations for evidence, or biometric payloads. + +## Backend boundary + +Django, Celery workers, storage/provider call paths, and the managed AI service use the shared redaction implementation in `backend/shared/logging_redaction.py`. + +The boundary is installed when Django starts, before Celery initializes its application logger, and before the AI service imports processing routes. It sanitizes every Python `LogRecord` after `extra` fields have been attached and before handlers format the record. This covers: + +- nested dictionaries/lists passed as structured logging context; +- positional and mapping logging arguments; +- bearer/JWT-like credentials and common credential assignments embedded in free text; +- common email, phone, and IP shapes in free text; +- exception messages and tracebacks; +- bytes and byte-like values, which are always replaced rather than rendered. + +Safe operational dimensions such as public request/verification IDs, operation names, status/reason codes, provider codes, durations, retry counts, and queue names may be logged when they do not themselves contain subject data. + +When a new secret, PII field, document field, or biometric representation is introduced, add its normalized key to the shared redaction corpus and add an adversarial test before using it in telemetry. + +## Frontend boundary + +Frontend code must use `safeLog` exported by `@identitycore/api-client`. The logger applies the same categories of key-aware and free-text redaction before calling the browser console. + +Direct `console.log`, `console.info`, `console.warn`, `console.error`, `console.debug`, or `console.trace` calls are rejected by the frontend lint gate for production source. This prevents a future component from bypassing the redaction helper with a raw API error, token, form state, capture payload, or server response. + +## What not to log + +Do not log: + +- authorization headers, API keys, passwords, cookies, refresh/session tokens, provider credentials, or private keys; +- names, email addresses, phone numbers, addresses, dates of birth, national/passport/document numbers, or tax identifiers; +- OCR/MRZ text, raw document images/bytes, evidence storage keys, selfies, face embeddings, liveness media, or biometric templates; +- complete request/response bodies from identity, provider, storage, or webhook operations. + +Prefer stable reason/error codes and public correlation identifiers over raw exception/request payloads. + +## Verification + +The CI suite includes adversarial tests for Django, Celery, managed AI, and frontend logging. Tests intentionally place sensitive values in nested structured fields, positional arguments, binary values, and exception text and assert that those values never reach formatted log output. Frontend lint also fails when production source introduces a direct console logging call. From 316a02175e5bf5102430743e5d9f452bb30a40d9 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:04:26 +0000 Subject: [PATCH 20/33] fix(logging): harden and optimize record sanitization --- backend/shared/logging_redaction.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/backend/shared/logging_redaction.py b/backend/shared/logging_redaction.py index 5322e1e5..d7822287 100644 --- a/backend/shared/logging_redaction.py +++ b/backend/shared/logging_redaction.py @@ -101,7 +101,9 @@ ) _BEARER_RE = re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._~+/=-]+") -_JWT_RE = re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b") +_JWT_RE = re.compile( + r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b" +) _EMAIL_RE = re.compile( r"(? A return value if isinstance(value, str): return redact_text(value) + if isinstance(value, BaseException): + return f"{value.__class__.__name__}: {redact_text(str(value))}" if isinstance(value, (bytes, bytearray, memoryview)): return REDACTED_BINARY if isinstance(value, Mapping): - return { - str(item_key): redact_value( + redacted_mapping = {} + for item_key, item_value in value.items(): + safe_key = redact_text(str(item_key)) + redacted_mapping[safe_key] = redact_value( item_value, key=item_key, _depth=_depth + 1, ) - for item_key, item_value in value.items() - } + return redacted_mapping if isinstance(value, tuple): return tuple(redact_value(item, _depth=_depth + 1) for item in value) if isinstance(value, list): return [redact_value(item, _depth=_depth + 1) for item in value] if isinstance(value, (set, frozenset)): return [redact_value(item, _depth=_depth + 1) for item in value] - return value + return redact_text(str(value)) def _redact_exception( @@ -203,10 +215,8 @@ def sanitize_log_record(record: logging.LogRecord) -> logging.LogRecord: else: record.args = tuple(redact_value(item) for item in record.args) - standard = set(logging.LogRecord(None, 0, "", 0, "", (), None).__dict__) - standard.update({"message", "asctime"}) for field, value in list(record.__dict__.items()): - if field in standard or field.startswith("_"): + if field in _STANDARD_LOG_RECORD_ATTRS or field.startswith("_"): continue record.__dict__[field] = redact_value(value, key=field) From 9f7fb3d2ea308658fd0080f477c94c6b0973c8dd Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:04:54 +0000 Subject: [PATCH 21/33] test(logging): prove provider storage and exception boundaries --- .../common/test_safe_logging_boundaries.py | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 backend/django/common/test_safe_logging_boundaries.py diff --git a/backend/django/common/test_safe_logging_boundaries.py b/backend/django/common/test_safe_logging_boundaries.py new file mode 100644 index 00000000..50e507af --- /dev/null +++ b/backend/django/common/test_safe_logging_boundaries.py @@ -0,0 +1,59 @@ +import io +import logging + +from django.test import SimpleTestCase + +from common.safe_logging import install_safe_logging + + +class SafeLoggingBoundaryTests(SimpleTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + install_safe_logging() + + def _capture(self, logger_name: str): + stream = io.StringIO() + handler = logging.StreamHandler(stream) + handler.setFormatter(logging.Formatter("%(message)s context=%(context)s")) + logger = logging.getLogger(logger_name) + logger.handlers = [handler] + logger.propagate = False + logger.setLevel(logging.INFO) + self.addCleanup(logger.handlers.clear) + return logger, stream + + def test_storage_and_provider_loggers_share_the_global_boundary(self): + for logger_name in ("common.storage", "apps.providers.services"): + with self.subTest(logger=logger_name): + logger, stream = self._capture(logger_name) + logger.info( + "operation completed", + extra={ + "context": { + "storage_key": "tenant/evidence/private.jpg", + "client_secret": "provider-secret", + "provider_code": "safe-provider-code", + } + }, + ) + output = stream.getvalue() + self.assertNotIn("tenant/evidence/private.jpg", output) + self.assertNotIn("provider-secret", output) + self.assertIn("safe-provider-code", output) + + def test_exception_objects_and_sensitive_mapping_keys_are_sanitized(self): + logger, stream = self._capture("identitycore.argument-redaction-test") + error = RuntimeError("token=exception-secret email=subject@example.test") + + logger.error( + "provider error: %s", + error, + extra={"context": {"subject@example.test": "lookup", "status": "failed"}}, + ) + + output = stream.getvalue() + self.assertNotIn("exception-secret", output) + self.assertNotIn("subject@example.test", output) + self.assertIn("RuntimeError", output) + self.assertIn("failed", output) From dc9bcb867af57256f9a02507c30a7467e06ae670 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:07:16 +0000 Subject: [PATCH 22/33] fix(logging): cover correlated identifiers and multiword PII --- backend/shared/logging_redaction.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/backend/shared/logging_redaction.py b/backend/shared/logging_redaction.py index d7822287..3c5a84c5 100644 --- a/backend/shared/logging_redaction.py +++ b/backend/shared/logging_redaction.py @@ -32,16 +32,21 @@ "session_token", "set_cookie", "token", - # Direct identifiers / PII. + # Direct identifiers / PII / correlation values that may identify a subject. "address", "birth_date", + "client_ip", "date_of_birth", + "device_fingerprint", "dob", "document_number", "email", + "external_reference", "first_name", "full_name", "ghana_card_number", + "ip", + "ip_address", "last_name", "middle_name", "national_id", @@ -49,8 +54,12 @@ "phone", "phone_number", "postal_address", + "remote_addr", + "subject_id", "tax_identification_number", "tin", + "user_agent", + "verification_subject_id", # Evidence locations and raw document / biometric material. "biometric_payload", "biometric_template", @@ -82,13 +91,16 @@ "_client_secret", "_credential", "_credentials", + "_fingerprint", "_password", "_private_key", "_refresh_token", "_secret", "_session_token", "_storage_key", + "_subject_id", "_token", + "_user_agent", ) _SENSITIVE_FRAGMENTS = ( @@ -114,9 +126,11 @@ r"(?i)\b(authorization|password|passcode|secret|token|api[_-]?key|client[_-]?secret|" r"access[_-]?key|refresh[_-]?token|session[_-]?token|cookie|email|phone(?:_number)?|" r"first[_-]?name|last[_-]?name|full[_-]?name|address|document[_-]?number|passport[_-]?number|" - r"national[_-]?id|date[_-]?of[_-]?birth|dob|selfie(?:_image)?|image[_-]?base64|ocr[_-]?text|mrz|" + r"national[_-]?id|date[_-]?of[_-]?birth|dob|external[_-]?reference|device[_-]?fingerprint|" + r"subject[_-]?id|verification[_-]?subject[_-]?id|client[_-]?ip|ip[_-]?address|remote[_-]?addr|" + r"user[_-]?agent|selfie(?:_image)?|image[_-]?base64|ocr[_-]?text|mrz|" r"biometric[_-]?(?:payload|template))\b" - r"\s*[:=]\s*([\"']?)([^\s,;\"'}]+)\2" + r"\s*[:=]\s*([\"']?)([^,;\n\r\"'}]+)\2" ) _AWS_ACCESS_KEY_RE = re.compile(r"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b") _STANDARD_LOG_RECORD_ATTRS = frozenset( From b1217a4874608c6be804f4b8bd2623dc94dad9e9 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:07:34 +0000 Subject: [PATCH 23/33] fix(logging): redact correlated frontend identifiers --- .../packages/api-client/src/safe-logging.ts | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/frontend/packages/api-client/src/safe-logging.ts b/frontend/packages/api-client/src/safe-logging.ts index 40997145..3d482ed4 100644 --- a/frontend/packages/api-client/src/safe-logging.ts +++ b/frontend/packages/api-client/src/safe-logging.ts @@ -12,17 +12,20 @@ const SENSITIVE_KEYS = new Set([ "biometric_payload", "biometric_template", "birth_date", + "client_ip", "client_secret", "cookie", "credentials", "csrf_token", "date_of_birth", + "device_fingerprint", "dob", "document_bytes", "document_image", "document_number", "document_storage_key", "email", + "external_reference", "face_embedding", "face_image", "first_name", @@ -32,6 +35,8 @@ const SENSITIVE_KEYS = new Set([ "image", "image_base64", "image_bytes", + "ip", + "ip_address", "last_name", "liveness_video", "middle_name", @@ -48,6 +53,7 @@ const SENSITIVE_KEYS = new Set([ "raw_image", "raw_ocr", "refresh_token", + "remote_addr", "secret", "secret_access_key", "selfie", @@ -56,9 +62,12 @@ const SENSITIVE_KEYS = new Set([ "session_token", "set_cookie", "storage_key", + "subject_id", "tax_identification_number", "tin", "token", + "user_agent", + "verification_subject_id", ]); const SENSITIVE_SUFFIXES = [ @@ -68,13 +77,16 @@ const SENSITIVE_SUFFIXES = [ "_client_secret", "_credential", "_credentials", + "_fingerprint", "_password", "_private_key", "_refresh_token", "_secret", "_session_token", "_storage_key", + "_subject_id", "_token", + "_user_agent", ]; const SENSITIVE_FRAGMENTS = [ @@ -113,7 +125,7 @@ export function redactLogText(value: string): string { ) .replace(/\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g, REDACTED) .replace( - /\b(authorization|password|passcode|secret|token|api[_-]?key|client[_-]?secret|access[_-]?key|refresh[_-]?token|session[_-]?token|cookie|email|phone(?:_number)?|first[_-]?name|last[_-]?name|full[_-]?name|address|document[_-]?number|passport[_-]?number|national[_-]?id|date[_-]?of[_-]?birth|dob|selfie(?:_image)?|image[_-]?base64|ocr[_-]?text|mrz|biometric[_-]?(?:payload|template))\b\s*[:=]\s*(["']?)([^\s,;"'}]+)\2/gi, + /\b(authorization|password|passcode|secret|token|api[_-]?key|client[_-]?secret|access[_-]?key|refresh[_-]?token|session[_-]?token|cookie|email|phone(?:_number)?|first[_-]?name|last[_-]?name|full[_-]?name|address|document[_-]?number|passport[_-]?number|national[_-]?id|date[_-]?of[_-]?birth|dob|external[_-]?reference|device[_-]?fingerprint|subject[_-]?id|verification[_-]?subject[_-]?id|client[_-]?ip|ip[_-]?address|remote[_-]?addr|user[_-]?agent|selfie(?:_image)?|image[_-]?base64|ocr[_-]?text|mrz|biometric[_-]?(?:payload|template))\b\s*[:=]\s*(["']?)([^,;\n\r"'}]+)\2/gi, (_match, label: string) => `${label}=${REDACTED}`, ) .replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, REDACTED) @@ -140,7 +152,11 @@ export function redactLogValue( if (typeof value === "object") { const output: Record = {}; for (const [entryKey, entryValue] of Object.entries(value)) { - output[entryKey] = redactLogValue(entryValue, entryKey, depth + 1); + output[redactLogText(entryKey)] = redactLogValue( + entryValue, + entryKey, + depth + 1, + ); } return output; } From 678cb3bda4ad34922886408617284a18a9e1eec6 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:07:56 +0000 Subject: [PATCH 24/33] test(logging): cover correlated and multiword PII --- .../packages/api-client/test/safe-logging.test.mjs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/frontend/packages/api-client/test/safe-logging.test.mjs b/frontend/packages/api-client/test/safe-logging.test.mjs index 2d94251f..2258c5ab 100644 --- a/frontend/packages/api-client/test/safe-logging.test.mjs +++ b/frontend/packages/api-client/test/safe-logging.test.mjs @@ -25,6 +25,10 @@ test("redacts nested credentials, PII, evidence, and binary values", () => { email: "ada@example.test", phone_number: "+233241234567", document_number: "GHA-123456789", + external_reference: "customer-4482", + device_fingerprint: "device-secret", + user_agent: "browser-fingerprint", + verification_subject_id: "vs_sensitive", safe_status: "pending_review", }, evidence: { @@ -39,6 +43,10 @@ test("redacts nested credentials, PII, evidence, and binary values", () => { assert.equal(redacted.profile.email, REDACTED); assert.equal(redacted.profile.phone_number, REDACTED); assert.equal(redacted.profile.document_number, REDACTED); + assert.equal(redacted.profile.external_reference, REDACTED); + assert.equal(redacted.profile.device_fingerprint, REDACTED); + assert.equal(redacted.profile.user_agent, REDACTED); + assert.equal(redacted.profile.verification_subject_id, REDACTED); assert.equal(redacted.profile.safe_status, "pending_review"); assert.equal(redacted.evidence.selfie_image, REDACTED); assert.equal(redacted.evidence.face_embedding, REDACTED); @@ -46,14 +54,14 @@ test("redacts nested credentials, PII, evidence, and binary values", () => { assert.equal(redacted.binary, REDACTED_BINARY); }); -test("redacts secrets and identifiers embedded in free text", () => { +test("redacts secrets and multiword identifiers embedded in free text", () => { const output = redactLogText( - "token=top-secret email=ada@example.test phone=+233241234567 document_number=GHA-123 Authorization: Bearer bearer-secret", + "full_name=Ada Lovelace; token=top-secret; email=ada@example.test; phone=+233241234567; external_reference=customer-4482; document_number=GHA-123; Authorization: Bearer bearer-secret", ); assert.doesNotMatch( output, - /top-secret|ada@example\.test|233241234567|GHA-123|bearer-secret/, + /Ada Lovelace|top-secret|ada@example\.test|233241234567|customer-4482|GHA-123|bearer-secret/, ); assert.match(output, /\[REDACTED\]/); }); From 2cfaece59b1ff83bd3d4b7d0507eb99cdf08fb41 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:08:25 +0000 Subject: [PATCH 25/33] test(logging): cover correlated backend identifiers --- .../common/test_safe_logging_boundaries.py | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/backend/django/common/test_safe_logging_boundaries.py b/backend/django/common/test_safe_logging_boundaries.py index 50e507af..7a47a221 100644 --- a/backend/django/common/test_safe_logging_boundaries.py +++ b/backend/django/common/test_safe_logging_boundaries.py @@ -33,6 +33,10 @@ def test_storage_and_provider_loggers_share_the_global_boundary(self): "context": { "storage_key": "tenant/evidence/private.jpg", "client_secret": "provider-secret", + "external_reference": "customer-4482", + "device_fingerprint": "device-secret", + "user_agent": "browser-fingerprint", + "verification_subject_id": "vs_sensitive", "provider_code": "safe-provider-code", } }, @@ -40,11 +44,15 @@ def test_storage_and_provider_loggers_share_the_global_boundary(self): output = stream.getvalue() self.assertNotIn("tenant/evidence/private.jpg", output) self.assertNotIn("provider-secret", output) + self.assertNotIn("customer-4482", output) + self.assertNotIn("device-secret", output) + self.assertNotIn("browser-fingerprint", output) + self.assertNotIn("vs_sensitive", output) self.assertIn("safe-provider-code", output) def test_exception_objects_and_sensitive_mapping_keys_are_sanitized(self): logger, stream = self._capture("identitycore.argument-redaction-test") - error = RuntimeError("token=exception-secret email=subject@example.test") + error = RuntimeError("token=exception-secret; email=subject@example.test") logger.error( "provider error: %s", @@ -57,3 +65,15 @@ def test_exception_objects_and_sensitive_mapping_keys_are_sanitized(self): self.assertNotIn("subject@example.test", output) self.assertIn("RuntimeError", output) self.assertIn("failed", output) + + def test_multiword_pii_in_free_text_is_fully_removed(self): + logger, stream = self._capture("identitycore.multiword-redaction-test") + logger.info( + "review full_name=Ada Lovelace; external_reference=customer-4482; status=pending", + extra={"context": {"status": "pending"}}, + ) + + output = stream.getvalue() + self.assertNotIn("Ada Lovelace", output) + self.assertNotIn("customer-4482", output) + self.assertIn("status=pending", output) From f3bb8888be28a80e6e369a07aa945b5d2d4c817a Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:09:10 +0000 Subject: [PATCH 26/33] fix(logging): redact rendered messages without breaking format args --- backend/shared/logging_redaction.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/backend/shared/logging_redaction.py b/backend/shared/logging_redaction.py index 3c5a84c5..1dcc1704 100644 --- a/backend/shared/logging_redaction.py +++ b/backend/shared/logging_redaction.py @@ -221,13 +221,15 @@ def _redact_exception( def sanitize_log_record(record: logging.LogRecord) -> logging.LogRecord: - """Sanitize message args, structured extras, stack text, and exception text in place.""" - record.msg = redact_value(record.msg) + """Sanitize rendered messages, structured extras, stack text, and exceptions.""" if record.args: - if isinstance(record.args, Mapping): - record.args = redact_value(record.args) - else: - record.args = tuple(redact_value(item) for item in record.args) + # Render once using Python logging's normal interpolation rules, then redact + # the complete result. Redacting the template before interpolation can remove + # placeholders and cause handlers to fail with formatting errors. + record.msg = redact_text(record.getMessage()) + record.args = () + else: + record.msg = redact_value(record.msg) for field, value in list(record.__dict__.items()): if field in _STANDARD_LOG_RECORD_ATTRS or field.startswith("_"): From 6cae92f0e0ac2394e464bd7ea45bc26d2cd50e88 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:10:26 +0000 Subject: [PATCH 27/33] fix(logging): contain malformed format strings safely --- backend/shared/logging_redaction.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/backend/shared/logging_redaction.py b/backend/shared/logging_redaction.py index 1dcc1704..b13527db 100644 --- a/backend/shared/logging_redaction.py +++ b/backend/shared/logging_redaction.py @@ -10,6 +10,7 @@ REDACTED = "[REDACTED]" REDACTED_BINARY = "[REDACTED_BINARY]" MAX_REDACTION_DEPTH = 12 +LOG_FORMAT_ERROR = "[LOG_FORMAT_ERROR]" _SENSITIVE_KEYS = frozenset( { @@ -224,9 +225,13 @@ def sanitize_log_record(record: logging.LogRecord) -> logging.LogRecord: """Sanitize rendered messages, structured extras, stack text, and exceptions.""" if record.args: # Render once using Python logging's normal interpolation rules, then redact - # the complete result. Redacting the template before interpolation can remove - # placeholders and cause handlers to fail with formatting errors. - record.msg = redact_text(record.getMessage()) + # the complete result. If the caller supplied a malformed format string, + # discard the args instead of letting logging break request/worker execution. + try: + rendered_message = record.getMessage() + except (TypeError, ValueError): + rendered_message = f"{redact_value(record.msg)} {LOG_FORMAT_ERROR}" + record.msg = redact_text(str(rendered_message)) record.args = () else: record.msg = redact_value(record.msg) @@ -264,6 +269,7 @@ def install_safe_logging() -> None: __all__ = [ + "LOG_FORMAT_ERROR", "REDACTED", "REDACTED_BINARY", "install_safe_logging", From 23961ed373683903e0d81d9d32ca0d9e9a99611e Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:10:49 +0000 Subject: [PATCH 28/33] test(logging): contain malformed format strings --- .../django/common/test_safe_logging_boundaries.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/backend/django/common/test_safe_logging_boundaries.py b/backend/django/common/test_safe_logging_boundaries.py index 7a47a221..94faf4d3 100644 --- a/backend/django/common/test_safe_logging_boundaries.py +++ b/backend/django/common/test_safe_logging_boundaries.py @@ -3,7 +3,7 @@ from django.test import SimpleTestCase -from common.safe_logging import install_safe_logging +from common.safe_logging import LOG_FORMAT_ERROR, install_safe_logging class SafeLoggingBoundaryTests(SimpleTestCase): @@ -77,3 +77,16 @@ def test_multiword_pii_in_free_text_is_fully_removed(self): self.assertNotIn("Ada Lovelace", output) self.assertNotIn("customer-4482", output) self.assertIn("status=pending", output) + + def test_malformed_format_string_does_not_raise_or_render_arguments(self): + logger, stream = self._capture("identitycore.format-error-test") + logger.info( + "provider failed without placeholder", + "secret-that-must-not-render", + extra={"context": {"status": "failed"}}, + ) + + output = stream.getvalue() + self.assertNotIn("secret-that-must-not-render", output) + self.assertIn(LOG_FORMAT_ERROR, output) + self.assertIn("failed", output) From e2691729ae9315553462683dcb540539cd33921a Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:11:01 +0000 Subject: [PATCH 29/33] fix(logging): keep Django safe logging exports aligned --- backend/django/common/safe_logging.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/django/common/safe_logging.py b/backend/django/common/safe_logging.py index bf9971bd..5c0ef2dc 100644 --- a/backend/django/common/safe_logging.py +++ b/backend/django/common/safe_logging.py @@ -8,6 +8,7 @@ sys.path.insert(0, str(_BACKEND_ROOT)) from shared.logging_redaction import ( # noqa: E402 + LOG_FORMAT_ERROR, REDACTED, REDACTED_BINARY, install_safe_logging, @@ -18,6 +19,7 @@ ) __all__ = [ + "LOG_FORMAT_ERROR", "REDACTED", "REDACTED_BINARY", "install_safe_logging", From be97df11efcb014c9af1660956da074f112aebc1 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:11:11 +0000 Subject: [PATCH 30/33] fix(logging): keep AI safe logging exports aligned --- backend/ai-service/app/core/safe_logging.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/ai-service/app/core/safe_logging.py b/backend/ai-service/app/core/safe_logging.py index 67d26ee5..3fa281c7 100644 --- a/backend/ai-service/app/core/safe_logging.py +++ b/backend/ai-service/app/core/safe_logging.py @@ -8,6 +8,7 @@ sys.path.insert(0, str(_BACKEND_ROOT)) from shared.logging_redaction import ( # noqa: E402 + LOG_FORMAT_ERROR, REDACTED, REDACTED_BINARY, install_safe_logging, @@ -18,6 +19,7 @@ ) __all__ = [ + "LOG_FORMAT_ERROR", "REDACTED", "REDACTED_BINARY", "install_safe_logging", From c72a71d0effb23f096b9fa5731520201f50675db Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:30:11 +0000 Subject: [PATCH 31/33] Fix managed AI redaction acceptance test --- backend/ai-service/tests/test_safe_logging.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/ai-service/tests/test_safe_logging.py b/backend/ai-service/tests/test_safe_logging.py index 702e0704..0942bea3 100644 --- a/backend/ai-service/tests/test_safe_logging.py +++ b/backend/ai-service/tests/test_safe_logging.py @@ -35,8 +35,9 @@ def test_managed_ai_uses_shared_nested_redaction_corpus(): assert redacted["request"]["email"] == REDACTED assert redacted["request"]["document_number"] == REDACTED assert redacted["request"]["safe_operation"] == "face_compare" - assert redacted["biometrics"]["face_embedding"] == REDACTED - assert redacted["biometrics"]["selfie_image"] == REDACTED + # A container explicitly named "biometrics" is sensitive as a whole. The + # redactor intentionally fails closed instead of retaining its structure. + assert redacted["biometrics"] == REDACTED def test_managed_ai_logger_redacts_structured_context_and_exception_text(): From 6668efb9ded9b1136fa0475ec0f2d06077423a62 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:30:22 +0000 Subject: [PATCH 32/33] Route verification portal errors through safe logging --- frontend/verification-portal/src/app/error.tsx | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/frontend/verification-portal/src/app/error.tsx b/frontend/verification-portal/src/app/error.tsx index e1b57b3a..008f7216 100644 --- a/frontend/verification-portal/src/app/error.tsx +++ b/frontend/verification-portal/src/app/error.tsx @@ -2,6 +2,7 @@ import { useEffect } from "react"; import { AlertTriangle } from "lucide-react"; +import { safeLog } from "@identitycore/api-client"; import { Button } from "@identitycore/ui"; import { VerificationShell } from "@/components/layout/verification-shell"; @@ -13,10 +14,11 @@ export default function GlobalError({ reset: () => void; }) { useEffect(() => { - // Errors here are boundary-level render failures, not the flow's own - // handled error state. Avoid logging identity evidence or session - // tokens; only the error/digest are safe to surface. - console.error(error); + safeLog("error", "verification_portal_render_error", { + error_name: error.name, + error_message: error.message, + digest: error.digest ?? "", + }); }, [error]); return ( From 24882cf5a7c0a6704a8a3ce34422c68a3c69f83c Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:30:34 +0000 Subject: [PATCH 33/33] Make safe logging lint AST-aware --- frontend/scripts/check-safe-logging.mjs | 50 ++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/frontend/scripts/check-safe-logging.mjs b/frontend/scripts/check-safe-logging.mjs index 4e1e3013..65fe4912 100644 --- a/frontend/scripts/check-safe-logging.mjs +++ b/frontend/scripts/check-safe-logging.mjs @@ -1,8 +1,16 @@ import { readFileSync, readdirSync } from "node:fs"; +import { createRequire } from "node:module"; import { dirname, extname, join, relative, resolve } from "node:path"; import { fileURLToPath } from "node:url"; const frontendRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const require = createRequire(import.meta.url); +const ts = require( + resolve( + frontendRoot, + "packages/api-client/node_modules/typescript/lib/typescript.js", + ), +); const allowedConsoleFile = resolve( frontendRoot, "packages/api-client/src/safe-logging.ts", @@ -16,7 +24,7 @@ const sourceRoots = [ "packages", ].map((path) => resolve(frontendRoot, path)); const extensions = new Set([".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx"]); -const directConsole = /\bconsole\.(?:debug|info|log|warn|error|trace)\s*\(/; +const consoleMethods = new Set(["debug", "info", "log", "warn", "error", "trace"]); const skipDirectories = new Set([ ".next", ".safe-logging-test", @@ -28,6 +36,42 @@ const skipDirectories = new Set([ "tests", ]); +function scriptKind(path) { + if (path.endsWith(".tsx")) return ts.ScriptKind.TSX; + if (path.endsWith(".jsx")) return ts.ScriptKind.JSX; + if (path.endsWith(".ts")) return ts.ScriptKind.TS; + return ts.ScriptKind.JS; +} + +function containsDirectConsoleCall(path, source) { + const sourceFile = ts.createSourceFile( + path, + source, + ts.ScriptTarget.Latest, + true, + scriptKind(path), + ); + let found = false; + + function visit(node) { + if (found) return; + if ( + ts.isCallExpression(node) && + ts.isPropertyAccessExpression(node.expression) && + ts.isIdentifier(node.expression.expression) && + node.expression.expression.text === "console" && + consoleMethods.has(node.expression.name.text) + ) { + found = true; + return; + } + ts.forEachChild(node, visit); + } + + visit(sourceFile); + return found; +} + function walk(path, findings) { for (const entry of readdirSync(path, { withFileTypes: true })) { if (entry.isDirectory() && skipDirectories.has(entry.name)) continue; @@ -40,7 +84,9 @@ function walk(path, findings) { if (/\.(?:spec|test)\.[cm]?[jt]sx?$/.test(entry.name)) continue; if (absolute === allowedConsoleFile) continue; const source = readFileSync(absolute, "utf8"); - if (directConsole.test(source)) findings.push(relative(frontendRoot, absolute)); + if (containsDirectConsoleCall(absolute, source)) { + findings.push(relative(frontendRoot, absolute)); + } } }