diff --git a/README.md b/README.md index 1cd96c8..1dd9447 100644 --- a/README.md +++ b/README.md @@ -166,8 +166,29 @@ This is the single source of truth for where fields end up in the final log outp | **Payload containers** | `payload`, `headers` | Root | Used in PII masking path | | **structlog internals** | `message`, `timestamp` | Root | Set by structlog processors | | **ECS event staging** | `ecs_event` | Root → renamed to `event` | Use `ecs_event` in log calls to avoid structlog's `event` message key conflict | +| **Service-configured root fields** | Keys named in `configure_root_fields()` / `ECSCTX_ROOT_FIELDS` | Root | Service-chosen additions to the allowlist (see below) | | **Everything else** | Any non-allowlisted key | `extra.*` | Auto-wrapped by `namespace_ecs_fields` | +### Configurable root fields + +A consuming service can promote additional keys to root (instead of `extra.*`) without +ecsctx hardcoding its domain schema. Configure in any of three ways (precedence: +explicit call > Django setting > env var): + +```python +# 1. Django settings.py — a list of keys +ECSCTX_ROOT_FIELDS = ["customer", "booking"] + +# 2. Framework-agnostic env var — comma-separated +# ECSCTX_ROOT_FIELDS="customer,booking" + +# 3. Programmatic, at startup +from ecsctx import configure_root_fields +configure_root_fields(extra_fields=["customer", "booking"]) +``` + +The built-in `ROOT_ALLOWLIST` is never reduced — configured fields only extend it. + **PII handling** (see [section 13](#13-pii-masking--tokenization) for full details): - **Automatic log masking**: `mask_sensitive_data` processor applies HMAC-SHA-256 tokenization (`ptok:v1:...`) and key-based redaction - **Explicit encryption API**: `protect()` encrypts (AES-256-GCM), `reveal()` decrypts, `tokenize()` produces deterministic HMAC tokens @@ -1333,6 +1354,7 @@ If you use a `common-logs` ingest pipeline, it can enforce ECS field types so ma | `PII_REFRESH_SECONDS` | Keyset refresh interval in seconds (vault provider) | `300` | No | | `PII_VAULT_TIMEOUT` | HTTP timeout for Vault requests in seconds | `10` | No | | `APP_VERSION` | Application version in `service.version` | `"0.0.0"` | No | +| `ECSCTX_ROOT_FIELDS` | Extra root-level log fields (CSV), extends `ROOT_ALLOWLIST` | — | No | | `SERVICE_TYPE` | Service type: `app`, `rq`, `celery` | Auto-detected from argv | No | | `PROJECT_NAME` | Project name in `project.name` + Vector data stream | `"connect"` | **Yes** | | `ENVIRONMENT` | Environment name for Vector data stream namespace | - | **Yes** | diff --git a/ecsctx/__init__.py b/ecsctx/__init__.py index b6e6c67..a5fc651 100644 --- a/ecsctx/__init__.py +++ b/ecsctx/__init__.py @@ -33,6 +33,8 @@ from ecsctx.processors import ( configure_masking, configure_masking_from_env, + configure_root_fields, + configure_root_fields_from_env, contextvars_injector, mask_sensitive_data, namespace_ecs_fields, @@ -61,6 +63,9 @@ "configure_masking", "configure_masking_from_env", "safe_tokenize", + # Root-fields config + "configure_root_fields", + "configure_root_fields_from_env", # PII "configure_pii", "configure_pii_from_env", diff --git a/ecsctx/contrib/django/processors.py b/ecsctx/contrib/django/processors.py index 1ddda70..423b70d 100644 --- a/ecsctx/contrib/django/processors.py +++ b/ecsctx/contrib/django/processors.py @@ -64,6 +64,47 @@ def _reset_masking_settings_flag() -> None: _mask_settings_attempted = False +_root_fields_settings_attempted = False + + +def _auto_configure_root_fields() -> None: + """Bridge the Django ``ECSCTX_ROOT_FIELDS`` setting into the core + root-fields config, lazily on first use. + + Same shape as _auto_configure_masking: runs at log time (settings fully + loaded), retries until settings are accessible, and preserves precedence — + an explicit configure_root_fields() call wins, then this setting, then the + ECSCTX_ROOT_FIELDS env var. + """ + global _root_fields_settings_attempted + if _root_fields_settings_attempted: + return + + from ecsctx.processors import configure_root_fields, root_fields_are_configured + + if root_fields_are_configured(): + _root_fields_settings_attempted = True + return + + try: + from django.conf import settings + + extra_fields = getattr(settings, "ECSCTX_ROOT_FIELDS", None) + except Exception: + # Settings not ready yet — retry on the next call (at real log time). + return + + _root_fields_settings_attempted = True + if extra_fields is not None: + configure_root_fields(extra_fields=list(extra_fields)) + + +def _reset_root_fields_settings_flag() -> None: + """Reset the settings-bridge guard. For testing only.""" + global _root_fields_settings_attempted + _root_fields_settings_attempted = False + + def _get_django_user_model(): """Get the Django User model from settings.""" try: @@ -122,9 +163,10 @@ def contextvars_injector(_logger, _method_name, event_dict): 6. Django User object serialization (NEW) """ - # 0. Auto-configure PII + masking exemptions on first call + # 0. Auto-configure PII + masking exemptions + root fields on first call _auto_configure_pii() _auto_configure_masking() + _auto_configure_root_fields() # 1. Inject from LoggingContext (decorators set this) event_dict = _inject_logging_context(event_dict) diff --git a/ecsctx/processors.py b/ecsctx/processors.py index e8d7c93..aec27ed 100644 --- a/ecsctx/processors.py +++ b/ecsctx/processors.py @@ -94,21 +94,67 @@ def _detect_service(): PRIMARY_KEYS = ROOT_ALLOWLIST +# --- Configurable root fields (mirrors the masking-exemption config pattern) --- +# A consuming service can promote additional keys to root (instead of `extra.*`) +# without ecsctx hardcoding its domain schema. Precedence: an explicit +# configure_root_fields() call wins, then the Django ECSCTX_ROOT_FIELDS setting +# (bridged lazily in contrib.django), then the ECSCTX_ROOT_FIELDS env var (CSV). +_custom_root_fields: frozenset | None = None +_root_fields_auto_configure_attempted: bool = False + + +def configure_root_fields(*, extra_fields: list[str] | None = None) -> None: + """Extend ROOT_ALLOWLIST with service-chosen root keys (highest precedence).""" + global _custom_root_fields, _root_fields_auto_configure_attempted + _custom_root_fields = frozenset(f for f in (extra_fields or []) if f) + _root_fields_auto_configure_attempted = True + + +def configure_root_fields_from_env() -> None: + """Load extra root fields from the ECSCTX_ROOT_FIELDS env var (CSV). Idempotent.""" + global _custom_root_fields, _root_fields_auto_configure_attempted + if _root_fields_auto_configure_attempted or _custom_root_fields is not None: + return + _root_fields_auto_configure_attempted = True + raw = os.environ.get("ECSCTX_ROOT_FIELDS", "") + _custom_root_fields = frozenset(p.strip() for p in raw.split(",") if p.strip()) + + +def root_fields_are_configured() -> bool: + """True if extra root fields have been explicitly set or env-loaded.""" + return _custom_root_fields is not None + + +def _get_root_allowlist() -> frozenset: + if _custom_root_fields is None: + configure_root_fields_from_env() + return ROOT_ALLOWLIST | (_custom_root_fields or frozenset()) + + +def _reset_root_fields() -> None: + """Reset root-fields config. For testing only.""" + global _custom_root_fields, _root_fields_auto_configure_attempted + _custom_root_fields = None + _root_fields_auto_configure_attempted = False + + def reshape_log_event(event_dict) -> dict: """Reshape log event: allowlisted keys stay at root, everything else goes into extra. - - Keys in ROOT_ALLOWLIST always stay at root. + - Keys in ROOT_ALLOWLIST (plus any configure_root_fields() additions) + always stay at root. - Non-allowlisted keys (scalars, lists, and dicts) are wrapped into ``extra``. """ if not isinstance(event_dict, dict): return event_dict + allowlist = _get_root_allowlist() reshaped = {} extra = {} for key, value in event_dict.items(): - if key in ROOT_ALLOWLIST or key.startswith("_") or key.startswith("event."): + if key in allowlist or key.startswith("_") or key.startswith("event."): reshaped[key] = value else: extra[key] = value diff --git a/tests/conftest.py b/tests/conftest.py index 54c980c..acf551d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,7 @@ import pytest from ecsctx.pii import _reset as _reset_pii -from ecsctx.processors import _reset_masking +from ecsctx.processors import _reset_masking, _reset_root_fields def _make_key_b64(length: int = 32) -> str: @@ -78,3 +78,10 @@ def _reset_masking_module(): """Reset masking exemption config between tests.""" yield _reset_masking() + + +@pytest.fixture(autouse=True) +def _reset_root_fields_module(): + """Reset configurable root-fields state between tests.""" + yield + _reset_root_fields() diff --git a/tests/test_django_processors.py b/tests/test_django_processors.py index c534917..b330f01 100644 --- a/tests/test_django_processors.py +++ b/tests/test_django_processors.py @@ -8,14 +8,21 @@ from ecsctx.contrib.django.processors import ( _auto_configure_masking, + _auto_configure_root_fields, _get_django_user_model, _is_django_user, _reset_masking_settings_flag, + _reset_root_fields_settings_flag, _serialize_django_user, contextvars_injector, ) from ecsctx.pii import configure_pii -from ecsctx.processors import _safe_dump_and_mask, masking_is_configured +from ecsctx.processors import ( + _safe_dump_and_mask, + masking_is_configured, + reshape_log_event, + root_fields_are_configured, +) User = get_user_model() @@ -159,3 +166,62 @@ def __getattr__(self, name): assert masking_is_configured() out = _safe_dump_and_mask({"payment_methods": [{"name": "KNET"}]}) assert out["payment_methods"][0]["name"] == "KNET" + + +class TestRootFieldsSettingsBridge: + @pytest.fixture(autouse=True) + def _reset_flag(self): + _reset_root_fields_settings_flag() + yield + _reset_root_fields_settings_flag() + + @override_settings(ECSCTX_ROOT_FIELDS=["customer"]) + def test_setting_applied_via_auto_configure(self): + _auto_configure_root_fields() + assert root_fields_are_configured() + result = reshape_log_event({"message": "hi", "customer": {"id": "c1"}}) + assert result["customer"] == {"id": "c1"} + assert "extra" not in result + + @override_settings(ECSCTX_ROOT_FIELDS=["customer"]) + def test_setting_applied_via_processor_first_call(self): + # A real log record through the Django contextvars_injector must + # trigger the settings bridge — no setup_logging() required. + contextvars_injector(None, None, {"event": "hello"}) + assert root_fields_are_configured() + result = reshape_log_event({"message": "hi", "customer": {"id": "c1"}}) + assert result["customer"] == {"id": "c1"} + + def test_absent_setting_is_noop(self): + _auto_configure_root_fields() + assert not root_fields_are_configured() + result = reshape_log_event({"message": "hi", "customer": {"id": "c1"}}) + assert result["extra"] == {"customer": {"id": "c1"}} + + @override_settings(ECSCTX_ROOT_FIELDS=["customer"]) + def test_explicit_configure_beats_setting(self): + from ecsctx.processors import configure_root_fields + + configure_root_fields(extra_fields=[]) # explicit empty wins + _auto_configure_root_fields() + result = reshape_log_event({"message": "hi", "customer": {"id": "c1"}}) + assert "customer" not in result + assert result["extra"] == {"customer": {"id": "c1"}} + + def test_retries_when_settings_not_ready(self): + """Bridge must not burn its one-shot flag if settings access raises + (e.g. during settings.py import) — it retries at real log time.""" + + class _NotReady: + def __getattr__(self, name): + raise RuntimeError("Requested setting, but settings are not configured.") + + with patch("django.conf.settings", _NotReady()): + _auto_configure_root_fields() + assert not root_fields_are_configured() + + with override_settings(ECSCTX_ROOT_FIELDS=["customer"]): + _auto_configure_root_fields() + assert root_fields_are_configured() + result = reshape_log_event({"message": "hi", "customer": {"id": "c1"}}) + assert result["customer"] == {"id": "c1"} diff --git a/tests/test_processors.py b/tests/test_processors.py index 59c3f1d..334bea4 100644 --- a/tests/test_processors.py +++ b/tests/test_processors.py @@ -7,7 +7,9 @@ _safe_dump_and_mask, safe_tokenize, configure_masking, + configure_root_fields, masking_is_configured, + root_fields_are_configured, namespace_ecs_fields, reshape_log_event, ) @@ -320,3 +322,42 @@ def test_empty_default_still_configured(self, token_keyset_path): out = _safe_dump_and_mask({"customer": {"name": "John"}}) assert out["customer"]["name"].startswith("ptok:v1:") assert masking_is_configured() + + +class TestRootFieldsConfig: + def test_default_non_allowlisted_goes_to_extra(self): + result = reshape_log_event({"message": "hi", "customer": {"id": "c1"}}) + assert "customer" not in result + assert result["extra"] == {"customer": {"id": "c1"}} + + def test_configured_field_stays_at_root(self): + configure_root_fields(extra_fields=["customer"]) + result = reshape_log_event({"message": "hi", "customer": {"id": "c1"}}) + assert result["customer"] == {"id": "c1"} + assert "extra" not in result + + def test_builtin_allowlist_unaffected_by_config(self): + configure_root_fields(extra_fields=["customer"]) + result = reshape_log_event({"message": "hi", "session_id": "s1", "other": 1}) + assert result["session_id"] == "s1" + assert result["extra"] == {"other": 1} + + def test_env_var_config(self, monkeypatch): + monkeypatch.setenv("ECSCTX_ROOT_FIELDS", "customer, booking") + result = reshape_log_event({ + "message": "hi", + "customer": {"id": "c1"}, + "booking": {"ref": "b1"}, + "other": 1, + }) + assert result["customer"] == {"id": "c1"} + assert result["booking"] == {"ref": "b1"} + assert result["extra"] == {"other": 1} + + def test_explicit_beats_env(self, monkeypatch): + monkeypatch.setenv("ECSCTX_ROOT_FIELDS", "customer") + configure_root_fields(extra_fields=[]) + result = reshape_log_event({"message": "hi", "customer": {"id": "c1"}}) + assert "customer" not in result + assert result["extra"] == {"customer": {"id": "c1"}} + assert root_fields_are_configured() diff --git a/uv.lock b/uv.lock index 02d9d57..b4a4e73 100644 --- a/uv.lock +++ b/uv.lock @@ -380,7 +380,7 @@ wheels = [ [[package]] name = "ecsctx" -version = "0.5.3" +version = "0.5.4" source = { editable = "." } dependencies = [ { name = "cryptography" },