feat(processors): configurable root fields via ECSCTX_ROOT_FIELDS

README.md

@@ -166,8 +166,29 @@ This is the single source of truth for where fields end up in the final log outp
 | **Payload containers** | `payload`, `headers` | Root | Used in PII masking path |
 | **structlog internals** | `message`, `timestamp` | Root | Set by structlog processors |
 | **ECS event staging** | `ecs_event` | Root → renamed to `event` | Use `ecs_event` in log calls to avoid structlog's `event` message key conflict |
+| **Service-configured root fields** | Keys named in `configure_root_fields()` / `ECSCTX_ROOT_FIELDS` | Root | Service-chosen additions to the allowlist (see below) |
 | **Everything else** | Any non-allowlisted key | `extra.*` | Auto-wrapped by `namespace_ecs_fields` |
 
+### Configurable root fields
+
+A consuming service can promote additional keys to root (instead of `extra.*`) without
+ecsctx hardcoding its domain schema. Configure in any of three ways (precedence:
+explicit call > Django setting > env var):
+
+```python
+# 1. Django settings.py — a list of keys
+ECSCTX_ROOT_FIELDS = ["customer", "booking"]
+
+# 2. Framework-agnostic env var — comma-separated
+#    ECSCTX_ROOT_FIELDS="customer,booking"
+
+# 3. Programmatic, at startup
+from ecsctx import configure_root_fields
+configure_root_fields(extra_fields=["customer", "booking"])
+```
+
+The built-in `ROOT_ALLOWLIST` is never reduced — configured fields only extend it.
+
 **PII handling** (see [section 13](#13-pii-masking--tokenization) for full details):
 - **Automatic log masking**: `mask_sensitive_data` processor applies HMAC-SHA-256 tokenization (`ptok:v1:...`) and key-based redaction
 - **Explicit encryption API**: `protect()` encrypts (AES-256-GCM), `reveal()` decrypts, `tokenize()` produces deterministic HMAC tokens
@@ -1333,6 +1354,7 @@ If you use a `common-logs` ingest pipeline, it can enforce ECS field types so ma
 | `PII_REFRESH_SECONDS` | Keyset refresh interval in seconds (vault provider) | `300` | No |
 | `PII_VAULT_TIMEOUT` | HTTP timeout for Vault requests in seconds | `10` | No |
 | `APP_VERSION` | Application version in `service.version` | `"0.0.0"` | No |
+| `ECSCTX_ROOT_FIELDS` | Extra root-level log fields (CSV), extends `ROOT_ALLOWLIST` | — | No |
 | `SERVICE_TYPE` | Service type: `app`, `rq`, `celery` | Auto-detected from argv | No |
 | `PROJECT_NAME` | Project name in `project.name` + Vector data stream | `"connect"` | **Yes** |
 | `ENVIRONMENT` | Environment name for Vector data stream namespace | - | **Yes** |

ecsctx/__init__.py

@@ -33,6 +33,8 @@
 from ecsctx.processors import (
     configure_masking,
     configure_masking_from_env,
+    configure_root_fields,
+    configure_root_fields_from_env,
     contextvars_injector,
     mask_sensitive_data,
     namespace_ecs_fields,
@@ -61,6 +63,9 @@
     "configure_masking",
     "configure_masking_from_env",
     "safe_tokenize",
+    # Root-fields config
+    "configure_root_fields",
+    "configure_root_fields_from_env",
     # PII
     "configure_pii",
     "configure_pii_from_env",

ecsctx/contrib/django/processors.py

@@ -64,6 +64,47 @@ def _reset_masking_settings_flag() -> None:
     _mask_settings_attempted = False
 
 
+_root_fields_settings_attempted = False
+
+
+def _auto_configure_root_fields() -> None:
+    """Bridge the Django ``ECSCTX_ROOT_FIELDS`` setting into the core
+    root-fields config, lazily on first use.
+
+    Same shape as _auto_configure_masking: runs at log time (settings fully
+    loaded), retries until settings are accessible, and preserves precedence —
+    an explicit configure_root_fields() call wins, then this setting, then the
+    ECSCTX_ROOT_FIELDS env var.
+    """
+    global _root_fields_settings_attempted
+    if _root_fields_settings_attempted:
+        return
+
+    from ecsctx.processors import configure_root_fields, root_fields_are_configured
+
+    if root_fields_are_configured():
+        _root_fields_settings_attempted = True
+        return
+
+    try:
+        from django.conf import settings
+
+        extra_fields = getattr(settings, "ECSCTX_ROOT_FIELDS", None)
+    except Exception:
+        # Settings not ready yet — retry on the next call (at real log time).
+        return
+
+    _root_fields_settings_attempted = True
+    if extra_fields is not None:
+        configure_root_fields(extra_fields=list(extra_fields))
+
+
+def _reset_root_fields_settings_flag() -> None:
+    """Reset the settings-bridge guard. For testing only."""
+    global _root_fields_settings_attempted
+    _root_fields_settings_attempted = False
+
+
 def _get_django_user_model():
     """Get the Django User model from settings."""
     try:
@@ -122,9 +163,10 @@ def contextvars_injector(_logger, _method_name, event_dict):
     6. Django User object serialization (NEW)
     """
 
-    # 0. Auto-configure PII + masking exemptions on first call
+    # 0. Auto-configure PII + masking exemptions + root fields on first call
     _auto_configure_pii()
     _auto_configure_masking()
+    _auto_configure_root_fields()
 
     # 1. Inject from LoggingContext (decorators set this)
     event_dict = _inject_logging_context(event_dict)

ecsctx/processors.py

@@ -94,21 +94,67 @@ def _detect_service():
 PRIMARY_KEYS = ROOT_ALLOWLIST
 
 
+# --- Configurable root fields (mirrors the masking-exemption config pattern) ---
+# A consuming service can promote additional keys to root (instead of `extra.*`)
+# without ecsctx hardcoding its domain schema. Precedence: an explicit
+# configure_root_fields() call wins, then the Django ECSCTX_ROOT_FIELDS setting
+# (bridged lazily in contrib.django), then the ECSCTX_ROOT_FIELDS env var (CSV).
+_custom_root_fields: frozenset | None = None
+_root_fields_auto_configure_attempted: bool = False
+
+
+def configure_root_fields(*, extra_fields: list[str] | None = None) -> None:
+    """Extend ROOT_ALLOWLIST with service-chosen root keys (highest precedence)."""
+    global _custom_root_fields, _root_fields_auto_configure_attempted
+    _custom_root_fields = frozenset(f for f in (extra_fields or []) if f)
+    _root_fields_auto_configure_attempted = True
+
+
+def configure_root_fields_from_env() -> None:
+    """Load extra root fields from the ECSCTX_ROOT_FIELDS env var (CSV). Idempotent."""
+    global _custom_root_fields, _root_fields_auto_configure_attempted
+    if _root_fields_auto_configure_attempted or _custom_root_fields is not None:
+        return
+    _root_fields_auto_configure_attempted = True
+    raw = os.environ.get("ECSCTX_ROOT_FIELDS", "")
+    _custom_root_fields = frozenset(p.strip() for p in raw.split(",") if p.strip())
+
+
+def root_fields_are_configured() -> bool:
+    """True if extra root fields have been explicitly set or env-loaded."""
+    return _custom_root_fields is not None
+
+
+def _get_root_allowlist() -> frozenset:
+    if _custom_root_fields is None:
+        configure_root_fields_from_env()
+    return ROOT_ALLOWLIST | (_custom_root_fields or frozenset())
+
+
+def _reset_root_fields() -> None:
+    """Reset root-fields config. For testing only."""
+    global _custom_root_fields, _root_fields_auto_configure_attempted
+    _custom_root_fields = None
+    _root_fields_auto_configure_attempted = False
+
+
 def reshape_log_event(event_dict) -> dict:
     """Reshape log event: allowlisted keys stay at root, everything else goes into extra.
 
-    - Keys in ROOT_ALLOWLIST always stay at root.
+    - Keys in ROOT_ALLOWLIST (plus any configure_root_fields() additions)
+      always stay at root.
     - Non-allowlisted keys (scalars, lists, and dicts) are wrapped into
       ``extra``.
     """
     if not isinstance(event_dict, dict):
         return event_dict
 
+    allowlist = _get_root_allowlist()
     reshaped = {}
     extra = {}
 
     for key, value in event_dict.items():
-        if key in ROOT_ALLOWLIST or key.startswith("_") or key.startswith("event."):
+        if key in allowlist or key.startswith("_") or key.startswith("event."):
             reshaped[key] = value
         else:
             extra[key] = value

tests/conftest.py

@@ -7,7 +7,7 @@
 import pytest
 
 from ecsctx.pii import _reset as _reset_pii
-from ecsctx.processors import _reset_masking
+from ecsctx.processors import _reset_masking, _reset_root_fields
 
 
 def _make_key_b64(length: int = 32) -> str:
@@ -78,3 +78,10 @@ def _reset_masking_module():
     """Reset masking exemption config between tests."""
     yield
     _reset_masking()
+
+
+@pytest.fixture(autouse=True)
+def _reset_root_fields_module():
+    """Reset configurable root-fields state between tests."""
+    yield
+    _reset_root_fields()

tests/test_django_processors.py

@@ -8,14 +8,21 @@
 
 from ecsctx.contrib.django.processors import (
     _auto_configure_masking,
+    _auto_configure_root_fields,
     _get_django_user_model,
     _is_django_user,
     _reset_masking_settings_flag,
+    _reset_root_fields_settings_flag,
     _serialize_django_user,
     contextvars_injector,
 )
 from ecsctx.pii import configure_pii
-from ecsctx.processors import _safe_dump_and_mask, masking_is_configured
+from ecsctx.processors import (
+    _safe_dump_and_mask,
+    masking_is_configured,
+    reshape_log_event,
+    root_fields_are_configured,
+)
 
 User = get_user_model()
 
@@ -159,3 +166,62 @@ def __getattr__(self, name):
         assert masking_is_configured()
         out = _safe_dump_and_mask({"payment_methods": [{"name": "KNET"}]})
         assert out["payment_methods"][0]["name"] == "KNET"
+
+
+class TestRootFieldsSettingsBridge:
+    @pytest.fixture(autouse=True)
+    def _reset_flag(self):
+        _reset_root_fields_settings_flag()
+        yield
+        _reset_root_fields_settings_flag()
+
+    @override_settings(ECSCTX_ROOT_FIELDS=["customer"])
+    def test_setting_applied_via_auto_configure(self):
+        _auto_configure_root_fields()
+        assert root_fields_are_configured()
+        result = reshape_log_event({"message": "hi", "customer": {"id": "c1"}})
+        assert result["customer"] == {"id": "c1"}
+        assert "extra" not in result
+
+    @override_settings(ECSCTX_ROOT_FIELDS=["customer"])
+    def test_setting_applied_via_processor_first_call(self):
+        # A real log record through the Django contextvars_injector must
+        # trigger the settings bridge — no setup_logging() required.
+        contextvars_injector(None, None, {"event": "hello"})
+        assert root_fields_are_configured()
+        result = reshape_log_event({"message": "hi", "customer": {"id": "c1"}})
+        assert result["customer"] == {"id": "c1"}
+
+    def test_absent_setting_is_noop(self):
+        _auto_configure_root_fields()
+        assert not root_fields_are_configured()
+        result = reshape_log_event({"message": "hi", "customer": {"id": "c1"}})
+        assert result["extra"] == {"customer": {"id": "c1"}}
+
+    @override_settings(ECSCTX_ROOT_FIELDS=["customer"])
+    def test_explicit_configure_beats_setting(self):
+        from ecsctx.processors import configure_root_fields
+
+        configure_root_fields(extra_fields=[])  # explicit empty wins
+        _auto_configure_root_fields()
+        result = reshape_log_event({"message": "hi", "customer": {"id": "c1"}})
+        assert "customer" not in result
+        assert result["extra"] == {"customer": {"id": "c1"}}
+
+    def test_retries_when_settings_not_ready(self):
+        """Bridge must not burn its one-shot flag if settings access raises
+        (e.g. during settings.py import) — it retries at real log time."""
+
+        class _NotReady:
+            def __getattr__(self, name):
+                raise RuntimeError("Requested setting, but settings are not configured.")
+
+        with patch("django.conf.settings", _NotReady()):
+            _auto_configure_root_fields()
+        assert not root_fields_are_configured()
+
+        with override_settings(ECSCTX_ROOT_FIELDS=["customer"]):
+            _auto_configure_root_fields()
+        assert root_fields_are_configured()
+        result = reshape_log_event({"message": "hi", "customer": {"id": "c1"}})
+        assert result["customer"] == {"id": "c1"}

tests/test_processors.py

@@ -7,7 +7,9 @@
     _safe_dump_and_mask,
     safe_tokenize,
     configure_masking,
+    configure_root_fields,
     masking_is_configured,
+    root_fields_are_configured,
     namespace_ecs_fields,
     reshape_log_event,
 )
@@ -320,3 +322,42 @@ def test_empty_default_still_configured(self, token_keyset_path):
         out = _safe_dump_and_mask({"customer": {"name": "John"}})
         assert out["customer"]["name"].startswith("ptok:v1:")
         assert masking_is_configured()
+
+
+class TestRootFieldsConfig:
+    def test_default_non_allowlisted_goes_to_extra(self):
+        result = reshape_log_event({"message": "hi", "customer": {"id": "c1"}})
+        assert "customer" not in result
+        assert result["extra"] == {"customer": {"id": "c1"}}
+
+    def test_configured_field_stays_at_root(self):
+        configure_root_fields(extra_fields=["customer"])
+        result = reshape_log_event({"message": "hi", "customer": {"id": "c1"}})
+        assert result["customer"] == {"id": "c1"}
+        assert "extra" not in result
+
+    def test_builtin_allowlist_unaffected_by_config(self):
+        configure_root_fields(extra_fields=["customer"])
+        result = reshape_log_event({"message": "hi", "session_id": "s1", "other": 1})
+        assert result["session_id"] == "s1"
+        assert result["extra"] == {"other": 1}
+
+    def test_env_var_config(self, monkeypatch):
+        monkeypatch.setenv("ECSCTX_ROOT_FIELDS", "customer, booking")
+        result = reshape_log_event({
+            "message": "hi",
+            "customer": {"id": "c1"},
+            "booking": {"ref": "b1"},
+            "other": 1,
+        })
+        assert result["customer"] == {"id": "c1"}
+        assert result["booking"] == {"ref": "b1"}
+        assert result["extra"] == {"other": 1}
+
+    def test_explicit_beats_env(self, monkeypatch):
+        monkeypatch.setenv("ECSCTX_ROOT_FIELDS", "customer")
+        configure_root_fields(extra_fields=[])
+        result = reshape_log_event({"message": "hi", "customer": {"id": "c1"}})
+        assert "customer" not in result
+        assert result["extra"] == {"customer": {"id": "c1"}}
+        assert root_fields_are_configured()