Fix yaml() Jinja filter returning NULL on Pillar containers

changelog/69218.fixed.md

@@ -0,0 +1,4 @@
+Fixed the ``yaml`` Jinja filter returning ``NULL`` when applied to Pillar
+lists or dicts. Pillar containers are wrapped in ``MaskedDict`` /
+``MaskedList`` for repr redaction; representers are now registered so the
+YAML dumper serializes them as their underlying list / dict.

salt/utils/yamldumper.py

@@ -16,6 +16,7 @@
 import salt.utils.context
 from salt.utils.datastructures import HashableOrderedDict
 from salt.utils.optsdict import DictProxy, ListProxy, OptsDict
+from salt.utils.secret import MaskedDict, MaskedList
 
 try:
     from yaml import CDumper as Dumper
@@ -117,13 +118,42 @@ def represent_listproxy(dumper, data):
 SafeOrderedDumper.add_representer(DictProxy, represent_dictproxy)
 OrderedDumper.add_representer(ListProxy, represent_listproxy)
 SafeOrderedDumper.add_representer(ListProxy, represent_listproxy)
+# Pillar containers are wrapped in MaskedDict / MaskedList for repr redaction;
+# they are still plain dict / list at the data level, so dump them as such
+# instead of falling through to represent_undefined (which would emit NULL).
+OrderedDumper.add_representer(
+    MaskedDict, yaml.representer.SafeRepresenter.represent_dict
+)
+SafeOrderedDumper.add_representer(
+    MaskedDict, yaml.representer.SafeRepresenter.represent_dict
+)
+IndentedSafeOrderedDumper.add_representer(
+    MaskedDict, yaml.representer.SafeRepresenter.represent_dict
+)
+OrderedDumper.add_representer(
+    MaskedList, yaml.representer.SafeRepresenter.represent_list
+)
+SafeOrderedDumper.add_representer(
+    MaskedList, yaml.representer.SafeRepresenter.represent_list
+)
+IndentedSafeOrderedDumper.add_representer(
+    MaskedList, yaml.representer.SafeRepresenter.represent_list
+)
 # Also register with base YAML dumpers for salt.utils.yaml.dump()
 yaml.Dumper.add_representer(OptsDict, represent_optsdict)
 yaml.SafeDumper.add_representer(OptsDict, represent_optsdict)
 yaml.Dumper.add_representer(DictProxy, represent_dictproxy)
 yaml.SafeDumper.add_representer(DictProxy, represent_dictproxy)
 yaml.Dumper.add_representer(ListProxy, represent_listproxy)
 yaml.SafeDumper.add_representer(ListProxy, represent_listproxy)
+yaml.Dumper.add_representer(MaskedDict, yaml.representer.SafeRepresenter.represent_dict)
+yaml.SafeDumper.add_representer(
+    MaskedDict, yaml.representer.SafeRepresenter.represent_dict
+)
+yaml.Dumper.add_representer(MaskedList, yaml.representer.SafeRepresenter.represent_list)
+yaml.SafeDumper.add_representer(
+    MaskedList, yaml.representer.SafeRepresenter.represent_list
+)
 
 OrderedDumper.add_representer(
     "tag:yaml.org,2002:timestamp", OrderedDumper.represent_scalar

tests/pytests/unit/utils/jinja/test_custom_extensions.py

@@ -149,6 +149,39 @@ def test_serialize_yaml_unicode():
     assert "str value" == rendered
 
 
+def test_serialize_yaml_masked_pillar_list():
+    """
+    Regression test for https://github.com/saltstack/salt/issues/69218
+
+    Pillar containers are wrapped in ``salt.utils.secret.MaskedList`` /
+    ``MaskedDict`` (``list`` / ``dict`` subclasses) so their repr can redact
+    secrets. The ``yaml`` Jinja filter must dump these as their underlying
+    list/dict rather than falling through to ``SafeOrderedDumper``'s
+    ``represent_undefined`` catch-all (which emits the scalar ``NULL``).
+    """
+    from salt.utils.secret import MaskedDict, MaskedList
+
+    dataset = MaskedList(["local", "local_async", "runner", "wheel"])
+    env = Environment(extensions=[SerializerExtension])
+
+    flow_rendered = env.from_string("{{ dataset|yaml }}").render(dataset=dataset)
+    assert flow_rendered != "NULL"
+    assert salt.utils.yaml.safe_load(flow_rendered) == list(dataset)
+
+    block_rendered = env.from_string("{{ dataset|yaml(False) }}").render(
+        dataset=dataset
+    )
+    assert block_rendered != "NULL"
+    assert salt.utils.yaml.safe_load(block_rendered) == list(dataset)
+
+    nested = MaskedDict({"salt": {"master": {"netapi_enable_clients": list(dataset)}}})
+    nested_rendered = env.from_string("{{ data|yaml }}").render(data=nested)
+    assert nested_rendered != "NULL"
+    assert salt.utils.yaml.safe_load(nested_rendered) == {
+        "salt": {"master": {"netapi_enable_clients": list(dataset)}}
+    }
+
+
 def test_serialize_python():
     dataset = {"foo": True, "bar": 42, "baz": [1, 2, 3], "qux": 2.0}
     env = Environment(extensions=[SerializerExtension])