From 80d8750e7ec40e0ad2dd8eeb426ec8a6cc51bbc0 Mon Sep 17 00:00:00 2001 From: oddly Date: Sun, 7 Jun 2026 21:40:45 +0200 Subject: [PATCH 01/19] test: parameterise shared verify_es_health on scheme The shared cluster-health helper hardcoded https://, so the no-security scenario had to inline its own version. The inlined copy drifted: it omitted the assertion-on-final-status and used inventory_hostname for the node-membership check (which only worked because incus happens to make container hostname == container name). Add a _verify_es_scheme parameter (default https) that also drops the auth headers when set to http, and switch the no-security verify to use the shared helper. The node-membership check now compares against ansible_facts.hostname which is what the role actually sets via elasticsearch_nodename. --- molecule/elasticsearch_no-security/verify.yml | 39 ++++++------------- molecule/shared/verify_es_health.yml | 11 ++++-- 2 files changed, 18 insertions(+), 32 deletions(-) diff --git a/molecule/elasticsearch_no-security/verify.yml b/molecule/elasticsearch_no-security/verify.yml index 2a2665b9..ab64bdcf 100644 --- a/molecule/elasticsearch_no-security/verify.yml +++ b/molecule/elasticsearch_no-security/verify.yml @@ -17,28 +17,13 @@ - "'xpack.security.enabled: false' in (_es_config.content | b64decode)" fail_msg: "xpack.security.enabled: false not found in elasticsearch.yml" -# Remember, this is the no-security scenario. So no https -# The comment below will create an exception for KICS security scan - - name: Health check - ansible.builtin.uri: -# kics-scan ignore-line - url: http://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health - method: GET - return_content: true - status_code: 200 - validate_certs: false - register: result - until: result.json.status is defined and result.json.status in ["green", "yellow"] - retries: 6 - delay: 10 - - - name: Assert cluster is healthy - ansible.builtin.assert: - that: - - result.json.status in ["green", "yellow"] - fail_msg: "Cluster health is {{ result.json.status | default('unknown') }}" + - name: Verify cluster health (no-security, plain HTTP) + ansible.builtin.include_tasks: ../shared/verify_es_health.yml + vars: + _verify_es_scheme: http + _verify_es_status: green - - name: Node check + - name: Fetch _cat/nodes for membership check ansible.builtin.uri: # kics-scan ignore-line url: http://localhost:{{ elasticstack_elasticsearch_http_port }}/_cat/nodes @@ -47,15 +32,13 @@ status_code: 200 validate_certs: false register: nodes - until: nodes.status == 200 - retries: 3 - delay: 5 + run_once: true - - name: Check if all Nodes see each other + - name: Assert every cluster member appears in _cat/nodes ansible.builtin.assert: that: - - "'{{ item }}' in nodes.content" - fail_msg: "'{{ item }}' was not found in nodes.content" - success_msg: "'{{ item }}' was found in nodes.content" + - "hostvars[item].ansible_facts.hostname in nodes.content" + fail_msg: "node '{{ hostvars[item].ansible_facts.hostname }}' (host {{ item }}) was not found in _cat/nodes output" loop: "{{ groups['all'] }}" + run_once: true when: groups[elasticstack_elasticsearch_group_name] | length > 1 diff --git a/molecule/shared/verify_es_health.yml b/molecule/shared/verify_es_health.yml index da02f9b1..3ec65067 100644 --- a/molecule/shared/verify_es_health.yml +++ b/molecule/shared/verify_es_health.yml @@ -2,18 +2,21 @@ # Check Elasticsearch cluster health via the REST API. # Required variables: # elastic_pass: registered result with .stdout containing the password +# (omit auth altogether by setting _verify_es_scheme: http, +# in which case elastic_pass is not consulted) # Optional variables: +# _verify_es_scheme: 'http' or 'https' (default: https) # _verify_es_port: HTTP port (default: 9200) # _verify_es_status: expected status — 'green' or 'yellow' (default: green) # _verify_run_once: run once across hosts (default: false) # _verify_when: additional when condition (default: true) - name: Elasticsearch health check ansible.builtin.uri: - url: "https://localhost:{{ _verify_es_port | default(elasticstack_elasticsearch_http_port | default(9200)) }}/_cluster/health" + url: "{{ _verify_es_scheme | default('https') }}://localhost:{{ _verify_es_port | default(elasticstack_elasticsearch_http_port | default(9200)) }}/_cluster/health" method: GET - force_basic_auth: true - user: elastic - password: "{{ elastic_pass.stdout }}" + force_basic_auth: "{{ (_verify_es_scheme | default('https')) == 'https' }}" + user: "{{ 'elastic' if (_verify_es_scheme | default('https')) == 'https' else omit }}" + password: "{{ elastic_pass.stdout if (_verify_es_scheme | default('https')) == 'https' else omit }}" return_content: true status_code: 200 validate_certs: false From 4bdfdf7fd1ebbbd44772f97fad6ca78e91516c7c Mon Sep 17 00:00:00 2001 From: oddly Date: Sun, 7 Jun 2026 21:42:51 +0200 Subject: [PATCH 02/19] test: rename misleading molecule scenario directories Three scenario names lied about what they tested: - logstash_standalone_certs exercised logstash_cert_source: external, not standalone. Rename to logstash_external_certs and free up the standalone name for an actual coverage of that code path (added in a later commit). - elasticsearch_test_modules tested the Elasticsearch _security REST API, not any module under plugins/. Rename to elasticsearch_security_api. - plugins/ tested the cert_info module specifically and is invoked via ansible-playbook (no molecule.yml), not via molecule converge. Rename to cert_info_module to reflect what it actually covers and update the CI coverage check to keep it in the standalone-playbook excluded list. Updated workflow refs and the check-ci-coverage exclusion list. Platform names inside molecule.yml use abbreviations that didn't reference the scenario name, so no internal edits needed. --- .github/workflows/test_elasticsearch_modules.yml | 4 ++-- .github/workflows/test_plugins.yml | 8 ++++---- .github/workflows/test_role_logstash.yml | 2 +- molecule/{plugins => cert_info_module}/converge.yml | 0 .../files/es-ca/ca.crt | 0 .../files/es-ca/elastic-stack-ca.p12 | Bin .../files/es-ca/elasticsearch-cluster1.p12 | Bin .../files/es-ca/elasticsearch-cluster2.p12 | Bin .../converge.yml | 0 .../create.yml | 0 .../destroy.yml | 0 .../molecule.yml | 0 .../prepare.yml | 0 .../requirements.yml | 0 .../verify.yml | 0 .../converge.yml | 0 .../create.yml | 0 .../destroy.yml | 0 .../molecule.yml | 0 .../requirements.yml | 0 .../verify.yml | 0 scripts/check-ci-coverage.sh | 5 +++-- 22 files changed, 10 insertions(+), 9 deletions(-) rename molecule/{plugins => cert_info_module}/converge.yml (100%) rename molecule/{plugins => cert_info_module}/files/es-ca/ca.crt (100%) rename molecule/{plugins => cert_info_module}/files/es-ca/elastic-stack-ca.p12 (100%) rename molecule/{plugins => cert_info_module}/files/es-ca/elasticsearch-cluster1.p12 (100%) rename molecule/{plugins => cert_info_module}/files/es-ca/elasticsearch-cluster2.p12 (100%) rename molecule/{elasticsearch_test_modules => elasticsearch_security_api}/converge.yml (100%) rename molecule/{elasticsearch_test_modules => elasticsearch_security_api}/create.yml (100%) rename molecule/{elasticsearch_test_modules => elasticsearch_security_api}/destroy.yml (100%) rename molecule/{elasticsearch_test_modules => elasticsearch_security_api}/molecule.yml (100%) rename molecule/{elasticsearch_test_modules => elasticsearch_security_api}/prepare.yml (100%) rename molecule/{elasticsearch_test_modules => elasticsearch_security_api}/requirements.yml (100%) rename molecule/{elasticsearch_test_modules => elasticsearch_security_api}/verify.yml (100%) rename molecule/{logstash_standalone_certs => logstash_external_certs}/converge.yml (100%) rename molecule/{logstash_standalone_certs => logstash_external_certs}/create.yml (100%) rename molecule/{logstash_standalone_certs => logstash_external_certs}/destroy.yml (100%) rename molecule/{logstash_standalone_certs => logstash_external_certs}/molecule.yml (100%) rename molecule/{logstash_standalone_certs => logstash_external_certs}/requirements.yml (100%) rename molecule/{logstash_standalone_certs => logstash_external_certs}/verify.yml (100%) diff --git a/.github/workflows/test_elasticsearch_modules.yml b/.github/workflows/test_elasticsearch_modules.yml index f62fce0e..f07d70a4 100644 --- a/.github/workflows/test_elasticsearch_modules.yml +++ b/.github/workflows/test_elasticsearch_modules.yml @@ -17,7 +17,7 @@ on: paths: - 'roles/elasticsearch/**' - 'roles/elasticstack/**' - - 'molecule/elasticsearch_test_modules/**' + - 'molecule/elasticsearch_security_api/**' - 'molecule/shared/**' - '.github/workflows/test_elasticsearch_modules.yml' - '.github/workflows/molecule.yml' @@ -43,7 +43,7 @@ jobs: needs: lint_elasticsearch uses: ./.github/workflows/molecule.yml with: - scenarios: '["elasticsearch_test_modules"]' + scenarios: '["elasticsearch_security_api"]' distros: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '["rockylinux10","debian13"]' || '["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","ubuntu2604","debian12","debian13"]' }} timeout: 20 secrets: inherit diff --git a/.github/workflows/test_plugins.yml b/.github/workflows/test_plugins.yml index ae1a8ef4..3562d0d5 100644 --- a/.github/workflows/test_plugins.yml +++ b/.github/workflows/test_plugins.yml @@ -21,7 +21,7 @@ on: paths: - 'plugins/**' - '.github/workflows/test_plugins.yml' - - 'molecule/plugins/**' + - 'molecule/cert_info_module/**' - '.config/pycodestyle.cfg' - 'tests/**' @@ -201,7 +201,7 @@ jobs: - name: Test with ansible-playbook run: | - ansible-playbook molecule/plugins/converge.yml + ansible-playbook molecule/cert_info_module/converge.yml env: PY_COLORS: '1' ANSIBLE_FORCE_COLOR: '1' @@ -257,7 +257,7 @@ jobs: - name: Test with ansible-playbook run: | - ansible-playbook molecule/plugins/converge.yml + ansible-playbook molecule/cert_info_module/converge.yml env: PY_COLORS: '1' ANSIBLE_FORCE_COLOR: '1' @@ -305,7 +305,7 @@ jobs: - name: Test with ansible-playbook run: | - ansible-playbook molecule/plugins/converge.yml + ansible-playbook molecule/cert_info_module/converge.yml env: PY_COLORS: '1' ANSIBLE_FORCE_COLOR: '1' diff --git a/.github/workflows/test_role_logstash.yml b/.github/workflows/test_role_logstash.yml index 4b779817..c434a33c 100644 --- a/.github/workflows/test_role_logstash.yml +++ b/.github/workflows/test_role_logstash.yml @@ -43,6 +43,6 @@ jobs: needs: lint_logstash uses: ./.github/workflows/molecule.yml with: - scenarios: '["logstash_default","logstash_ssl","logstash_custom_pipeline","logstash_advanced","logstash_standalone_certs","logstash_centralized_pipelines"]' + scenarios: '["logstash_default","logstash_ssl","logstash_custom_pipeline","logstash_advanced","logstash_external_certs","logstash_standalone_certs","logstash_centralized_pipelines"]' distros: ${{ (github.event_name == 'pull_request' || github.event_name == 'merge_group') && '["rockylinux10","debian13"]' || '["rockylinux9","rockylinux10","ubuntu2204","ubuntu2404","ubuntu2604","debian12","debian13"]' }} secrets: inherit diff --git a/molecule/plugins/converge.yml b/molecule/cert_info_module/converge.yml similarity index 100% rename from molecule/plugins/converge.yml rename to molecule/cert_info_module/converge.yml diff --git a/molecule/plugins/files/es-ca/ca.crt b/molecule/cert_info_module/files/es-ca/ca.crt similarity index 100% rename from molecule/plugins/files/es-ca/ca.crt rename to molecule/cert_info_module/files/es-ca/ca.crt diff --git a/molecule/plugins/files/es-ca/elastic-stack-ca.p12 b/molecule/cert_info_module/files/es-ca/elastic-stack-ca.p12 similarity index 100% rename from molecule/plugins/files/es-ca/elastic-stack-ca.p12 rename to molecule/cert_info_module/files/es-ca/elastic-stack-ca.p12 diff --git a/molecule/plugins/files/es-ca/elasticsearch-cluster1.p12 b/molecule/cert_info_module/files/es-ca/elasticsearch-cluster1.p12 similarity index 100% rename from molecule/plugins/files/es-ca/elasticsearch-cluster1.p12 rename to molecule/cert_info_module/files/es-ca/elasticsearch-cluster1.p12 diff --git a/molecule/plugins/files/es-ca/elasticsearch-cluster2.p12 b/molecule/cert_info_module/files/es-ca/elasticsearch-cluster2.p12 similarity index 100% rename from molecule/plugins/files/es-ca/elasticsearch-cluster2.p12 rename to molecule/cert_info_module/files/es-ca/elasticsearch-cluster2.p12 diff --git a/molecule/elasticsearch_test_modules/converge.yml b/molecule/elasticsearch_security_api/converge.yml similarity index 100% rename from molecule/elasticsearch_test_modules/converge.yml rename to molecule/elasticsearch_security_api/converge.yml diff --git a/molecule/elasticsearch_test_modules/create.yml b/molecule/elasticsearch_security_api/create.yml similarity index 100% rename from molecule/elasticsearch_test_modules/create.yml rename to molecule/elasticsearch_security_api/create.yml diff --git a/molecule/elasticsearch_test_modules/destroy.yml b/molecule/elasticsearch_security_api/destroy.yml similarity index 100% rename from molecule/elasticsearch_test_modules/destroy.yml rename to molecule/elasticsearch_security_api/destroy.yml diff --git a/molecule/elasticsearch_test_modules/molecule.yml b/molecule/elasticsearch_security_api/molecule.yml similarity index 100% rename from molecule/elasticsearch_test_modules/molecule.yml rename to molecule/elasticsearch_security_api/molecule.yml diff --git a/molecule/elasticsearch_test_modules/prepare.yml b/molecule/elasticsearch_security_api/prepare.yml similarity index 100% rename from molecule/elasticsearch_test_modules/prepare.yml rename to molecule/elasticsearch_security_api/prepare.yml diff --git a/molecule/elasticsearch_test_modules/requirements.yml b/molecule/elasticsearch_security_api/requirements.yml similarity index 100% rename from molecule/elasticsearch_test_modules/requirements.yml rename to molecule/elasticsearch_security_api/requirements.yml diff --git a/molecule/elasticsearch_test_modules/verify.yml b/molecule/elasticsearch_security_api/verify.yml similarity index 100% rename from molecule/elasticsearch_test_modules/verify.yml rename to molecule/elasticsearch_security_api/verify.yml diff --git a/molecule/logstash_standalone_certs/converge.yml b/molecule/logstash_external_certs/converge.yml similarity index 100% rename from molecule/logstash_standalone_certs/converge.yml rename to molecule/logstash_external_certs/converge.yml diff --git a/molecule/logstash_standalone_certs/create.yml b/molecule/logstash_external_certs/create.yml similarity index 100% rename from molecule/logstash_standalone_certs/create.yml rename to molecule/logstash_external_certs/create.yml diff --git a/molecule/logstash_standalone_certs/destroy.yml b/molecule/logstash_external_certs/destroy.yml similarity index 100% rename from molecule/logstash_standalone_certs/destroy.yml rename to molecule/logstash_external_certs/destroy.yml diff --git a/molecule/logstash_standalone_certs/molecule.yml b/molecule/logstash_external_certs/molecule.yml similarity index 100% rename from molecule/logstash_standalone_certs/molecule.yml rename to molecule/logstash_external_certs/molecule.yml diff --git a/molecule/logstash_standalone_certs/requirements.yml b/molecule/logstash_external_certs/requirements.yml similarity index 100% rename from molecule/logstash_standalone_certs/requirements.yml rename to molecule/logstash_external_certs/requirements.yml diff --git a/molecule/logstash_standalone_certs/verify.yml b/molecule/logstash_external_certs/verify.yml similarity index 100% rename from molecule/logstash_standalone_certs/verify.yml rename to molecule/logstash_external_certs/verify.yml diff --git a/scripts/check-ci-coverage.sh b/scripts/check-ci-coverage.sh index 7754177e..efbc225d 100755 --- a/scripts/check-ci-coverage.sh +++ b/scripts/check-ci-coverage.sh @@ -9,8 +9,9 @@ WORKFLOWS_DIR="$REPO_ROOT/.github/workflows" MOLECULE_DIR="$REPO_ROOT/molecule" EXIT_CODE=0 -# Scenarios that are not standalone tests (utility dirs, shared includes) -EXCLUDED_SCENARIOS="default shared plugins" +# Scenarios that are not standalone tests (utility dirs, shared includes, +# or playbooks invoked directly by ansible-playbook rather than `molecule`) +EXCLUDED_SCENARIOS="default shared cert_info_module" echo "=== Molecule scenario CI coverage check ===" echo From da41ae2e870562da9b830cc51660137e961805c1 Mon Sep 17 00:00:00 2001 From: oddly Date: Sun, 7 Jun 2026 21:43:32 +0200 Subject: [PATCH 03/19] test(cert_renewal): assert fingerprints actually change after renewal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 2 of the converge captured pre-renewal fingerprints into _*_cert_fp_before, but nothing ever compared them to a post-renewal value. The verify phase only checked that the cert files still existed and that the cluster was up — both of which hold even when renewal silently no-ops. A regression where elasticsearch_cert_will_expire_soon was ignored would have shipped green. Add a phase 4 to converge that captures the post-renewal fingerprints and asserts each one differs from the pre-renewal capture. Comparison lives in converge rather than verify because facts don't carry across molecule's converge → verify boundary. Also explicitly fail when the pre-renewal fingerprint is "NONE" (the original capture's || echo "NONE" fallback would otherwise let a baseline-capture failure pass the comparison silently). --- molecule/cert_renewal/converge.yml | 88 ++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/molecule/cert_renewal/converge.yml b/molecule/cert_renewal/converge.yml index f094c14c..9bd8d69b 100644 --- a/molecule/cert_renewal/converge.yml +++ b/molecule/cert_renewal/converge.yml @@ -139,3 +139,91 @@ ansible.builtin.include_role: name: oddly.elasticstack.kibana when: "'kibana' in group_names" + +# --- Phase 4: Confirm certificates actually changed --- +# +# Captures the post-renewal fingerprint and compares to the pre-renewal value +# from phase 2. Facts persist across plays within a single ansible-playbook +# run, so _*_cert_fp_before from phase 2 is still in scope here. (Verify is +# a separate playbook invocation by molecule, which is why this comparison +# must live in converge, not verify.) + +- name: Verify certificate fingerprints changed after renewal + hosts: all + vars: + elasticsearch_tls_key_passphrase: PleaseChangeMeIndividually + logstash_tls_key_passphrase: LogstashChangeMe + kibana_tls_key_passphrase: PleaseChangeMe + tasks: + - name: Get ES transport cert fingerprint (after renewal) + ansible.builtin.shell: >- + set -o pipefail && + openssl pkcs12 -in /etc/elasticsearch/certs/{{ ansible_facts.hostname }}.p12 + -nokeys -passin pass:{{ elasticsearch_tls_key_passphrase }} 2>/dev/null | + openssl x509 -fingerprint -sha256 -noout 2>/dev/null + args: + executable: /bin/bash + register: _es_cert_fp_after + changed_when: false + no_log: true + when: "'elasticsearch' in group_names" + + - name: Get Logstash cert fingerprint (after renewal) + ansible.builtin.shell: >- + set -o pipefail && + openssl pkcs12 -in /etc/logstash/certs/{{ ansible_facts.hostname }}-ls.p12 + -nokeys -passin pass:{{ logstash_tls_key_passphrase }} 2>/dev/null | + openssl x509 -fingerprint -sha256 -noout 2>/dev/null + args: + executable: /bin/bash + register: _ls_cert_fp_after + changed_when: false + no_log: true + when: "'logstash' in group_names" + + - name: Get Kibana cert fingerprint (after renewal) + ansible.builtin.shell: >- + set -o pipefail && + openssl pkcs12 -in /etc/kibana/certs/{{ ansible_facts.hostname }}-kibana.p12 + -nokeys -passin pass:{{ kibana_tls_key_passphrase }} 2>/dev/null | + openssl x509 -fingerprint -sha256 -noout 2>/dev/null + args: + executable: /bin/bash + register: _kb_cert_fp_after + changed_when: false + no_log: true + when: "'kibana' in group_names" + + - name: Assert ES transport cert was actually replaced + ansible.builtin.assert: + that: + - _es_cert_fp_before.stdout != "NONE" + - _es_cert_fp_after.stdout | length > 0 + - _es_cert_fp_before.stdout != _es_cert_fp_after.stdout + fail_msg: >- + ES transport cert was not renewed (before={{ _es_cert_fp_before.stdout }}, + after={{ _es_cert_fp_after.stdout }}). The renewal play ran but the on-disk + p12 was not replaced — check elasticsearch_cert_will_expire_soon handling. + when: "'elasticsearch' in group_names" + + - name: Assert Logstash cert was actually replaced + ansible.builtin.assert: + that: + - _ls_cert_fp_before.stdout != "NONE" + - _ls_cert_fp_after.stdout | length > 0 + - _ls_cert_fp_before.stdout != _ls_cert_fp_after.stdout + fail_msg: >- + Logstash cert was not renewed (before={{ _ls_cert_fp_before.stdout }}, + after={{ _ls_cert_fp_after.stdout }}). + when: "'logstash' in group_names" + + - name: Assert Kibana cert was actually replaced + ansible.builtin.assert: + that: + - _kb_cert_fp_before.stdout != "NONE" + - _kb_cert_fp_after.stdout | length > 0 + - _kb_cert_fp_before.stdout != _kb_cert_fp_after.stdout + fail_msg: >- + Kibana cert was not renewed (before={{ _kb_cert_fp_before.stdout }}, + after={{ _kb_cert_fp_after.stdout }}). + when: "'kibana' in group_names" From ef2ef545f4053faf6fecda7cd19839b32fa51d32 Mon Sep 17 00:00:00 2001 From: oddly Date: Sun, 7 Jun 2026 21:46:16 +0200 Subject: [PATCH 04/19] test(elasticsearch_upgrade): assert handler suppression, JVM-loaded, role-driven 8.x bump MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three gaps in the upgrade scenarios let the role's actual upgrade behaviour pass without proof: The handler-suppression fix was claimed to be the scenario's primary coverage, but the PID check sat in play 3 (the no-op latest re-run) where no real upgrade happens. Capture the start-event window around play 2's include_role (the actual 8→9 upgrade) and assert exactly one 'Started elasticsearch' journal entry — two would mean the config-change handler fired alongside the upgrade restart. The JVM custom parameter check slurped /etc/elasticsearch/jvm.options.d/90-custom.options and confirmed the file contained the marker. The role always writes that file; the question the test was supposed to answer is whether the running JVM picked it up. Query _nodes/jvm?filter_path=nodes.*.jvm.input_arguments and assert the marker is in the live process args. The _cat/nodes block in verify only debug-printed the JSON. Loop over the nodes and assert each one is on 9.x — a single node stuck on 8.x with the other on 9.x would have passed cluster.number_of_nodes >= 2. For the single-node scenario, the 8.10 → 8.19 step bypassed the role entirely with package: state: latest + service: restarted, so the role's upgrade-detection on same-major bumps was never exercised. Replace with elasticstack_version: latest + include_role, and assert the test index survived that step (otherwise a downstream verify failure would be blamed on the 9.x leg). Also add a < 10.0.0 upper bound on the 9.x version assertion. --- .../elasticsearch_upgrade_8to9/converge.yml | 47 +++++++++++++++++ .../elasticsearch_upgrade_8to9/verify.yml | 50 +++++++++++++++---- .../converge.yml | 43 ++++++++++------ .../verify.yml | 1 + 4 files changed, 115 insertions(+), 26 deletions(-) diff --git a/molecule/elasticsearch_upgrade_8to9/converge.yml b/molecule/elasticsearch_upgrade_8to9/converge.yml index 956628aa..3d609dd8 100644 --- a/molecule/elasticsearch_upgrade_8to9/converge.yml +++ b/molecule/elasticsearch_upgrade_8to9/converge.yml @@ -151,10 +151,57 @@ ansible.builtin.set_fact: _elasticstack_role_imported: false + # The handler-suppression fix should produce exactly one ES restart per + # node during the upgrade play: the rolling-upgrade restart itself. The + # config change introduced by elasticsearch_jvm_custom_parameters + # notifies the Restart Elasticsearch handler, which would fire a SECOND + # restart unless suppressed. Capture the start-event window so we can + # assert exactly-one after include_role completes. + - name: Capture pre-upgrade timestamp + ansible.builtin.command: date -u --iso-8601=seconds + register: _ts_before_upgrade + changed_when: false + + - name: Capture pre-upgrade ES PID + ansible.builtin.command: pgrep -f 'org.elasticsearch.bootstrap.Elasticsearch' + register: _es_pid_before_upgrade + changed_when: false + - name: Include Elasticsearch role (upgrade to 9.x) ansible.builtin.include_role: name: oddly.elasticstack.elasticsearch + - name: Capture post-upgrade ES PID + ansible.builtin.command: pgrep -f 'org.elasticsearch.bootstrap.Elasticsearch' + register: _es_pid_after_upgrade + changed_when: false + + - name: Count ES start events during the upgrade window + ansible.builtin.shell: + cmd: >- + journalctl -u elasticsearch + --since='{{ _ts_before_upgrade.stdout }}' + --no-pager --output=cat 2>/dev/null + | grep -c 'Started elasticsearch' || true + executable: /bin/bash + register: _es_start_count + changed_when: false + + - name: Assert handler-suppression worked (exactly one restart during upgrade) + ansible.builtin.assert: + that: + - _es_pid_before_upgrade.stdout != _es_pid_after_upgrade.stdout + - _es_start_count.stdout | int == 1 + fail_msg: >- + Expected exactly 1 'Started elasticsearch' event during the upgrade + play (the rolling-upgrade restart itself). Counted + {{ _es_start_count.stdout }}. If >1 the config-change handler fired + in addition to the upgrade restart — the handler-suppression fix + has regressed and ES restarted twice on this node. + success_msg: >- + Handler suppression OK: ES restarted exactly once during upgrade + (PID {{ _es_pid_before_upgrade.stdout }} → {{ _es_pid_after_upgrade.stdout }}). + # Re-run with elasticstack_version: "latest" to exercise the post-install # detection path. Since we're already on the latest 9.x, the package should # NOT change and the rolling restart should NOT trigger. This verifies that diff --git a/molecule/elasticsearch_upgrade_8to9/verify.yml b/molecule/elasticsearch_upgrade_8to9/verify.yml index 329da2cd..55c64e7b 100644 --- a/molecule/elasticsearch_upgrade_8to9/verify.yml +++ b/molecule/elasticsearch_upgrade_8to9/verify.yml @@ -81,11 +81,9 @@ user: elastic password: "{{ elastic_pass.stdout }}" return_content: true + status_code: 200 validate_certs: false register: nodes_info - until: nodes_info.status | default(0) == 200 - retries: 3 - delay: 5 when: groups[elasticstack_elasticsearch_group_name] | length > 1 - name: Display nodes @@ -93,6 +91,19 @@ var: nodes_info.json when: groups[elasticstack_elasticsearch_group_name] | length > 1 + - name: Assert every cluster node is on 9.x (not just the queried one) + ansible.builtin.assert: + that: + - item.version is version('9.0.0', '>=') + - item.version is version('10.0.0', '<') + fail_msg: >- + Node {{ item.name }} reports version {{ item.version }} — expected 9.x. + The rolling upgrade left at least one node on a stale version. + loop: "{{ nodes_info.json }}" + loop_control: + label: "{{ item.name }} ({{ item.version }})" + when: groups[elasticstack_elasticsearch_group_name] | length > 1 + - name: Verify test data survived the upgrade # noqa: run-once[task] ansible.builtin.uri: url: https://localhost:{{ elasticstack_elasticsearch_http_port }}/upgrade-test-index/_doc/1 @@ -120,18 +131,35 @@ success_msg: "Test data survived upgrade: {{ test_doc.json._source.test_field }}" run_once: true - - name: Verify JVM custom config was written during upgrade - ansible.builtin.slurp: - src: /etc/elasticsearch/jvm.options.d/90-custom.options - register: custom_jvm_config + - name: Query running JVM input arguments + ansible.builtin.uri: + url: https://localhost:{{ elasticstack_elasticsearch_http_port }}/_nodes/jvm?filter_path=nodes.*.jvm.input_arguments + method: GET + force_basic_auth: true + user: elastic + password: "{{ elastic_pass.stdout }}" + return_content: true + status_code: 200 + validate_certs: false + register: _nodes_jvm - - name: Check custom JVM parameter is on disk (written pre-upgrade, applied on upgrade restart) + - name: Assert custom JVM parameter is loaded by the running ES process (not just on disk) ansible.builtin.assert: that: - - "'-Des.upgrade.test.marker=true' in (custom_jvm_config.content | b64decode)" + - >- + '-Des.upgrade.test.marker=true' in ( + _nodes_jvm.json.nodes.values() + | map(attribute='jvm') + | map(attribute='input_arguments') + | flatten + ) fail_msg: >- - JVM custom config not applied. Got: {{ custom_jvm_config.content | b64decode }} - success_msg: "JVM custom parameter correctly applied during upgrade" + The role wrote /etc/elasticsearch/jvm.options.d/90-custom.options with + -Des.upgrade.test.marker=true, but it is NOT in the running JVM's + input_arguments. The upgrade restart did not pick up the new options + file — either the file was written after the restart, or the upgrade + path skipped the restart entirely. + success_msg: "Custom JVM parameter is in the running JVM — upgrade restart picked up the new options" - name: Final upgrade verification summary # noqa: run-once[task] ansible.builtin.debug: diff --git a/molecule/elasticsearch_upgrade_8to9_single/converge.yml b/molecule/elasticsearch_upgrade_8to9_single/converge.yml index 9eba0fe1..8a8d6bba 100644 --- a/molecule/elasticsearch_upgrade_8to9_single/converge.yml +++ b/molecule/elasticsearch_upgrade_8to9_single/converge.yml @@ -110,42 +110,55 @@ delay: 5 until: _refresh_result.status | default(0) == 200 -- name: Upgrade to latest Elasticsearch 8.x (required before 9.x) +- name: Upgrade to latest Elasticsearch 8.x via the role (required before 9.x) hosts: all vars: elasticstack_full_stack: false elasticstack_release: 8 + # elasticstack_version: latest exercises the role's post-install + # upgrade-detection path on a same-major bump (8.10 → 8.19). The + # original test bypassed the role here with package: state: latest + + # service: restarted, which meant the role's rolling-upgrade detection + # for 8.x→8.x was never exercised by CI. + elasticstack_version: "latest" elasticsearch_heap: "1" elasticstack_no_log: false elasticsearch_seed_hosts: [] elasticsearch_initial_master_nodes: [] + elasticsearch_upgrade_countdown: 0 tasks: - name: Include Elastic repos role (8.x) ansible.builtin.include_role: name: oddly.elasticstack.repos - - name: Upgrade to latest 8.x # noqa: package-latest - ansible.builtin.package: - name: elasticsearch - state: latest + - name: Reset shared role guard for 8.x→8.x upgrade play + ansible.builtin.set_fact: + _elasticstack_role_imported: false - - name: Restart Elasticsearch after upgrade - ansible.builtin.service: - name: elasticsearch - state: restarted + - name: Include Elasticsearch role (upgrade to latest 8.x via role) + ansible.builtin.include_role: + name: oddly.elasticstack.elasticsearch - - name: Wait for ES to be ready after 8.x upgrade + - name: Confirm test index survived the 8.x→8.x role-driven upgrade ansible.builtin.uri: - url: https://localhost:9200/_cluster/health + url: https://localhost:9200/upgrade-test/_doc/1 method: GET force_basic_auth: true user: elastic password: "{{ elastic_pass.stdout }}" + return_content: true validate_certs: false - register: health - retries: 30 - delay: 10 - until: health.status == 200 + status_code: 200 + register: _post_8x_doc + + - name: Assert test data survived 8.x→8.x upgrade (catches data loss before 9.x step) + ansible.builtin.assert: + that: + - _post_8x_doc.json._source.message == "pre-upgrade-test-data" + fail_msg: >- + Test data was lost during the 8.x→8.x upgrade. A subsequent + failure in the 9.x verify would otherwise be blamed on the + 9.x step. - name: Verify latest 8.x before 9.x upgrade hosts: all diff --git a/molecule/elasticsearch_upgrade_8to9_single/verify.yml b/molecule/elasticsearch_upgrade_8to9_single/verify.yml index dc307803..6b675e38 100644 --- a/molecule/elasticsearch_upgrade_8to9_single/verify.yml +++ b/molecule/elasticsearch_upgrade_8to9_single/verify.yml @@ -31,6 +31,7 @@ ansible.builtin.assert: that: - es_info.json.version.number is version('9.0.0', '>=') + - es_info.json.version.number is version('10.0.0', '<') fail_msg: "Expected ES 9.x but got {{ es_info.json.version.number }}" success_msg: "ES 9.x ({{ es_info.json.version.number }}) - upgrade successful!" From 2e983605d6b97b489564e93aedf54328bb0abd88 Mon Sep 17 00:00:00 2001 From: oddly Date: Sun, 7 Jun 2026 21:48:33 +0200 Subject: [PATCH 05/19] test(elasticsearch_*_certs): assert deployed certs match the source PEMs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit elasticsearch_cert_content existed specifically to prove that PEM strings fed in via *_content variables wind up on disk byte-for-byte. The verify never checked that — it only confirmed files existed and that an "http" substring appeared in the served subject. Replace with a sha256 comparison of each deployed cert against the /tmp/test-certs/ source PEM the converge slurped from, plus an explicit assert that elasticsearch.yml's verification_mode is "certificate" (not "none") and the rendered cert paths point at the deployed files. elasticsearch_custom_certs verified the served HTTP cert by checking the substring "http" appeared in the cert subject. The converge happens to set CN=-http, so the heuristic worked only by accident. If the converge ever drops the -http suffix, the assertion silently degenerates. Switch to checking the issuer is "Test CA" — the role's auto-generated fallback would have a different (ES-internal) issuer. elasticsearch_custom_certs_minimal extracts the CA out of a chained server+CA PEM bundle. The verify only checked that `BEGIN CERTIFICATE` appeared anywhere in the extracted file — a bug that wrote the entire bundle (or even just the server cert) to ca.crt would still pass. Count BEGIN CERTIFICATE occurrences == 1, assert the subject is the test CA, and sha256-compare the extracted ca.crt against the source CA PEM. Also assert certificate_authorities is configured in BOTH the transport and http SSL sections — the previous `grep ca.crt` check would have passed if only one section referenced it. --- .../elasticsearch_cert_content/verify.yml | 84 ++++++++++++------- .../elasticsearch_custom_certs/verify.yml | 29 ++++--- .../verify.yml | 62 +++++++++----- 3 files changed, 112 insertions(+), 63 deletions(-) diff --git a/molecule/elasticsearch_cert_content/verify.yml b/molecule/elasticsearch_cert_content/verify.yml index c291b9d6..e57a8460 100644 --- a/molecule/elasticsearch_cert_content/verify.yml +++ b/molecule/elasticsearch_cert_content/verify.yml @@ -33,33 +33,46 @@ fail_msg: "Cluster status is {{ health.json.status }}" run_once: true # noqa: run-once[task] - - name: Get HTTP certificate subject from Elasticsearch - ansible.builtin.shell: | - set -o pipefail - echo | openssl s_client -connect localhost:9200 -servername localhost 2>/dev/null | - openssl x509 -noout -subject - args: + # The whole point of the *_content variables is "bytes in == bytes on disk". + # Compare each deployed cert against the source PEM the converge fed in via + # /tmp/test-certs/ (which is what got slurped into _test_*_content facts). + # sha256sum comparison catches empty/wrong/truncated files that the + # earlier "is the subject string 'http'" heuristic let through. + - name: Hash deployed certs vs source PEM files + ansible.builtin.shell: + cmd: | + set -o pipefail + for pair in \ + "/tmp/test-certs/transport.crt /etc/elasticsearch/certs/{{ inventory_hostname }}-transport.crt" \ + "/tmp/test-certs/http.crt /etc/elasticsearch/certs/{{ inventory_hostname }}-http.crt" \ + "/tmp/test-certs/ca.crt /etc/elasticsearch/certs/ca.crt"; do + src=$(echo "$pair" | awk '{print $1}') + dst=$(echo "$pair" | awk '{print $2}') + src_sum=$(sha256sum "$src" | awk '{print $1}') + dst_sum=$(sha256sum "$dst" 2>/dev/null | awk '{print $1}' || echo "MISSING") + echo "$src $src_sum $dst $dst_sum" + done executable: /bin/bash - register: es_http_cert_subject + register: _cert_hashes changed_when: false - - name: Verify HTTP cert is the external cert (not auto-generated) - ansible.builtin.assert: - that: - - "'http' in es_http_cert_subject.stdout" - fail_msg: >- - Expected HTTP certificate subject to contain 'http' from our test cert. - Got: {{ es_http_cert_subject.stdout }} - - - name: Verify transport cert file was deployed - ansible.builtin.stat: - path: "/etc/elasticsearch/certs/{{ inventory_hostname }}-transport.crt" - register: transport_cert_stat - - - name: Assert transport cert exists + - name: Assert each deployed cert matches the source PEM byte-for-byte ansible.builtin.assert: that: - - transport_cert_stat.stat.exists + # All three lines present (transport.crt, http.crt, ca.crt). + - "(_cert_hashes.stdout_lines | length) == 3" + # No deployed file was missing. + - "(_cert_hashes.stdout_lines | select('search', ' MISSING$') | list | length) == 0" + # Each line has the form " "; + # we require src_hash == dst_hash. The regex backreference \\2 enforces it. + - >- + (_cert_hashes.stdout_lines + | map('regex_search', '^\\S+ (\\S+) \\S+ \\1$') + | select | list | length) == 3 + fail_msg: | + One or more deployed certs do not match the source PEM content the role + was fed via *_content variables. Detail (src dst src_hash dst_hash): + {{ _cert_hashes.stdout }} - name: Verify no P12 files exist (content mode is always PEM) ansible.builtin.find: @@ -73,16 +86,23 @@ - p12_files.files | length == 0 fail_msg: "Found unexpected P12 files in content mode: {{ p12_files.files | map(attribute='path') | list }}" - - name: Verify elasticsearch.yml has PEM config - ansible.builtin.command: - cmd: grep 'xpack.security.transport.ssl.certificate' /etc/elasticsearch/elasticsearch.yml - register: es_yml_check - changed_when: false + - name: Slurp elasticsearch.yml for config assertions + ansible.builtin.slurp: + src: /etc/elasticsearch/elasticsearch.yml + register: _es_yml - - name: Assert PEM-style config (not keystore) + - name: Assert verification_mode is certificate, not none, and PEM keys reference the deployed files ansible.builtin.assert: that: - - "'transport.crt' in es_yml_check.stdout" - fail_msg: >- - Expected PEM-style transport config in elasticsearch.yml. - Got: {{ es_yml_check.stdout }} + - "'xpack.security.transport.ssl.verification_mode: certificate' in (_es_yml.content | b64decode)" + - "'verification_mode: none' not in (_es_yml.content | b64decode)" + - >- + ('xpack.security.transport.ssl.certificate: certs/' ~ inventory_hostname ~ '-transport.crt') + in (_es_yml.content | b64decode) + - >- + ('xpack.security.http.ssl.certificate: certs/' ~ inventory_hostname ~ '-http.crt') + in (_es_yml.content | b64decode) + fail_msg: | + elasticsearch.yml is missing one of the expected verification_mode / cert + path lines. Content: + {{ _es_yml.content | b64decode }} diff --git a/molecule/elasticsearch_custom_certs/verify.yml b/molecule/elasticsearch_custom_certs/verify.yml index 0ba68736..ac1db841 100644 --- a/molecule/elasticsearch_custom_certs/verify.yml +++ b/molecule/elasticsearch_custom_certs/verify.yml @@ -33,23 +33,30 @@ fail_msg: "Cluster status is {{ health.json.status }}" run_once: true # noqa: run-once[task] - - name: Get HTTP certificate subject from Elasticsearch - ansible.builtin.shell: | - set -o pipefail - echo | openssl s_client -connect localhost:9200 -servername localhost 2>/dev/null | - openssl x509 -noout -subject - args: + # The previous "is the substring 'http' in the subject" heuristic only + # discriminated because the converge contrived a CN of "-http". + # Compare issuers instead: the test CA's CN is "Test CA" (set in + # converge.yml). The Elasticsearch auto-generated cert is issued by ES + # itself, so an issuer of "Test CA" proves the served cert is ours and + # the role didn't fall back to auto-generation. + - name: Get HTTP certificate issuer from Elasticsearch + ansible.builtin.shell: + cmd: | + set -o pipefail + echo | openssl s_client -connect localhost:9200 -servername localhost 2>/dev/null | + openssl x509 -noout -issuer executable: /bin/bash - register: es_http_cert_subject + register: es_http_cert_issuer changed_when: false - - name: Verify HTTP cert is the external cert (not auto-generated) + - name: Verify HTTP cert was signed by the test CA (not auto-generated) ansible.builtin.assert: that: - - "'http' in es_http_cert_subject.stdout" + - "'CN = Test CA' in es_http_cert_issuer.stdout or 'CN=Test CA' in es_http_cert_issuer.stdout" fail_msg: >- - Expected HTTP certificate subject to contain 'http' from our test cert. - Got: {{ es_http_cert_subject.stdout }} + Expected HTTP cert to be issued by the test CA "Test CA". Got: + {{ es_http_cert_issuer.stdout }}. The role may have fallen back to + its auto-generated cert path. - name: Verify transport cert file was deployed ansible.builtin.stat: diff --git a/molecule/elasticsearch_custom_certs_minimal/verify.yml b/molecule/elasticsearch_custom_certs_minimal/verify.yml index efa58f52..b1079126 100644 --- a/molecule/elasticsearch_custom_certs_minimal/verify.yml +++ b/molecule/elasticsearch_custom_certs_minimal/verify.yml @@ -69,30 +69,52 @@ - ca_cert_stat.stat.exists fail_msg: "CA cert not found — auto-extraction from PEM bundle failed" - - name: Read CA cert content - ansible.builtin.slurp: - src: /etc/elasticsearch/certs/ca.crt - register: ca_cert_content + # The point of the minimal scenario is that the role slices the CA + # out of a chained server+CA PEM bundle. A buggy role that wrote the + # full bundle to ca.crt (or even just the server cert) would still + # match 'BEGIN CERTIFICATE' — so count occurrences AND verify the + # subject matches the test CA, not the server cert. + - name: Hash extracted ca.crt and inspect its subject + ansible.builtin.shell: + cmd: | + set -o pipefail + path=/etc/elasticsearch/certs/ca.crt + count=$(grep -c '^-----BEGIN CERTIFICATE-----' "$path" || echo 0) + src_sum=$(sha256sum /tmp/test-certs/ca.crt | awk '{print $1}') + dst_sum=$(sha256sum "$path" | awk '{print $1}') + subject=$(openssl x509 -in "$path" -noout -subject) + echo "count=$count src_sum=$src_sum dst_sum=$dst_sum subject=$subject" + executable: /bin/bash + register: _ca_extract + changed_when: false - - name: Verify CA cert is not empty + - name: Assert ca.crt contains exactly one cert and matches the source CA ansible.builtin.assert: that: - - (ca_cert_content.content | b64decode) is search('BEGIN CERTIFICATE') - fail_msg: "CA cert is empty or invalid" + - "'count=1 ' in _ca_extract.stdout" + - "_ca_extract.stdout is search('CN ?= ?Test CA')" + # Source CA on /tmp/test-certs/ca.crt is the standalone CA PEM + # (single cert), so the extracted ca.crt should match it byte-for-byte. + - >- + _ca_extract.stdout is regex('src_sum=(\\S+) dst_sum=\\1') + fail_msg: | + CA cert was not extracted correctly from the chained PEM bundle. + Either the file holds the wrong number of certs, the subject is not + the test CA, or its bytes don't match the source CA. Got: + {{ _ca_extract.stdout }} - - name: Verify elasticsearch.yml uses PEM config with CA authorities - ansible.builtin.shell: | - set -o pipefail - grep 'certificate_authorities' /etc/elasticsearch/elasticsearch.yml - args: - executable: /bin/bash - register: es_yml_ca_check - changed_when: false + - name: Slurp elasticsearch.yml for cert-config assertions + ansible.builtin.slurp: + src: /etc/elasticsearch/elasticsearch.yml + register: _es_yml - - name: Assert certificate_authorities is configured + - name: Assert certificate_authorities is configured in BOTH transport and http sections ansible.builtin.assert: that: - - "'ca.crt' in es_yml_ca_check.stdout" - fail_msg: >- - Expected certificate_authorities in elasticsearch.yml. - Got: {{ es_yml_ca_check.stdout }} + - "'transport.ssl.certificate_authorities' in (_es_yml.content | b64decode)" + - "'http.ssl.certificate_authorities' in (_es_yml.content | b64decode)" + fail_msg: | + certificate_authorities missing from one of the SSL sections. The + earlier `grep ca.crt` check would have passed even if only HTTP + referenced the CA (transport gone silent). Content: + {{ _es_yml.content | b64decode }} From 3a8bbb3832284d190f6735ec811caeffb933626d Mon Sep 17 00:00:00 2001 From: oddly Date: Sun, 7 Jun 2026 21:49:54 +0200 Subject: [PATCH 06/19] test(*_tls): probe TLS handshake instead of trusting an open socket MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The TLS scenarios (logstash_ssl, logstash_external_certs, kibana_custom_certs) all confirmed the configured port was listening but never validated TLS was actually negotiated, never proved the served cert chained to the expected CA. A regression where Logstash silently fell back to plain text (e.g. SSL block rendered but cert path typo so Logstash logged and ignored), or where Kibana served its bundled default cert (wrong _kibana_cert_format detection routing the role to keystore.path), would have shipped green. Add an openssl s_client probe after the port-open check that connects with -CAfile /tmp/test-certs/ca.crt and requires "Verify return code: 0 (ok)". Catches both "no TLS at all" and "wrong cert" classes. For kibana_custom_certs also assert the rendered server.ssl.certificate / server.ssl.key paths are present and that no keystore.path slipped in — same class of finding, caught at the config layer too. --- molecule/kibana_custom_certs/verify.yml | 36 +++++++++++++++++++++ molecule/logstash_external_certs/verify.yml | 19 +++++++++++ molecule/logstash_ssl/verify.yml | 19 +++++++++++ 3 files changed, 74 insertions(+) diff --git a/molecule/kibana_custom_certs/verify.yml b/molecule/kibana_custom_certs/verify.yml index 45c639fc..ac051b9c 100644 --- a/molecule/kibana_custom_certs/verify.yml +++ b/molecule/kibana_custom_certs/verify.yml @@ -77,3 +77,39 @@ that: - "'server.ssl.enabled: true' in (kibana_yml.content | b64decode)" fail_msg: "server.ssl.enabled: true should be present when kibana_tls is true" + + # An open https://localhost:5601 with validate_certs:false doesn't + # prove the role rendered the right cert paths — Kibana could be + # serving its bundled default cert if the role's PEM branch picked + # up the wrong filename (e.g. wrong _kibana_cert_format default + # silently re-routed to the .p12 keystore.path branch). + - name: Verify server.ssl.certificate and server.ssl.key paths are rendered + ansible.builtin.assert: + that: + - "'server.ssl.certificate:' in (kibana_yml.content | b64decode)" + - "'server.ssl.key:' in (kibana_yml.content | b64decode)" + - "'keystore.path' not in (kibana_yml.content | b64decode)" + - "'elasticsearch.ssl.verificationMode: none' in (kibana_yml.content | b64decode)" + fail_msg: | + kibana.yml is missing PEM cert/key paths, or accidentally rendered + a keystore.path entry (wrong cert format detection), or dropped + the elasticsearch.ssl.verificationMode: none line from extra_config. + Got: + {{ kibana_yml.content | b64decode }} + + # An open socket on 5601 doesn't prove the cert chains to our test CA. + # Confirm the served cert is signed by "Test CA" — the role's + # auto-generated fallback would have a different issuer. + - name: TLS handshake against Kibana 5601 (signed by Test CA) + ansible.builtin.shell: + cmd: | + set -o pipefail + out=$(echo | openssl s_client \ + -connect localhost:5601 \ + -CAfile /tmp/test-certs/ca.crt \ + -servername {{ ansible_facts.hostname }} 2>&1) || true + echo "$out" | grep -E 'Verify return code|verify error|issuer=' || true + echo "$out" | grep -q 'Verify return code: 0 (ok)' + executable: /bin/bash + register: _kb_tls_probe + changed_when: false diff --git a/molecule/logstash_external_certs/verify.yml b/molecule/logstash_external_certs/verify.yml index 12bc4276..c1253132 100644 --- a/molecule/logstash_external_certs/verify.yml +++ b/molecule/logstash_external_certs/verify.yml @@ -95,6 +95,25 @@ register: port_check ignore_errors: true + # An open socket doesn't prove TLS — Logstash could be serving plain text + # on 5044 if the SSL block silently failed to load. Run an actual TLS + # handshake against the port and verify the served cert chains to the + # external CA we deployed. + - name: TLS handshake against Logstash 5044 (signed by Test CA) + ansible.builtin.shell: + cmd: | + set -o pipefail + out=$(echo | openssl s_client \ + -connect localhost:5044 \ + -CAfile /tmp/test-certs/ca.crt \ + -servername {{ ansible_facts.hostname }} 2>&1) || true + echo "$out" | grep -E 'Verify return code|verify error|issuer=' || true + echo "$out" | grep -q 'Verify return code: 0 (ok)' + executable: /bin/bash + register: _ls_tls_probe + changed_when: false + when: port_check is not failed + - name: Show diagnostics on port check failure when: port_check is failed block: diff --git a/molecule/logstash_ssl/verify.yml b/molecule/logstash_ssl/verify.yml index 2c9eff06..945c3dda 100644 --- a/molecule/logstash_ssl/verify.yml +++ b/molecule/logstash_ssl/verify.yml @@ -103,6 +103,25 @@ register: port_check ignore_errors: true + # An open socket doesn't prove TLS — Logstash could be serving plain text + # on 5044 if the SSL block silently failed to load. Run an actual TLS + # handshake against the port and verify the served cert chains to our + # test CA. + - name: TLS handshake against Logstash 5044 (signed by Test CA) + ansible.builtin.shell: + cmd: | + set -o pipefail + out=$(echo | openssl s_client \ + -connect localhost:5044 \ + -CAfile /tmp/test-certs/ca.crt \ + -servername {{ ansible_facts.hostname }} 2>&1) || true + echo "$out" | grep -E 'Verify return code|verify error|issuer=' || true + echo "$out" | grep -q 'Verify return code: 0 (ok)' + executable: /bin/bash + register: _ls_tls_probe + changed_when: false + when: port_check is not failed + - name: Show diagnostics on port check failure when: port_check is failed block: From a33ae18e3d3885c300f08df081fff43ed84f9e09 Mon Sep 17 00:00:00 2001 From: oddly Date: Sun, 7 Jun 2026 21:51:35 +0200 Subject: [PATCH 07/19] test(beats_security, elasticstack_default): assert end-to-end ingest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two ingest-flavoured scenarios both verified configuration in isolation but never proved data actually flowed through the full pipeline. beats_security only checked that filebeat.yml mentions ssl / certificate_authorities / output.elasticsearch. None of those substrings would catch a wrong password, a wrong CA path, or a silently-failing TLS handshake. Emit a unique syslog marker on the beats node and assert ES returns a hit for that marker under filebeat-*. elasticstack_default proved Beats → Logstash via Logstash's events.out counter but never confirmed Logstash → ES. Query ES for any document in logstash-* / logs-* / ecs-* indices and assert at least one has docs.count > 0. The leg from Logstash to ES is exactly the one that fails first when an ES output password or CA path is wrong, and the config-only checks above don't see it. --- molecule/beats_security/verify.yml | 44 +++++++++++++++++++++++ molecule/elasticstack_default/verify.yml | 46 ++++++++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/molecule/beats_security/verify.yml b/molecule/beats_security/verify.yml index b1160c6f..035a3c53 100644 --- a/molecule/beats_security/verify.yml +++ b/molecule/beats_security/verify.yml @@ -86,3 +86,47 @@ - "'certificate_authorities' in (filebeat_yml.content | b64decode)" - "'output.elasticsearch' in (filebeat_yml.content | b64decode)" fail_msg: "SSL configuration not found in filebeat.yml" + + # End-to-end ingest test: emit a uniquely-tagged line via syslog on + # the beats node, then query ES for that exact string. Proves + # Filebeat actually authenticates to ES with the configured TLS + # client and that documents arrive — the substring checks above + # would all pass against a config that silently drops events + # (wrong CA path, wrong password, broken handshake). + - name: Emit a unique marker line via syslog + ansible.builtin.command: + cmd: logger -t molecule-beats-security 'MARKER {{ ansible_facts.hostname }} molecule-beats-security-test' + changed_when: false + + # The ingest assertion runs on the ES host because filebeat-* lives in + # ES, not on the beats node. + - name: Confirm Filebeat shipped the marker to Elasticsearch + when: "'elasticsearch' in group_names" + block: + - name: Wait for filebeat-* index to receive the marker + ansible.builtin.uri: + url: "https://localhost:{{ elasticstack_elasticsearch_http_port }}/filebeat-*/_search?q=message:molecule-beats-security-test" + method: GET + force_basic_auth: true + user: elastic + password: "{{ elastic_pass.stdout }}" + return_content: true + status_code: [200, 404] + validate_certs: false + register: _filebeat_search + until: (_filebeat_search.json.hits.total.value | default(0)) > 0 + retries: 30 + delay: 10 + run_once: true # noqa: run-once[task] + + - name: Assert at least one document landed in filebeat-* + ansible.builtin.assert: + that: + - _filebeat_search.json.hits.total.value > 0 + fail_msg: >- + No matching document found in filebeat-*. Filebeat is running + and the config looks right (SSL block, output.elasticsearch + present), but no events made it from the beats node to ES. + Likely causes: wrong elastic password, wrong CA path on the + beats node, or TLS handshake failing silently. + run_once: true # noqa: run-once[task] diff --git a/molecule/elasticstack_default/verify.yml b/molecule/elasticstack_default/verify.yml index ebe980ea..055bc7fa 100644 --- a/molecule/elasticstack_default/verify.yml +++ b/molecule/elasticstack_default/verify.yml @@ -77,6 +77,52 @@ msg: "Logstash events — in: {{ logstash_stats.json.pipelines.main.events.in }}, out: {{ logstash_stats.json.pipelines.main.events.out }}" when: "'logstash' in group_names" + # End-to-end: Beats → Logstash is proven by events.out > 0 above. To + # prove Logstash → ES (the gap a config-only verify silently misses), + # query ES for any non-system index with documents. The flagship scenario + # has Filebeat+Auditbeat+Metricbeat shipping to Logstash, which outputs + # to ES with the default `logstash-*` / data-stream `logs-*` naming — + # either appears here. + - name: Confirm Logstash output index landed in Elasticsearch + ansible.builtin.uri: + url: "https://localhost:{{ elasticstack_elasticsearch_http_port }}/_cat/indices/logstash-*,logs-*,ecs-*?h=index,docs.count&format=json&expand_wildcards=open" + method: GET + force_basic_auth: true + user: elastic + password: "{{ elastic_pass.stdout }}" + return_content: true + status_code: [200, 404] + validate_certs: false + register: _ls_es_indices + until: >- + (_ls_es_indices.json | default([]) + | selectattr('docs.count', 'defined') + | selectattr('docs.count', 'ne', '0') + | list | length) > 0 + retries: 18 + delay: 10 + delegate_to: "{{ elasticsearch_ca }}" + run_once: true + when: "'logstash' in group_names" # noqa: run-once[task] + + - name: Assert at least one Logstash output index has documents in ES + ansible.builtin.assert: + that: + - >- + (_ls_es_indices.json | default([]) + | selectattr('docs.count', 'defined') + | selectattr('docs.count', 'ne', '0') + | list | length) > 0 + fail_msg: >- + Logstash reported events.out > 0 to its internal counter, but no + documents landed in ES under logstash-*, logs-*, or ecs-* indices. + The Beats → Logstash leg works but Logstash → ES does not — likely + a wrong elastic password in the ES output, a TLS chain failure to + ES, or the output plugin silently dropping events. + run_once: true + delegate_to: "{{ elasticsearch_ca }}" + when: "'logstash' in group_names" # noqa: run-once[task] + - name: Run Kibana checks when: "'kibana' in group_names" block: From ca1942e4397ea0a2fcc70830f8a73db276a8e039 Mon Sep 17 00:00:00 2001 From: oddly Date: Sun, 7 Jun 2026 21:52:14 +0200 Subject: [PATCH 08/19] test(cert_info_module): register and assert on every negative-path call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All five negative tests for the cert_info module previously called the module with failed_when: false and never registered the result. A regression that started swallowing bad inputs — missing path, non-existent file, wrong passphrase, no passphrase against an encrypted p12, no parameters at all — would have passed every test silently. Register each call, assert `is failed`, and provide a fail_msg that shows what the module did return when the assertion fires. Also strengthen the happy-path: assert the returned dict has the expected keys and that the date parses to a real timestamp (a bug returning {} would otherwise only surface as a confusing Jinja error in the days-until-expiry debug below). --- molecule/cert_info_module/converge.yml | 103 ++++++++++++++++++++----- 1 file changed, 84 insertions(+), 19 deletions(-) diff --git a/molecule/cert_info_module/converge.yml b/molecule/cert_info_module/converge.yml index 1a9d8e60..128c438e 100644 --- a/molecule/cert_info_module/converge.yml +++ b/molecule/cert_info_module/converge.yml @@ -1,42 +1,107 @@ --- -# The workaround for arbitrarily named role directory is important because the git repo has one name and the role within it another -# Found at: https://github.com/ansible-community/molecule/issues/1567#issuecomment-436876722 +# Tests the oddly.elasticstack.cert_info module across happy and failure paths. +# Invoked directly via `ansible-playbook` from .github/workflows/test_plugins.yml +# (this is not a molecule scenario — there is no molecule.yml). - name: Converge hosts: localhost tasks: - # - # Test modules - # - - name: Test + + # --- Happy path --- + + - name: cert_info on a valid PKCS#12 with the right passphrase oddly.elasticstack.cert_info: path: files/es-ca/elastic-stack-ca.p12 passphrase: PleaseChangeMe - register: test - - name: Debug + register: _ci_ok + + - name: Show happy-path result ansible.builtin.debug: - msg: "{{ test }}" - - name: Test required parameters (missing path) # noqa: args[module] + var: _ci_ok + + - name: Assert happy-path returned the expected fields + ansible.builtin.assert: + that: + - _ci_ok is not failed + - _ci_ok.not_valid_after is defined + - _ci_ok.not_valid_before is defined + - _ci_ok.subject is defined + # An empty/None response would fail the date arithmetic below in + # surprising ways; check the date parses to a real timestamp. + - (_ci_ok.not_valid_after | regex_replace('[+-]\d{2}:\d{2}$', '') | to_datetime('%Y-%m-%d %H:%M:%S')).year >= 2020 + fail_msg: "cert_info happy-path did not return the expected dict shape: {{ _ci_ok }}" + + - name: Compute days until expiry (smoke-test the typical caller pattern) + ansible.builtin.debug: + msg: >- + Cert expires in + {{ ((_ci_ok.not_valid_after | regex_replace('[+-]\d{2}:\d{2}$', '') | to_datetime('%Y-%m-%d %H:%M:%S')) - + (ansible_facts.date_time.date | to_datetime('%Y-%m-%d'))).days }} + days + + # --- Negative paths --- + # + # Each negative case must explicitly fail. The previous version used + # `failed_when: false` with no register/assertion, so a regression where + # cert_info started swallowing bad inputs (e.g. missing passphrase + # falling back to "" and succeeding) would have passed silently. + + - name: cert_info with missing path argument # noqa: args[module] oddly.elasticstack.cert_info: passphrase: PleaseChangeMe + register: _ci_missing_path failed_when: false - - name: Test wrong path + + - name: Assert missing-path call failed + ansible.builtin.assert: + that: + - _ci_missing_path is failed + fail_msg: "cert_info should have failed on missing path, but returned: {{ _ci_missing_path }}" + + - name: cert_info with a non-existent path oddly.elasticstack.cert_info: - path: es-ca-wrong + path: does-not-exist.p12 passphrase: PleaseChangeMe + register: _ci_bad_path failed_when: false - - name: Debug with to_datetime() - strip timezone for naive datetime arithmetic - ansible.builtin.debug: - msg: >- - "{{ ((test.not_valid_after | regex_replace('[+-]\d{2}:\d{2}$', '') | to_datetime('%Y-%m-%d %H:%M:%S')) - (ansible_facts.date_time.date | to_datetime('%Y-%m-%d'))).days }}" - - name: Test wrong passphrase + + - name: Assert bad-path call failed + ansible.builtin.assert: + that: + - _ci_bad_path is failed + fail_msg: "cert_info should have failed on non-existent path, but returned: {{ _ci_bad_path }}" + + - name: cert_info with the wrong passphrase oddly.elasticstack.cert_info: path: files/es-ca/elastic-stack-ca.p12 passphrase: PleaseChangeMe-wrong + register: _ci_bad_pass failed_when: false - - name: Test no passphrase + + - name: Assert wrong-passphrase call failed + ansible.builtin.assert: + that: + - _ci_bad_pass is failed + fail_msg: "cert_info should have failed on wrong passphrase, but returned: {{ _ci_bad_pass }}" + + - name: cert_info with no passphrase against an encrypted PKCS#12 oddly.elasticstack.cert_info: path: files/es-ca/elastic-stack-ca.p12 + register: _ci_no_pass failed_when: false - - name: Test no parameters # noqa: args[module] + + - name: Assert no-passphrase call failed + ansible.builtin.assert: + that: + - _ci_no_pass is failed + fail_msg: "cert_info should have failed when no passphrase given for an encrypted p12, but returned: {{ _ci_no_pass }}" + + - name: cert_info with no parameters at all # noqa: args[module] oddly.elasticstack.cert_info: + register: _ci_no_params failed_when: false + + - name: Assert no-parameters call failed + ansible.builtin.assert: + that: + - _ci_no_params is failed + fail_msg: "cert_info should have failed with no parameters, but returned: {{ _ci_no_params }}" From cee1950784a04876f0a9e1abdc5bb8f388fb6052 Mon Sep 17 00:00:00 2001 From: oddly Date: Sun, 7 Jun 2026 21:52:45 +0200 Subject: [PATCH 09/19] test(elasticsearch_default): don't skip the rotation guard on its own regression MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "auto-generated password no longer works" block guards against the role overwriting initial_passwords with the user-supplied password. The API call was gated `when: _old_pass.stdout != "TestPassword123!"` — meaning the assertion silently skipped the moment the regression actually happened (because then _old_pass.stdout would equal TestPassword123!). Drop the skip clause and add an explicit assertion that initial_passwords does NOT hold the user password. If the role ever flips that behaviour, downstream services that read from initial_passwords (Kibana via fetch_password.yml, Logstash, Beats) will silently end up with the wrong credentials — exactly the failure mode the user-supplied-password feature is supposed to avoid. --- molecule/elasticsearch_default/verify.yml | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/molecule/elasticsearch_default/verify.yml b/molecule/elasticsearch_default/verify.yml index f31fc177..314d68bd 100644 --- a/molecule/elasticsearch_default/verify.yml +++ b/molecule/elasticsearch_default/verify.yml @@ -34,6 +34,26 @@ changed_when: false run_once: true + # The role MUST preserve the auto-generated password in initial_passwords + # when the user supplies a custom one — overwriting it (to the user's + # password) is the exact regression this block exists to catch. The + # previous version skipped the API call when _old_pass.stdout matched + # the user password, which silently turned the regression-guard into a + # no-op the moment the regression actually happened. + - name: Assert initial_passwords still holds the auto-generated password (not overwritten) # noqa: run-once[task] + ansible.builtin.assert: + that: + - _old_pass.stdout | length > 0 + - _old_pass.stdout != "TestPassword123!" + fail_msg: >- + initial_passwords was overwritten with the user-defined password. + The role is supposed to preserve the auto-generated value here + and only switch to the user password for API calls. If + initial_passwords now contains the user value, the role has + regressed and downstream services (Kibana, Logstash) that read + from this file will end up with the wrong credentials. + run_once: true + - name: Try API with old password (should fail) # noqa: run-once[task] ansible.builtin.uri: url: "https://localhost:9200/" @@ -43,7 +63,6 @@ validate_certs: false status_code: 401 run_once: true - when: _old_pass.stdout != "TestPassword123!" - name: Read cluster settings # noqa: run-once[task] ansible.builtin.uri: From a451248863d40ed7970d315d9cf6124fb161a1d5 Mon Sep 17 00:00:00 2001 From: oddly Date: Sun, 7 Jun 2026 21:53:07 +0200 Subject: [PATCH 10/19] test(kibana_custom): add negative sniff_on_start assertion for 9.x The existing assertion only fires on pre-9.x, where it confirms sniffOnStart: true is rendered. Default CI runs on 9.x, so the assertion was always skipped. A template regression that restored sniffOnStart on 9.x would only show up as a Kibana startup warning, not a test failure. Add the symmetric negative-case assertion so the 9.x path is actually exercised. --- molecule/kibana_custom/verify.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/molecule/kibana_custom/verify.yml b/molecule/kibana_custom/verify.yml index de04ef16..b5aca810 100644 --- a/molecule/kibana_custom/verify.yml +++ b/molecule/kibana_custom/verify.yml @@ -79,6 +79,21 @@ fail_msg: "Sniff on start not found in kibana.yml" when: (lookup('env', 'ELASTIC_RELEASE') | default('9', true) | int) < 9 + # Negative test on 9.x: even with kibana_sniff_on_start: true the + # template should NOT emit sniffOnStart (Kibana 9.x removed the + # sniffing setting). Without this assert, a regression that restored + # the line on 9.x would only show up under default-release CI as a + # confusing Kibana startup warning, not a test failure. + - name: Verify sniff on start is NOT emitted on 9.x + ansible.builtin.assert: + that: + - "'sniffOnStart' not in (kibana_yml.content | b64decode)" + fail_msg: >- + kibana_sniff_on_start: true should be silently dropped on + Kibana 9.x, but sniffOnStart appears in kibana.yml. Kibana + 9.x will log a warning at startup and ignore the value. + when: (lookup('env', 'ELASTIC_RELEASE') | default('9', true) | int) >= 9 + - name: Check for kibana.yml backup files ansible.builtin.find: paths: /etc/kibana From 8f24cf454bc9cb510f2b2b595f78b1f7070ef4ad Mon Sep 17 00:00:00 2001 From: oddly Date: Sun, 7 Jun 2026 21:53:41 +0200 Subject: [PATCH 11/19] test(logstash_centralized_pipelines): confirm Logstash actually starts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scenario only inspected /etc/logstash/logstash.yml — never checked Logstash service was running, port 9600 was open, or central pipeline retrieval was succeeding. If xpack.management.* was misconfigured (broken cert path, wrong URL, missing required field) Logstash would crash on startup and every config assertion above would still pass. Add a service-running check, wait for port 9600, and scrape the journal for the concrete error messages Logstash emits when xpack.management can't connect or load a pipeline. The feature the scenario claims to test now actually gets exercised. Also promote the "warn if local pipeline configs exist" debug task to an assert — it had changed_when: true on a debug, which does nothing on failure. logstash_no_pipelines: true is in the converge, so an unexpected /etc/logstash/conf.d/main directory should fail the test, not whisper at it. --- .../logstash_centralized_pipelines/verify.yml | 63 +++++++++++++++++-- 1 file changed, 58 insertions(+), 5 deletions(-) diff --git a/molecule/logstash_centralized_pipelines/verify.yml b/molecule/logstash_centralized_pipelines/verify.yml index ada80fa7..b7114039 100644 --- a/molecule/logstash_centralized_pipelines/verify.yml +++ b/molecule/logstash_centralized_pipelines/verify.yml @@ -108,8 +108,61 @@ path: /etc/logstash/conf.d/main register: pipeline_dir - - name: Warn if local pipeline configs exist - ansible.builtin.debug: - msg: "Local pipeline configs should not exist with centralized management" - when: pipeline_dir.stat.exists and (not pipeline_dir.stat.isdir or pipeline_dir.stat.size > 0) - changed_when: true + - name: Assert no local pipeline configs exist with centralized management + ansible.builtin.assert: + that: + - >- + (not pipeline_dir.stat.exists) or + (pipeline_dir.stat.isdir and pipeline_dir.stat.size == 0) + fail_msg: >- + /etc/logstash/conf.d/main exists with content despite + logstash_no_pipelines: true. The role should not have written + local pipeline files when xpack.management is configured. + + # --- Confirm Logstash actually starts --- + # + # The previous version of this verify only inspected /etc/logstash/logstash.yml + # — never confirmed Logstash came up. If xpack.management.* was misconfigured + # (broken cert path, wrong URL, missing required field) Logstash crashes on + # startup but every assertion above still passes because they only test + # rendered config. + + - name: Check Logstash service is running + ansible.builtin.service: + name: logstash + state: started + check_mode: true + register: _ls_svc + failed_when: _ls_svc.changed + + - name: Wait for Logstash monitoring API + ansible.builtin.wait_for: + port: 9600 + timeout: 180 + state: started + register: _ls_api_wait + + - name: Scrape recent Logstash log for xpack.management bootstrap errors + ansible.builtin.shell: + cmd: | + journalctl -u logstash --no-pager -n 200 2>/dev/null \ + || tail -200 /var/log/logstash/logstash-plain.log 2>/dev/null \ + || true + executable: /bin/bash + register: _ls_log + changed_when: false + + - name: Assert no xpack.management configuration-retrieval failures in the log + ansible.builtin.assert: + that: + # Concrete error messages Logstash emits when xpack.management + # can't connect or load a pipeline. If any of these appear, + # Logstash is up but central-pipeline retrieval is broken — the + # scenario claims to test the feature, so this matters. + - "'Could not load the configuration' not in _ls_log.stdout" + - "'Configuration retrieval failed' not in _ls_log.stdout" + - "'Failed to fetch X-Pack information' not in _ls_log.stdout" + fail_msg: | + Logstash started but xpack.management failed to retrieve the + centralized pipeline. Recent log: + {{ _ls_log.stdout }} From 50ef908bbfb3243eb731918cf9888b9a0917d830 Mon Sep 17 00:00:00 2001 From: oddly Date: Sun, 7 Jun 2026 21:54:13 +0200 Subject: [PATCH 12/19] test(elasticsearch_diagnostics): require real diagnostic content, not just the header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two weaknesses in the diagnostic-recovery check: The bad-config bait used "bogus.nonexistent.setting: true". ES does reject unknown root settings on most versions, but some versions silently archive them as "archived.bogus..." instead of failing. If that ever happens (newer ES version, or a future config change), the rescue block doesn't fire and the test reports "did not fail on bad config" while actual diagnostic regressions ship through. Switch to discovery.type: bogus-invalid-discovery-type which ES guarantees to reject. The diagnostic assertion only required the literal substring "Recent log output" to be in the failure message — the role hardcodes that header string regardless of whether journalctl returned anything. A regression where the diagnostics machinery silently emitted an empty journal block (e.g. journalctl command typo, missing journal in the container) would still pass. Require the message be substantially longer than the header itself and contain at least one token that real ES startup-failure log content would carry (elasticsearch.service / Elasticsearch / exit code). --- .../elasticsearch_diagnostics/converge.yml | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/molecule/elasticsearch_diagnostics/converge.yml b/molecule/elasticsearch_diagnostics/converge.yml index 4d0461eb..b7752285 100644 --- a/molecule/elasticsearch_diagnostics/converge.yml +++ b/molecule/elasticsearch_diagnostics/converge.yml @@ -30,10 +30,15 @@ - name: Test bad config produces fast failure with diagnostics block: - - name: Inject bad setting into elasticsearch.yml + # Use a setting ES guarantees to reject (malformed YAML at the + # top level). The previous "bogus.nonexistent.setting: true" + # bait relied on ES rejecting unknown root settings, which some + # versions silently archive as "archived.bogus..." — the rescue + # below would then not fire and the test would mis-report. + - name: Inject malformed setting that ES is guaranteed to reject ansible.builtin.lineinfile: path: /etc/elasticsearch/elasticsearch.yml - line: "bogus.nonexistent.setting: true" + line: "discovery.type: bogus-invalid-discovery-type" - name: Attempt restart with bad config (should fail with diagnostics) ansible.builtin.include_role: @@ -45,13 +50,27 @@ msg: "restart_and_verify_elasticsearch.yml did not fail on bad config" rescue: - - name: Assert failure includes diagnostic log output + - name: Assert failure includes non-empty diagnostic log output ansible.builtin.assert: that: - ansible_failed_result.msg is defined - "'Recent log output' in ansible_failed_result.msg" + # The previous check only required the literal header string + # 'Recent log output' to appear — the role hardcodes that + # string, so the assertion would pass even if journalctl + # returned an empty stdout (broken diagnostics). Require + # enough trailing content that real journal output must be + # present (200+ chars beyond the header). + - ansible_failed_result.msg | length > (('Recent log output:' | length) + 200) + # And require at least one of the systemd/ES tokens that any + # real ES startup-failure log would contain. + - >- + ('elasticsearch.service' in ansible_failed_result.msg) + or ('Elasticsearch' in ansible_failed_result.msg) + or ('exit code' in ansible_failed_result.msg) fail_msg: >- - Expected failure message with 'Recent log output', got: + Expected a substantial diagnostic message containing real + journal output, got: {{ ansible_failed_result.msg | default('no message') }} - name: Restore good config From eaf53e2099715dd5fc325784918e705cca881b52 Mon Sep 17 00:00:00 2001 From: oddly Date: Sun, 7 Jun 2026 21:57:30 +0200 Subject: [PATCH 13/19] test(logstash_standalone_certs): add scenario for the actually-uncovered standalone cert path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous scenario named logstash_standalone_certs actually exercised cert_source: external (renamed to logstash_external_certs in the rename commit). The real "standalone" code path in roles/logstash/tasks/logstash-security.yml lines 328-449 — where the role generates its own self-signed CA, server cert, and PKCS12 keystore via openssl shell-outs and community.crypto.openssl_privatekey — had zero molecule coverage. A regression to any of the openssl invocations, the PKCS#8 conversion, or the PKCS12 export would ship undetected. The scenario is a single-host Logstash deployment (no ES). The role's standalone block still calls cert_distribute for an ES ca.crt and runs fetch_password against initial_passwords, so prepare.yml seeds /opt/es-ca/ca.crt and /usr/share/elasticsearch/initial_passwords on the same host. converge sets logstash_create_user/role: false to keep the unreachable ES API calls suppressed. verify covers: every standalone-generated artefact exists with mode 0640, the standalone-ca.crt subject matches the role's hardcoded /CN=Logstash Standalone CA, openssl verify confirms the server cert chains to it, the server key is unencrypted PKCS#8 PEM (Logstash beats input requirement), the PKCS12 keystore unlocks with the configured passphrase AND rejects a wrong one, and Logstash actually serves TLS on 5044 with a cert that verifies against the standalone CA. --- .../logstash_standalone_certs/converge.yml | 31 ++++ molecule/logstash_standalone_certs/create.yml | 1 + .../logstash_standalone_certs/destroy.yml | 1 + .../logstash_standalone_certs/molecule.yml | 28 +++ .../logstash_standalone_certs/prepare.yml | 68 ++++++++ .../requirements.yml | 4 + molecule/logstash_standalone_certs/verify.yml | 161 ++++++++++++++++++ 7 files changed, 294 insertions(+) create mode 100644 molecule/logstash_standalone_certs/converge.yml create mode 120000 molecule/logstash_standalone_certs/create.yml create mode 120000 molecule/logstash_standalone_certs/destroy.yml create mode 100644 molecule/logstash_standalone_certs/molecule.yml create mode 100644 molecule/logstash_standalone_certs/prepare.yml create mode 100644 molecule/logstash_standalone_certs/requirements.yml create mode 100644 molecule/logstash_standalone_certs/verify.yml diff --git a/molecule/logstash_standalone_certs/converge.yml b/molecule/logstash_standalone_certs/converge.yml new file mode 100644 index 00000000..d8f10816 --- /dev/null +++ b/molecule/logstash_standalone_certs/converge.yml @@ -0,0 +1,31 @@ +--- +# Exercises logstash_cert_source: standalone — the path where Logstash +# generates its own self-signed CA + server cert and PKCS12 keystore. +# Previously uncovered by any molecule scenario (the directory named +# logstash_standalone_certs actually tested cert_source: external). +- name: Converge + hosts: all + vars: + elasticstack_release: "{{ lookup('env', 'ELASTIC_RELEASE') | default('9', true) | int }}" + elasticstack_full_stack: false + elasticstack_no_log: false + logstash_output_elasticsearch: false + logstash_create_user: false + logstash_create_role: false + logstash_pipeline_unsafe_shutdown: true + logstash_heap: "512m" + logstash_input_beats: true + logstash_input_beats_ssl: true + logstash_input_beats_port: 5044 + logstash_cert_source: standalone + logstash_tls_key_passphrase: StandaloneTestPass + logstash_extra_outputs: | + stdout { codec => rubydebug } + tasks: + - name: Include Elastic repos role + ansible.builtin.include_role: + name: oddly.elasticstack.repos + + - name: Include Logstash role + ansible.builtin.include_role: + name: oddly.elasticstack.logstash diff --git a/molecule/logstash_standalone_certs/create.yml b/molecule/logstash_standalone_certs/create.yml new file mode 120000 index 00000000..138c2c2f --- /dev/null +++ b/molecule/logstash_standalone_certs/create.yml @@ -0,0 +1 @@ +../shared/create.yml \ No newline at end of file diff --git a/molecule/logstash_standalone_certs/destroy.yml b/molecule/logstash_standalone_certs/destroy.yml new file mode 120000 index 00000000..d18e9a34 --- /dev/null +++ b/molecule/logstash_standalone_certs/destroy.yml @@ -0,0 +1 @@ +../shared/destroy.yml \ No newline at end of file diff --git a/molecule/logstash_standalone_certs/molecule.yml b/molecule/logstash_standalone_certs/molecule.yml new file mode 100644 index 00000000..4f4abd03 --- /dev/null +++ b/molecule/logstash_standalone_certs/molecule.yml @@ -0,0 +1,28 @@ +--- +prerun: false +dependency: + name: galaxy + options: + requirements-file: requirements.yml +driver: + name: default +platforms: + - name: "ls-standalone-${MOLECULE_DISTRO:-debian12}-r${ELASTIC_RELEASE:-9}${MOLECULE_RUN_SUFFIX}" + groups: + - logstash + distro: "${MOLECULE_DISTRO:-debian12}" + memory_mb: 2048 +provisioner: + name: ansible + env: + ANSIBLE_LOG_PATH: /var/log/ansible.log + connection_options: + ansible_connection: ssh + ansible_user: root + ansible_ssh_common_args: "-o StrictHostKeyChecking=no" + inventory: + group_vars: + all: + ansible_python_interpreter: /usr/bin/python3 +verifier: + name: ansible diff --git a/molecule/logstash_standalone_certs/prepare.yml b/molecule/logstash_standalone_certs/prepare.yml new file mode 100644 index 00000000..4684fe4e --- /dev/null +++ b/molecule/logstash_standalone_certs/prepare.yml @@ -0,0 +1,68 @@ +--- +- name: Prepare + hosts: all + tasks: + - name: Populate /etc/hosts with molecule instances + ansible.builtin.lineinfile: + path: /etc/hosts + regexp: ".*{{ item }}$" + line: "{{ hostvars[item]['ansible_host'] }} {{ item }}" + loop: "{{ groups['all'] }}" + when: groups['all'] | length > 1 + + - name: Common prepare tasks + ansible.builtin.include_tasks: ../shared/prepare_common.yml + + # The standalone cert_source path in roles/logstash still distributes + # a CA file from elasticstack_ca_host/elasticstack_ca_dir to the + # Logstash node (line 434 of logstash-security.yml). This scenario + # has no ES, so we fake the CA host: elasticstack_ca_host defaults + # to inventory_hostname when no elasticsearch group exists, so + # pre-seed /opt/es-ca/ca.crt on the Logstash node with a placeholder + # cert that cert_distribute can fetch from itself. + - name: Create faux ES CA directory + ansible.builtin.file: + path: /opt/es-ca + state: directory + owner: root + group: root + mode: "0700" + + - name: Generate placeholder ES CA key + ansible.builtin.command: + cmd: openssl genrsa -out /opt/es-ca/ca.key 2048 + creates: /opt/es-ca/ca.key + + - name: Generate placeholder ES CA cert + ansible.builtin.command: + cmd: >- + openssl req -x509 -new -nodes + -key /opt/es-ca/ca.key + -sha256 -days 365 + -subj "/CN=Faux ES CA for standalone test" + -out /opt/es-ca/ca.crt + creates: /opt/es-ca/ca.crt + + # logstash-security.yml's "Manage Elasticsearch user and role" block runs + # unconditionally for cert_mode == 'standalone' and calls fetch_password + # against elasticstack_initial_passwords (default /usr/share/elasticsearch/ + # initial_passwords). With no ES installed, the file doesn't exist and + # the role fails before reaching the actual user/role creation tasks + # (which we suppress via logstash_create_user/role: false in converge). + # Seed the file with a dummy elastic password so the fetch succeeds. + - name: Create faux ES install directory + ansible.builtin.file: + path: /usr/share/elasticsearch + state: directory + owner: root + group: root + mode: "0755" + + - name: Seed faux initial_passwords for fetch_password.yml + ansible.builtin.copy: + dest: /usr/share/elasticsearch/initial_passwords + content: | + PASSWORD elastic = dummy-password-for-standalone-test + owner: root + group: root + mode: "0600" diff --git a/molecule/logstash_standalone_certs/requirements.yml b/molecule/logstash_standalone_certs/requirements.yml new file mode 100644 index 00000000..554b93f4 --- /dev/null +++ b/molecule/logstash_standalone_certs/requirements.yml @@ -0,0 +1,4 @@ +--- +collections: + - community.general + - community.crypto diff --git a/molecule/logstash_standalone_certs/verify.yml b/molecule/logstash_standalone_certs/verify.yml new file mode 100644 index 00000000..9c1ed8ea --- /dev/null +++ b/molecule/logstash_standalone_certs/verify.yml @@ -0,0 +1,161 @@ +--- +- name: Verify Logstash standalone certs + hosts: all + tasks: + - name: Check Logstash service is running + ansible.builtin.service: + name: logstash + state: started + check_mode: true + register: _ls_svc + failed_when: _ls_svc.changed + + # --- Standalone-generated files exist --- + + - name: Stat each standalone artefact + ansible.builtin.stat: + path: "{{ item }}" + register: _ls_artefacts + loop: + - /etc/logstash/certs/standalone-ca.crt + - /etc/logstash/certs/standalone-ca.key + - /etc/logstash/certs/{{ inventory_hostname }}-server.crt + - /etc/logstash/certs/{{ inventory_hostname }}.key + - /etc/logstash/certs/keystore.pfx + - /etc/logstash/certs/ca.crt + + - name: Assert every standalone-generated cert/key/keystore exists + ansible.builtin.assert: + that: + - item.stat.exists + fail_msg: "{{ item.item }} not generated by standalone cert_source path" + loop: "{{ _ls_artefacts.results }}" + loop_control: + label: "{{ item.item }}" + + - name: Assert standalone cert permissions are 0640 + ansible.builtin.assert: + that: + - item.stat.mode == "0640" + fail_msg: "{{ item.item }} has mode {{ item.stat.mode }}, expected 0640" + loop: "{{ _ls_artefacts.results }}" + loop_control: + label: "{{ item.item }}" + + # --- Standalone CA subject matches the hardcoded role value --- + + - name: Inspect standalone CA subject + ansible.builtin.command: + cmd: openssl x509 -in /etc/logstash/certs/standalone-ca.crt -noout -subject + register: _ca_subject + changed_when: false + + - name: Assert standalone CA was self-signed with the expected CN + ansible.builtin.assert: + that: + - "'CN = Logstash Standalone CA' in _ca_subject.stdout or 'CN=Logstash Standalone CA' in _ca_subject.stdout" + fail_msg: | + standalone-ca.crt subject is unexpected. The role hardcodes + /CN=Logstash Standalone CA. Got: {{ _ca_subject.stdout }} + + # --- Server cert chains to the standalone CA --- + + - name: Verify server cert was signed by the standalone CA + ansible.builtin.shell: + cmd: | + openssl verify -CAfile /etc/logstash/certs/standalone-ca.crt \ + /etc/logstash/certs/{{ inventory_hostname }}-server.crt + executable: /bin/bash + register: _server_verify + changed_when: false + + - name: Assert verify returned "OK" + ansible.builtin.assert: + that: + - "'OK' in _server_verify.stdout" + fail_msg: "openssl verify did not return OK: {{ _server_verify.stdout }}" + + # --- Server key is PKCS#8 PEM (Logstash requirement) --- + + - name: Read server key header + ansible.builtin.command: + cmd: head -1 /etc/logstash/certs/{{ inventory_hostname }}.key + register: _server_key_header + changed_when: false + + - name: Assert server key is unencrypted PKCS#8 PEM + ansible.builtin.assert: + that: + - _server_key_header.stdout == '-----BEGIN PRIVATE KEY-----' + fail_msg: >- + Expected PKCS#8 PEM (-----BEGIN PRIVATE KEY-----), got + {{ _server_key_header.stdout }}. The beats input plugin requires PKCS#8. + + # --- Keystore unlocks with the configured passphrase --- + + - name: Confirm PKCS12 keystore unlocks with the test passphrase + ansible.builtin.command: + cmd: >- + openssl pkcs12 -in /etc/logstash/certs/keystore.pfx + -nokeys -passin pass:StandaloneTestPass + register: _pfx_check + changed_when: false + failed_when: _pfx_check.rc != 0 + + - name: Confirm PKCS12 keystore REJECTS a wrong passphrase + ansible.builtin.command: + cmd: >- + openssl pkcs12 -in /etc/logstash/certs/keystore.pfx + -nokeys -passin pass:wrong-passphrase + register: _pfx_bad_check + changed_when: false + failed_when: false + + - name: Assert wrong-passphrase open failed + ansible.builtin.assert: + that: + - _pfx_bad_check.rc != 0 + fail_msg: >- + keystore.pfx accepted a wrong passphrase — the role generated + an unencrypted (or null-password) keystore. + + # --- Listening + TLS handshake --- + + - name: Wait for Logstash beats input on 5044 + ansible.builtin.wait_for: + port: 5044 + timeout: 180 + state: started + register: _port_check + ignore_errors: true + + - name: TLS handshake against Logstash 5044 (signed by standalone CA) + ansible.builtin.shell: + cmd: | + set -o pipefail + out=$(echo | openssl s_client \ + -connect localhost:5044 \ + -CAfile /etc/logstash/certs/standalone-ca.crt \ + -servername {{ ansible_facts.hostname }} 2>&1) || true + echo "$out" | grep -E 'Verify return code|verify error|issuer=' || true + echo "$out" | grep -q 'Verify return code: 0 (ok)' + executable: /bin/bash + register: _ls_tls_probe + changed_when: false + when: _port_check is not failed + + - name: Show diagnostics if port did not open + when: _port_check is failed + block: + - name: Show recent Logstash log + ansible.builtin.shell: + cmd: >- + journalctl -u logstash --no-pager -n 100 2>/dev/null + || tail -100 /var/log/logstash/logstash-plain.log 2>/dev/null + || echo "no logs" + executable: /bin/bash + register: _ls_log + changed_when: false + - name: Fail with log context + ansible.builtin.fail: + msg: "Port 5044 did not open. Recent log:\n{{ _ls_log.stdout }}" From 43e66056e8c28c24211f2fb2b040b3490efbba1b Mon Sep 17 00:00:00 2001 From: oddly Date: Sun, 7 Jun 2026 22:19:16 +0200 Subject: [PATCH 14/19] test: address ansible-lint findings in the audit-fix commits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three errors flagged by ansible-lint on the audit-fixes branch: risky-shell-pipe in elasticsearch_upgrade_8to9/converge.yml — the journalctl|grep pipe needed set -o pipefail. run-once[task] (×2) in elasticsearch_no-security/verify.yml — the two new run_once tasks need the # noqa: run-once[task] suppression the rest of the verify files in this collection already use for the same construct. Six name[casing] warnings in cert_info_module/converge.yml — task names I renamed/added that started with the lowercase module name. Prefixed each with "Run " so the first letter is uppercase per the rule. --- molecule/cert_info_module/converge.yml | 12 ++++++------ molecule/elasticsearch_no-security/verify.yml | 4 ++-- molecule/elasticsearch_upgrade_8to9/converge.yml | 11 ++++++----- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/molecule/cert_info_module/converge.yml b/molecule/cert_info_module/converge.yml index 128c438e..3f05a196 100644 --- a/molecule/cert_info_module/converge.yml +++ b/molecule/cert_info_module/converge.yml @@ -8,7 +8,7 @@ # --- Happy path --- - - name: cert_info on a valid PKCS#12 with the right passphrase + - name: Run cert_info on a valid PKCS#12 with the right passphrase oddly.elasticstack.cert_info: path: files/es-ca/elastic-stack-ca.p12 passphrase: PleaseChangeMe @@ -45,7 +45,7 @@ # cert_info started swallowing bad inputs (e.g. missing passphrase # falling back to "" and succeeding) would have passed silently. - - name: cert_info with missing path argument # noqa: args[module] + - name: Run cert_info with missing path argument # noqa: args[module] oddly.elasticstack.cert_info: passphrase: PleaseChangeMe register: _ci_missing_path @@ -57,7 +57,7 @@ - _ci_missing_path is failed fail_msg: "cert_info should have failed on missing path, but returned: {{ _ci_missing_path }}" - - name: cert_info with a non-existent path + - name: Run cert_info with a non-existent path oddly.elasticstack.cert_info: path: does-not-exist.p12 passphrase: PleaseChangeMe @@ -70,7 +70,7 @@ - _ci_bad_path is failed fail_msg: "cert_info should have failed on non-existent path, but returned: {{ _ci_bad_path }}" - - name: cert_info with the wrong passphrase + - name: Run cert_info with the wrong passphrase oddly.elasticstack.cert_info: path: files/es-ca/elastic-stack-ca.p12 passphrase: PleaseChangeMe-wrong @@ -83,7 +83,7 @@ - _ci_bad_pass is failed fail_msg: "cert_info should have failed on wrong passphrase, but returned: {{ _ci_bad_pass }}" - - name: cert_info with no passphrase against an encrypted PKCS#12 + - name: Run cert_info with no passphrase against an encrypted PKCS#12 oddly.elasticstack.cert_info: path: files/es-ca/elastic-stack-ca.p12 register: _ci_no_pass @@ -95,7 +95,7 @@ - _ci_no_pass is failed fail_msg: "cert_info should have failed when no passphrase given for an encrypted p12, but returned: {{ _ci_no_pass }}" - - name: cert_info with no parameters at all # noqa: args[module] + - name: Run cert_info with no parameters at all # noqa: args[module] oddly.elasticstack.cert_info: register: _ci_no_params failed_when: false diff --git a/molecule/elasticsearch_no-security/verify.yml b/molecule/elasticsearch_no-security/verify.yml index ab64bdcf..0c3f25c5 100644 --- a/molecule/elasticsearch_no-security/verify.yml +++ b/molecule/elasticsearch_no-security/verify.yml @@ -23,7 +23,7 @@ _verify_es_scheme: http _verify_es_status: green - - name: Fetch _cat/nodes for membership check + - name: Fetch _cat/nodes for membership check # noqa: run-once[task] ansible.builtin.uri: # kics-scan ignore-line url: http://localhost:{{ elasticstack_elasticsearch_http_port }}/_cat/nodes @@ -34,7 +34,7 @@ register: nodes run_once: true - - name: Assert every cluster member appears in _cat/nodes + - name: Assert every cluster member appears in _cat/nodes # noqa: run-once[task] ansible.builtin.assert: that: - "hostvars[item].ansible_facts.hostname in nodes.content" diff --git a/molecule/elasticsearch_upgrade_8to9/converge.yml b/molecule/elasticsearch_upgrade_8to9/converge.yml index 3d609dd8..e99ec754 100644 --- a/molecule/elasticsearch_upgrade_8to9/converge.yml +++ b/molecule/elasticsearch_upgrade_8to9/converge.yml @@ -178,11 +178,12 @@ - name: Count ES start events during the upgrade window ansible.builtin.shell: - cmd: >- - journalctl -u elasticsearch - --since='{{ _ts_before_upgrade.stdout }}' - --no-pager --output=cat 2>/dev/null - | grep -c 'Started elasticsearch' || true + cmd: | + set -o pipefail + journalctl -u elasticsearch \ + --since='{{ _ts_before_upgrade.stdout }}' \ + --no-pager --output=cat 2>/dev/null \ + | grep -c 'Started elasticsearch' || true executable: /bin/bash register: _es_start_count changed_when: false From bfffafd5a90f36aa99894fd5ebfb2fb159e67d30 Mon Sep 17 00:00:00 2001 From: oddly Date: Sun, 7 Jun 2026 23:09:49 +0200 Subject: [PATCH 15/19] ci: memory-aware admission gate for the molecule runner pool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With 20 statically-installed runners on incus-ci, the host can be busy in CPU/IO but underused in RAM because the cap is the number of runners, not memory. Bumping the runner count would help throughput on light workloads but risk OOM when multiple heavy scenarios (elasticstack_default at 20 GB, cert_renewal at ~11 GB) hit at once. Add a coordinated admission gate. Each molecule job acquires a reservation before converge and releases it after (or always() on failure). Reservations live under /var/lib/molecule-gate/r.${RUNNER_NAME}, summed inside an flock'd critical section so concurrent runners see each other's claims. A job is admitted only when MemAvailable minus the pending reservations exceeds the scenario's memory_mb plus a 2 GB buffer. Stale reservations (>1h) are GC'd by the next acquire under the same lock — covers crashed jobs that skipped the release step. The gate times out at 30 min and proceeds without a reservation rather than failing the job. The reasoning: a single OOM is better than blocking the whole queue if some unrelated runaway process holds the host memory. memory_mb-per-scenario lookup is hardcoded in the script and matches molecule//molecule.yml at the time of writing. Update both together when a scenario changes. --- .github/workflows/molecule.yml | 15 +++ scripts/wait-for-memory.sh | 162 +++++++++++++++++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100755 scripts/wait-for-memory.sh diff --git a/.github/workflows/molecule.yml b/.github/workflows/molecule.yml index 11fbb743..b7b40b59 100644 --- a/.github/workflows/molecule.yml +++ b/.github/workflows/molecule.yml @@ -105,6 +105,14 @@ jobs: mkdir -p ~/.ssh ssh-keyscan -t ed25519 ${{ secrets.INCUS_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true + - name: Acquire memory slot + # Coordinated admission gate. Takes an flock'd reservation on the + # runner host's RAM budget so the pool can scale past the point where + # every concurrent slot fits at peak memory_mb. Times out and proceeds + # without a reservation rather than failing — better one OOM than a + # stuck queue. + run: bash scripts/wait-for-memory.sh acquire "${{ matrix.scenario }}" 1800 + - name: Converge run: molecule converge -s ${{ matrix.scenario }} env: @@ -143,6 +151,13 @@ jobs: MOLECULE_SSH_KEY: ${{ runner.temp }}/molecule_id_ed25519 DISTRO_CACHE_URL: http://${{ secrets.REGISTRY_HOST }}:8081 + - name: Release memory slot + # Runs whether converge/verify/idempotence passed or failed, so the + # reservation is freed promptly. Stale reservations (>1h) are also + # GC'd by the next acquire under the same lock as a safety net. + if: always() + run: bash scripts/wait-for-memory.sh release + - name: Collect and upload diagnostics if: failure() uses: ./.github/actions/collect-diagnostics diff --git a/scripts/wait-for-memory.sh b/scripts/wait-for-memory.sh new file mode 100755 index 00000000..37670f60 --- /dev/null +++ b/scripts/wait-for-memory.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +# Coordinated memory admission gate for the molecule runner pool. +# +# Each scenario reserves its memory_mb on the runner host before converge, +# and releases the reservation in a post-step. A flock-serialised reader +# sums in-flight reservations and only admits a new job when: +# +# MemAvailable - sum(reservations) >= scenario_mb + buffer +# +# Reservations are files under $GATE_DIR keyed by RUNNER_NAME (so each runner +# instance can hold at most one). Stale files older than $TTL_SEC are +# garbage-collected during the lock window — covers crashed jobs that +# never reach the release step. +# +# Usage: +# wait-for-memory.sh acquire [timeout_s] +# wait-for-memory.sh release +# +# Env: +# RUNNER_NAME (auto-set by GitHub Actions; required) +# MOLECULE_GATE_DIR (default /var/lib/molecule-gate, falls back to /tmp/molecule-gate) +# MOLECULE_GATE_TTL (default 3600 — drop reservations older than this) +# WAIT_FOR_MEMORY_BUFFER_MB (default 2048 — extra headroom over scenario_mb) + +set -euo pipefail + +action="${1:?usage: $0 [args...]}" + +GATE_DIR="${MOLECULE_GATE_DIR:-/var/lib/molecule-gate}" +mkdir -p "$GATE_DIR" 2>/dev/null || { + GATE_DIR=/tmp/molecule-gate + mkdir -p "$GATE_DIR" +} +chmod 0777 "$GATE_DIR" 2>/dev/null || true + +LOCK="$GATE_DIR/.lock" +TTL_SEC="${MOLECULE_GATE_TTL:-3600}" +buffer_mb="${WAIT_FOR_MEMORY_BUFFER_MB:-2048}" + +runner="${RUNNER_NAME:-runner-$$}" +my_resv="$GATE_DIR/r.${runner}" + +# --- Required MB per scenario (sum of memory_mb across all platforms) --- +# Update this table when a scenario's memory_mb changes. +declare -A REQ=( + [beats_advanced]=2048 + [beats_default]=4096 + [beats_peculiar]=2048 + [beats_security]=6144 + [cert_renewal]=10752 + [elasticsearch_cert_content]=4096 + [elasticsearch_custom]=4096 + [elasticsearch_custom_certs]=4096 + [elasticsearch_custom_certs_minimal]=4096 + [elasticsearch_default]=8192 + [elasticsearch_diagnostics]=4096 + [elasticsearch_no-security]=8192 + [elasticsearch_roles_calculation]=16384 + [elasticsearch_security_api]=4096 + [elasticsearch_upgrade_8to9]=8192 + [elasticsearch_upgrade_8to9_single]=4096 + [elasticstack_default]=20480 + [es_kibana]=13824 + [kibana_custom]=8192 + [kibana_custom_certs]=8192 + [kibana_default]=4096 + [logstash_advanced]=2048 + [logstash_centralized_pipelines]=2048 + [logstash_custom_pipeline]=2048 + [logstash_default]=2048 + [logstash_elasticsearch]=8192 + [logstash_external_certs]=2048 + [logstash_standalone_certs]=2048 + [logstash_ssl]=2048 + [repos_default]=1024 +) + +# --- Helpers --- + +# Sum reservations under exclusive lock; garbage-collect stale entries. +# Echoes "" to stdout. Caller must hold the lock or call this +# inside the locked section. +sum_reservations() { + local now total=0 mtime mb + now=$(date +%s) + shopt -s nullglob + for f in "$GATE_DIR"/r.*; do + mtime=$(stat -c %Y "$f" 2>/dev/null || echo 0) + if [ $(( now - mtime )) -gt "$TTL_SEC" ]; then + rm -f "$f" + continue + fi + mb=$(awk '{print $1+0; exit}' "$f" 2>/dev/null || echo 0) + total=$(( total + mb )) + done + shopt -u nullglob + echo "$total" +} + +case "$action" in + acquire) + scenario="${2:?usage: $0 acquire [timeout_s]}" + timeout_s="${3:-1800}" + required="${REQ[$scenario]:-4096}" + threshold=$(( required + buffer_mb )) + + printf 'molecule-gate[%s]: acquire scenario=%s required=%dMB buffer=%dMB threshold=%dMB timeout=%ds\n' \ + "$runner" "$scenario" "$required" "$buffer_mb" "$threshold" "$timeout_s" + + deadline=$(( $(date +%s) + timeout_s )) + attempt=0 + while :; do + attempt=$(( attempt + 1 )) + # Critical section: read MemAvailable, sum reservations, optionally claim. + exec 9>"$LOCK" + flock 9 + reserved=$(sum_reservations) + available=$(awk '/^MemAvailable:/{print int($2/1024)}' /proc/meminfo) + effective=$(( available - reserved )) + if [ "$effective" -ge "$threshold" ]; then + printf '%d %s\n' "$required" "$scenario" > "$my_resv" + flock -u 9 + printf 'molecule-gate[%s]: ADMITTED MemAvailable=%dMB reserved=%dMB effective=%dMB required=%dMB (attempt %d)\n' \ + "$runner" "$available" "$reserved" "$effective" "$threshold" "$attempt" + exit 0 + fi + flock -u 9 + + now=$(date +%s) + if [ "$now" -ge "$deadline" ]; then + # Take the slot anyway under the same lock — better one OOM than a stuck queue. + exec 9>"$LOCK" + flock 9 + printf '%d %s\n' "$required" "$scenario" > "$my_resv" + flock -u 9 + printf 'molecule-gate[%s]: TIMEOUT after %ds (effective=%dMB needed=%dMB) — proceeding without headroom\n' \ + "$runner" "$timeout_s" "$effective" "$threshold" >&2 + exit 0 + fi + printf 'molecule-gate[%s]: waiting (available=%dMB reserved=%dMB effective=%dMB needed=%dMB, attempt %d)\n' \ + "$runner" "$available" "$reserved" "$effective" "$threshold" "$attempt" + sleep 30 + done + ;; + + release) + exec 9>"$LOCK" + flock 9 + if [ -f "$my_resv" ]; then + rm -f "$my_resv" + printf 'molecule-gate[%s]: released\n' "$runner" + else + printf 'molecule-gate[%s]: nothing to release\n' "$runner" + fi + flock -u 9 + ;; + + *) + echo "usage: $0 [args...]" >&2 + exit 2 + ;; +esac From ed71aafed58ef3f239274d85e55f861b4c33e5b6 Mon Sep 17 00:00:00 2001 From: oddly Date: Mon, 8 Jun 2026 09:17:59 +0200 Subject: [PATCH 16/19] test: fix five scenarios surfaced by CI on the audit-fixes branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI on the audit-fixes commit surfaced five real test failures (excluding ~31 pre-runner-PATH-fix flakes). Each one is a different class of mistake in my own changes from the earlier commits; not in the role code. logstash_centralized_pipelines kept "Wait for Logstash monitoring API on port 9600" and a journal-scrape for xpack.management errors. Both fail for this scenario by design: the converge points xpack.management at fake ES hosts (es-001/es-002.example.com), so Logstash never binds 9600 and the log fills with the very error strings the scrape was checking against. Keep only the "service active" check; comment why we don't probe further. elasticsearch_upgrade_8to9 counted journal start events with `grep -c 'Started elasticsearch'` — case-sensitive. systemd actually emits "Started Elasticsearch." with a capital E (the unit description), so the count came back as 0 and the assertion fired with the wrong message. Switch to `grep -ic`. elasticstack_default's "Confirm Logstash output index landed in Elasticsearch" was retrying 18 × 10s = 3 min. On slower runners the Beats → Logstash → ES warmup is longer than that. Bump to 60 × 10s = 10 min, which covers the slowest observed startups and still fails fast on real breakage. elasticsearch_upgrade_8to9_single tried to drive the 8.10 → 8.19 step via the role with elasticstack_version: latest. The role's elasticsearch-upgrade-detection only flags rolling upgrades on a pinned-higher version OR a release change — same-major bumps with `latest` aren't detected. On top of that, the chiark apt/yum mirror's cached metadata didn't surface newer 8.x patches without a forced refresh, so the role-driven path silently kept ES at 8.10. Revert to the direct package + restart approach (with explicit cache refresh) and leave a comment noting that adding a same-major `latest` detection path to the role is a worthwhile follow-up but a role change, not a test change. cert_renewal's renewal play set elasticstack_ca_will_expire_soon and friends as play vars to trigger renewal. Those facts live in each role's vars/main.yml — higher precedence than play vars — so the play-supplied value gets overridden when the role loads. Replace with the *_expiration_buffer overrides (99999 days), which are role defaults (lower precedence, overridable) and feed into the cert_check_expiry task that then sets the will_expire_soon fact via set_fact — the supported way to trigger renewal from outside. --- molecule/cert_renewal/converge.yml | 18 ++++-- .../elasticsearch_upgrade_8to9/converge.yml | 5 +- .../converge.yml | 57 ++++++++++++++----- molecule/elasticstack_default/verify.yml | 6 +- .../logstash_centralized_pipelines/verify.yml | 44 +++----------- 5 files changed, 72 insertions(+), 58 deletions(-) diff --git a/molecule/cert_renewal/converge.yml b/molecule/cert_renewal/converge.yml index 9bd8d69b..73f716dc 100644 --- a/molecule/cert_renewal/converge.yml +++ b/molecule/cert_renewal/converge.yml @@ -114,11 +114,19 @@ logstash_heap: "512m" logstash_pipeline_unsafe_shutdown: true kibana_tls: true - # Force certificate renewal - elasticstack_ca_will_expire_soon: true - elasticsearch_cert_will_expire_soon: true - kibana_cert_will_expire_soon: true - logstash_cert_will_expire_soon: true + # Force certificate renewal. + # + # The *_cert_will_expire_soon flags live in each role's vars/main.yml + # (precedence higher than play vars), so setting them here would be + # overridden when the role loads. Instead set the expiration buffer to + # a value larger than the cert's validity period — the role's + # cert_check_expiry task then computes "expires within buffer days" as + # true and flips the will_expire_soon fact via set_fact, which DOES + # propagate correctly to the renewal gates. + elasticstack_ca_expiration_buffer: 99999 + elasticsearch_cert_expiration_buffer: 99999 + kibana_cert_expiration_buffer: 99999 + logstash_cert_expiration_buffer: 99999 logstash_cert_force_regenerate: true tasks: - name: Reset shared role import guard for re-run diff --git a/molecule/elasticsearch_upgrade_8to9/converge.yml b/molecule/elasticsearch_upgrade_8to9/converge.yml index e99ec754..32ec779b 100644 --- a/molecule/elasticsearch_upgrade_8to9/converge.yml +++ b/molecule/elasticsearch_upgrade_8to9/converge.yml @@ -180,10 +180,13 @@ ansible.builtin.shell: cmd: | set -o pipefail + # systemd emits "Started Elasticsearch." with a capital E (the + # unit's Description=Elasticsearch). Use case-insensitive grep so + # we don't depend on the exact casing across distros. journalctl -u elasticsearch \ --since='{{ _ts_before_upgrade.stdout }}' \ --no-pager --output=cat 2>/dev/null \ - | grep -c 'Started elasticsearch' || true + | grep -ic 'started elasticsearch' || true executable: /bin/bash register: _es_start_count changed_when: false diff --git a/molecule/elasticsearch_upgrade_8to9_single/converge.yml b/molecule/elasticsearch_upgrade_8to9_single/converge.yml index 8a8d6bba..2960019b 100644 --- a/molecule/elasticsearch_upgrade_8to9_single/converge.yml +++ b/molecule/elasticsearch_upgrade_8to9_single/converge.yml @@ -110,36 +110,63 @@ delay: 5 until: _refresh_result.status | default(0) == 200 -- name: Upgrade to latest Elasticsearch 8.x via the role (required before 9.x) +- name: Upgrade to latest Elasticsearch 8.x (required before 9.x) hosts: all vars: elasticstack_full_stack: false elasticstack_release: 8 - # elasticstack_version: latest exercises the role's post-install - # upgrade-detection path on a same-major bump (8.10 → 8.19). The - # original test bypassed the role here with package: state: latest + - # service: restarted, which meant the role's rolling-upgrade detection - # for 8.x→8.x was never exercised by CI. - elasticstack_version: "latest" elasticsearch_heap: "1" elasticstack_no_log: false elasticsearch_seed_hosts: [] elasticsearch_initial_master_nodes: [] - elasticsearch_upgrade_countdown: 0 tasks: - name: Include Elastic repos role (8.x) ansible.builtin.include_role: name: oddly.elasticstack.repos - - name: Reset shared role guard for 8.x→8.x upgrade play - ansible.builtin.set_fact: - _elasticstack_role_imported: false + # The role's elasticstack_version: latest path does NOT trigger the + # rolling-upgrade detection on same-major bumps — only pinned-higher + # version OR release change qualifies (see + # roles/elasticsearch/tasks/elasticsearch-upgrade-detection.yml). + # And the chiark apt/yum mirror's cached metadata may not surface + # newer 8.x patches without a forced refresh. Drive the package + + # restart directly here, as the original test did. Worth a follow-up + # to add a "latest-mode same-major upgrade detection" path to the + # role, but that's a role change, not a test change. + - name: Force apt cache update before 8.x upgrade + ansible.builtin.apt: + update_cache: true + when: ansible_facts.os_family == 'Debian' + + - name: Force dnf metadata refresh before 8.x upgrade + ansible.builtin.command: dnf clean expire-cache + changed_when: false + when: ansible_facts.os_family == 'RedHat' - - name: Include Elasticsearch role (upgrade to latest 8.x via role) - ansible.builtin.include_role: - name: oddly.elasticstack.elasticsearch + - name: Upgrade to latest 8.x # noqa: package-latest + ansible.builtin.package: + name: elasticsearch + state: latest + + - name: Restart Elasticsearch after upgrade + ansible.builtin.service: + name: elasticsearch + state: restarted + + - name: Wait for ES to be ready after 8.x upgrade + ansible.builtin.uri: + url: https://localhost:9200/_cluster/health + method: GET + force_basic_auth: true + user: elastic + password: "{{ elastic_pass.stdout }}" + validate_certs: false + register: health + retries: 30 + delay: 10 + until: health.status == 200 - - name: Confirm test index survived the 8.x→8.x role-driven upgrade + - name: Confirm test index survived the 8.x→8.x upgrade ansible.builtin.uri: url: https://localhost:9200/upgrade-test/_doc/1 method: GET diff --git a/molecule/elasticstack_default/verify.yml b/molecule/elasticstack_default/verify.yml index 055bc7fa..d708ef8b 100644 --- a/molecule/elasticstack_default/verify.yml +++ b/molecule/elasticstack_default/verify.yml @@ -99,7 +99,11 @@ | selectattr('docs.count', 'defined') | selectattr('docs.count', 'ne', '0') | list | length) > 0 - retries: 18 + # Generous retry budget: in CI the Beats → Logstash → ES chain needs + # cluster green + Logstash plugin warmup + ES create-index latency. + # 18×10s was not enough on slow runners; 60×10s = 10 min covers the + # slowest observed startups while still failing on real breakage. + retries: 60 delay: 10 delegate_to: "{{ elasticsearch_ca }}" run_once: true diff --git a/molecule/logstash_centralized_pipelines/verify.yml b/molecule/logstash_centralized_pipelines/verify.yml index b7114039..9fedd35d 100644 --- a/molecule/logstash_centralized_pipelines/verify.yml +++ b/molecule/logstash_centralized_pipelines/verify.yml @@ -123,10 +123,14 @@ # # The previous version of this verify only inspected /etc/logstash/logstash.yml # — never confirmed Logstash came up. If xpack.management.* was misconfigured - # (broken cert path, wrong URL, missing required field) Logstash crashes on - # startup but every assertion above still passes because they only test - # rendered config. - + # (malformed YAML, missing required field) Logstash crashes on startup and + # every assertion above still passes because they only test rendered config. + # + # We can ONLY check the service is active here. The scenario's ES hosts + # (es-001.example.com, es-002.example.com) are intentionally fake to test + # config rendering without a real cluster, so the Logstash monitoring API + # may never bind and the log will contain xpack.management connection + # errors — those are EXPECTED here and not a regression signal. - name: Check Logstash service is running ansible.builtin.service: name: logstash @@ -134,35 +138,3 @@ check_mode: true register: _ls_svc failed_when: _ls_svc.changed - - - name: Wait for Logstash monitoring API - ansible.builtin.wait_for: - port: 9600 - timeout: 180 - state: started - register: _ls_api_wait - - - name: Scrape recent Logstash log for xpack.management bootstrap errors - ansible.builtin.shell: - cmd: | - journalctl -u logstash --no-pager -n 200 2>/dev/null \ - || tail -200 /var/log/logstash/logstash-plain.log 2>/dev/null \ - || true - executable: /bin/bash - register: _ls_log - changed_when: false - - - name: Assert no xpack.management configuration-retrieval failures in the log - ansible.builtin.assert: - that: - # Concrete error messages Logstash emits when xpack.management - # can't connect or load a pipeline. If any of these appear, - # Logstash is up but central-pipeline retrieval is broken — the - # scenario claims to test the feature, so this matters. - - "'Could not load the configuration' not in _ls_log.stdout" - - "'Configuration retrieval failed' not in _ls_log.stdout" - - "'Failed to fetch X-Pack information' not in _ls_log.stdout" - fail_msg: | - Logstash started but xpack.management failed to retrieve the - centralized pipeline. Recent log: - {{ _ls_log.stdout }} From 1dcdaaed978953273611d84c87aad3a0b56344f6 Mon Sep 17 00:00:00 2001 From: oddly Date: Mon, 8 Jun 2026 12:05:36 +0200 Subject: [PATCH 17/19] test: fix three more scenarios surfaced on the latest CI run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit elasticsearch_upgrade_8to9's handler-suppression assertion still fired with "Counted 0" after the case-sensitivity fix. Root cause was the timestamp format: date -u --iso-8601=seconds emits "2026-06-08T07:23:45+00:00" and journalctl --since parses the +00:00 offset inconsistently — the resulting filter rejects every record and the count comes back as zero. Switch to date -u "+%Y-%m-%d %H:%M:%S", the format journalctl --since prefers. elasticstack_default's "Confirm Logstash output index landed in Elasticsearch" failed with 1473 documents in .ds-logs-generic-default-... because the until clause uses selectattr('docs.count', ...) — Jinja parses that as a chained attribute access (item.docs.count), not a single dict key with a literal dot, so the selectattr never matches even when docs.count is "1473". Switch the query to /_count which returns a flat {"count": N} and assert on that directly. Also widen expand_wildcards so hidden data-stream backing indices count. beats_security's "Wait for filebeat-* index to receive the marker" returned hits.total.value=0 because Filebeat 8+ writes to data streams whose backing indices (.ds-filebeat-...) are hidden, and the default expand_wildcards=open excludes them. Add expand_wildcards=open,hidden + ignore_unavailable + allow_no_indices to the search URL, bump retries to 60 to cover slow flush+create-index, and tighten the integer cast on the until. --- molecule/beats_security/verify.yml | 19 ++++++++++---- .../elasticsearch_upgrade_8to9/converge.yml | 5 +++- molecule/elasticstack_default/verify.yml | 26 +++++++------------ 3 files changed, 28 insertions(+), 22 deletions(-) diff --git a/molecule/beats_security/verify.yml b/molecule/beats_security/verify.yml index 035a3c53..0c309b5f 100644 --- a/molecule/beats_security/verify.yml +++ b/molecule/beats_security/verify.yml @@ -103,26 +103,35 @@ - name: Confirm Filebeat shipped the marker to Elasticsearch when: "'elasticsearch' in group_names" block: + # Filebeat 8+ writes to data streams (filebeat-) whose + # backing indices (.ds-filebeat-...) are hidden. The default + # expand_wildcards=open excludes them, so the search needs + # expand_wildcards=open,hidden, ignore_unavailable and + # allow_no_indices so we don't 400 if the data stream hasn't + # rolled over yet. - name: Wait for filebeat-* index to receive the marker ansible.builtin.uri: - url: "https://localhost:{{ elasticstack_elasticsearch_http_port }}/filebeat-*/_search?q=message:molecule-beats-security-test" + url: "https://localhost:{{ elasticstack_elasticsearch_http_port }}/filebeat-*/_search?q=message:molecule-beats-security-test&expand_wildcards=open,hidden&ignore_unavailable=true&allow_no_indices=true" method: GET force_basic_auth: true user: elastic password: "{{ elastic_pass.stdout }}" return_content: true - status_code: [200, 404] + status_code: 200 validate_certs: false register: _filebeat_search - until: (_filebeat_search.json.hits.total.value | default(0)) > 0 - retries: 30 + until: (_filebeat_search.json.hits.total.value | default(0) | int) > 0 + # Filebeat's default flush interval + ES create-index latency can + # exceed 5 min on slow CI runners; 60×10s = 10 min covers the slow + # tail while still failing fast on real breakage. + retries: 60 delay: 10 run_once: true # noqa: run-once[task] - name: Assert at least one document landed in filebeat-* ansible.builtin.assert: that: - - _filebeat_search.json.hits.total.value > 0 + - (_filebeat_search.json.hits.total.value | default(0) | int) > 0 fail_msg: >- No matching document found in filebeat-*. Filebeat is running and the config looks right (SSL block, output.elasticsearch diff --git a/molecule/elasticsearch_upgrade_8to9/converge.yml b/molecule/elasticsearch_upgrade_8to9/converge.yml index 32ec779b..009ac262 100644 --- a/molecule/elasticsearch_upgrade_8to9/converge.yml +++ b/molecule/elasticsearch_upgrade_8to9/converge.yml @@ -158,7 +158,10 @@ # restart unless suppressed. Capture the start-event window so we can # assert exactly-one after include_role completes. - name: Capture pre-upgrade timestamp - ansible.builtin.command: date -u --iso-8601=seconds + # journalctl --since wants 'YYYY-MM-DD HH:MM:SS' (UTC implied via -u). + # Don't use --iso-8601=seconds: that emits '+00:00' which journalctl + # parses inconsistently and silently filters out every match. + ansible.builtin.command: date -u "+%Y-%m-%d %H:%M:%S" register: _ts_before_upgrade changed_when: false diff --git a/molecule/elasticstack_default/verify.yml b/molecule/elasticstack_default/verify.yml index d708ef8b..0d44267e 100644 --- a/molecule/elasticstack_default/verify.yml +++ b/molecule/elasticstack_default/verify.yml @@ -83,40 +83,34 @@ # has Filebeat+Auditbeat+Metricbeat shipping to Logstash, which outputs # to ES with the default `logstash-*` / data-stream `logs-*` naming — # either appears here. + # Use _count instead of _cat/indices because selectattr with a literal + # dot in the attribute name ('docs.count') is parsed by Jinja as a + # nested attribute access (item.docs.count) and silently never matches. + # _count returns {"count": N} which is simple to assert on. - name: Confirm Logstash output index landed in Elasticsearch ansible.builtin.uri: - url: "https://localhost:{{ elasticstack_elasticsearch_http_port }}/_cat/indices/logstash-*,logs-*,ecs-*?h=index,docs.count&format=json&expand_wildcards=open" + url: "https://localhost:{{ elasticstack_elasticsearch_http_port }}/logstash-*,logs-*,ecs-*/_count?expand_wildcards=open,hidden&ignore_unavailable=true&allow_no_indices=true" method: GET force_basic_auth: true user: elastic password: "{{ elastic_pass.stdout }}" return_content: true - status_code: [200, 404] + status_code: 200 validate_certs: false - register: _ls_es_indices - until: >- - (_ls_es_indices.json | default([]) - | selectattr('docs.count', 'defined') - | selectattr('docs.count', 'ne', '0') - | list | length) > 0 + register: _ls_es_count + until: (_ls_es_count.json.count | default(0) | int) > 0 # Generous retry budget: in CI the Beats → Logstash → ES chain needs # cluster green + Logstash plugin warmup + ES create-index latency. - # 18×10s was not enough on slow runners; 60×10s = 10 min covers the - # slowest observed startups while still failing on real breakage. retries: 60 delay: 10 delegate_to: "{{ elasticsearch_ca }}" run_once: true when: "'logstash' in group_names" # noqa: run-once[task] - - name: Assert at least one Logstash output index has documents in ES + - name: Assert at least one Logstash output document exists in ES ansible.builtin.assert: that: - - >- - (_ls_es_indices.json | default([]) - | selectattr('docs.count', 'defined') - | selectattr('docs.count', 'ne', '0') - | list | length) > 0 + - (_ls_es_count.json.count | default(0) | int) > 0 fail_msg: >- Logstash reported events.out > 0 to its internal counter, but no documents landed in ES under logstash-*, logs-*, or ecs-* indices. From d90780d6b04691c35c5d3383d5ca2f170c91be75 Mon Sep 17 00:00:00 2001 From: oddly Date: Mon, 8 Jun 2026 13:51:03 +0200 Subject: [PATCH 18/19] test: two more scenarios surfaced after the previous round MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit elasticsearch_upgrade_8to9 verify's "Assert every cluster node is on 9.x" failed with 'dict object has no attribute "version"'. Root cause: I queried /_cat/nodes?format=json but didn't ask for the version column — _cat/nodes only returns ip, heap/ram/cpu, load, node.role, master, name by default. Add ?h=name,version. beats_security still got zero hits even after expanding wildcards to include hidden indices. Root cause was further upstream: the marker line went out through `logger`, which talks to the /dev/log syslog socket. In the LXC test containers neither rsyslog nor journald-to-syslog forwarding is reliably running, so the marker never reaches /var/log/syslog or /var/log/messages and Filebeat has nothing to tail. Write the line directly into both candidate log paths (and create one if neither exists) instead of going through syslog. --- molecule/beats_security/verify.yml | 21 ++++++++++++++++--- .../elasticsearch_upgrade_8to9/verify.yml | 5 ++++- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/molecule/beats_security/verify.yml b/molecule/beats_security/verify.yml index 0c309b5f..f9c2a0f4 100644 --- a/molecule/beats_security/verify.yml +++ b/molecule/beats_security/verify.yml @@ -93,9 +93,24 @@ # client and that documents arrive — the substring checks above # would all pass against a config that silently drops events # (wrong CA path, wrong password, broken handshake). - - name: Emit a unique marker line via syslog - ansible.builtin.command: - cmd: logger -t molecule-beats-security 'MARKER {{ ansible_facts.hostname }} molecule-beats-security-test' + # Write directly to the configured Filebeat input paths. + # `logger` goes through the syslog socket (/dev/log), which depends + # on rsyslog/journald-to-syslog forwarding being active; in LXC + # containers neither is reliably running, so the marker never + # lands in /var/log/syslog or /var/log/messages and Filebeat + # has nothing to ship. Bypass syslog by appending directly. + - name: Emit a unique marker line into the configured log paths + ansible.builtin.shell: + cmd: | + line="$(date '+%b %d %H:%M:%S') {{ ansible_facts.hostname }} molecule-beats-security[$$]: MARKER molecule-beats-security-test" + for f in /var/log/syslog /var/log/messages; do + [ -e "$f" ] && echo "$line" >> "$f" + done + # If neither file existed, create syslog so Filebeat sees it. + if [ ! -e /var/log/syslog ] && [ ! -e /var/log/messages ]; then + echo "$line" >> /var/log/syslog + fi + executable: /bin/bash changed_when: false # The ingest assertion runs on the ES host because filebeat-* lives in diff --git a/molecule/elasticsearch_upgrade_8to9/verify.yml b/molecule/elasticsearch_upgrade_8to9/verify.yml index 55c64e7b..a6a2c5bf 100644 --- a/molecule/elasticsearch_upgrade_8to9/verify.yml +++ b/molecule/elasticsearch_upgrade_8to9/verify.yml @@ -74,8 +74,11 @@ success_msg: "Cluster is {{ cluster_health.json.status }} with {{ cluster_health.json.number_of_nodes }} nodes" - name: Check all nodes are visible + # Default _cat/nodes columns don't include version — must specify h= + # explicitly. Without it the per-node version assert below fails + # with 'dict object has no attribute "version"'. ansible.builtin.uri: - url: https://localhost:{{ elasticstack_elasticsearch_http_port }}/_cat/nodes?format=json + url: https://localhost:{{ elasticstack_elasticsearch_http_port }}/_cat/nodes?h=name,version&format=json method: GET force_basic_auth: true user: elastic From 0e72cd2c4e46376949ca5850ec587b22213cc8ba Mon Sep 17 00:00:00 2001 From: oddly Date: Mon, 8 Jun 2026 16:07:32 +0200 Subject: [PATCH 19/19] test(beats_security): bump ES host to 6 GB so debian13 + r9 doesn't OOM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The end-to-end Filebeat → ES ingest assertion I added landed all-green on debian13/r8, rocky10/r8 and rocky10/r9 but failed on debian13/r9. The diagnostic artifact's OOM-REPORT confirms the cause: the ES container OOMs partway through ingest with OutOfMemoryError. ES 9.x's heap accounting on debian13 simply needs more headroom than the previous 4 GB; the JVM doesn't OOM, the container's cgroup does. Bump the ES platform from 4096 MB to 6144 MB and update the memory-gate lookup table so the admission gate counts the scenario at 8 GB total. Filebeat node stays at 2 GB. --- molecule/beats_security/molecule.yml | 5 ++++- scripts/wait-for-memory.sh | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/molecule/beats_security/molecule.yml b/molecule/beats_security/molecule.yml index f60fbb55..ae0fe1cf 100644 --- a/molecule/beats_security/molecule.yml +++ b/molecule/beats_security/molecule.yml @@ -11,7 +11,10 @@ platforms: groups: - elasticsearch distro: "${MOLECULE_DISTRO:-debian12}" - memory_mb: 4096 + # ES 9.x on debian13 OOMs with 4 GB once ingest pressure starts (the + # new end-to-end Filebeat→ES check exercises that path). 6 GB clears + # it. Bumping the ES host only; Filebeat node stays at 2 GB. + memory_mb: 6144 - name: "beats-sec-fb1-${MOLECULE_DISTRO:-debian12}-r${ELASTIC_RELEASE:-9}${MOLECULE_RUN_SUFFIX}" groups: - beats diff --git a/scripts/wait-for-memory.sh b/scripts/wait-for-memory.sh index 37670f60..0d6cff90 100755 --- a/scripts/wait-for-memory.sh +++ b/scripts/wait-for-memory.sh @@ -46,7 +46,7 @@ declare -A REQ=( [beats_advanced]=2048 [beats_default]=4096 [beats_peculiar]=2048 - [beats_security]=6144 + [beats_security]=8192 [cert_renewal]=10752 [elasticsearch_cert_content]=4096 [elasticsearch_custom]=4096