diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 48ed7fea..1f5b484b 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -297,13 +297,14 @@ jobs: ######## SPIN UP PLATFORM BACKEND ############# - name: Check out and start up platform with deps/containers id: run-platform - uses: opentdf/platform/test/start-up-with-containers@11af44a5d4826ed281bf2e0e4e31d6ff6154b393 # pqc-enabled + uses: opentdf/platform/test/start-up-with-containers@8ccc608d2947f7f868d63e3741d3e78dc0ee88ce # DSPX-3397-platform-go-sdk with: platform-ref: ${{ fromJSON(needs.resolve-versions.outputs.platform-tag-to-sha)[matrix.platform-tag] }} ec-tdf-enabled: true extra-keys: ${{ steps.load-extra-keys.outputs.EXTRA_KEYS }} log-type: json pqc-enabled: ${{ steps.pqc-check.outputs.supported == 'true' }} + dpop-challenge-enabled: true - name: Install uv uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 @@ -591,7 +592,7 @@ jobs: - name: Start additional kas id: kas-alpha if: ${{ steps.multikas.outputs.supported == 'true' }} - uses: opentdf/platform/test/start-additional-kas@11af44a5d4826ed281bf2e0e4e31d6ff6154b393 # pqc-enabled + uses: opentdf/platform/test/start-additional-kas@8ccc608d2947f7f868d63e3741d3e78dc0ee88ce # DSPX-3397-platform-go-sdk: require_nonce on additional KAS with: ec-tdf-enabled: true kas-name: alpha @@ -599,11 +600,12 @@ jobs: log-type: json pqc-enabled: ${{ steps.pqc-check.outputs.supported == 'true' }} root-key: ${{ steps.km-check.outputs.root_key }} + dpop-challenge-enabled: true - name: Start additional kas id: kas-beta if: ${{ steps.multikas.outputs.supported == 'true' }} - uses: opentdf/platform/test/start-additional-kas@11af44a5d4826ed281bf2e0e4e31d6ff6154b393 # pqc-enabled + uses: opentdf/platform/test/start-additional-kas@8ccc608d2947f7f868d63e3741d3e78dc0ee88ce # DSPX-3397-platform-go-sdk: require_nonce on additional KAS with: ec-tdf-enabled: true kas-name: beta @@ -611,11 +613,12 @@ jobs: log-type: json pqc-enabled: ${{ steps.pqc-check.outputs.supported == 'true' }} root-key: ${{ steps.km-check.outputs.root_key }} + dpop-challenge-enabled: true - name: Start additional kas id: kas-gamma if: ${{ steps.multikas.outputs.supported == 'true' }} - uses: opentdf/platform/test/start-additional-kas@11af44a5d4826ed281bf2e0e4e31d6ff6154b393 # pqc-enabled + uses: opentdf/platform/test/start-additional-kas@8ccc608d2947f7f868d63e3741d3e78dc0ee88ce # DSPX-3397-platform-go-sdk: require_nonce on additional KAS with: ec-tdf-enabled: true kas-name: gamma @@ -623,11 +626,12 @@ jobs: log-type: json pqc-enabled: ${{ steps.pqc-check.outputs.supported == 'true' }} root-key: ${{ steps.km-check.outputs.root_key }} + dpop-challenge-enabled: true - name: Start additional kas id: kas-delta if: ${{ steps.multikas.outputs.supported == 'true' }} - uses: opentdf/platform/test/start-additional-kas@11af44a5d4826ed281bf2e0e4e31d6ff6154b393 # pqc-enabled + uses: opentdf/platform/test/start-additional-kas@8ccc608d2947f7f868d63e3741d3e78dc0ee88ce # DSPX-3397-platform-go-sdk: require_nonce on additional KAS with: ec-tdf-enabled: true kas-port: 8484 @@ -635,11 +639,12 @@ jobs: log-type: json pqc-enabled: ${{ steps.pqc-check.outputs.supported == 'true' }} root-key: ${{ steps.km-check.outputs.root_key }} + dpop-challenge-enabled: true - name: Start additional KM kas (km1) id: kas-km1 if: ${{ steps.multikas.outputs.supported == 'true' }} - uses: opentdf/platform/test/start-additional-kas@11af44a5d4826ed281bf2e0e4e31d6ff6154b393 # pqc-enabled + uses: opentdf/platform/test/start-additional-kas@8ccc608d2947f7f868d63e3741d3e78dc0ee88ce # DSPX-3397-platform-go-sdk: require_nonce on additional KAS with: ec-tdf-enabled: true key-management: ${{ steps.km-check.outputs.supported }} @@ -648,11 +653,12 @@ jobs: log-type: json pqc-enabled: ${{ steps.pqc-check.outputs.supported == 'true' }} root-key: ${{ steps.km-check.outputs.root_key }} + dpop-challenge-enabled: true - name: Start additional KM kas (km2) id: kas-km2 if: ${{ steps.multikas.outputs.supported == 'true' }} - uses: opentdf/platform/test/start-additional-kas@11af44a5d4826ed281bf2e0e4e31d6ff6154b393 # pqc-enabled + uses: opentdf/platform/test/start-additional-kas@8ccc608d2947f7f868d63e3741d3e78dc0ee88ce # DSPX-3397-platform-go-sdk: require_nonce on additional KAS with: ec-tdf-enabled: true kas-name: km2 @@ -661,6 +667,7 @@ jobs: log-type: json pqc-enabled: ${{ steps.pqc-check.outputs.supported == 'true' }} root-key: ${{ steps.km-check.outputs.root_key }} + dpop-challenge-enabled: true - name: Run attribute based configuration tests if: ${{ steps.multikas.outputs.supported == 'true' }} diff --git a/xtest/sdk/java/cli.sh b/xtest/sdk/java/cli.sh index cece9630..d8013a08 100755 --- a/xtest/sdk/java/cli.sh +++ b/xtest/sdk/java/cli.sh @@ -42,6 +42,29 @@ else exit 1 fi +# Cache `java -jar cmdline.jar help [...]` output to avoid paying JVM startup +# (typically 150-500ms) for the capability probes on every encrypt/decrypt. +# Keyed by the jar's mtime so a reinstall invalidates the cache. stderr is +# discarded to keep JVM warnings (reflective-access, agent notices) out of logs. +jar_help() { + local jar="$SCRIPT_DIR/cmdline.jar" + local mtime + mtime=$(stat -c %Y "$jar" 2>/dev/null || stat -f %m "$jar" 2>/dev/null || echo 0) + local key + key=$(printf '%s' "$*" | tr -c 'a-zA-Z0-9' '_') + local uid + uid=$(id -u 2>/dev/null || echo default) + local cache="${TMPDIR:-/tmp}/xtest-java-help-${uid}-${mtime}-${key}" + if [[ ! -f "$cache" ]]; then + # Write to a process-unique temp file, then rename: concurrent xdist + # workers see either no cache or the complete file, never a partial read. + local tmp="${cache}.$$" + java -jar "$jar" help "$@" >"$tmp" 2>/dev/null + mv -f "$tmp" "$cache" + fi + cat "$cache" +} + if [ "$1" == "supports" ]; then case "$2" in autoconfigure | ns_grants) @@ -118,8 +141,8 @@ if [ "$1" == "supports" ]; then exit $? ;; dpop_nonce_challenge) - echo "dpop_nonce_challenge not supported" - exit 1 + java -jar "$SCRIPT_DIR"/cmdline.jar supports dpop_nonce_challenge + exit $? ;; *) echo "Unknown feature: $2" @@ -135,7 +158,7 @@ args=( ) # when we added support for KAS allowlist, we changed the platform endpoint format to require scheme -if java -jar "$SCRIPT_DIR"/cmdline.jar help decrypt | grep kas-allowlist; then +if jar_help decrypt | grep -q kas-allowlist; then args+=("--platform-endpoint=$PLATFORMURL") else args+=("--platform-endpoint=$PLATFORMENDPOINT") @@ -197,5 +220,9 @@ if [ -n "$XT_WITH_TARGET_MODE" ]; then args+=(--with-target-mode "$XT_WITH_TARGET_MODE") fi +if jar_help | grep -q -- '--verbose'; then + args+=(--verbose) +fi + echo java -jar "$SCRIPT_DIR"/cmdline.jar "${args[@]}" --file="$2" ">" "$3" java -jar "$SCRIPT_DIR"/cmdline.jar "${args[@]}" --file="$2" >"$3" diff --git a/xtest/sdk/js/cli.sh b/xtest/sdk/js/cli.sh index d3160ca3..11c1dfe1 100755 --- a/xtest/sdk/js/cli.sh +++ b/xtest/sdk/js/cli.sh @@ -18,6 +18,10 @@ # XT_WITH_ATTRIBUTES [string] - Attributes to be used for encryption # XT_WITH_MIME_TYPE [string] - MIME type for the encrypted file # XT_WITH_TARGET_MODE [string] - Target spec mode for the encrypted file +# XT_WITH_DPOP [string] - Enable DPoP token binding; value selects algorithm (e.g. ES256) +# XT_WITH_DPOP_KEY [string] - Path to PEM-encoded PKCS8 private key for DPoP signing +# CLIENTID [string] - Override OIDC client ID (default: opentdf) +# CLIENTSECRET [string] - Override OIDC client secret (default: secret) # SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) @@ -124,9 +128,17 @@ if [ "$XTEST_DIR" = "/" ]; then exit 1 fi +# Capture any caller-set overrides before test.env unconditionally resets them. +_pre_clientid="${CLIENTID:-}" +_pre_clientsecret="${CLIENTSECRET:-}" + # shellcheck disable=SC1091 source "$XTEST_DIR"/test.env +# Restore caller overrides (e.g. from pytest monkeypatch for DPoP client). +[[ -n "$_pre_clientid" ]] && CLIENTID="$_pre_clientid" +[[ -n "$_pre_clientsecret" ]] && CLIENTSECRET="$_pre_clientsecret" + src_file=$(realpath "$2") dst_file=$(realpath "$(dirname "$3")")/$(basename "$3") @@ -134,9 +146,16 @@ args=( --output "$dst_file" --kasEndpoint "$KASURL" --oidcEndpoint "$KCFULLURL" - --auth opentdf:secret + --auth "${CLIENTID:-opentdf}:${CLIENTSECRET:-secret}" ) +if [ -n "$XT_WITH_DPOP" ]; then + args+=(--dpop "$XT_WITH_DPOP") +fi +if [ -n "$XT_WITH_DPOP_KEY" ]; then + args+=(--dpop-key "$XT_WITH_DPOP_KEY") +fi + args+=(--containerType tdf3) if [ -n "$XT_WITH_ATTRIBUTES" ]; then @@ -185,6 +204,24 @@ if ! cd "$SCRIPT_DIR"; then exit 1 fi +# Echo a CLI invocation with the --auth secret masked, so CI logs never capture +# client credentials. The real (unmasked) args are still used for execution. +echo_redacted() { + local out=() a mask_next=0 + for a in "$@"; do + if [[ "$mask_next" == 1 ]]; then + out+=("${a%%:*}:***") + mask_next=0 + elif [[ "$a" == "--auth" ]]; then + out+=("$a") + mask_next=1 + else + out+=("$a") + fi + done + echo "${out[@]}" +} + if [ "$1" == "encrypt" ]; then if npx $CTL help | grep autoconfigure; then args+=(--policyEndpoint "$PLATFORMURL" --autoconfigure true) @@ -205,7 +242,7 @@ if [ "$1" == "encrypt" ]; then args+=(--tdfSpecVersion "$XT_WITH_TARGET_MODE") fi - echo npx $CTL encrypt "$src_file" "${args[@]}" + echo_redacted npx $CTL encrypt "$src_file" "${args[@]}" npx $CTL encrypt "$src_file" "${args[@]}" elif [ "$1" == "decrypt" ]; then if [ "$XT_WITH_VERIFY_ASSERTIONS" == 'false' ]; then @@ -227,7 +264,7 @@ elif [ "$1" == "decrypt" ]; then args+=(--ignoreAllowList) fi - echo npx $CTL decrypt "$src_file" "${args[@]}" + echo_redacted npx $CTL decrypt "$src_file" "${args[@]}" npx $CTL decrypt "$src_file" "${args[@]}" else echo "Incorrect argument provided" diff --git a/xtest/test_dpop.py b/xtest/test_dpop.py index 085f71fc..18844774 100644 --- a/xtest/test_dpop.py +++ b/xtest/test_dpop.py @@ -305,6 +305,37 @@ def _post_rewrap( ) +def _post_rewrap_with_nonce_retry( + call: RewrapCall, + key: DPoPKey, + *, + access_token: str, + auth_scheme: str = "DPoP", +) -> requests.Response: + """POST a rewrap, transparently satisfying a `require_nonce` challenge. + + Mints a fresh DPoP proof (no nonce) and sends it. If the KAS replies with a + `401` carrying a `DPoP-Nonce` header (the `use_dpop_nonce` challenge), the + proof is re-minted with that nonce and the request is retried once, keeping + the same `auth_scheme`. This makes callers agnostic to whether + `require_nonce` is enabled on the target KAS: with it off the first request + is returned as-is; with it on the second (nonce-bearing) request is. + """ + proof = key.sign_dpop_proof(htm="POST", htu=call.url, access_token=access_token) + response = _post_rewrap( + call, access_token=access_token, dpop_proof=proof, auth_scheme=auth_scheme + ) + nonce = response.headers.get("DPoP-Nonce") + if response.status_code == 401 and nonce: + proof = key.sign_dpop_proof( + htm="POST", htu=call.url, access_token=access_token, nonce=nonce + ) + response = _post_rewrap( + call, access_token=access_token, dpop_proof=proof, auth_scheme=auth_scheme + ) + return response + + def _assert_unauthorized(response: requests.Response) -> None: assert response.status_code == 401, response.text # Confirm the rejection is actually a DPoP-related challenge so a 401 @@ -323,9 +354,11 @@ def _skip_unless_dpop_enabled(encrypt_sdk: tdfs.SDK, in_focus: set[tdfs.SDK]) -> @pytest.fixture(autouse=True) def _dpop_client_env(monkeypatch: pytest.MonkeyPatch) -> None: - # SDK CLI shims read CLIENTID from the environment; tests in this module - # must use the DPoP-bound client provisioned by `service provision keycloak`. + # SDK CLI shims read CLIENTID/XT_WITH_DPOP from the environment; tests in + # this module must use the DPoP-bound client provisioned by + # `service provision keycloak` and enable DPoP proof generation. monkeypatch.setenv("CLIENTID", "opentdf-dpop") + monkeypatch.setenv("XT_WITH_DPOP", "ES256") def test_dpop_happy_path_roundtrip( @@ -426,16 +459,14 @@ def test_dpop_bearer_scheme_warns_but_accepted_for_dpop_token( dpop_access = _get_dpop_access_token() rewrap_call = _signed_rewrap_request(ct_file, dpop_access.key) - bearer_proof = dpop_access.key.sign_dpop_proof( - htm="POST", - htu=rewrap_call.url, - access_token=dpop_access.token, - ) + # Both calls go through the nonce-retry helper so the test passes whether or + # not the target KAS has `require_nonce` enabled: when it is, the lenient + # accept (and the WARN) only happen after the nonce challenge is satisfied. mark = audit_logs.mark("before_bearer_scheme_request") - bearer_response = _post_rewrap( + bearer_response = _post_rewrap_with_nonce_retry( rewrap_call, + dpop_access.key, access_token=dpop_access.token, - dpop_proof=bearer_proof, auth_scheme="Bearer", ) @@ -447,16 +478,10 @@ def test_dpop_bearer_scheme_warns_but_accepted_for_dpop_token( ) # Compliant path control: same proof key, same token, just the right scheme. - # Distinct jti via fresh proof so the server's replay cache doesn't reject it. - dpop_proof = dpop_access.key.sign_dpop_proof( - htm="POST", - htu=rewrap_call.url, - access_token=dpop_access.token, - ) - dpop_response = _post_rewrap( + dpop_response = _post_rewrap_with_nonce_retry( rewrap_call, + dpop_access.key, access_token=dpop_access.token, - dpop_proof=dpop_proof, auth_scheme="DPoP", ) assert dpop_response.status_code == 200, dpop_response.text