From 96244baa91cfcbdb897f3bf0e5698a6eb85c8a9c Mon Sep 17 00:00:00 2001
From: Richard Zowalla <rzo1@apache.org>
Date: Mon, 4 May 2026 20:46:19 +0200
Subject: [PATCH 1/3] feat(playwright): add JsRenderingDetector parse filter

Heuristically flags URLs whose content looks JavaScript-rendered by
inspecting SPA framework fingerprints, noscript blocks, empty hydration
roots, and a thin-content fallback. Sets a routing metadata key so
DelegatorProtocol can dispatch subsequent fetches to Playwright while
the bulk of the crawl stays on a cheap HTTP client.
---
 docs/src/main/asciidoc/configuration.adoc     |  54 ++++
 external/playwright/README.md                 |  74 ++++++
 .../parsefilter/JsRenderingDetector.java      | 243 ++++++++++++++++++
 .../parsefilter/JsRenderingDetectorTest.java  | 148 +++++++++++
 4 files changed, 519 insertions(+)
 create mode 100644 external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetector.java
 create mode 100644 external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetectorTest.java

diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc
index 7526546c3..b0e5147d0 100644
--- a/docs/src/main/asciidoc/configuration.adoc
+++ b/docs/src/main/asciidoc/configuration.adoc
@@ -494,6 +494,60 @@ See the link:https://github.com/apache/stormcrawler/tree/main/external/playwrigh
 | playwright.load.event | - | Page load event to wait for (e.g., "domcontentloaded", "networkidle").
 |===
 
+===== JS rendering detection
+
+Browser fetching is much more expensive than a plain HTTP fetch, so most operators only want
+Playwright on URLs that actually need it. The `JsRenderingDetector` parse filter inspects the
+parsed page from a cheap fetch and sets a metadata flag (default `fetch.with=playwright`) on URLs
+that look JavaScript-rendered. Pair it with link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/DelegatorProtocol.java[DelegatorProtocol]
+to route subsequent fetches of those URLs to the Playwright protocol while leaving everything else
+on a fast HTTP client.
+
+Detection signals (cheapest first, short-circuiting):
+
+* SPA framework fingerprints in raw HTML — `data-reactroot`, `ng-version=`, `__NEXT_DATA__`,
+  `window.__NUXT__`, `data-svelte-h=`, `data-vue-app`, `data-astro-cid`, `<router-outlet`.
+* `<noscript>` blocks containing language like _"enable JavaScript"_.
+* Empty SPA hydration roots: `<div id="root"></div>` / `#app` / `#__next` / `#__nuxt`.
+* Outcome-based fallback: at least one `<script>` is present and both `text.length` and the
+  outlink count are below configurable thresholds.
+
+Detection is skipped when `playwright.protocol.end` is already on the URL (i.e. it was just
+fetched by Playwright) or when the routing key is already set, so the filter is idempotent.
+
+Register the filter in `parsefilters.json`:
+
+[source,json]
+----
+{
+  "class": "org.apache.stormcrawler.protocol.playwright.parsefilter.JsRenderingDetector",
+  "name": "js-rendering-detector",
+  "params": { "minTextLength": 200, "minOutlinks": 2 }
+}
+----
+
+And route on the metadata key it sets:
+
+[source,yaml]
+----
+http.protocol.implementation:  "org.apache.stormcrawler.protocol.DelegatorProtocol"
+https.protocol.implementation: "org.apache.stormcrawler.protocol.DelegatorProtocol"
+protocol.delegator.config:
+  - className: "org.apache.stormcrawler.protocol.playwright.HttpProtocol"
+    filters:
+      "fetch.with": "playwright"
+  - className: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
+----
+
+The dotted metadata key is quoted in the YAML above for readability; SnakeYAML accepts the
+unquoted form too. Note that `DelegatorProtocol` requires the *last* entry in
+`protocol.delegator.config` to have no `filters:` — it acts as the fallback, so keep the cheap
+protocol at the bottom of the list.
+
+The first fetch on an unknown URL goes through the cheap protocol; once the detector flags it the
+metadata sticks via the status backend and the next fetch is dispatched to Playwright. The full
+parameter list and tuning notes are in the link:https://github.com/apache/stormcrawler/tree/main/external/playwright[playwright module README].
+
 ==== Language ID
 
 Language identification for crawled documents using the lang-detect library.
diff --git a/external/playwright/README.md b/external/playwright/README.md
index 725bea647..912b0529d 100644
--- a/external/playwright/README.md
+++ b/external/playwright/README.md
@@ -47,3 +47,77 @@ Per-URL metadata triggers:
 |---|---|
 | `playwright.trace` | If present on the input metadata, a Playwright trace zip is recorded for the navigation and its path is returned in the response metadata under the same key. |
 
+## JS rendering detection
+
+Browser-based fetching is expensive — typically 10–50× slower than a plain HTTP fetch and limited by how many browsers a host can run concurrently. Most operators only want Playwright on the URLs that actually need it. The `JsRenderingDetector` parse filter solves the routing question without adding new infrastructure: it inspects the parsed page from a cheap fetch and, when the content looks JS-rendered, sets a metadata flag that `DelegatorProtocol` (already part of `core`) routes on.
+
+### How detection works
+
+The filter applies four heuristics, cheapest-first, and short-circuits on the first hit:
+
+1. **SPA framework fingerprints** in raw HTML — `data-reactroot`, `ng-version=`, `__NEXT_DATA__`, `window.__NUXT__`, `data-svelte-h=`, `data-vue-app`, `data-astro-cid`, `<router-outlet`. Defaults are overridable via the `fingerprints` parameter.
+2. **`<noscript>` blocks** that explicitly request JavaScript — match patterns like _"enable JavaScript"_, _"requires JavaScript"_, _"JavaScript is disabled"_.
+3. **Empty SPA hydration roots** — `<div id="root"></div>` / `#app` / `#__next` / `#__nuxt` with no children. IDs override­able via `emptyRootIds`.
+4. **Outcome-based fallback** — when at least one `<script>` is present and both `text.length < minTextLength` (default 200) and `outlinks.size() < minOutlinks` (default 2), the URL is flagged as a thin SPA. The `<script>` gate keeps the filter from flagging static error stubs.
+
+### What the filter sets
+
+| Metadata key | Value | Notes |
+|---|---|---|
+| `fetch.with` | `playwright` | Routing key, override­able via `metadataKey` / `metadataValue`. |
+| `fetch.with.reason` | e.g. `fingerprint:data-reactroot`, `noscript-js-required`, `empty-root:root`, `thin-content:text=12,outlinks=0` | Diagnostic — set unless `recordReason: false`. |
+
+### Loop guards
+
+- Detection is skipped when `playwright.protocol.end` is already present on the URL — i.e. the URL was just fetched by Playwright; reapplying the heuristic would just reflag it. Override the watch key via `skipIfMetadataPresent`.
+- Detection is also skipped when the routing key is already set, so the filter is idempotent and safe to leave permanently in `parsefilters.json`.
+
+### Parameters
+
+| Name | Type | Default | Notes |
+|---|---|---|---|
+| `metadataKey` | string | `fetch.with` | Routing key set on a hit. |
+| `metadataValue` | string | `playwright` | Value to set. |
+| `minTextLength` | int | `200` | Outcome-based threshold for visible text. |
+| `minOutlinks` | int | `2` | Outcome-based threshold for extracted outlinks. |
+| `fingerprints` | string array | _see above_ | Substrings searched in raw HTML; replaces defaults when set. |
+| `emptyRootIds` | string array | `["root","app","__next","__nuxt"]` | Element IDs treated as empty SPA hydration roots. |
+| `skipIfMetadataPresent` | string | `playwright.protocol.end` | Short-circuit when this metadata key is set. Empty string disables. |
+| `recordReason` | bool | `true` | Also set `metadataKey + ".reason"` describing which signal fired. |
+
+### Wiring
+
+Add the filter to your `parsefilters.json`:
+
+```json
+{
+  "class": "org.apache.stormcrawler.protocol.playwright.parsefilter.JsRenderingDetector",
+  "name": "js-rendering-detector",
+  "params": { "minTextLength": 200, "minOutlinks": 2 }
+}
+```
+
+Route on the metadata key it sets via `DelegatorProtocol`:
+
+```yaml
+http.protocol.implementation:  "org.apache.stormcrawler.protocol.DelegatorProtocol"
+https.protocol.implementation: "org.apache.stormcrawler.protocol.DelegatorProtocol"
+protocol.delegator.config:
+  - className: "org.apache.stormcrawler.protocol.playwright.HttpProtocol"
+    filters:
+      "fetch.with": "playwright"
+  - className: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
+```
+
+A few wiring notes:
+
+- The dotted metadata key (`fetch.with`) is quoted in the YAML above to make it unambiguous to a human reader; SnakeYAML treats unquoted `fetch.with: "playwright"` as the same single-key scalar, so either parses correctly.
+- `DelegatorProtocol` requires the **last** entry in `protocol.delegator.config` to have no `filters:` — it acts as the fallback. Keep OkHttp (or whichever cheap protocol you pick) at the bottom of the list.
+- The first fetch on an unknown URL goes through the cheap protocol; if the detector flags it, the metadata sticks via the status backend and the next fetch is dispatched to Playwright. Sibling URLs on the same host don't inherit the flag — that requires a host-keyed metadata transfer scheme and is intentionally out of scope.
+
+### When _not_ to use it
+
+- **Operator allowlist suffices.** If you already know which hosts need a browser, add them as a `urlPatterns` rule on the Playwright leg of `DelegatorProtocol` and skip the filter.
+- **Anti-bot / WAF challenge pages.** Cloudflare, DataDome, and Akamai challenge fingerprints aren't covered here; those usually need a stealth-mode browser, not just rendering.
+- **Aggressively first-fetch-sensitive crawls.** The first fetch on an unknown SPA host is always wasted (you get a stub document) before the filter learns about the host. If that's unacceptable, prefer the operator allowlist.
+
diff --git a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetector.java b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetector.java
new file mode 100644
index 000000000..9243a678d
--- /dev/null
+++ b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetector.java
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stormcrawler.protocol.playwright.parsefilter;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.parse.ParseData;
+import org.apache.stormcrawler.parse.ParseFilter;
+import org.apache.stormcrawler.parse.ParseResult;
+import org.apache.stormcrawler.protocol.playwright.HttpProtocol;
+import org.jetbrains.annotations.NotNull;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.DocumentFragment;
+
+/**
+ * Heuristic parse filter that flags URLs whose content looks like it requires JavaScript rendering,
+ * intended to be paired with {@link org.apache.stormcrawler.protocol.DelegatorProtocol} so that
+ * subsequent fetches of those URLs are routed to the Playwright protocol.
+ *
+ * <p>The filter sets a metadata key (default {@code fetch.with=playwright}) on URLs that match any
+ * of the following signals, in cheapest-first order:
+ *
+ * <ol>
+ *   <li>SPA framework fingerprints found in the raw HTML (Angular {@code ng-version}, React
+ *       {@code data-reactroot}, Next.js {@code __NEXT_DATA__}, Nuxt {@code window.__NUXT__},
+ *       Svelte {@code data-svelte-h}, Astro islands, Angular {@code <router-outlet>}, ...).
+ *   <li>{@code <noscript>} blocks containing language like "enable JavaScript".
+ *   <li>An empty SPA hydration root: {@code <div id="root"></div>} / {@code #app} / {@code #__next}
+ *       / {@code #__nuxt} with no children.
+ *   <li>Outcome-based fallback: very short text and few outlinks despite at least one
+ *       {@code <script>} tag being present.
+ * </ol>
+ *
+ * <p>To avoid loops, detection is skipped when the URL was just fetched by Playwright (recognised
+ * by the presence of {@link HttpProtocol#MD_KEY_END} in its metadata) or when the routing key has
+ * already been set on the URL.
+ *
+ * <h3>Wiring</h3>
+ *
+ * Register the filter in {@code parsefilters.json}:
+ *
+ * <pre>{@code
+ * {
+ *   "class": "org.apache.stormcrawler.protocol.playwright.parsefilter.JsRenderingDetector",
+ *   "name": "js-rendering-detector",
+ *   "params": { "minTextLength": 200, "minOutlinks": 2 }
+ * }
+ * }</pre>
+ *
+ * Route on the metadata key it sets via {@code DelegatorProtocol}:
+ *
+ * <pre>{@code
+ * http.protocol.implementation: "org.apache.stormcrawler.protocol.DelegatorProtocol"
+ * https.protocol.implementation: "org.apache.stormcrawler.protocol.DelegatorProtocol"
+ * protocol.delegator.config:
+ *   - className: "org.apache.stormcrawler.protocol.playwright.HttpProtocol"
+ *     filters:
+ *       fetch.with: "playwright"
+ *   - className: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
+ * }</pre>
+ *
+ * <p>Decisions stick on the URL via metadata persistence in the status backend, so the second time
+ * the URL is fetched it is dispatched to Playwright. For host-wide propagation (so siblings of the
+ * detected URL also get Playwright on first fetch) pair this with a host-keyed metadata transfer
+ * scheme — that's intentionally out of scope here.
+ *
+ * <h3>Parameters</h3>
+ *
+ * <ul>
+ *   <li>{@code metadataKey} (string, default {@code fetch.with}) — routing key set on hits
+ *   <li>{@code metadataValue} (string, default {@code playwright}) — value to set
+ *   <li>{@code minTextLength} (int, default 200) — outcome-based threshold for visible text
+ *   <li>{@code minOutlinks} (int, default 2) — outcome-based threshold for extracted outlinks
+ *   <li>{@code fingerprints} (string array) — substrings searched in raw HTML; replaces defaults
+ *   <li>{@code emptyRootIds} (string array) — element IDs treated as empty SPA hydration roots
+ *   <li>{@code skipIfMetadataPresent} (string, default {@link HttpProtocol#MD_KEY_END}) — short-circuit
+ *       when this metadata key is set; defeat with empty string to always re-evaluate
+ *   <li>{@code recordReason} (bool, default true) — also set {@code metadataKey + ".reason"}
+ *       describing which signal fired (useful for triage)
+ * </ul>
+ */
+public class JsRenderingDetector extends ParseFilter {
+
+    private static final org.slf4j.Logger LOG =
+            LoggerFactory.getLogger(JsRenderingDetector.class);
+
+    private static final List<String> DEFAULT_FINGERPRINTS =
+            List.of(
+                    "ng-version=",
+                    "data-reactroot",
+                    "__NEXT_DATA__",
+                    "window.__NUXT__",
+                    "data-svelte-h=",
+                    "data-vue-app",
+                    "data-astro-cid",
+                    "<router-outlet");
+
+    private static final List<String> DEFAULT_EMPTY_ROOT_IDS =
+            List.of("root", "app", "__next", "__nuxt");
+
+    private static final Pattern NOSCRIPT_JS_REQUIRED =
+            Pattern.compile(
+                    "<noscript[^>]*>[\\s\\S]*?(enable\\s+JavaScript|requires?\\s+JavaScript|JavaScript\\s+is\\s+disabled)[\\s\\S]*?</noscript>",
+                    Pattern.CASE_INSENSITIVE);
+
+    private static final Pattern HAS_SCRIPT_TAG =
+            Pattern.compile("<script\\b", Pattern.CASE_INSENSITIVE);
+
+    private String metadataKey = "fetch.with";
+    private String metadataValue = "playwright";
+    private int minTextLength = 200;
+    private int minOutlinks = 2;
+    private List<String> fingerprints = DEFAULT_FINGERPRINTS;
+    private List<String> emptyRootIds = DEFAULT_EMPTY_ROOT_IDS;
+    private String skipIfMetadataPresent = HttpProtocol.MD_KEY_END;
+    private boolean recordReason = true;
+
+    private final List<Pattern> emptyRootPatterns = new ArrayList<>();
+
+    @Override
+    public void configure(
+            @NotNull final Map<String, Object> stormConf, @NotNull final JsonNode params) {
+        if (params != null && !params.isMissingNode() && !params.isNull()) {
+            if (params.has("metadataKey")) this.metadataKey = params.get("metadataKey").asText();
+            if (params.has("metadataValue"))
+                this.metadataValue = params.get("metadataValue").asText();
+            if (params.has("minTextLength"))
+                this.minTextLength = params.get("minTextLength").asInt(this.minTextLength);
+            if (params.has("minOutlinks"))
+                this.minOutlinks = params.get("minOutlinks").asInt(this.minOutlinks);
+            if (params.has("fingerprints"))
+                this.fingerprints = readStringArray(params.get("fingerprints"));
+            if (params.has("emptyRootIds"))
+                this.emptyRootIds = readStringArray(params.get("emptyRootIds"));
+            if (params.has("skipIfMetadataPresent"))
+                this.skipIfMetadataPresent = params.get("skipIfMetadataPresent").asText();
+            if (params.has("recordReason"))
+                this.recordReason = params.get("recordReason").asBoolean(this.recordReason);
+        }
+        emptyRootPatterns.clear();
+        for (final String id : emptyRootIds) {
+            // <div|main|section ... id="ID" ...>\s*</tag> — empty hydration root
+            emptyRootPatterns.add(
+                    Pattern.compile(
+                            "<(?:div|main|section)\\b[^>]*\\bid\\s*=\\s*[\"']?"
+                                    + Pattern.quote(id)
+                                    + "[\"']?[^>]*>\\s*</(?:div|main|section)>",
+                            Pattern.CASE_INSENSITIVE));
+        }
+    }
+
+    @Override
+    public boolean needsDOM() {
+        return false;
+    }
+
+    @Override
+    public void filter(
+            final String url,
+            final byte[] content,
+            final DocumentFragment doc,
+            final ParseResult parse) {
+        if (content == null || content.length == 0) return;
+
+        final Metadata md = parse.get(url).getMetadata();
+        if (skipIfMetadataPresent != null
+                && !skipIfMetadataPresent.isEmpty()
+                && md.containsKey(skipIfMetadataPresent)) {
+            return;
+        }
+        if (md.containsKey(metadataKey)) return;
+
+        final String reason = detectReason(url, content, parse);
+        if (reason != null) {
+            md.setValue(metadataKey, metadataValue);
+            if (recordReason) {
+                md.setValue(metadataKey + ".reason", reason);
+            }
+            LOG.debug("Flagged {} for JS rendering ({})", url, reason);
+        }
+    }
+
+    private String detectReason(
+            final String url, final byte[] content, final ParseResult parse) {
+        final String html = new String(content, StandardCharsets.UTF_8);
+
+        // 1. SPA framework fingerprints
+        for (final String fp : fingerprints) {
+            if (html.contains(fp)) return "fingerprint:" + fp;
+        }
+
+        // 2. <noscript> with explicit JS-required language
+        if (NOSCRIPT_JS_REQUIRED.matcher(html).find()) {
+            return "noscript-js-required";
+        }
+
+        // 3. Empty SPA hydration root
+        for (int i = 0; i < emptyRootPatterns.size(); i++) {
+            if (emptyRootPatterns.get(i).matcher(html).find()) {
+                return "empty-root:" + emptyRootIds.get(i);
+            }
+        }
+
+        // 4. Outcome-based: gated on at least one <script> being present so we don't flag
+        //    plain HTML stubs (e.g. very short error pages).
+        if (!HAS_SCRIPT_TAG.matcher(html).find()) return null;
+
+        final ParseData data = parse.get(url);
+        final String text = data.getText();
+        final int textLen = text == null ? 0 : text.trim().length();
+        final int outlinks = parse.getOutlinks() == null ? 0 : parse.getOutlinks().size();
+        if (textLen < minTextLength && outlinks < minOutlinks) {
+            return "thin-content:text=" + textLen + ",outlinks=" + outlinks;
+        }
+        return null;
+    }
+
+    private static List<String> readStringArray(final JsonNode node) {
+        if (node == null || !node.isArray()) return List.of();
+        final List<String> list = new ArrayList<>(node.size());
+        node.forEach(n -> list.add(n.asText()));
+        return list;
+    }
+}
diff --git a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetectorTest.java b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetectorTest.java
new file mode 100644
index 000000000..67bf8894b
--- /dev/null
+++ b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetectorTest.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.stormcrawler.protocol.playwright.parsefilter;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+import org.apache.stormcrawler.parse.ParseResult;
+import org.apache.stormcrawler.protocol.playwright.HttpProtocol;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class JsRenderingDetectorTest {
+
+    private static final ObjectMapper MAPPER = new ObjectMapper();
+
+    private JsRenderingDetector detector(final String paramJson) throws Exception {
+        final JsRenderingDetector d = new JsRenderingDetector();
+        d.configure(Map.of(), MAPPER.readTree(paramJson));
+        return d;
+    }
+
+    private ParseResult applyTo(final JsRenderingDetector d, final String url, final String html) {
+        final ParseResult parse = new ParseResult();
+        d.filter(url, html.getBytes(StandardCharsets.UTF_8), null, parse);
+        return parse;
+    }
+
+    @Test
+    void detectsReactByDataReactRoot() throws Exception {
+        final JsRenderingDetector d = detector("{}");
+        final ParseResult p = applyTo(d, "u", "<html><body><div data-reactroot></div></body></html>");
+        Assertions.assertEquals("playwright", p.get("u").getMetadata().getFirstValue("fetch.with"));
+        Assertions.assertTrue(
+                p.get("u").getMetadata().getFirstValue("fetch.with.reason").startsWith("fingerprint:"));
+    }
+
+    @Test
+    void detectsAngularByNgVersion() throws Exception {
+        final JsRenderingDetector d = detector("{}");
+        final ParseResult p =
+                applyTo(d, "u", "<html><body><app-root ng-version=\"17.0.0\"></app-root></body></html>");
+        Assertions.assertEquals("playwright", p.get("u").getMetadata().getFirstValue("fetch.with"));
+    }
+
+    @Test
+    void detectsNoscriptJsRequired() throws Exception {
+        final JsRenderingDetector d = detector("{}");
+        final String html =
+                "<html><body><noscript>Please enable JavaScript to view this site.</noscript></body></html>";
+        final ParseResult p = applyTo(d, "u", html);
+        Assertions.assertEquals(
+                "noscript-js-required", p.get("u").getMetadata().getFirstValue("fetch.with.reason"));
+    }
+
+    @Test
+    void detectsEmptySpaRoot() throws Exception {
+        final JsRenderingDetector d = detector("{}");
+        final ParseResult p =
+                applyTo(d, "u", "<html><body><div id=\"root\"></div><script src=\"/app.js\"></script></body></html>");
+        Assertions.assertTrue(
+                p.get("u").getMetadata().getFirstValue("fetch.with.reason").startsWith("empty-root:"));
+    }
+
+    @Test
+    void thinContentWithScriptIsFlagged() throws Exception {
+        final JsRenderingDetector d = detector("{\"minTextLength\":50,\"minOutlinks\":1}");
+        // No fingerprints, no empty-root pattern; only the outcome-based path fires
+        final String html =
+                "<html><body><p>Hi</p><script>console.log('app')</script></body></html>";
+        final ParseResult p = applyTo(d, "u", html);
+        Assertions.assertNotNull(p.get("u").getMetadata().getFirstValue("fetch.with"));
+        Assertions.assertTrue(
+                p.get("u").getMetadata().getFirstValue("fetch.with.reason").startsWith("thin-content:"));
+    }
+
+    @Test
+    void plainHtmlIsNotFlagged() throws Exception {
+        final JsRenderingDetector d = detector("{}");
+        final String html =
+                "<html><body><p>"
+                        + "A".repeat(1000)
+                        + "</p><a href='/x'>x</a><a href='/y'>y</a><a href='/z'>z</a></body></html>";
+        final ParseResult p = applyTo(d, "u", html);
+        Assertions.assertNull(p.get("u").getMetadata().getFirstValue("fetch.with"));
+    }
+
+    @Test
+    void shortPageWithoutScriptIsNotFlagged() throws Exception {
+        // outcome-based fallback is gated on at least one <script>
+        final JsRenderingDetector d = detector("{}");
+        final ParseResult p = applyTo(d, "u", "<html><body><p>404 Not Found</p></body></html>");
+        Assertions.assertNull(p.get("u").getMetadata().getFirstValue("fetch.with"));
+    }
+
+    @Test
+    void skipsIfAlreadyFetchedByPlaywright() throws Exception {
+        final JsRenderingDetector d = detector("{}");
+        final ParseResult parse = new ParseResult();
+        // simulate metadata coming from a Playwright fetch
+        parse.get("u").getMetadata().setValue(HttpProtocol.MD_KEY_END, "2026-05-04T00:00:00Z");
+        d.filter(
+                "u",
+                "<div data-reactroot></div>".getBytes(StandardCharsets.UTF_8),
+                null,
+                parse);
+        Assertions.assertNull(parse.get("u").getMetadata().getFirstValue("fetch.with"));
+    }
+
+    @Test
+    void skipsIfAlreadyFlagged() throws Exception {
+        final JsRenderingDetector d = detector("{}");
+        final ParseResult parse = new ParseResult();
+        parse.get("u").getMetadata().setValue("fetch.with", "playwright");
+        d.filter(
+                "u",
+                "<html><body><p>some content</p></body></html>".getBytes(StandardCharsets.UTF_8),
+                null,
+                parse);
+        // not overwritten, no reason added
+        Assertions.assertEquals("playwright", parse.get("u").getMetadata().getFirstValue("fetch.with"));
+        Assertions.assertNull(parse.get("u").getMetadata().getFirstValue("fetch.with.reason"));
+    }
+
+    @Test
+    void customMetadataKeyAndValue() throws Exception {
+        final JsRenderingDetector d =
+                detector("{\"metadataKey\":\"render\",\"metadataValue\":\"yes\"}");
+        final ParseResult p =
+                applyTo(d, "u", "<html><body><div data-reactroot></div></body></html>");
+        Assertions.assertEquals("yes", p.get("u").getMetadata().getFirstValue("render"));
+        Assertions.assertNotNull(p.get("u").getMetadata().getFirstValue("render.reason"));
+    }
+}

From e80b3821fd3190edd08392358c94801399b7b1f2 Mon Sep 17 00:00:00 2001
From: Richard Zowalla <rzo1@apache.org>
Date: Mon, 4 May 2026 20:57:34 +0200
Subject: [PATCH 2/3] feat(playwright): add JsRenderingRedirectionBolt and
 free-form match list

Pairs JsRenderingDetector with a bolt that reads the routing flag and
emits to StatusStreamName for an immediate refetch through Playwright,
suppressing the cheap fetch's stub from the index. Adds a requiredMessages
parameter to the detector for free-form JS-required / loader / cookie
prompts that don't fit the noscript pattern.
---
 docs/src/main/asciidoc/configuration.adoc     |   8 +-
 external/playwright/README.md                 |  37 ++++-
 .../bolt/JsRenderingRedirectionBolt.java      | 134 +++++++++++++++
 .../parsefilter/JsRenderingDetector.java      |  61 +++++--
 .../bolt/JsRenderingRedirectionBoltTest.java  | 152 ++++++++++++++++++
 .../parsefilter/JsRenderingDetectorTest.java  |  33 ++++
 6 files changed, 409 insertions(+), 16 deletions(-)
 create mode 100644 external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/bolt/JsRenderingRedirectionBolt.java
 create mode 100644 external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/bolt/JsRenderingRedirectionBoltTest.java

diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc
index b0e5147d0..de2d39c97 100644
--- a/docs/src/main/asciidoc/configuration.adoc
+++ b/docs/src/main/asciidoc/configuration.adoc
@@ -544,8 +544,12 @@ unquoted form too. Note that `DelegatorProtocol` requires the *last* entry in
 `protocol.delegator.config` to have no `filters:` — it acts as the fallback, so keep the cheap
 protocol at the bottom of the list.
 
-The first fetch on an unknown URL goes through the cheap protocol; once the detector flags it the
-metadata sticks via the status backend and the next fetch is dispatched to Playwright. The full
+The parse filter alone does **not** trigger an immediate refetch — it only sets the metadata flag
+on the current fetch, and `DefaultScheduler` reschedules the URL according to the FETCHED interval
+(`fetchInterval.default`, 24h by default). For faster turnaround, either add a per-metadata-key
+fetch interval (`fetchInterval.fetch.with=playwright: 5`) or drop `JsRenderingRedirectionBolt`
+between the parser and indexer. The bolt reads the routing flag and, on hit, emits only to the
+status stream with `Status.FETCHED` so the stub document never reaches the index. The full
 parameter list and tuning notes are in the link:https://github.com/apache/stormcrawler/tree/main/external/playwright[playwright module README].
 
 ==== Language ID
diff --git a/external/playwright/README.md b/external/playwright/README.md
index 912b0529d..1b1867fa3 100644
--- a/external/playwright/README.md
+++ b/external/playwright/README.md
@@ -82,6 +82,7 @@ The filter applies four heuristics, cheapest-first, and short-circuits on the fi
 | `minOutlinks` | int | `2` | Outcome-based threshold for extracted outlinks. |
 | `fingerprints` | string array | _see above_ | Substrings searched in raw HTML; replaces defaults when set. |
 | `emptyRootIds` | string array | `["root","app","__next","__nuxt"]` | Element IDs treated as empty SPA hydration roots. |
+| `requiredMessages` | string array | _empty_ | Additional substrings that, when found anywhere in the HTML, flag the URL. Use for site-specific JS-required prompts and loader text that don't fit the noscript pattern (e.g. `"Loading..."`, `"[object Object]"`, `"Please enable cookies"`). |
 | `skipIfMetadataPresent` | string | `playwright.protocol.end` | Short-circuit when this metadata key is set. Empty string disables. |
 | `recordReason` | bool | `true` | Also set `metadataKey + ".reason"` describing which signal fired. |
 
@@ -113,7 +114,41 @@ A few wiring notes:
 
 - The dotted metadata key (`fetch.with`) is quoted in the YAML above to make it unambiguous to a human reader; SnakeYAML treats unquoted `fetch.with: "playwright"` as the same single-key scalar, so either parses correctly.
 - `DelegatorProtocol` requires the **last** entry in `protocol.delegator.config` to have no `filters:` — it acts as the fallback. Keep OkHttp (or whichever cheap protocol you pick) at the bottom of the list.
-- The first fetch on an unknown URL goes through the cheap protocol; if the detector flags it, the metadata sticks via the status backend and the next fetch is dispatched to Playwright. Sibling URLs on the same host don't inherit the flag — that requires a host-keyed metadata transfer scheme and is intentionally out of scope.
+- The filter alone does **not** trigger an immediate refetch. It only sets the metadata; the URL is rescheduled by `DefaultScheduler` according to the FETCHED interval (`fetchInterval.default`, 24h by default), and `DelegatorProtocol` picks Playwright on the next scheduled fetch. To get faster turnaround, either drop in the `JsRenderingRedirectionBolt` described below, or add a per-metadata-key fetch interval to your YAML: `fetchInterval.fetch.with=playwright: 5` (refetch flagged URLs in 5 minutes instead of 24 hours).
+- Sibling URLs on the same host don't inherit the flag — that requires a host-keyed metadata transfer scheme and is intentionally out of scope.
+
+### Forcing an immediate refetch — `JsRenderingRedirectionBolt`
+
+The detector flags URLs but doesn't, on its own, prevent the cheap fetch's stub document from flowing downstream into the parser, indexer, and outlink emission. For most crawls that's fine — the next scheduled fetch replaces the stub with the rendered version. If you want the stub to be discarded and the URL refetched immediately through Playwright, drop `JsRenderingRedirectionBolt` between the parser and the indexer. The bolt:
+
+- reads the routing flag set by the detector (or any other upstream component),
+- on hit, emits **only** to `StatusStreamName` with `Status.FETCHED` so the URL is rescheduled and the stub never reaches the index,
+- on miss, passes the tuple through unchanged,
+- short-circuits when `playwright.protocol.end` is already on the URL — the loop guard.
+
+The bolt has no detection logic of its own; it just acts on the metadata flag. That keeps the heuristics in one place (the parse filter) and lets you swap or extend the bolt independently.
+
+Topology fragment:
+
+```text
+... -> JSoupParserBolt -> JsRenderingRedirectionBolt -> IndexerBolt -> ...
+                                                    \-> StatusStream
+```
+
+YAML:
+
+```yaml
+# refetch flagged URLs in 5 minutes rather than 24 hours
+fetchInterval.fetch.with=playwright: 5
+```
+
+Configuration keys:
+
+| Key | Default | Notes |
+|---|---|---|
+| `playwright.redirect.metadata.key` | `fetch.with` | Routing key the bolt watches for. |
+| `playwright.redirect.metadata.value` | `playwright` | Value the bolt watches for. |
+| `playwright.redirect.skip.if.metadata.present` | `playwright.protocol.end` | Loop guard — pass through unchanged when this key is set on the URL. Empty string disables. |
 
 ### When _not_ to use it
 
diff --git a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/bolt/JsRenderingRedirectionBolt.java b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/bolt/JsRenderingRedirectionBolt.java
new file mode 100644
index 000000000..3dbc047e3
--- /dev/null
+++ b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/bolt/JsRenderingRedirectionBolt.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.stormcrawler.protocol.playwright.bolt;
+
+import java.util.Map;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.topology.OutputFieldsDeclarer;
+import org.apache.storm.topology.base.BaseRichBolt;
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.tuple.Tuple;
+import org.apache.storm.tuple.Values;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.protocol.playwright.HttpProtocol;
+import org.apache.stormcrawler.util.ConfUtils;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Bolt that consumes the routing flag set by {@link
+ * org.apache.stormcrawler.protocol.playwright.parsefilter.JsRenderingDetector} (or any other
+ * upstream component) and forces an immediate refetch through Playwright instead of letting the
+ * cheap fetch's stub document propagate downstream.
+ *
+ * <p>Pipeline placement: between the parser bolt (which produces tuples of
+ * {@code (url, content, metadata, text)}) and the indexer / persistence bolts. On hit, the bolt
+ * emits only to the {@link Constants#StatusStreamName} with status {@link Status#FETCHED}, so the
+ * URL is rescheduled and the stub never reaches the index. On miss, the tuple passes through to
+ * the default stream unchanged.
+ *
+ * <p>Pair this with a per-metadata-key fetch interval to control how soon the refetch happens —
+ * by default {@code Status.FETCHED} reschedules at {@code fetchInterval.default} (24h):
+ *
+ * <pre>{@code
+ * # refetch flagged URLs in 5 minutes rather than 24 hours
+ * fetchInterval.fetch.with=playwright: 5
+ * }</pre>
+ *
+ * <h3>Configuration</h3>
+ *
+ * <ul>
+ *   <li>{@code playwright.redirect.metadata.key} (default {@code fetch.with})
+ *   <li>{@code playwright.redirect.metadata.value} (default {@code playwright})
+ *   <li>{@code playwright.redirect.skip.if.metadata.present} (default
+ *       {@link HttpProtocol#MD_KEY_END}) — passes the tuple through unchanged when this metadata key
+ *       is already set, preventing loops with content that came back from Playwright. Set to empty
+ *       to disable the loop guard.
+ * </ul>
+ */
+public class JsRenderingRedirectionBolt extends BaseRichBolt {
+
+    private static final org.slf4j.Logger LOG =
+            LoggerFactory.getLogger(JsRenderingRedirectionBolt.class);
+
+    public static final String CONF_METADATA_KEY = "playwright.redirect.metadata.key";
+    public static final String CONF_METADATA_VALUE = "playwright.redirect.metadata.value";
+    public static final String CONF_SKIP_IF_METADATA_PRESENT =
+            "playwright.redirect.skip.if.metadata.present";
+
+    public static final String DEFAULT_METADATA_KEY = "fetch.with";
+    public static final String DEFAULT_METADATA_VALUE = "playwright";
+
+    private OutputCollector collector;
+    private String routingKey;
+    private String routingValue;
+    private String skipIfMetadataPresent;
+
+    @Override
+    public void prepare(
+            final Map<String, Object> conf,
+            final TopologyContext context,
+            final OutputCollector collector) {
+        this.collector = collector;
+        this.routingKey = ConfUtils.getString(conf, CONF_METADATA_KEY, DEFAULT_METADATA_KEY);
+        this.routingValue = ConfUtils.getString(conf, CONF_METADATA_VALUE, DEFAULT_METADATA_VALUE);
+        this.skipIfMetadataPresent =
+                ConfUtils.getString(conf, CONF_SKIP_IF_METADATA_PRESENT, HttpProtocol.MD_KEY_END);
+    }
+
+    @Override
+    public void execute(final Tuple tuple) {
+        final String url = tuple.getStringByField("url");
+        final byte[] content = tuple.getBinaryByField("content");
+        final Metadata metadata = (Metadata) tuple.getValueByField("metadata");
+        final String text = tuple.getStringByField("text");
+
+        if (shouldRedirect(metadata)) {
+            LOG.debug("Redirecting {} to Playwright (status stream)", url);
+            collector.emit(
+                    Constants.StatusStreamName,
+                    tuple,
+                    new Values(url, metadata, Status.FETCHED));
+        } else {
+            collector.emit(tuple, new Values(url, content, metadata, text));
+        }
+        collector.ack(tuple);
+    }
+
+    private boolean shouldRedirect(final Metadata metadata) {
+        if (metadata == null) {
+            return false;
+        }
+        if (skipIfMetadataPresent != null
+                && !skipIfMetadataPresent.isEmpty()
+                && metadata.containsKey(skipIfMetadataPresent)) {
+            // already came back from Playwright — don't loop
+            return false;
+        }
+        return metadata.containsKeyWithValue(routingKey, routingValue);
+    }
+
+    @Override
+    public void declareOutputFields(final OutputFieldsDeclarer declarer) {
+        declarer.declare(new Fields("url", "content", "metadata", "text"));
+        declarer.declareStream(
+                Constants.StatusStreamName, new Fields("url", "metadata", "status"));
+    }
+}
diff --git a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetector.java b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetector.java
index 9243a678d..382ef4c16 100644
--- a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetector.java
+++ b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetector.java
@@ -14,6 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.stormcrawler.protocol.playwright.parsefilter;
 
 import com.fasterxml.jackson.databind.JsonNode;
@@ -92,6 +93,10 @@
  *   <li>{@code minOutlinks} (int, default 2) — outcome-based threshold for extracted outlinks
  *   <li>{@code fingerprints} (string array) — substrings searched in raw HTML; replaces defaults
  *   <li>{@code emptyRootIds} (string array) — element IDs treated as empty SPA hydration roots
+ *   <li>{@code requiredMessages} (string array, default empty) — additional substrings that, when
+ *       found anywhere in the HTML, flag the URL. Use this for site-specific JS-required prompts
+ *       and loader text that don't fit the noscript pattern (e.g. {@code "Loading..."},
+ *       {@code "[object Object]"}, {@code "Please enable cookies"}).
  *   <li>{@code skipIfMetadataPresent} (string, default {@link HttpProtocol#MD_KEY_END}) — short-circuit
  *       when this metadata key is set; defeat with empty string to always re-evaluate
  *   <li>{@code recordReason} (bool, default true) — also set {@code metadataKey + ".reason"}
@@ -131,6 +136,7 @@ public class JsRenderingDetector extends ParseFilter {
     private int minOutlinks = 2;
     private List<String> fingerprints = DEFAULT_FINGERPRINTS;
     private List<String> emptyRootIds = DEFAULT_EMPTY_ROOT_IDS;
+    private List<String> requiredMessages = List.of();
     private String skipIfMetadataPresent = HttpProtocol.MD_KEY_END;
     private boolean recordReason = true;
 
@@ -140,21 +146,33 @@ public class JsRenderingDetector extends ParseFilter {
     public void configure(
             @NotNull final Map<String, Object> stormConf, @NotNull final JsonNode params) {
         if (params != null && !params.isMissingNode() && !params.isNull()) {
-            if (params.has("metadataKey")) this.metadataKey = params.get("metadataKey").asText();
-            if (params.has("metadataValue"))
+            if (params.has("metadataKey")) {
+                this.metadataKey = params.get("metadataKey").asText();
+            }
+            if (params.has("metadataValue")) {
                 this.metadataValue = params.get("metadataValue").asText();
-            if (params.has("minTextLength"))
+            }
+            if (params.has("minTextLength")) {
                 this.minTextLength = params.get("minTextLength").asInt(this.minTextLength);
-            if (params.has("minOutlinks"))
+            }
+            if (params.has("minOutlinks")) {
                 this.minOutlinks = params.get("minOutlinks").asInt(this.minOutlinks);
-            if (params.has("fingerprints"))
+            }
+            if (params.has("fingerprints")) {
                 this.fingerprints = readStringArray(params.get("fingerprints"));
-            if (params.has("emptyRootIds"))
+            }
+            if (params.has("emptyRootIds")) {
                 this.emptyRootIds = readStringArray(params.get("emptyRootIds"));
-            if (params.has("skipIfMetadataPresent"))
+            }
+            if (params.has("requiredMessages")) {
+                this.requiredMessages = readStringArray(params.get("requiredMessages"));
+            }
+            if (params.has("skipIfMetadataPresent")) {
                 this.skipIfMetadataPresent = params.get("skipIfMetadataPresent").asText();
-            if (params.has("recordReason"))
+            }
+            if (params.has("recordReason")) {
                 this.recordReason = params.get("recordReason").asBoolean(this.recordReason);
+            }
         }
         emptyRootPatterns.clear();
         for (final String id : emptyRootIds) {
@@ -179,7 +197,9 @@ public void filter(
             final byte[] content,
             final DocumentFragment doc,
             final ParseResult parse) {
-        if (content == null || content.length == 0) return;
+        if (content == null || content.length == 0) {
+            return;
+        }
 
         final Metadata md = parse.get(url).getMetadata();
         if (skipIfMetadataPresent != null
@@ -187,7 +207,9 @@ public void filter(
                 && md.containsKey(skipIfMetadataPresent)) {
             return;
         }
-        if (md.containsKey(metadataKey)) return;
+        if (md.containsKey(metadataKey)) {
+            return;
+        }
 
         final String reason = detectReason(url, content, parse);
         if (reason != null) {
@@ -205,7 +227,9 @@ private String detectReason(
 
         // 1. SPA framework fingerprints
         for (final String fp : fingerprints) {
-            if (html.contains(fp)) return "fingerprint:" + fp;
+            if (html.contains(fp)) {
+                return "fingerprint:" + fp;
+            }
         }
 
         // 2. <noscript> with explicit JS-required language
@@ -213,6 +237,13 @@ private String detectReason(
             return "noscript-js-required";
         }
 
+        // 2b. Free-form JS-required / loader / cookie-required messages
+        for (final String msg : requiredMessages) {
+            if (!msg.isEmpty() && html.contains(msg)) {
+                return "required-message:" + msg;
+            }
+        }
+
         // 3. Empty SPA hydration root
         for (int i = 0; i < emptyRootPatterns.size(); i++) {
             if (emptyRootPatterns.get(i).matcher(html).find()) {
@@ -222,7 +253,9 @@ private String detectReason(
 
         // 4. Outcome-based: gated on at least one <script> being present so we don't flag
         //    plain HTML stubs (e.g. very short error pages).
-        if (!HAS_SCRIPT_TAG.matcher(html).find()) return null;
+        if (!HAS_SCRIPT_TAG.matcher(html).find()) {
+            return null;
+        }
 
         final ParseData data = parse.get(url);
         final String text = data.getText();
@@ -235,7 +268,9 @@ private String detectReason(
     }
 
     private static List<String> readStringArray(final JsonNode node) {
-        if (node == null || !node.isArray()) return List.of();
+        if (node == null || !node.isArray()) {
+            return List.of();
+        }
         final List<String> list = new ArrayList<>(node.size());
         node.forEach(n -> list.add(n.asText()));
         return list;
diff --git a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/bolt/JsRenderingRedirectionBoltTest.java b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/bolt/JsRenderingRedirectionBoltTest.java
new file mode 100644
index 000000000..e3463acbe
--- /dev/null
+++ b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/bolt/JsRenderingRedirectionBoltTest.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.stormcrawler.protocol.playwright.bolt;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.tuple.Tuple;
+import org.apache.stormcrawler.Constants;
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.TestOutputCollector;
+import org.apache.stormcrawler.persistence.Status;
+import org.apache.stormcrawler.protocol.playwright.HttpProtocol;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+class JsRenderingRedirectionBoltTest {
+
+    private JsRenderingRedirectionBolt bolt;
+    private TestOutputCollector output;
+
+    @BeforeEach
+    void setUp() {
+        bolt = new JsRenderingRedirectionBolt();
+        output = new TestOutputCollector();
+    }
+
+    private void prepare(final Map<String, Object> conf) {
+        final TopologyContext context = mock(TopologyContext.class);
+        bolt.prepare(conf, context, new OutputCollector(output));
+    }
+
+    private Tuple tupleWith(final String url, final byte[] content, final Metadata metadata) {
+        final Tuple t = mock(Tuple.class);
+        when(t.getStringByField("url")).thenReturn(url);
+        when(t.getBinaryByField("content")).thenReturn(content);
+        when(t.getValueByField("metadata")).thenReturn(metadata);
+        when(t.getStringByField("text")).thenReturn("some text");
+        return t;
+    }
+
+    @Test
+    void redirectsToStatusStreamWhenFlagPresent() {
+        prepare(new HashMap<>());
+        final Metadata md = new Metadata();
+        md.setValue("fetch.with", "playwright");
+        bolt.execute(tupleWith("http://example.com/", new byte[] {1, 2, 3}, md));
+
+        final List<List<Object>> status = output.getEmitted(Constants.StatusStreamName);
+        Assertions.assertEquals(1, status.size(), "should redirect to status stream");
+        Assertions.assertEquals("http://example.com/", status.get(0).get(0));
+        Assertions.assertSame(md, status.get(0).get(1));
+        Assertions.assertEquals(Status.FETCHED, status.get(0).get(2));
+
+        Assertions.assertEquals(0, output.getEmitted().size(), "default stream should be empty");
+        Assertions.assertEquals(1, output.getAckedTuples().size());
+    }
+
+    @Test
+    void passesThroughWhenFlagAbsent() {
+        prepare(new HashMap<>());
+        final Metadata md = new Metadata();
+        bolt.execute(tupleWith("http://example.com/", new byte[] {1, 2, 3}, md));
+
+        final List<List<Object>> def = output.getEmitted();
+        Assertions.assertEquals(1, def.size(), "should pass through default stream");
+        Assertions.assertEquals(4, def.get(0).size(), "default emission has url/content/metadata/text");
+
+        Assertions.assertEquals(
+                0,
+                output.getEmitted(Constants.StatusStreamName).size(),
+                "status stream should be empty");
+        Assertions.assertEquals(1, output.getAckedTuples().size());
+    }
+
+    @Test
+    void skipsLoopWhenAlreadyFetchedByPlaywright() {
+        prepare(new HashMap<>());
+        // both flags set: detector flagged it AND it just came back from Playwright
+        final Metadata md = new Metadata();
+        md.setValue("fetch.with", "playwright");
+        md.setValue(HttpProtocol.MD_KEY_END, "2026-05-04T00:00:00Z");
+        bolt.execute(tupleWith("http://example.com/", new byte[] {1, 2, 3}, md));
+
+        Assertions.assertEquals(1, output.getEmitted().size());
+        Assertions.assertEquals(0, output.getEmitted(Constants.StatusStreamName).size());
+    }
+
+    @Test
+    void honoursCustomMetadataKeyAndValue() {
+        final Map<String, Object> conf = new HashMap<>();
+        conf.put(JsRenderingRedirectionBolt.CONF_METADATA_KEY, "render");
+        conf.put(JsRenderingRedirectionBolt.CONF_METADATA_VALUE, "yes");
+        prepare(conf);
+
+        final Metadata md = new Metadata();
+        md.setValue("render", "yes");
+        bolt.execute(tupleWith("http://example.com/", new byte[] {1}, md));
+
+        Assertions.assertEquals(1, output.getEmitted(Constants.StatusStreamName).size());
+        Assertions.assertEquals(0, output.getEmitted().size());
+    }
+
+    @Test
+    void wrongValueDoesNotRedirect() {
+        prepare(new HashMap<>());
+        final Metadata md = new Metadata();
+        // key set but to a different routing target
+        md.setValue("fetch.with", "selenium");
+        bolt.execute(tupleWith("http://example.com/", new byte[] {1}, md));
+
+        Assertions.assertEquals(1, output.getEmitted().size());
+        Assertions.assertEquals(0, output.getEmitted(Constants.StatusStreamName).size());
+    }
+
+    @Test
+    void emptySkipKeyDisablesLoopGuard() {
+        final Map<String, Object> conf = new HashMap<>();
+        conf.put(JsRenderingRedirectionBolt.CONF_SKIP_IF_METADATA_PRESENT, "");
+        prepare(conf);
+
+        final Metadata md = new Metadata();
+        md.setValue("fetch.with", "playwright");
+        md.setValue(HttpProtocol.MD_KEY_END, "2026-05-04T00:00:00Z");
+        bolt.execute(tupleWith("http://example.com/", new byte[] {1}, md));
+
+        // loop guard disabled → still redirects despite the playwright marker
+        Assertions.assertEquals(1, output.getEmitted(Constants.StatusStreamName).size());
+        Assertions.assertEquals(0, output.getEmitted().size());
+    }
+}
diff --git a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetectorTest.java b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetectorTest.java
index 67bf8894b..2e488cb88 100644
--- a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetectorTest.java
+++ b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetectorTest.java
@@ -14,6 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.stormcrawler.protocol.playwright.parsefilter;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -145,4 +146,36 @@ void customMetadataKeyAndValue() throws Exception {
         Assertions.assertEquals("yes", p.get("u").getMetadata().getFirstValue("render"));
         Assertions.assertNotNull(p.get("u").getMetadata().getFirstValue("render.reason"));
     }
+
+    @Test
+    void requiredMessageMatchesAnywhere() throws Exception {
+        // free-form list, fires outside <noscript> on a substring match
+        final JsRenderingDetector d =
+                detector(
+                        "{\"requiredMessages\":[\"Loading...\",\"[object Object]\","
+                                + "\"This page requires JavaScript\"]}");
+        final ParseResult p =
+                applyTo(
+                        d,
+                        "u",
+                        "<html><body><div id=\"app\"><span>Loading...</span></div></body></html>");
+        Assertions.assertEquals(
+                "playwright", p.get("u").getMetadata().getFirstValue("fetch.with"));
+        Assertions.assertEquals(
+                "required-message:Loading...",
+                p.get("u").getMetadata().getFirstValue("fetch.with.reason"));
+    }
+
+    @Test
+    void requiredMessageNoHitDoesNotFlag() throws Exception {
+        final JsRenderingDetector d = detector("{\"requiredMessages\":[\"Loading...\"]}");
+        final ParseResult p =
+                applyTo(
+                        d,
+                        "u",
+                        "<html><body><p>"
+                                + "A".repeat(1000)
+                                + "</p><a href='/x'>x</a><a href='/y'>y</a><a href='/z'>z</a></body></html>");
+        Assertions.assertNull(p.get("u").getMetadata().getFirstValue("fetch.with"));
+    }
 }

From f8b4d68a0ed05636bc02c891e645368b9225c8a4 Mon Sep 17 00:00:00 2001
From: Richard Zowalla <rzo1@apache.org>
Date: Mon, 4 May 2026 21:02:39 +0200
Subject: [PATCH 3/3] style(playwright): apply google-java-format to detector
 and bolt

---
 .../bolt/JsRenderingRedirectionBolt.java      | 28 ++++++------
 .../parsefilter/JsRenderingDetector.java      | 24 +++++------
 .../bolt/JsRenderingRedirectionBoltTest.java  |  3 +-
 .../parsefilter/JsRenderingDetectorTest.java  | 43 ++++++++++++-------
 4 files changed, 53 insertions(+), 45 deletions(-)

diff --git a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/bolt/JsRenderingRedirectionBolt.java b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/bolt/JsRenderingRedirectionBolt.java
index 3dbc047e3..8c1917e2b 100644
--- a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/bolt/JsRenderingRedirectionBolt.java
+++ b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/bolt/JsRenderingRedirectionBolt.java
@@ -38,14 +38,13 @@
  * upstream component) and forces an immediate refetch through Playwright instead of letting the
  * cheap fetch's stub document propagate downstream.
  *
- * <p>Pipeline placement: between the parser bolt (which produces tuples of
- * {@code (url, content, metadata, text)}) and the indexer / persistence bolts. On hit, the bolt
- * emits only to the {@link Constants#StatusStreamName} with status {@link Status#FETCHED}, so the
- * URL is rescheduled and the stub never reaches the index. On miss, the tuple passes through to
- * the default stream unchanged.
+ * <p>Pipeline placement: between the parser bolt (which produces tuples of {@code (url, content,
+ * metadata, text)}) and the indexer / persistence bolts. On hit, the bolt emits only to the {@link
+ * Constants#StatusStreamName} with status {@link Status#FETCHED}, so the URL is rescheduled and the
+ * stub never reaches the index. On miss, the tuple passes through to the default stream unchanged.
  *
- * <p>Pair this with a per-metadata-key fetch interval to control how soon the refetch happens —
- * by default {@code Status.FETCHED} reschedules at {@code fetchInterval.default} (24h):
+ * <p>Pair this with a per-metadata-key fetch interval to control how soon the refetch happens — by
+ * default {@code Status.FETCHED} reschedules at {@code fetchInterval.default} (24h):
  *
  * <pre>{@code
  * # refetch flagged URLs in 5 minutes rather than 24 hours
@@ -57,10 +56,10 @@
  * <ul>
  *   <li>{@code playwright.redirect.metadata.key} (default {@code fetch.with})
  *   <li>{@code playwright.redirect.metadata.value} (default {@code playwright})
- *   <li>{@code playwright.redirect.skip.if.metadata.present} (default
- *       {@link HttpProtocol#MD_KEY_END}) — passes the tuple through unchanged when this metadata key
- *       is already set, preventing loops with content that came back from Playwright. Set to empty
- *       to disable the loop guard.
+ *   <li>{@code playwright.redirect.skip.if.metadata.present} (default {@link
+ *       HttpProtocol#MD_KEY_END}) — passes the tuple through unchanged when this metadata key is
+ *       already set, preventing loops with content that came back from Playwright. Set to empty to
+ *       disable the loop guard.
  * </ul>
  */
 public class JsRenderingRedirectionBolt extends BaseRichBolt {
@@ -103,9 +102,7 @@ public void execute(final Tuple tuple) {
         if (shouldRedirect(metadata)) {
             LOG.debug("Redirecting {} to Playwright (status stream)", url);
             collector.emit(
-                    Constants.StatusStreamName,
-                    tuple,
-                    new Values(url, metadata, Status.FETCHED));
+                    Constants.StatusStreamName, tuple, new Values(url, metadata, Status.FETCHED));
         } else {
             collector.emit(tuple, new Values(url, content, metadata, text));
         }
@@ -128,7 +125,6 @@ private boolean shouldRedirect(final Metadata metadata) {
     @Override
     public void declareOutputFields(final OutputFieldsDeclarer declarer) {
         declarer.declare(new Fields("url", "content", "metadata", "text"));
-        declarer.declareStream(
-                Constants.StatusStreamName, new Fields("url", "metadata", "status"));
+        declarer.declareStream(Constants.StatusStreamName, new Fields("url", "metadata", "status"));
     }
 }
diff --git a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetector.java b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetector.java
index 382ef4c16..8be245e81 100644
--- a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetector.java
+++ b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetector.java
@@ -41,14 +41,14 @@
  * of the following signals, in cheapest-first order:
  *
  * <ol>
- *   <li>SPA framework fingerprints found in the raw HTML (Angular {@code ng-version}, React
- *       {@code data-reactroot}, Next.js {@code __NEXT_DATA__}, Nuxt {@code window.__NUXT__},
- *       Svelte {@code data-svelte-h}, Astro islands, Angular {@code <router-outlet>}, ...).
+ *   <li>SPA framework fingerprints found in the raw HTML (Angular {@code ng-version}, React {@code
+ *       data-reactroot}, Next.js {@code __NEXT_DATA__}, Nuxt {@code window.__NUXT__}, Svelte {@code
+ *       data-svelte-h}, Astro islands, Angular {@code <router-outlet>}, ...).
  *   <li>{@code <noscript>} blocks containing language like "enable JavaScript".
  *   <li>An empty SPA hydration root: {@code <div id="root"></div>} / {@code #app} / {@code #__next}
  *       / {@code #__nuxt} with no children.
- *   <li>Outcome-based fallback: very short text and few outlinks despite at least one
- *       {@code <script>} tag being present.
+ *   <li>Outcome-based fallback: very short text and few outlinks despite at least one {@code
+ *       <script>} tag being present.
  * </ol>
  *
  * <p>To avoid loops, detection is skipped when the URL was just fetched by Playwright (recognised
@@ -95,18 +95,17 @@
  *   <li>{@code emptyRootIds} (string array) — element IDs treated as empty SPA hydration roots
  *   <li>{@code requiredMessages} (string array, default empty) — additional substrings that, when
  *       found anywhere in the HTML, flag the URL. Use this for site-specific JS-required prompts
- *       and loader text that don't fit the noscript pattern (e.g. {@code "Loading..."},
- *       {@code "[object Object]"}, {@code "Please enable cookies"}).
- *   <li>{@code skipIfMetadataPresent} (string, default {@link HttpProtocol#MD_KEY_END}) — short-circuit
- *       when this metadata key is set; defeat with empty string to always re-evaluate
+ *       and loader text that don't fit the noscript pattern (e.g. {@code "Loading..."}, {@code
+ *       "[object Object]"}, {@code "Please enable cookies"}).
+ *   <li>{@code skipIfMetadataPresent} (string, default {@link HttpProtocol#MD_KEY_END}) —
+ *       short-circuit when this metadata key is set; defeat with empty string to always re-evaluate
  *   <li>{@code recordReason} (bool, default true) — also set {@code metadataKey + ".reason"}
  *       describing which signal fired (useful for triage)
  * </ul>
  */
 public class JsRenderingDetector extends ParseFilter {
 
-    private static final org.slf4j.Logger LOG =
-            LoggerFactory.getLogger(JsRenderingDetector.class);
+    private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(JsRenderingDetector.class);
 
     private static final List<String> DEFAULT_FINGERPRINTS =
             List.of(
@@ -221,8 +220,7 @@ public void filter(
         }
     }
 
-    private String detectReason(
-            final String url, final byte[] content, final ParseResult parse) {
+    private String detectReason(final String url, final byte[] content, final ParseResult parse) {
         final String html = new String(content, StandardCharsets.UTF_8);
 
         // 1. SPA framework fingerprints
diff --git a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/bolt/JsRenderingRedirectionBoltTest.java b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/bolt/JsRenderingRedirectionBoltTest.java
index e3463acbe..fbc2562dc 100644
--- a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/bolt/JsRenderingRedirectionBoltTest.java
+++ b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/bolt/JsRenderingRedirectionBoltTest.java
@@ -85,7 +85,8 @@ void passesThroughWhenFlagAbsent() {
 
         final List<List<Object>> def = output.getEmitted();
         Assertions.assertEquals(1, def.size(), "should pass through default stream");
-        Assertions.assertEquals(4, def.get(0).size(), "default emission has url/content/metadata/text");
+        Assertions.assertEquals(
+                4, def.get(0).size(), "default emission has url/content/metadata/text");
 
         Assertions.assertEquals(
                 0,
diff --git a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetectorTest.java b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetectorTest.java
index 2e488cb88..2b88c76c8 100644
--- a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetectorTest.java
+++ b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetectorTest.java
@@ -44,17 +44,24 @@ private ParseResult applyTo(final JsRenderingDetector d, final String url, final
     @Test
     void detectsReactByDataReactRoot() throws Exception {
         final JsRenderingDetector d = detector("{}");
-        final ParseResult p = applyTo(d, "u", "<html><body><div data-reactroot></div></body></html>");
+        final ParseResult p =
+                applyTo(d, "u", "<html><body><div data-reactroot></div></body></html>");
         Assertions.assertEquals("playwright", p.get("u").getMetadata().getFirstValue("fetch.with"));
         Assertions.assertTrue(
-                p.get("u").getMetadata().getFirstValue("fetch.with.reason").startsWith("fingerprint:"));
+                p.get("u")
+                        .getMetadata()
+                        .getFirstValue("fetch.with.reason")
+                        .startsWith("fingerprint:"));
     }
 
     @Test
     void detectsAngularByNgVersion() throws Exception {
         final JsRenderingDetector d = detector("{}");
         final ParseResult p =
-                applyTo(d, "u", "<html><body><app-root ng-version=\"17.0.0\"></app-root></body></html>");
+                applyTo(
+                        d,
+                        "u",
+                        "<html><body><app-root ng-version=\"17.0.0\"></app-root></body></html>");
         Assertions.assertEquals("playwright", p.get("u").getMetadata().getFirstValue("fetch.with"));
     }
 
@@ -65,16 +72,23 @@ void detectsNoscriptJsRequired() throws Exception {
                 "<html><body><noscript>Please enable JavaScript to view this site.</noscript></body></html>";
         final ParseResult p = applyTo(d, "u", html);
         Assertions.assertEquals(
-                "noscript-js-required", p.get("u").getMetadata().getFirstValue("fetch.with.reason"));
+                "noscript-js-required",
+                p.get("u").getMetadata().getFirstValue("fetch.with.reason"));
     }
 
     @Test
     void detectsEmptySpaRoot() throws Exception {
         final JsRenderingDetector d = detector("{}");
         final ParseResult p =
-                applyTo(d, "u", "<html><body><div id=\"root\"></div><script src=\"/app.js\"></script></body></html>");
+                applyTo(
+                        d,
+                        "u",
+                        "<html><body><div id=\"root\"></div><script src=\"/app.js\"></script></body></html>");
         Assertions.assertTrue(
-                p.get("u").getMetadata().getFirstValue("fetch.with.reason").startsWith("empty-root:"));
+                p.get("u")
+                        .getMetadata()
+                        .getFirstValue("fetch.with.reason")
+                        .startsWith("empty-root:"));
     }
 
     @Test
@@ -86,7 +100,10 @@ void thinContentWithScriptIsFlagged() throws Exception {
         final ParseResult p = applyTo(d, "u", html);
         Assertions.assertNotNull(p.get("u").getMetadata().getFirstValue("fetch.with"));
         Assertions.assertTrue(
-                p.get("u").getMetadata().getFirstValue("fetch.with.reason").startsWith("thin-content:"));
+                p.get("u")
+                        .getMetadata()
+                        .getFirstValue("fetch.with.reason")
+                        .startsWith("thin-content:"));
     }
 
     @Test
@@ -114,11 +131,7 @@ void skipsIfAlreadyFetchedByPlaywright() throws Exception {
         final ParseResult parse = new ParseResult();
         // simulate metadata coming from a Playwright fetch
         parse.get("u").getMetadata().setValue(HttpProtocol.MD_KEY_END, "2026-05-04T00:00:00Z");
-        d.filter(
-                "u",
-                "<div data-reactroot></div>".getBytes(StandardCharsets.UTF_8),
-                null,
-                parse);
+        d.filter("u", "<div data-reactroot></div>".getBytes(StandardCharsets.UTF_8), null, parse);
         Assertions.assertNull(parse.get("u").getMetadata().getFirstValue("fetch.with"));
     }
 
@@ -133,7 +146,8 @@ void skipsIfAlreadyFlagged() throws Exception {
                 null,
                 parse);
         // not overwritten, no reason added
-        Assertions.assertEquals("playwright", parse.get("u").getMetadata().getFirstValue("fetch.with"));
+        Assertions.assertEquals(
+                "playwright", parse.get("u").getMetadata().getFirstValue("fetch.with"));
         Assertions.assertNull(parse.get("u").getMetadata().getFirstValue("fetch.with.reason"));
     }
 
@@ -159,8 +173,7 @@ void requiredMessageMatchesAnywhere() throws Exception {
                         d,
                         "u",
                         "<html><body><div id=\"app\"><span>Loading...</span></div></body></html>");
-        Assertions.assertEquals(
-                "playwright", p.get("u").getMetadata().getFirstValue("fetch.with"));
+        Assertions.assertEquals("playwright", p.get("u").getMetadata().getFirstValue("fetch.with"));
         Assertions.assertEquals(
                 "required-message:Loading...",
                 p.get("u").getMetadata().getFirstValue("fetch.with.reason"));