[4] refactor ADTK anomaly detector to use utils package

influxdata/library/plugin_library.json

@@ -122,8 +122,8 @@
                         "path": "influxdata/notifier/notifier_plugin.py"
                     }
                 ],
-                "required_libraries": ["requests", "adtk", "pandas"],
-                "last_update": "2025-06-16",
+                "required_libraries": ["influxdata-plugin-utils>=0.3.0", "requests", "adtk", "pandas<3"],
+                "last_update": "2026-08-16",
                 "trigger_types_supported": ["scheduler"]
             },
             {

influxdata/stateless_adtk_detector/README.md

@@ -25,15 +25,33 @@ This plugin includes a JSON metadata schema in its docstring that defines suppor
 | `field`           | string | required | Numeric field to evaluate                                                                   |
 | `detectors`       | string | required | Dot-separated list of advanced ADTK detectors for different anomaly types                   |
 | `detector_params` | string | required | Base64-encoded JSON parameters for each detector                                            |
-| `window`          | string | required | Data analysis window with flexible scheduling. Format: `<number><unit>` (e.g., "1h", "30m") |
+| `window`          | string | required | Data analysis window. Format: `<number><unit>` (e.g., "1h", "30min"). Must be positive      |
 | `senders`         | string | required | Dot-separated notification channels with multi-channel notification support                 |
 
+Duration units: `us`, `ms`, `s`, `min`, `h`, `d`, `w`.
+
 ### Advanced parameters
 
-| Parameter                | Type   | Default | Description                                                                                  |
-|--------------------------|--------|---------|----------------------------------------------------------------------------------------------|
-| `min_consensus`          | number | 1       | Minimum detectors required to agree for consensus-based filtering to reduce false positives  |
-| `min_condition_duration` | string | "0s"    | Minimum duration for configurable anomaly persistence before alerting                        |
+| Parameter                   | Type    | Default  | Description                                                                                                |
+|-----------------------------|---------|----------|------------------------------------------------------------------------------------------------------------|
+| `min_consensus`             | number  | 1        | Minimum detectors required to agree for consensus-based filtering to reduce false positives (1 or greater) |
+| `min_condition_duration`    | string  | "0s"     | Minimum duration for configurable anomaly persistence before alerting                                      |
+| `group_by_tags`             | bool    | false    | Analyze every tag combination as its own time series                                                       |
+| `max_notifications_per_run` | number  | 20       | Maximum number of notifications a single run may send                                                      |
+
+#### Analyzing tagged measurements
+
+By default the whole window forms a single time series. When a measurement holds several tag combinations (for example `host=server1` and `host=server2`), those rows share timestamps, and ADTK keeps only the first value of each timestamp — so one arbitrary series is analyzed and the rest of the data is ignored.
+
+Set `group_by_tags=true` to analyze each tag combination separately. Every series then gets its own detector run, its own consensus evaluation, and its own debounce state, so a measurement with N tag combinations can produce up to N notifications per run.
+
+#### Notification behavior
+
+The timestamp of the last alerted point is remembered per series, so a `window` longer than the trigger interval does not resend anomalies that earlier runs already reported.
+
+A single run sends at most `max_notifications_per_run` notifications. Anomalies beyond the limit are counted in a warning and are not resent by later runs — raise the limit if a run legitimately produces more alerts.
+
+Points that have no value for `field` are dropped before detection and reported in the log; without this a single NULL makes every detector fail.
 
 ### Notification parameters
 
@@ -52,6 +70,8 @@ This plugin includes a JSON metadata schema in its docstring that defines suppor
 
 *To use a TOML configuration file, set the `PLUGIN_DIR` environment variable and specify the `config_file_path` in the trigger arguments.* This is in addition to the `--plugin-dir` flag when starting InfluxDB 3.
 
+When a config file is given, it replaces the inline trigger arguments entirely. In TOML, `detectors` and `senders` accept either a list (`["QuantileAD", "PersistAD"]`) or a dot-separated string, and detector parameters may be given as a `[detector_params]` table or as a base64-encoded JSON string.
+
 #### Example TOML configuration
 
 [adtk_anomaly_config_scheduler.toml](adtk_anomaly_config_scheduler.toml)
@@ -74,10 +94,17 @@ For more information on using TOML configuration files, see the Using TOML Confi
 ## Software Requirements
 
 - **InfluxDB 3 Core/Enterprise**: with the Processing Engine enabled.
+- **Python 3.11+**
 - **Python packages**:
+  - `influxdata-plugin-utils>=0.3.0` (for configuration loading, parsing, and schema introspection)
   - `adtk` (for anomaly detection)
-  - `pandas` (for data manipulation)
+  - `pandas<3` (for data manipulation)
   - `requests` (for HTTP notifications)
+
+`pandas` must stay below 3.0. Window-based detectors (`LevelShiftAD`, `VolatilityShiftAD`, `PersistAD`)
+return `NaN` for the first `window` points, which pandas 3 refuses to store in a boolean result. With
+pandas 3 installed those three detectors fail with `Invalid value 'nan' for dtype 'bool'`, are skipped
+with a warning, and stop contributing to the consensus.
 - **Notification Sender Plugin** *(optional)*: Required if using the `senders` parameter. See the [influxdata/notifier plugin](../notifier/README.md).
 
 ### Installation steps
@@ -95,9 +122,10 @@ For more information on using TOML configuration files, see the Using TOML Confi
 2. Install required Python packages:
 
    ```bash
+   influxdb3 install package "influxdata-plugin-utils>=0.3.0"
    influxdb3 install package requests
    influxdb3 install package adtk
-   influxdb3 install package pandas
+   influxdb3 install package "pandas<3"
    ```
 
 3. *(Optional)* For notifications, install the [influxdata/notifier plugin](../notifier/README.md) and create an HTTP trigger for it.
@@ -113,7 +141,7 @@ influxdb3 create trigger \
   --database mydb \
   --path "gh:influxdata/stateless_adtk_detector/adtk_anomaly_detection_plugin.py" \
   --trigger-spec "every:10m" \
-  --trigger-arguments "measurement=cpu,field=usage,detectors=QuantileAD.LevelShiftAD,detector_params=eyJRdWFudGlsZUFKIjogeyJsb3ciOiAwLjA1LCAiaGlnaCI6IDAuOTV9LCAiTGV2ZWxTaGlmdEFKIjogeyJ3aW5kb3ciOiA1fX0=,window=10m,senders=slack,slack_webhook_url=$SLACK_WEBHOOK_URL" \
+  --trigger-arguments "measurement=cpu,field=usage,detectors=QuantileAD.LevelShiftAD,detector_params=eyJRdWFudGlsZUFKIjogeyJsb3ciOiAwLjA1LCAiaGlnaCI6IDAuOTV9LCAiTGV2ZWxTaGlmdEFKIjogeyJ3aW5kb3ciOiA1fX0=,window=10min,senders=slack,slack_webhook_url=$SLACK_WEBHOOK_URL" \
   anomaly_detector
 ```
 
@@ -157,7 +185,7 @@ influxdb3 create trigger \
   --database monitoring \
   --path "gh:influxdata/stateless_adtk_detector/adtk_anomaly_detection_plugin.py" \
   --trigger-spec "every:15m" \
-  --trigger-arguments "measurement=cpu_metrics,field=utilization,detectors=QuantileAD.LevelShiftAD,detector_params=eyJRdWFudGlsZUFEIjogeyJsb3ciOiAwLjEsICJoaWdoIjogMC45fSwgIkxldmVsU2hpZnRBRCI6IHsid2luZG93IjogMTB9fQ==,min_consensus=2,window=30m,senders=discord,discord_webhook_url=$DISCORD_WEBHOOK_URL" \
+  --trigger-arguments "measurement=cpu_metrics,field=utilization,detectors=QuantileAD.LevelShiftAD,detector_params=eyJRdWFudGlsZUFEIjogeyJsb3ciOiAwLjEsICJoaWdoIjogMC45fSwgIkxldmVsU2hpZnRBRCI6IHsid2luZG93IjogMTB9fQ==,min_consensus=2,window=30min,senders=discord,discord_webhook_url=$DISCORD_WEBHOOK_URL" \
   cpu_consensus_detector
 ```
 
@@ -175,7 +203,7 @@ influxdb3 create trigger \
   --database trading \
   --path "gh:influxdata/stateless_adtk_detector/adtk_anomaly_detection_plugin.py" \
   --trigger-spec "every:1m" \
-  --trigger-arguments "measurement=stock_prices,field=price,detectors=VolatilityShiftAD,detector_params=eyJWb2xhdGlsaXR5U2hpZnRBRCI6IHsid2luZG93IjogMjB9fQ==,window=1h,min_condition_duration=5m,senders=sms,twilio_from_number=+1234567890,twilio_to_number=+0987654321" \
+  --trigger-arguments "measurement=stock_prices,field=price,detectors=VolatilityShiftAD,detector_params=eyJWb2xhdGlsaXR5U2hpZnRBRCI6IHsid2luZG93IjogMjB9fQ==,window=1h,min_condition_duration=5min,senders=sms,twilio_from_number=+1234567890,twilio_to_number=+0987654321" \
   volatility_detector
 ```
 
@@ -186,6 +214,9 @@ influxdb3 create trigger \
 
 - `adtk_anomaly_detection_plugin.py`: The main plugin code containing the scheduled handler for anomaly detection
 - `adtk_anomaly_config_scheduler.toml`: Example TOML configuration file
+- `test_adtk_anomaly_detection.py`: Pytest suite (49 tests, runs without a live InfluxDB 3 server)
+- `requirements.txt`: Runtime dependencies (`influxdata-plugin-utils>=0.3.0`, `requests`, `adtk`, `pandas<3`)
+- `requirements-dev.txt`: Development dependencies (`pytest`)
 
 ### Logging
 
@@ -209,6 +240,14 @@ Key operations:
 4. Evaluates consensus across detectors
 5. Sends notifications when anomalies are confirmed
 
+#### `parse_detectors(influxdb3_local, config, task_id)`
+
+Resolves the detectors to apply together with their parameters. Detectors that are unknown, have no entry in `detector_params`, or miss a parameter required to construct them are skipped with a warning and do not count toward `min_consensus`.
+
+#### `split_by_tags(df, tags, group_by_tags)`
+
+Splits query results into one frame per tag combination when `group_by_tags` is enabled, so detectors never mix values written under different tag sets.
+
 ## Troubleshooting
 
 ### Common issues
@@ -221,9 +260,25 @@ Key operations:
 
 **Solution**: Increase `min_consensus` to require more detectors to agree. Add `min_condition_duration` to require anomalies to persist. Adjust detector-specific thresholds in `detector_params`.
 
+#### Issue: A newly created measurement is reported as not found
+
+**Solution**: Table and tag names are cached for one hour per trigger. Wait for the cache to expire, or recreate the trigger to clear it.
+
+#### Issue: Anomalies of some tag combinations are never detected
+
+**Solution**: Set `group_by_tags=true`. Without it, rows of different tag combinations share timestamps and only the first series survives.
+
+#### Issue: A detector is skipped with a warning
+
+**Solution**: The warning names the reason: the detector is not in the supported list (check the spelling), has no entry in `detector_params`, or misses a required parameter (`window` for `LevelShiftAD` and `VolatilityShiftAD`). `Invalid value 'nan' for dtype 'bool'` means pandas 3 is installed — downgrade to `pandas<3`.
+
+#### Issue: No anomalies are ever reported
+
+**Solution**: Check the warnings. `min_consensus` must not exceed the number of detectors that were actually applied — skipped detectors reduce that count. `min_condition_duration` must be shorter than `window`, otherwise no anomaly can persist long enough within a single query window.
+
 #### Issue: Missing dependencies
 
-**Solution**: Install required packages: `adtk`, `pandas`, `requests`. Ensure the Notifier Plugin is installed for notifications.
+**Solution**: Install required packages: `influxdata-plugin-utils`, `adtk`, `pandas`, `requests`. Ensure the Notifier Plugin is installed for notifications.
 
 #### Issue: Data quality issues
 

influxdata/stateless_adtk_detector/adtk_anomaly_config_scheduler.toml

@@ -16,25 +16,34 @@ field = "your_field"  # e.g., "usage", "value", "temp"
 
 # ADTK detectors to use
 # Supported detectors: GeneralizedESDTestAD, InterQuartileRangeAD, ThresholdAD, QuantileAD, LevelShiftAD, VolatilityShiftAD, PersistAD, SeasonalAD
-# Specify a list of detector names (strings)
+# Specify a list of detector names (strings), or a dot-separated string
 detectors = ["your_detector"]  # e.g., ["QuantileAD"], ["GeneralizedESDTestAD", "InterQuartileRangeAD"]
 
 # Time window for analysis
-# Format: <number><unit>, where unit is s (seconds), min (minutes), h (hours), d (days), w (weeks)
-window = "your_window"  # e.g., "15m", "24h"
+# Format: <number><unit>, where unit is us, ms, s (seconds), min (minutes), h (hours), d (days), w (weeks)
+# Must be a positive duration
+window = "your_window"  # e.g., "15min", "24h"
 
 # Notification channels
-# Specify a list of notification channels (strings)
+# Specify a list of notification channels (strings), or a dot-separated string
 senders = ["your_channel"]  # e.g., ["slack"], ["http", "sms"]
 
 ########## Optional Parameters ##########
 # Minimum number of detectors that must agree on an anomaly
 # Specify an integer ≥ 1; default is 1
 #min_consensus = 1  # e.g., 2
 
+# Analyze every tag combination as its own time series
+# Without it the whole window is one series and rows sharing a timestamp are collapsed
+#group_by_tags = true  # default is false
+
+# Maximum number of notifications a single run may send
+# Anomalies beyond the limit are reported in a warning and not resent later
+#max_notifications_per_run = 20  # e.g., 50
+
 # Minimum duration an anomaly condition must persist before notifying
-# Format: <number><unit>, where unit is s (seconds), min (minutes), h (hours), d (days), w (weeks)
-#min_condition_duration = "your_duration"  # e.g., "1m", "10m"
+# Format: <number><unit>, where unit is us, ms, s (seconds), min (minutes), h (hours), d (days), w (weeks)
+#min_condition_duration = "your_duration"  # e.g., "1min", "10min"
 
 # InfluxDB 3 API token
 # Specify the token (string); can also be provided via INFLUXDB3_AUTH_TOKEN environment variable
@@ -84,14 +93,12 @@ senders = ["your_channel"]  # e.g., ["slack"], ["http", "sms"]
 #twilio_to_number = "your_twilio_to_number"  # e.g., "+0987654321"
 
 # --- WhatsApp (via Twilio) ---
-# WhatsApp sender number (required for WhatsApp, format: +1234567890)
-#whatsapp_from_number = "your_whatsapp_from_number"  # e.g., "+1234567890"
-# WhatsApp recipient number (required for WhatsApp, format: +0987654321)
-#whatsapp_to_number = "your_whatsapp_to_number"  # e.g., "+0987654321"
+# WhatsApp uses the same twilio_* settings as SMS above
 
 # Detector parameters (Required)
 # Format: {"DetectorName": {param1: value, ...}, ...}
 # Specify parameters for each detector listed in detectors
+# A base64-encoded JSON string is also accepted: detector_params = "eyJRdWFudGlsZUFEIjogey4uLn19"
 [detector_params]
 your_detector = { param1 = "value1", param2 = "value2" }  # e.g., QuantileAD = { low = 0.05, high = 0.95 }