diff --git a/influxdata/library/plugin_library.json b/influxdata/library/plugin_library.json index 7370d99..635843f 100644 --- a/influxdata/library/plugin_library.json +++ b/influxdata/library/plugin_library.json @@ -57,8 +57,8 @@ "path": "influxdata/notifier/notifier_plugin.py" } ], - "required_libraries": ["requests"], - "last_update": "2025-06-16", + "required_libraries": ["influxdata-plugin-utils>=0.3.0", "requests"], + "last_update": "2026-08-18", "trigger_types_supported": ["data_writes"] }, { @@ -106,8 +106,8 @@ "path": "influxdata/notifier/notifier_plugin.py" } ], - "required_libraries": ["requests"], - "last_update": "2025-06-16", + "required_libraries": ["influxdata-plugin-utils>=0.3.0", "requests"], + "last_update": "2026-08-17", "trigger_types_supported": ["scheduler", "data_writes"] }, { @@ -122,8 +122,8 @@ "path": "influxdata/notifier/notifier_plugin.py" } ], - "required_libraries": ["requests", "adtk", "pandas"], - "last_update": "2025-06-16", + "required_libraries": ["influxdata-plugin-utils>=0.3.0", "requests", "adtk", "pandas<3"], + "last_update": "2026-08-16", "trigger_types_supported": ["scheduler"] }, { @@ -336,8 +336,8 @@ "author": "Synthefy", "docs_file_link": "https://github.com/influxdata/influxdb3_plugins/blob/main/influxdata/synthefy_forecasting/README.md", "required_plugins": [], - "required_libraries": ["pandas", "requests"], - "last_update": "2026-01-08", + "required_libraries": ["pandas", "requests", "influxdata-plugin-utils>=0.3.0"], + "last_update": "2026-08-10", "trigger_types_supported": ["http"] }, { @@ -380,8 +380,8 @@ "author": "InfluxData", "docs_file_link": "https://github.com/influxdata/influxdb3_plugins/blob/main/influxdata/stock_plugin/README.md", "required_plugins": [], - "required_libraries": ["yfinance", "pandas_market_calendars"], - "last_update": "2026-06-10", + "required_libraries": ["yfinance", "pandas_market_calendars", "influxdata-plugin-utils>=0.3.0"], + "last_update": "2026-08-12", "trigger_types_supported": ["scheduler"] }, { diff --git a/influxdata/mad_check/README.md b/influxdata/mad_check/README.md index f4c2bfe..d3765ba 100644 --- a/influxdata/mad_check/README.md +++ b/influxdata/mad_check/README.md @@ -26,25 +26,29 @@ This plugin includes a JSON metadata schema in its docstring that defines suppor ### MAD threshold parameters -| Component | Description | Example | -|----------------|------------------------------------------------|-------------| -| `field_name` | The numeric field to monitor | `temp` | -| `k` | MAD multiplier for anomaly threshold | `2.5` | -| `window_count` | Number of recent points for MAD computation | `20` | -| `threshold` | Count (integer) or duration (e.g., "2m", "1h") | `5` or `2m` | +| Component | Description | Example | +|----------------|----------------------------------------------------------------|---------------| +| `field_name` | The numeric field to monitor | `temp` | +| `k` | MAD multiplier for the anomaly cutoff (float, ≥ 0) | `2.5` | +| `window_count` | Number of recent points for MAD computation (integer, 2–10000) | `20` | +| `threshold` | Consecutive outliers (integer, ≥ 1) or a duration | `5` or `2min` | -Multiple thresholds are separated by `@`: `temp:2.5:20:5@load:3:10:2m` +Multiple thresholds are separated by `@`: `temp:2.5:20:5@load:3:10:2min` + +Durations use the format ``, where unit is `us` (microseconds), `ms` (milliseconds), `s` (seconds), `min` (minutes), `h` (hours), `d` (days), or `w` (weeks). + +Thresholds that share a field and `window_count` share one MAD window, so you can combine a count-based and a duration-based alert on the same detector: `temp:2.5:20:5@temp:2.5:20:2min`. Invalid thresholds are skipped with a warning; if none remain, the plugin logs an error and stops. Repeated identical thresholds are also skipped with a warning, because they would share one counter. ### Optional parameters -| Parameter | Type | Default | Description | -|---------------------------|--------|--------------------------------------|-------------------------------------------------------------------------------------------| -| `influxdb3_auth_token` | string | env var | API token for InfluxDB 3 (or use INFLUXDB3_AUTH_TOKEN env var) | -| `state_change_count` | string | "0" | Maximum allowed value flips before suppressing notifications | -| `notification_count_text` | string | see *Default notification templates* | Template for count-based alerts with variables: $table, $field, $threshold_count, $tags | -| `notification_time_text` | string | see *Default notification templates* | Template for duration-based alerts with variables: $table, $field, $threshold_time, $tags | -| `notification_path` | string | "notify" | URL path for the notification sending plugin | -| `port_override` | string | "8181" | Port number where InfluxDB accepts requests | +| Parameter | Type | Default | Description | +|---------------------------|--------|--------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `influxdb3_auth_token` | string | env var | API token for InfluxDB 3 (or use INFLUXDB3_AUTH_TOKEN env var) | +| `state_change_count` | string | "0" | Number of transitions between normal and outlier state, within the MAD window, at which notifications are suppressed. Use 2 or greater; `1` is treated as `0`. See *Flip Detection* | +| `notification_count_text` | string | see *Default notification templates* | Template for count-based alerts with variables: $table, $field, $threshold_count, $tags | +| `notification_time_text` | string | see *Default notification templates* | Template for duration-based alerts with variables: $table, $field, $threshold_time, $tags | +| `notification_path` | string | "notify" | URL path for the notification sending plugin | +| `port_override` | string | "8181" | Port number where InfluxDB accepts requests | #### Default notification templates @@ -89,7 +93,11 @@ Multiple thresholds are separated by `@`: `temp:2.5:20:5@load:3:10:2m` |--------------------|--------|---------|----------------------------------------------------------------------------------| | `config_file_path` | string | none | TOML config file path relative to `PLUGIN_DIR` (required for TOML configuration) | -*To use a TOML configuration file, set the `PLUGIN_DIR` environment variable and specify the `config_file_path` in the trigger arguments.* This is in addition to the `--plugin-dir` flag when starting InfluxDB 3. +*To use a TOML configuration file, set the `PLUGIN_DIR` environment variable and specify the `config_file_path` in the trigger arguments.* This is in addition to the `--plugin-dir` flag when starting InfluxDB 3. Relative paths are resolved against the first directory that is set: `PLUGIN_DIR`, then `INFLUXDB3_PLUGIN_DIR`, then the parent of `VIRTUAL_ENV`. Only that directory is used — the file is not looked up in the remaining ones. + +When `config_file_path` is set, the TOML file provides the whole configuration and inline trigger arguments are ignored. `INFLUXDB3_AUTH_TOKEN` from the environment still applies when `influxdb3_auth_token` is not set in the file. In TOML, `senders` and `mad_thresholds` use native structures (a list and a list of entries) instead of the inline string formats, though the inline strings are also accepted. + +The plugin caches the loaded configuration for 10 minutes to keep the write path fast, so configuration changes take effect within that window. #### Example TOML configuration @@ -101,6 +109,7 @@ For more information on using TOML configuration files, see the Using TOML Confi - **InfluxDB 3 Core/Enterprise**: with the Processing Engine enabled. - **Python packages**: + - `influxdata-plugin-utils>=0.3.0` (configuration loading, parsing, and schema introspection) - `requests` (for notification delivery) - **Notification Sender Plugin** *(optional)*: Required if using the `senders` parameter. See the [influxdata/notifier plugin](../notifier/README.md). @@ -119,6 +128,7 @@ For more information on using TOML configuration files, see the Using TOML Confi 2. Install required Python packages: ```bash + influxdb3 install package influxdata-plugin-utils influxdb3 install package requests ``` @@ -139,7 +149,7 @@ influxdb3 create trigger \ --database mydb \ --path "gh:influxdata/mad_check/mad_check_plugin.py" \ --trigger-spec "all_tables" \ - --trigger-arguments 'measurement=cpu,mad_thresholds="temp:2.5:20:5@load:3:10:2m",senders=slack,slack_webhook_url="$SLACK_WEBHOOK_URL"' \ + --trigger-arguments 'measurement=cpu,mad_thresholds="temp:2.5:20:5@load:3:10:2min",senders=slack,slack_webhook_url="$SLACK_WEBHOOK_URL"' \ mad_anomaly_detector ``` @@ -192,7 +202,7 @@ influxdb3 create trigger \ --database monitoring \ --path "gh:influxdata/mad_check/mad_check_plugin.py" \ --trigger-spec "all_tables" \ - --trigger-arguments 'measurement=system_metrics,mad_thresholds="cpu_load:3:30:2m@memory_used:2.5:30:5m",senders=slack.discord,slack_webhook_url="$SLACK_WEBHOOK_URL",discord_webhook_url="$DISCORD_WEBHOOK_URL"' \ + --trigger-arguments 'measurement=system_metrics,mad_thresholds="cpu_load:3:30:2min@memory_used:2.5:30:5min",senders=slack.discord,slack_webhook_url="$SLACK_WEBHOOK_URL",discord_webhook_url="$DISCORD_WEBHOOK_URL"' \ system_anomaly_detector ``` @@ -224,7 +234,7 @@ Set `HTTP_WEBHOOK_URL` to your HTTP webhook endpoint. **Expected output** - Detects vibration anomalies exceeding 2 MADs for 10 consecutive points -- If values flip between normal/anomalous more than 3 times in the 50-point window, suppresses notifications +- Suppresses notifications once the value has switched between normal and outlier state 3 times within the 50-point window, so two switches are still tolerated - Sends custom formatted message to HTTP endpoint ## Using TOML Configuration Files @@ -258,7 +268,7 @@ This plugin supports using TOML configuration files to specify all plugin argume ```toml # Required parameters measurement = "cpu" - mad_thresholds = "temp:2.5:20:5@load:3:10:2m" + mad_thresholds = "temp:2.5:20:5@load:3:10:2min" senders = "slack" # Notification settings @@ -285,6 +295,9 @@ This plugin supports using TOML configuration files to specify all plugin argume - `mad_check_plugin.py`: The main plugin code containing the handler for data write triggers - `mad_anomaly_config_data_writes.toml`: Example TOML configuration file +- `test_mad_check.py`: Pytest suite, runs without a live InfluxDB 3 server +- `requirements.txt`: Runtime dependencies (`influxdata-plugin-utils>=0.3.0`, `requests`) +- `requirements-dev.txt`: Development dependencies (`pytest`) ### Logging @@ -326,9 +339,15 @@ threshold = k * mad is_anomaly = abs(value - median) > threshold ``` +When more than half of the values in the window are identical, `mad` is `0` and the bounds collapse onto the median, so any different value counts as an outlier no matter how large `k` is. This affects flat signals: a stable sensor, a metric that is usually `0`, or a low-resolution integer field. The effect also works in reverse — once outliers fill more than half of the window they become the new median and stop being detected. + #### Flip Detection -Counts transitions between normal and anomalous states within the window to prevent alert fatigue from rapidly changing values. +The plugin keeps the recent outlier flags of each threshold in a deque the size of `window_count` and counts transitions between normal and outlier state. Once the number of transitions reaches `state_change_count`, the alert is computed as usual but not delivered, and a warning is logged instead. This prevents alert fatigue from values that switch in and out of the outlier state. + +An alert that follows normal data always records one normal-to-outlier transition, so `state_change_count` must be 2 or greater to leave sustained anomalies alone. A value of `1` would suppress every alert; the plugin logs a warning and treats it as `0`. + +A count threshold needs consecutive outliers, so when it fires the last `threshold` flags are all outliers and only `window_count - threshold` transitions can remain in the window. Suppression therefore requires `window_count >= threshold + state_change_count`; otherwise the plugin logs a `Flip suppression never triggers` warning naming the field. Duration thresholds have no such limit. ## Troubleshooting @@ -344,13 +363,18 @@ Counts transitions between normal and anomalous states within the window to prev ` 3. Ensure notification channel parameters are provided for selected senders -#### Issue: "Invalid MAD thresholds format" error +#### Issue: "No valid MAD thresholds provided" error -**Solution**: Check threshold format is correct: +**Solution**: Each invalid threshold is logged as a warning naming the part that failed. Check the format: -- Count-based: `field:k:window:count` (e.g., `temp:2.5:20:5`) -- Duration-based: `field:k:window:duration` (e.g., `temp:2.5:20:2m`) +- Count-based: `field:k:window_count:count` (e.g., `temp:2.5:20:5`) +- Duration-based: `field:k:window_count:duration` (e.g., `temp:2.5:20:2min`) - Multiple thresholds separated by `@` +- `k` must not be negative, `window_count` must be 2 or greater, the count must be 1 or greater + +#### Issue: Alerts are logged but never delivered + +**Solution**: Look for `Suppressed count alert` or `Suppressed duration alert` warnings. They mean flip suppression is active. Raise `state_change_count`, or remove it to disable suppression. #### Issue: Too many false positive alerts @@ -361,6 +385,8 @@ Counts transitions between normal and anomalous states within the window to prev 3. Enable flip suppression with `state_change_count` 4. Increase the window size for more stable statistics +If the log line reports `mad=0.000`, the window has no spread and `k` has no effect. Require the change to persist with a count or duration threshold instead. + #### Issue: Missing anomalies (false negatives) **Solution**: @@ -371,26 +397,28 @@ Counts transitions between normal and anomalous states within the window to prev ### Debugging tips -1. **Monitor deque sizes**: +1. **Check whether windows are still filling up**: ```bash - influxdb3 query --database YOUR_DATABASE "SELECT * FROM system.processing_engine_logs WHERE log_text LIKE '%Deque%'" + influxdb3 query --database YOUR_DATABASE "SELECT * FROM system.processing_engine_logs WHERE log_text LIKE '%Waiting for%points for MAD%'" ``` -2. **Check MAD calculations**: +2. **Check MAD calculations** (logged for detected outliers only): ```bash - influxdb3 query --database YOUR_DATABASE "SELECT * FROM system.processing_engine_logs WHERE log_text LIKE '%MAD:%'" + influxdb3 query --database YOUR_DATABASE "SELECT * FROM system.processing_engine_logs WHERE log_text LIKE '%MAD calculation%'" ``` 3. **Test with known anomalies**: Write test data with obvious outliers to verify detection ### Performance considerations -- **Memory usage**: Each field maintains a deque of `window_count` values +- **Memory usage**: Each field and series maintains a deque of `window_count` values - **Computation**: MAD is computed on every data write for monitored fields -- **Caching**: Measurement and tag names are cached for 1 hour -- **Notification retries**: Failed notifications retry up to 3 times with exponential backoff +- **Caching**: Measurement and tag names are cached for 1 hour, the loaded configuration for 10 minutes +- **Early exit**: Writes that contain no rows of the configured measurement return before thresholds, senders and tags are parsed; the configuration and the table list come from the cache +- **Notification delivery**: Each alert is sent in a single attempt with a 5-second timeout; retries would hold up the write path +- **Logging**: MAD calculations are logged only for points detected as outliers, so a calm table produces two log lines per write ## Questions/Comments diff --git a/influxdata/mad_check/mad_anomaly_config_data_writes.toml b/influxdata/mad_check/mad_anomaly_config_data_writes.toml index cb3a3c0..9462ad7 100644 --- a/influxdata/mad_check/mad_anomaly_config_data_writes.toml +++ b/influxdata/mad_check/mad_anomaly_config_data_writes.toml @@ -12,14 +12,15 @@ measurement = "your_measurement" # e.g., "cpu", "temperature", "home" # MAD threshold conditions for anomaly detection # Format: [[field, k, window_count, threshold], ...] # - field: numeric field name (string) -# - k: multiplier of MAD for cutoff (float) -# - window_count: number of recent points to compute median and MAD (integer) -# - threshold: integer (count-based) or duration string (e.g., "2m") for duration-based triggering -# Supported units for duration: s (seconds), min (minutes), h (hours), d (days), w (weeks) -mad_thresholds = [["field1", 2.0, 5, "your_threshold"]] # e.g., [["temp", 2.0, 5, 1], ["load", 3.5, 10, "2m"]] +# - k: multiplier of MAD for cutoff (float >= 0) +# - window_count: number of recent points to compute median and MAD (integer, 2-10000) +# - threshold: integer >= 1 (consecutive outliers) or duration string (e.g., "2min") +# Supported units for duration: us (microseconds), ms (milliseconds), s (seconds), +# min (minutes), h (hours), d (days), w (weeks) +mad_thresholds = [["field1", 2.0, 5, "your_threshold"]] # e.g., [["temp", 2.0, 5, 1], ["load", 3.5, 10, "2min"]] # Notification channels -# Specify a dot-separated list of notification channels (strings) +# Specify a list of notification channels (strings) senders = ["your_channel"] # e.g., ["slack"], ["http", "sms"] ########## Optional Parameters ########## @@ -27,8 +28,10 @@ senders = ["your_channel"] # e.g., ["slack"], ["http", "sms"] # Specify the token (string); can also be provided via INFLUXDB3_AUTH_TOKEN environment variable #influxdb3_auth_token = "your_api_token" # e.g., "apiv3_AuHk_8LYFHTa1QMccT..." -# Maximum allowed flips in recent values before suppressing notifications -# Specify an integer ≥ 0; default is 0 (disabled) +# Number of transitions between normal and outlier state, within the MAD window, +# at which notifications are suppressed +# Specify an integer ≥ 2; default is 0 (suppression disabled), and 1 is treated as 0 +# Requires window_count >= threshold + state_change_count for count-based thresholds #state_change_count = 2 # e.g., 2 # Template for count-based notifications diff --git a/influxdata/mad_check/mad_check_plugin.py b/influxdata/mad_check/mad_check_plugin.py index ed61e3e..8fde081 100644 --- a/influxdata/mad_check/mad_check_plugin.py +++ b/influxdata/mad_check/mad_check_plugin.py @@ -10,8 +10,8 @@ }, { "name": "mad_thresholds", - "example": "temp:'2.5':20:5@load:3:10:2m", - "description": "Threshold conditions for MAD-based anomaly detection (e.g., field:k:window_count:threshold). Multiple conditions separated by '@'.", + "example": "temp:2.5:20:5@load:3:10:2min", + "description": "Threshold conditions for MAD-based anomaly detection in the form 'field:k:window_count:threshold', separated by '@'. window_count is between 2 and 10000. The threshold is either a count of consecutive outliers or a duration such as 30s, 5min, 2h, 1d.", "required": true }, { @@ -29,7 +29,7 @@ { "name": "state_change_count", "example": "2", - "description": "Maximum allowed flips (changes) in recent values before suppressing notifications. If 0, suppression is disabled. Default: 0.", + "description": "Number of transitions between normal and outlier state, within the MAD window, at which notifications are suppressed. Use 2 or greater; 1 would suppress every alert and is treated as 0. Default: 0 (suppression disabled).", "required": false }, { @@ -119,7 +119,7 @@ { "name": "config_file_path", "example": "config.toml", - "description": "Path to config file to override args. Format: 'config.toml'.", + "description": "Path to a TOML config file that replaces the trigger arguments entirely. Format: 'config.toml'.", "required": false } ] @@ -128,19 +128,22 @@ import json import os -import random import re -import time -import tomllib import uuid from collections import defaultdict, deque from datetime import datetime, timedelta, timezone -from pathlib import Path from statistics import median from string import Template from urllib.parse import urlparse import requests +from influxdata_plugin_utils.config import Validator, load_plugin_config +from influxdata_plugin_utils.introspection import get_table_names, get_tag_names +from influxdata_plugin_utils.parsing import ( + parse_delimited_list, + parse_int, + parse_timedelta, +) # Supported sender types with their required arguments AVAILABLE_SENDERS = { @@ -159,112 +162,131 @@ # List of keywords to exclude from argument validation in AVAILABLE_SENDERS EXCLUDED_KEYWORDS = ["headers", "token", "sid"] - -def get_all_measurements(influxdb3_local) -> list[str]: +_DEFAULT_COUNT_TEXT = ( + "MAD count alert: Field $field in $table outlier for $threshold_count " + "consecutive points. Tags: $tags" +) +_DEFAULT_TIME_TEXT = ( + "MAD duration alert: Field $field in $table outlier for $threshold_time. " + "Tags: $tags" +) + + +_WRITES_VALIDATORS = [ + Validator("measurement", required=True, cast=str), + Validator("mad_thresholds", required=True), + Validator("senders", required=True), + Validator( + "port_override", + default=8181, + cast=lambda raw: parse_int(raw, minimum=1, maximum=65535), + ), + Validator("notification_path", default="notify", cast=str), + Validator( + "state_change_count", default=0, cast=lambda raw: parse_int(raw, minimum=0) + ), + Validator("notification_count_text", default=_DEFAULT_COUNT_TEXT, cast=str), + Validator("notification_time_text", default=_DEFAULT_TIME_TEXT, cast=str), +] + +_WRITES_CONFIG_CACHE_KEY = "mad_check:writes_config" +_WRITES_CONFIG_TTL_SECONDS = 10 * 60 + +# window_count bounds: below two points the MAD is always zero, and one deque of +# _MAX_WINDOW_COUNT values is kept per series +_MIN_WINDOW_COUNT = 2 +_MAX_WINDOW_COUNT = 10_000 + + +def _load_config( + influxdb3_local, args: dict | None, validators: list, task_id: str +) -> dict | None: """ - Retrieves a list of all tables of type 'BASE TABLE' from cache or the current InfluxDB database. + Load the plugin configuration, applying defaults and type casts. Args: influxdb3_local: InfluxDB client instance. + args (dict | None): Runtime arguments of the trigger. + validators (list): Validators providing defaults and casts. + task_id (str): Unique task identifier. Returns: - list[str]: List of table names (e.g., ["cpu", "memory", "disk"]). - """ - # check cache first - measurements: list = influxdb3_local.cache.get("measurements") - if measurements: - return measurements - - # if not in cache, query the database - result: list = influxdb3_local.query("SHOW TABLES") - measurements = [ - row["table_name"] for row in result if row.get("table_type") == "BASE TABLE" - ] - - # cache the result for 1 hour - influxdb3_local.cache.put(f"measurements", measurements, 60 * 60) - - return measurements - - -def get_tag_names(influxdb3_local, measurement: str, task_id: str) -> list[str]: - """ - Retrieves the list of tag names for a measurement from cache or the database. - - Args: - influxdb3_local: InfluxDB client instance. - measurement (str): Name of the measurement to query. - task_id (str): The task ID. - - Returns: - list[str]: List of tag names with 'Dictionary(Int32, Utf8)' data type. - """ - # check cache first - tags: list = influxdb3_local.cache.get(f"{measurement}_tags") - if tags: - return tags - - # if not in cache, query the database - query = """ - SELECT column_name - FROM information_schema.columns - WHERE table_name = $measurement - AND data_type = 'Dictionary(Int32, Utf8)' + dict | None: Config values keyed by lower-case name, or None if loading failed. """ - res: list[dict] = influxdb3_local.query(query, {"measurement": measurement}) - - if not res: - influxdb3_local.info( - f"[{task_id}] No tags found for measurement '{measurement}'." + args = args or {} + config_file_path = args.get("config_file_path") + if config_file_path and not str(config_file_path).endswith(".toml"): + influxdb3_local.error( + f"[{task_id}] Invalid config file format: expected a .toml file" ) - return [] + return None - tag_names: list[str] = [tag["column_name"] for tag in res] - - # cache the result for 1 hour - influxdb3_local.cache.put(f"{measurement}_tags", tag_names, 60 * 60) + try: + loaded = load_plugin_config( + args, + validators=validators, + env_keys=["INFLUXDB3_AUTH_TOKEN"], + source="toml" if config_file_path else "args", + ) + except Exception as e: + influxdb3_local.error(f"[{task_id}] Failed to load configuration: {e}") + return None - return tag_names + return {key.lower(): value for key, value in loaded.as_dict().items()} def generate_cache_key( measurement: str, field: str, - k: float | int | str, + discriminator: float | int | str, suffix: str, tags: list[str], row: dict, ) -> str: """ - Generate a consistent cache key string combining measurement, field, k, suffix, and tag values. + Generate a consistent cache key from the measurement, field, suffix, and tag values. Args: measurement (str): Measurement (table) name. field (str): Field name being checked. - k (float|int|str): Multiplier or identifier used in key. - suffix (str): Identifier (e.g., "count-time", "time-time", "deque", "values"). + discriminator (float|int|str): Value separating keys of different thresholds. + suffix (str): Identifier (e.g., "count-count", "time-time", "deque", "flips"). tags (list[str]): List of tag column names to include. row (dict): Current row data; used to extract tag values. Returns: - str: Formatted key, e.g. "cpu:temp:2.0:count-time:host=server1:region=us-west". + str: Formatted key, e.g. "cpu:temp:2.0-20:count-count:host=server1:region=us-west". """ - base = f"{measurement}:{field}:{k}:{suffix}" + base = f"{measurement}:{field}:{discriminator}:{suffix}" for tag in sorted(tags): tag_val = row.get(tag, "None") base += f":{tag}={tag_val}" return base -def parse_senders(influxdb3_local, args: dict, task_id: str) -> dict: +def read_counter(influxdb3_local, cache_key: str) -> int: + """Read an outlier counter, treating a missing or non-numeric entry as zero.""" + try: + return int(influxdb3_local.cache.get(cache_key)) + except (TypeError, ValueError): + return 0 + + +def read_window(influxdb3_local, cache_key: str, window_count: int) -> deque: + """Read a cached deque, replacing it when it is missing or sized differently.""" + window = influxdb3_local.cache.get(cache_key, default=deque(maxlen=window_count)) + if not isinstance(window, deque) or window.maxlen != window_count: + window = deque(maxlen=window_count) + return window + + +def parse_senders(influxdb3_local, config: dict, task_id: str) -> dict: """ - Parse and validate sender configurations from input arguments. + Parse and validate sender configurations from the loaded config. Args: influxdb3_local: InfluxDB client instance. - args (dict): Input arguments containing: - - "senders": dot-separated list of sender types (e.g., "slack.http"). - - For each sender, its own required keys (see AVAILABLE_SENDERS). + config (dict): Loaded config containing "senders" and related settings. task_id (str): Unique task identifier used for logging context. Returns: @@ -282,39 +304,31 @@ def parse_senders(influxdb3_local, args: dict, task_id: str) -> dict: Exception: If no valid senders are found after parsing. """ senders_config: defaultdict = defaultdict(dict) - - senders: str | list = args.get("senders") - if args["use_config_file"]: - if not isinstance(senders, list): - raise Exception( - f"[{task_id}] 'senders' must be a list when using config file" - ) - else: - senders = senders.split(".") + senders: list = parse_delimited_list(config["senders"], sep=".") for sender in senders: if sender not in AVAILABLE_SENDERS: influxdb3_local.warn(f"[{task_id}] Invalid sender type: {sender}") continue for key in AVAILABLE_SENDERS[sender]: - if key not in args and not any(ex in key for ex in EXCLUDED_KEYWORDS): + if key not in config and not any(ex in key for ex in EXCLUDED_KEYWORDS): influxdb3_local.warn( f"[{task_id}] Required key '{key}' missing for sender '{sender}'" ) senders_config.pop(sender, None) break if "url" in key and not validate_webhook_url( - influxdb3_local, sender, args[key], task_id + influxdb3_local, sender, config[key], task_id ): senders_config.pop(sender, None) break - if key not in args: + if key not in config: continue - senders_config[sender][key] = args[key] + senders_config[sender][key] = config[key] if not senders_config: - raise Exception(f"[{task_id}] No valid senders configured") + raise Exception("No valid senders configured") return senders_config @@ -322,8 +336,7 @@ def send_notification( influxdb3_local, port: int, path: str, token: str, payload: dict, task_id: str ) -> None: """ - Send a JSON POST to the given InfluxDB 3 webhook endpoint, with up to - 3 retry attempts and randomized backoff delays between attempts. + Send a JSON POST to the given InfluxDB 3 webhook endpoint. Args: influxdb3_local: InfluxDB client instance. @@ -332,72 +345,25 @@ def send_notification( token (str): API v3 token string (without the "Bearer " prefix). payload (dict): Dict to serialize as JSON in the POST body. task_id (str): Unique task identifier. - - Raises: - requests.RequestException: If all retries fail or a non-2xx response is received. """ url: str = f"http://localhost:{port}/api/v3/engine/{path}" headers: dict = { "Content-Type": "application/json", "Authorization": f"Bearer {token}", } - data: str = json.dumps(payload) - - max_retries: int = 3 - timeout: float = 5.0 - - for attempt in range(1, max_retries + 1): - try: - resp = requests.post(url, headers=headers, data=data, timeout=timeout) - resp.raise_for_status() # raises on 4xx/5xx - influxdb3_local.info( - f"[{task_id}] Alert sent to notification plugin with results: {resp.json()['results']}" - ) - break - except requests.RequestException as e: - influxdb3_local.warn( - f"[{task_id}] [Attempt {attempt}/{max_retries}] Error sending alert to notification plugin: {e}" - ) - if attempt < max_retries: - wait = random.uniform(1, 4) - influxdb3_local.info( - f"[{task_id}] Retrying sending alert to notification plugin in {wait:.1f} seconds." - ) - time.sleep(wait) - else: - influxdb3_local.error( - f"[{task_id}] Failed to send alert to notification plugin after {max_retries} attempts: {e}" - ) - - -def parse_port_override(args: dict, task_id: str) -> int: - """ - Parse and validate the 'port_override' argument, converting it from string to int. - - Args: - args (dict): Runtime arguments containing 'port_override'. - task_id (str): Unique task identifier for logging context. - - Returns: - int: Parsed port number (1–65535), or 8181 if not provided. - - Raises: - Exception: If 'port_override' is provided but is not a valid integer in the range 1–65535. - """ - raw: str | int = args.get("port_override", 8181) try: - port = int(raw) - except (TypeError, ValueError): - raise Exception(f"[{task_id}] Invalid port_override, not an integer: {raw!r}") - - # Validate port range - if not (1 <= port <= 65535): - raise Exception( - f"[{task_id}] Invalid port_override, must be between 1 and 65535: {port}" + resp = requests.post( + url, headers=headers, data=json.dumps(payload), timeout=5.0 + ) + resp.raise_for_status() # raises on 4xx/5xx + influxdb3_local.info( + f"[{task_id}] Alert sent to notification plugin with results: {resp.json()['results']}" + ) + except requests.RequestException as e: + influxdb3_local.error( + f"[{task_id}] Failed to send alert to notification plugin: {e}" ) - - return port def validate_webhook_url(influxdb3_local, service: str, url: str, task_id: str) -> bool: @@ -442,357 +408,367 @@ def interpolate_notification_text(text: str, row_data: dict) -> str: return Template(text).safe_substitute(row_data) -def _coerce_value(raw: str) -> str | int | float | bool: - """ - Convert a raw string value into int, float, bool, or str. - """ - raw = raw.strip() - # Quoted string - if (raw.startswith('"') and raw.endswith('"')) or ( - raw.startswith("'") and raw.endswith("'") - ): - raw = raw[1:-1] - # Boolean - if raw.lower() in ("true", "false"): - return raw.lower() == "true" - # Integer - if re.fullmatch(r"-?\d+", raw): - return int(raw) - # Float - if re.fullmatch(r"-?\d+\.\d*", raw): - return float(raw) - # Plain string - return raw - - -def parse_mad_thresholds(influxdb3_local, args: dict, task_id: str) -> list[tuple]: +def _strip_quotes(raw) -> str: + """Remove one pair of surrounding quotes from a value.""" + text: str = str(raw).strip() + if len(text) >= 2 and text[0] == text[-1] and text[0] in ("'", '"'): + return text[1:-1] + return text + + +def _parse_threshold_param( + influxdb3_local, raw, task_id: str +) -> int | timedelta | None: """ - Parse MAD-based threshold definitions from args into structured tuples or use values from config file. + Parse the fourth part of a threshold into a consecutive count or a duration. - Args: - influxdb3_local: InfluxDB client for logging. - args (dict): Must include "mad_thresholds" key, a string of '@'-separated segments. - task_id (str): Unique identifier for logging. - - Each segment has the form: - field_name:k:window_count:threshold - where: - - field_name (str): Name of the numeric field. - - k (float): Multiplier for MAD. - - window_count (int): Number of recent points to compute median/MAD. - - threshold: - • If integer → count-based: trigger after this many consecutive outliers. - • If duration string (e.g., "2m", "30s") → duration-based. + A bare integer is a count of consecutive outliers; anything else is a duration + such as '30s' or '2h'. Returns: - list[list[str, float, int, int|timedelta]]: - Each list: [field_name, k, window_count, threshold_param]. + int | timedelta | None: The parsed threshold, or None when it is invalid. + """ + if isinstance(raw, bool): + influxdb3_local.warn(f"[{task_id}] Invalid threshold parameter: {raw!r}") + return None - Raises: - Exception: If no valid segments are parsed. + if isinstance(raw, int) or re.fullmatch(r"-?\d+", str(raw).strip()): + count = int(raw) + if count < 1: + influxdb3_local.warn( + f"[{task_id}] Invalid threshold count {count}, must be 1 or greater" + ) + return None + return count + + try: + duration: timedelta = parse_timedelta(raw) + except ValueError as e: + influxdb3_local.warn(f"[{task_id}] Invalid threshold duration {raw!r}: {e}") + return None + + if duration <= timedelta(0): + influxdb3_local.warn( + f"[{task_id}] Invalid threshold duration {raw!r}, must be positive" + ) + return None + return duration + + +def _parse_mad_entry(influxdb3_local, entry, task_id: str) -> tuple | None: """ - valid_units: dict = { - "s": "seconds", - "min": "minutes", - "h": "hours", - "d": "days", - "w": "weeks", - } - raw_input: str | list = args.get("mad_thresholds") - results: list = [] + Validate one [field, k, window_count, threshold] definition. + + Returns: + tuple | None: (field_name, k, window_count, threshold_param), or None when any + part is invalid. + """ + field_name: str = str(entry[0]).strip() + if not field_name: + influxdb3_local.warn(f"[{task_id}] Invalid threshold {entry}: empty field name") + return None + + try: + k: float = float(_strip_quotes(entry[1])) + except (TypeError, ValueError): + influxdb3_local.warn(f"[{task_id}] Invalid k in threshold {entry}") + return None + if k < 0: + influxdb3_local.warn( + f"[{task_id}] Invalid k {k} in threshold {entry}, must not be negative" + ) + return None - if args["use_config_file"]: - if not isinstance(raw_input, list): - raise Exception( - f"[{task_id}] 'mad_thresholds' must be a list when using config file" + try: + window_count: int = parse_int( + entry[2], minimum=_MIN_WINDOW_COUNT, maximum=_MAX_WINDOW_COUNT + ) + except ValueError as e: + influxdb3_local.warn( + f"[{task_id}] Invalid window_count in threshold {entry}: {e}" + ) + return None + + threshold_param = _parse_threshold_param(influxdb3_local, entry[3], task_id) + if threshold_param is None: + return None + + return field_name, k, window_count, threshold_param + + +def _mad_thresholds_from_entries(influxdb3_local, entries: list, task_id: str) -> list: + """Parse thresholds given as [field, k, window_count, threshold] entries.""" + thresholds: list = [] + + for entry in entries: + if not isinstance(entry, (list, tuple)) or len(entry) != 4: + influxdb3_local.warn( + f"[{task_id}] Invalid threshold '{entry}', expected " + f"[field, k, window_count, threshold]" ) - for threshold in raw_input: - try: - field_name: str = str(threshold[0]) - k: float = float(threshold[1]) - window_count: int = int(threshold[2]) - threshold_input: int | str = threshold[3] - if isinstance(threshold_input, str): - num_part, unit_part = "", "" - for unit in sorted(valid_units.keys(), key=len, reverse=True): - if threshold_input.endswith(unit): - num_part = threshold_input[: -len(unit)] - unit_part = unit - break - if not num_part or unit_part not in valid_units: - influxdb3_local.warn( - f"[{task_id}] Invalid threshold format '{threshold_input}'" - ) - continue - try: - num = int(num_part) - except ValueError: - influxdb3_local.warn( - f"[{task_id}] Invalid number in threshold '{threshold_input}'" - ) - continue - threshold_param = timedelta(**{valid_units[unit_part]: num}) - elif isinstance(threshold_input, int): - threshold_param = threshold_input - else: - influxdb3_local.warn( - f"[{task_id}] Invalid threshold format '{threshold_input}'" - ) - continue - results.append([field_name, k, window_count, threshold_param]) - except Exception: - influxdb3_local.warn( - f"[{task_id}] Invalid threshold definition: {threshold}, skipping" - ) - return results + continue + parsed = _parse_mad_entry(influxdb3_local, entry, task_id) + if parsed is not None: + thresholds.append(parsed) + + return thresholds - segments: list = [seg.strip() for seg in raw_input.split("@") if seg.strip()] - for seg in segments: - parts = seg.split(":") + +def _mad_thresholds_from_string(influxdb3_local, raw: str, task_id: str) -> list: + """Parse thresholds given as ':::' joined by '@'.""" + thresholds: list = [] + + for segment in parse_delimited_list(raw, sep="@"): + parts: list[str] = segment.split(":") if len(parts) != 4: influxdb3_local.warn( - f"[{task_id}] Invalid segment '{seg}'; expected 4 parts delimited by ':'" + f"[{task_id}] Skipping invalid threshold '{segment}' – expected 4 parts " + f"delimited by ':'" ) continue + parsed = _parse_mad_entry(influxdb3_local, parts, task_id) + if parsed is not None: + thresholds.append(parsed) - field_name = parts[0].strip() - try: - k: str | float = parts[1].strip() - if k[0] == k[-1] and k[0] in ("'", '"'): - k = k[1:-1] - k = float(k) - except ValueError: - influxdb3_local.warn(f"[{task_id}] Invalid k in segment '{seg}'") - continue + return thresholds - try: - window_count: int = int(parts[2].strip()) - except ValueError: - influxdb3_local.warn(f"[{task_id}] Invalid window_count in '{seg}'") - continue - raw_thresh = parts[3].strip() - if re.fullmatch(r"-?\d+", raw_thresh): - threshold_param: int | timedelta = int(raw_thresh) - else: - num_part, unit_part = "", "" - for unit in sorted(valid_units.keys(), key=len, reverse=True): - if raw_thresh.endswith(unit): - num_part = raw_thresh[: -len(unit)] - unit_part = unit - break - if not num_part or unit_part not in valid_units: - influxdb3_local.warn( - f"[{task_id}] Invalid threshold format '{raw_thresh}'" - ) - continue - try: - num = int(num_part) - except ValueError: - influxdb3_local.warn( - f"[{task_id}] Invalid number in threshold '{raw_thresh}'" - ) - continue - threshold_param = timedelta(**{valid_units[unit_part]: num}) +def parse_mad_thresholds(influxdb3_local, config: dict, task_id: str) -> list: + """ + Parse MAD threshold definitions into structured tuples. + + Thresholds come either as entries of [field, k, window_count, threshold] (TOML) or + as a string of ':::' expressions separated by '@'. + + Args: + influxdb3_local: InfluxDB client instance. + config (dict): Loaded config containing "mad_thresholds". + task_id (str): Unique task identifier. + + Returns: + list[tuple]: Tuples of (field_name, k, window_count, count_or_duration). + + Example: + 'temp:2.5:20:5@load:3:10:2min' + [ + ("temp", 2.5, 20, 5), + ("load", 3.0, 10, datetime.timedelta(minutes=2)), + ] + + Raises: + Exception: If no valid thresholds are parsed. + """ + raw: str | list = config["mad_thresholds"] + + if isinstance(raw, (list, tuple)): + thresholds = _mad_thresholds_from_entries(influxdb3_local, raw, task_id) + elif isinstance(raw, str): + thresholds = _mad_thresholds_from_string(influxdb3_local, raw, task_id) + else: + raise Exception( + "'mad_thresholds' must be a list of entries or a string, " + f"got {type(raw).__name__}" + ) - results.append([field_name, k, window_count, threshold_param]) + # Repeated definitions share one cache key, so each one would advance the same + # counter and reach the threshold ahead of time + unique_thresholds: list = [] + for threshold in thresholds: + if threshold in unique_thresholds: + influxdb3_local.warn( + f"[{task_id}] Skipping duplicate threshold {threshold}" + ) + continue + unique_thresholds.append(threshold) - if not results: - raise Exception(f"[{task_id}] No valid MAD threshold segments in '{raw_input}'") - return results + if not unique_thresholds: + raise Exception("No valid MAD thresholds provided.") + return unique_thresholds -def check_state_changes(cached_values: deque, max_flips: int) -> bool: +def check_state_changes(outlier_flags: deque, state_change_count: int) -> bool: """ - Count how many times the value changes in a deque; suppress if flips exceed max_flips. + Count transitions between normal and outlier state in the window. Args: - cached_values (deque): Recent field values (size = state_change_window). - max_flips (int): Maximum allowed flips in that window. + outlier_flags (deque): Recent outlier flags of one field. + state_change_count (int): Number of transitions at which notifications are + suppressed. 0 disables suppression. Returns: - bool: True if actual flips ≤ max_flips; False otherwise. + bool: True while the number of transitions stays below state_change_count. """ - if len(cached_values) < 2 or max_flips == 0: + if len(outlier_flags) < 2 or state_change_count == 0: return True flips: int = 0 - prev = None - first = True - for v in cached_values: - if first: - prev = v - first = False - continue - if v != prev: + previous = outlier_flags[0] + for flag in list(outlier_flags)[1:]: + if flag != previous: flips += 1 - if flips >= max_flips: + if flips >= state_change_count: return False - prev = v + previous = flag return True +def normalize_state_change_count( + influxdb3_local, state_change_count: int, task_id: str +) -> int: + """Treat 1 as disabled: an alert after normal data always records one transition.""" + if state_change_count != 1: + return state_change_count + + influxdb3_local.warn( + f"[{task_id}] state_change_count=1 would suppress every alert, treating it as 0 " + f"(disabled); use 2 or greater to suppress flapping" + ) + return 0 + + +def warn_on_inert_suppression( + influxdb3_local, mad_thresholds: list, state_change_count: int, task_id: str +) -> None: + """Warn about count thresholds whose window leaves no room for enough transitions.""" + if state_change_count == 0: + return + + for field_name, _k, window_count, threshold_param in mad_thresholds: + if isinstance(threshold_param, timedelta): + continue + if state_change_count > window_count - threshold_param: + influxdb3_local.warn( + f"[{task_id}] Flip suppression never triggers for '{field_name}' with " + f"count threshold {threshold_param}: set window_count to " + f"{threshold_param + state_change_count} or more" + ) + + def process_writes(influxdb3_local, table_batches: list, args: dict | None = None): """ WAL-Flush trigger applying MAD-based anomaly detection on fields without querying data repeatedly. - Uses in-memory deques in cache to maintain the last N values per field+series, computing median/MAD - incrementally. Supports both count- and duration-based triggers, plus flip-detection suppression. + Uses in-memory deques in cache to maintain the last N values per field and series, + computing median/MAD incrementally. Supports both count- and duration-based + thresholds, plus suppression of alerts on data that flips in and out of the + outlier state. Args: influxdb3_local: InfluxDB client for logging, cache, and minimal queries. table_batches (list): Each element is {"table_name": str, "rows": [dict, ...]}. - args (dict): - Required: - - measurement (str): Measurement name to monitor. - - mad_thresholds (str): '@'-separated segments "field:k:window:threshold". - - senders (str): Dot-separated notification channels. - Optional: - - config_file_path (str): path to config file to override args. - - state_change_count (int): Max flips allowed before suppressing. - - port_override (int): HTTP port for notification plugin (default 8181). - - influxdb3_auth_token (str): API v3 token (or via ENV var). - - notification_path (str): Path on engine (default "notify"). - - notification_count_text (str): Template for count-based messages. - - notification_time_text (str): Template for duration-based messages. + args (dict): Runtime arguments of the trigger. Exceptions: All exceptions are caught and logged via influxdb3_local.error. """ + if not table_batches: + return + task_id: str = str(uuid.uuid4()) - influxdb3_local.info(f"[{task_id}] Starting writes processing with args: {args}") - - # Override args with config file if specified - if args: - if path := args.get("config_file_path", None): - if not path.endswith(".toml"): - influxdb3_local.error( - f"[{task_id}] Invalid config file format: expected a .toml file" - ) - return - try: - plugin_dir_var: str | None = os.getenv("PLUGIN_DIR", None) - if plugin_dir_var: - file_path = Path(plugin_dir_var) / path - else: - # Fallbacks for servers where the operator has not exported PLUGIN_DIR: - # - INFLUXDB3_PLUGIN_DIR: set when the server is configured via env var - # - VIRTUAL_ENV: exported by the processing engine; default venv is /.venv - candidates: list[str] = [] - if influxdb3_plugin_dir := os.environ.get("INFLUXDB3_PLUGIN_DIR"): - candidates.append(influxdb3_plugin_dir) - if virtual_env := os.environ.get("VIRTUAL_ENV"): - candidates.append(str(Path(virtual_env).parent)) - - resolved = None - for base in candidates: - candidate = Path(base) / path - if candidate.exists(): - resolved = candidate - break - - if resolved is None: - candidates_str = ", ".join(candidates) if candidates else "none available" - influxdb3_local.error( - f"[{task_id}] PLUGIN_DIR env var not set and config file path " - f"'{path}' was not found via fallbacks (tried: {candidates_str})" - ) - return - file_path = resolved - influxdb3_local.info(f"[{task_id}] Reading config file {file_path}") - with open(file_path, "rb") as f: - args = tomllib.load(f) - args["use_config_file"] = True - influxdb3_local.info(f"[{task_id}] New args content: {args}") - except Exception: - influxdb3_local.error(f"[{task_id}] Failed to read config file") - return - else: - args["use_config_file"] = False - - if ( - not args - or "measurement" not in args - or "mad_thresholds" not in args - or "senders" not in args - ): + config: dict | None = influxdb3_local.cache.get(_WRITES_CONFIG_CACHE_KEY) + if config is None: + config = _load_config(influxdb3_local, args, _WRITES_VALIDATORS, task_id) + if config is None: + return + influxdb3_local.cache.put( + _WRITES_CONFIG_CACHE_KEY, config, _WRITES_CONFIG_TTL_SECONDS + ) + + measurement: str = config["measurement"] + if measurement not in get_table_names(influxdb3_local): influxdb3_local.error( - f"[{task_id}] Missing required arguments: measurement, mad_thresholds, or senders" + f"[{task_id}] Measurement '{measurement}' not found in database" ) return - measurement: str = args["measurement"] - all_measurements: list = get_all_measurements(influxdb3_local) - if measurement not in all_measurements: - influxdb3_local.error(f"[{task_id}] Measurement '{measurement}' not found") + monitored_batches: list = [ + table_batch + for table_batch in table_batches + if table_batch.get("table_name") == measurement + ] + if not monitored_batches: return + influxdb3_local.info(f"[{task_id}] Starting writes process") + try: - # Parse configuration - mad_thresholds: list = parse_mad_thresholds(influxdb3_local, args, task_id) - senders_config: dict = parse_senders(influxdb3_local, args, task_id) - tags: list = get_tag_names(influxdb3_local, measurement, task_id) - port_override: int = parse_port_override(args, task_id) - state_change_count: int = int(args.get("state_change_count", 0)) - notification_path: str = args.get("notification_path", "notify") - influxdb3_auth_token: str = args.get("influxdb3_auth_token") or os.getenv( - "INFLUXDB3_AUTH_TOKEN" + mad_thresholds: list = parse_mad_thresholds(influxdb3_local, config, task_id) + influxdb3_local.info(f"[{task_id}] MAD thresholds: {mad_thresholds}") + + senders_config: dict = parse_senders(influxdb3_local, config, task_id) + port_override: int = config["port_override"] + state_change_count: int = normalize_state_change_count( + influxdb3_local, config["state_change_count"], task_id + ) + notification_path: str = config["notification_path"] + influxdb3_auth_token: str = ( + config.get("influxdb3_auth_token") + or os.getenv("INFLUXDB3_AUTH_TOKEN") + or "" ) if not influxdb3_auth_token: - influxdb3_local.error(f"[{task_id}] Missing influxdb3_auth_token") + influxdb3_local.error( + f"[{task_id}] Missing required argument: influxdb3_auth_token" + ) return - notification_count_tpl: str = args.get( - "notification_count_text", - "MAD count alert: Field $field in $table outlier for $threshold_count consecutive points. Tags: $tags", - ) - notification_time_tpl: str = args.get( - "notification_time_text", - "MAD duration alert: Field $field in $table outlier for $threshold_time. Tags: $tags", + notification_count_tpl: str = config["notification_count_text"] + notification_time_tpl: str = config["notification_time_text"] + + warn_on_inert_suppression( + influxdb3_local, mad_thresholds, state_change_count, task_id ) - # Process each batch of newly written rows - for batch in table_batches: - if batch.get("table_name") != measurement: - continue + tags: list = get_tag_names(influxdb3_local, measurement) - for row in batch.get("rows", []): + for batch in monitored_batches: + for row in batch["rows"]: tag_str: str = ", ".join(f"{t}={row.get(t, 'None')}" for t in tags) + # A MAD window depends only on the field and its size, so thresholds + # sharing one window update it once per row. + row_windows: dict = {} + for field_name, k, window_count, threshold_param in mad_thresholds: - # Extract current field value + is_duration: bool = isinstance(threshold_param, timedelta) + state_suffix: str = "time-time" if is_duration else "count-count" + reset_value: str = "" if is_duration else "0" + threshold_label: str = ( + f"{threshold_param.total_seconds()}s" + if is_duration + else str(threshold_param) + ) + threshold_id: str = f"{k}-{window_count}-{threshold_label}" + state_key: str = generate_cache_key( + measurement, field_name, threshold_id, state_suffix, tags, row + ) + current_val = row.get(field_name) if current_val is None or not isinstance(current_val, (int, float)): - influxdb3_local.info( - f"[{task_id}] Field '{field_name}' missing or non-numeric → reset" - ) - # Reset any running state - count_key = generate_cache_key( - measurement, field_name, k, "count-count", tags, row - ) - time_key = generate_cache_key( - measurement, field_name, k, "time-time", tags, row - ) - influxdb3_local.cache.put(count_key, "0") - influxdb3_local.cache.put(time_key, "") + if ( + influxdb3_local.cache.get(state_key, default=reset_value) + != reset_value + ): + influxdb3_local.info( + f"[{task_id}] Field '{field_name}' missing or non-numeric, resetting state for tags: {tag_str}" + ) + influxdb3_local.cache.put(state_key, reset_value) continue now: datetime = datetime.now(timezone.utc) - # Manage deque of size window_count for median/MAD - deque_key: str = generate_cache_key( - measurement, field_name, k, "deque", tags, row - ) - window_deque = influxdb3_local.cache.get( - deque_key, default=deque(maxlen=window_count) - ) - if ( - not isinstance(window_deque, deque) - or window_deque.maxlen != window_count - ): - window_deque = deque(maxlen=window_count) - - window_deque.append(current_val) - influxdb3_local.cache.put(deque_key, window_deque) + # Deque of the last window_count values, used for median/MAD + window_id: tuple = (field_name, window_count) + if window_id not in row_windows: + deque_key: str = generate_cache_key( + measurement, field_name, window_count, "deque", tags, row + ) + window_deque = read_window( + influxdb3_local, deque_key, window_count + ) + window_deque.append(current_val) + influxdb3_local.cache.put(deque_key, window_deque) + row_windows[window_id] = window_deque + window_deque = row_windows[window_id] # Wait until deque is full before computing MAD if len(window_deque) < window_count: @@ -810,28 +786,32 @@ def process_writes(influxdb3_local, table_batches: list, args: dict | None = Non is_outlier: bool = (current_val < lower) or (current_val > upper) - influxdb3_local.info( - f"[{task_id}] MAD calculation for {field_name}: median={med:.3f}, mad={mad:.3f}, " - f"thresholds=({lower:.3f}, {upper:.3f}), current={current_val:.3f}, outlier={is_outlier}, tags: {tag_str}" - ) + if is_outlier: + influxdb3_local.info( + f"[{task_id}] MAD calculation for {field_name}: median={med:.3f}, mad={mad:.3f}, " + f"thresholds=({lower:.3f}, {upper:.3f}), current={current_val:.3f}, outlier=True, tags: {tag_str}" + ) - # Flip-detection deque (size = state_change_window) + # Suppress alerts when the outlier state flips too often + flips_key: str = generate_cache_key( + measurement, field_name, threshold_id, "flips", tags, row + ) + outlier_flags = read_window( + influxdb3_local, flips_key, window_count + ) + outlier_flags.append(is_outlier) + influxdb3_local.cache.put(flips_key, outlier_flags) can_send: bool = check_state_changes( - window_deque, state_change_count + outlier_flags, state_change_count ) # Count-based mode - if not isinstance(threshold_param, timedelta): - count_key: str = generate_cache_key( - measurement, field_name, k, "count-count", tags, row - ) - count_so_far: int = int( - influxdb3_local.cache.get(count_key, default="0") - ) + if not is_duration: + count_so_far: int = read_counter(influxdb3_local, state_key) if is_outlier: count_so_far += 1 - influxdb3_local.cache.put(count_key, str(count_so_far)) + influxdb3_local.cache.put(state_key, str(count_so_far)) influxdb3_local.info( f"[{task_id}] Count-based outlier {count_so_far}/{threshold_param} for {field_name}, tags: {tag_str}" ) @@ -862,26 +842,15 @@ def process_writes(influxdb3_local, table_batches: list, args: dict | None = Non ) else: influxdb3_local.warn( - f"[{task_id}] Suppressed count alert due to flips > {state_change_count}" + f"[{task_id}] Suppressed count alert for {field_name}: outlier state flipped at least {state_change_count} times in the last {window_count} points" ) - influxdb3_local.cache.put(count_key, "0") - - else: - influxdb3_local.warn( - f"[{task_id}] MAD count threshold reached for {measurement}.{field_name} (k={k}) for the {count_so_far}/{threshold_param} time. tags: {tag_str}" - ) + influxdb3_local.cache.put(state_key, "0") else: - influxdb3_local.info( - f"[{task_id}] Count-based outlier cleared for {field_name}, tags: {tag_str}" - ) - influxdb3_local.cache.put(count_key, "0") + influxdb3_local.cache.put(state_key, "0") # Duration-based mode else: - time_key: str = generate_cache_key( - measurement, field_name, k, "time-time", tags, row - ) - start_iso = influxdb3_local.cache.get(time_key, default="") + start_iso = influxdb3_local.cache.get(state_key, default="") if start_iso: try: @@ -893,7 +862,7 @@ def process_writes(influxdb3_local, table_batches: list, args: dict | None = Non if is_outlier: if not start_dt: - influxdb3_local.cache.put(time_key, now.isoformat()) + influxdb3_local.cache.put(state_key, now.isoformat()) influxdb3_local.warn( f"[{task_id}] Duration-based outlier started for {field_name} at {now.isoformat()} (k={k}), tags: {tag_str}" ) @@ -926,9 +895,9 @@ def process_writes(influxdb3_local, table_batches: list, args: dict | None = Non ) else: influxdb3_local.warn( - f"[{task_id}] Suppressed time alert due to flips > {state_change_count}" + f"[{task_id}] Suppressed duration alert for {field_name}: outlier state flipped at least {state_change_count} times in the last {window_count} points" ) - influxdb3_local.cache.put(time_key, "") + influxdb3_local.cache.put(state_key, "") else: influxdb3_local.info( f"[{task_id}] MAD outlier ongoing for {field_name}, elapsed {elapsed}, threshold {threshold_param}, tags: {tag_str}" @@ -938,7 +907,7 @@ def process_writes(influxdb3_local, table_batches: list, args: dict | None = Non influxdb3_local.info( f"[{task_id}] MAD outlier cleared for {field_name}, tags: {tag_str}; resetting" ) - influxdb3_local.cache.put(time_key, "") + influxdb3_local.cache.put(state_key, "") except Exception as e: influxdb3_local.error(f"[{task_id}] Unexpected error: {e}") diff --git a/influxdata/mad_check/manifest.toml b/influxdata/mad_check/manifest.toml index 4f09e6d..1a3d9a3 100644 --- a/influxdata/mad_check/manifest.toml +++ b/influxdata/mad_check/manifest.toml @@ -2,7 +2,7 @@ manifest_schema_version = "1.3" [plugin] name = "mad_check" -version = "1.2.0" +version = "1.3.0" description = "Provides Median Absolute Deviation (MAD)-based anomaly detection using data writes trigger. Maintains in-memory deques for efficient computation and supports count-based and duration-based thresholds." triggers = ["process_writes"] homepage = "https://www.influxdata.com/" @@ -16,7 +16,7 @@ exclude = [ [dependencies] database_version = ">=3.0.0" -python = ["requests"] +python = ["influxdata-plugin-utils>=0.3.0", "requests"] [[dependencies.plugins]] index_url = "https://github.com/influxdata/influxdb3_plugins/releases/download/registry/index.json" diff --git a/influxdata/mad_check/requirements-dev.txt b/influxdata/mad_check/requirements-dev.txt new file mode 100644 index 0000000..57bfbef --- /dev/null +++ b/influxdata/mad_check/requirements-dev.txt @@ -0,0 +1,3 @@ +pytest +influxdata-plugin-utils>=0.3.0 +requests \ No newline at end of file diff --git a/influxdata/mad_check/requirements.txt b/influxdata/mad_check/requirements.txt index 663bd1f..3349a3f 100644 --- a/influxdata/mad_check/requirements.txt +++ b/influxdata/mad_check/requirements.txt @@ -1 +1,2 @@ +influxdata-plugin-utils>=0.3.0 requests \ No newline at end of file diff --git a/influxdata/mad_check/test_mad_check.py b/influxdata/mad_check/test_mad_check.py new file mode 100644 index 0000000..0702814 --- /dev/null +++ b/influxdata/mad_check/test_mad_check.py @@ -0,0 +1,577 @@ +import json +from collections import deque +from datetime import datetime, timedelta, timezone + +import pytest + +import mad_check_plugin as plugin + +TOKEN = "apiv3_secret_token_value" +WEBHOOK = "https://example.com/hook" + +# Four calm values: the next written row completes a window of five +WARMUP = [20.0, 20.5, 21.0, 20.5] + + +class FakeCache: + def __init__(self): + self.store = {} + self.ttls = {} + + def get(self, key, default=None, use_global=None): + return self.store.get(key, default) + + def put(self, key, value, ttl=None, use_global=None): + self.store[key] = value + self.ttls[key] = ttl + + def delete(self, key, use_global=None): + return self.store.pop(key, None) is not None + + +class FakeInfluxdb3Local: + """Stub of the runtime client: logging, trigger-local cache and queries.""" + + def __init__(self, tables=("home",), tags=("host",)): + self.cache = FakeCache() + self.logs = [] + self.tables = list(tables) + self.tags = list(tags) + self.queries = [] + + def info(self, message): + self.logs.append(("info", message)) + + def warn(self, message): + self.logs.append(("warn", message)) + + def error(self, message): + self.logs.append(("error", message)) + + def query(self, query, params=None): + self.queries.append(query) + if "SHOW TABLES" in query: + return [{"table_name": t, "table_type": "BASE TABLE"} for t in self.tables] + return [{"column_name": tag} for tag in self.tags] + + def messages(self, level=None): + return [m for lvl, m in self.logs if level is None or lvl == level] + + +class FakeResponse: + def raise_for_status(self): + return None + + def json(self): + return {"results": "recorded"} + + +@pytest.fixture +def sent(monkeypatch): + """Collect notification payloads instead of posting them.""" + posts = [] + + def fake_post(url, headers=None, data=None, timeout=None): + posts.append({"url": url, "headers": headers, "payload": json.loads(data)}) + return FakeResponse() + + monkeypatch.setattr(plugin.requests, "post", fake_post) + return posts + + +@pytest.fixture +def plugin_dir(monkeypatch, tmp_path): + monkeypatch.setenv("PLUGIN_DIR", str(tmp_path)) + monkeypatch.delenv("INFLUXDB3_AUTH_TOKEN", raising=False) + return tmp_path + + +WRITES_ARGS = { + "measurement": "home", + "mad_thresholds": "temp:2:5:2", + "senders": "http", + "http_webhook_url": WEBHOOK, + "influxdb3_auth_token": TOKEN, +} + + +def batch(rows, table="home"): + return [{"table_name": table, "rows": rows}] + + +def rows(*values, host="a", field="temp"): + return [{"host": host, field: value} for value in values] + + +def state_key(suffix, field="temp", k=2.0, window_count=5, threshold="2", host="a"): + return plugin.generate_cache_key( + "home", + field, + f"{k}-{window_count}-{threshold}", + suffix, + ["host"], + {"host": host}, + ) + + +def window_key(field="temp", window_count=5, host="a"): + return plugin.generate_cache_key( + "home", field, window_count, "deque", ["host"], {"host": host} + ) + + +def seed_window(influxdb3_local, values=WARMUP, field="temp", window_count=5, host="a"): + """Pre-fill the MAD window so the next written row completes it.""" + influxdb3_local.cache.put( + window_key(field, window_count, host), deque(values, maxlen=window_count) + ) + + +# --- parsing ---------------------------------------------------------------- + + +def test_mad_thresholds_from_string(): + assert plugin.parse_mad_thresholds( + FakeInfluxdb3Local(), {"mad_thresholds": "temp:2.5:20:5@load:3:10:2min"}, "tid" + ) == [("temp", 2.5, 20, 5), ("load", 3.0, 10, timedelta(minutes=2))] + + +def test_mad_thresholds_from_toml_entries(): + assert plugin.parse_mad_thresholds( + FakeInfluxdb3Local(), + {"mad_thresholds": [["temp", 2.0, 5, 1], ["load", 3.5, 10, "500ms"]]}, + "tid", + ) == [("temp", 2.0, 5, 1), ("load", 3.5, 10, timedelta(milliseconds=500))] + + +def test_mad_thresholds_accepts_quoted_k_and_dotted_field(): + assert plugin.parse_mad_thresholds( + FakeInfluxdb3Local(), {"mad_thresholds": "disk.used:'2.5':20:5"}, "tid" + ) == [("disk.used", 2.5, 20, 5)] + + +@pytest.mark.parametrize( + "invalid", + [ + "temp::20:5", # regression: an empty k used to abort the whole flush + "temp:abc:20:5", + "temp:-2:20:5", + "temp:2.5:0:5", + "temp:2.5:1:5", + "temp:2.5:-3:5", + "temp:2.5:20:0", + "temp:2.5:20:-4", + "temp:2.5:20:2x", + "temp:2.5:20", + "temp:2.5:20000:5", # window_count above _MAX_WINDOW_COUNT + ], +) +def test_mad_thresholds_skips_invalid_segments(invalid): + influxdb3_local = FakeInfluxdb3Local() + + thresholds = plugin.parse_mad_thresholds( + influxdb3_local, {"mad_thresholds": f"{invalid}@load:3:10:2min"}, "tid" + ) + + assert thresholds == [("load", 3.0, 10, timedelta(minutes=2))] + assert influxdb3_local.messages("warn") + + +def test_mad_thresholds_skips_entries_of_wrong_length(): + influxdb3_local = FakeInfluxdb3Local() + + thresholds = plugin.parse_mad_thresholds( + influxdb3_local, + {"mad_thresholds": [["temp", 2.0, 5], ["load", 3.0, 10, 2]]}, + "tid", + ) + + assert thresholds == [("load", 3.0, 10, 2)] + assert any("expected [field, k" in m for m in influxdb3_local.messages("warn")) + + +def test_mad_thresholds_drops_duplicates(): + influxdb3_local = FakeInfluxdb3Local() + + thresholds = plugin.parse_mad_thresholds( + influxdb3_local, {"mad_thresholds": "temp:2:5:4@temp:2:5:4"}, "tid" + ) + + assert thresholds == [("temp", 2.0, 5, 4)] + assert any("duplicate threshold" in m for m in influxdb3_local.messages("warn")) + + +def test_mad_thresholds_without_valid_segments_raises(): + with pytest.raises(Exception, match="No valid MAD thresholds"): + plugin.parse_mad_thresholds( + FakeInfluxdb3Local(), {"mad_thresholds": "temp:2.5:20:0"}, "tid" + ) + + +def test_mad_thresholds_rejects_unsupported_type(): + with pytest.raises(Exception, match="must be a list of entries or a string"): + plugin.parse_mad_thresholds(FakeInfluxdb3Local(), {"mad_thresholds": 42}, "tid") + + +@pytest.mark.parametrize( + "senders, expected", + [ + ("http", {"http": {"http_webhook_url": WEBHOOK}}), + (["http"], {"http": {"http_webhook_url": WEBHOOK}}), + ], +) +def test_parse_senders_accepts_string_and_list(senders, expected): + config = {"senders": senders, "http_webhook_url": WEBHOOK} + + assert plugin.parse_senders(FakeInfluxdb3Local(), config, "tid") == expected + + +@pytest.mark.parametrize( + "config", + [ + {"senders": "telegram"}, + {"senders": "http"}, + {"senders": "http", "http_webhook_url": "ftp://example.com"}, + ], +) +def test_parse_senders_rejects_unusable_channels(config): + with pytest.raises(Exception, match="No valid senders configured"): + plugin.parse_senders(FakeInfluxdb3Local(), config, "tid") + + +# --- configuration ---------------------------------------------------------- + + +def test_load_config_applies_defaults(plugin_dir): + config = plugin._load_config( + FakeInfluxdb3Local(), dict(WRITES_ARGS), plugin._WRITES_VALIDATORS, "tid" + ) + + assert config["port_override"] == 8181 + assert config["notification_path"] == "notify" + assert config["state_change_count"] == 0 + + +def test_load_config_reads_toml(plugin_dir): + (plugin_dir / "writes.toml").write_text( + "measurement = 'home'\n" + "senders = ['http']\n" + "http_webhook_url = 'https://example.com/hook'\n" + "influxdb3_auth_token = 'tok'\n" + "mad_thresholds = [['temp', 2.0, 5, '2min']]\n" + "state_change_count = 3\n" + ) + + config = plugin._load_config( + FakeInfluxdb3Local(), + {"config_file_path": "writes.toml"}, + plugin._WRITES_VALIDATORS, + "tid", + ) + + assert config["senders"] == ["http"] + assert config["mad_thresholds"] == [["temp", 2.0, 5, "2min"]] + assert config["state_change_count"] == 3 + + +def test_load_config_rejects_non_toml_path(plugin_dir): + influxdb3_local = FakeInfluxdb3Local() + + config = plugin._load_config( + influxdb3_local, + {"config_file_path": "writes.yaml"}, + plugin._WRITES_VALIDATORS, + "tid", + ) + + assert config is None + assert any("expected a .toml file" in m for m in influxdb3_local.messages("error")) + + +@pytest.mark.parametrize( + "override", + [ + {"measurement": None}, + {"port_override": "0"}, + {"port_override": "99999"}, + {"state_change_count": "-1"}, + ], +) +def test_load_config_reports_validation_failures(plugin_dir, override): + influxdb3_local = FakeInfluxdb3Local() + args = {**WRITES_ARGS, **override} + args = {key: value for key, value in args.items() if value is not None} + + config = plugin._load_config( + influxdb3_local, args, plugin._WRITES_VALIDATORS, "tid" + ) + + assert config is None + assert any( + "Failed to load configuration" in m for m in influxdb3_local.messages("error") + ) + + +def test_load_config_uses_token_from_environment(plugin_dir, monkeypatch): + monkeypatch.setenv("INFLUXDB3_AUTH_TOKEN", "env-token") + args = { + key: value + for key, value in WRITES_ARGS.items() + if key != "influxdb3_auth_token" + } + + config = plugin._load_config( + FakeInfluxdb3Local(), args, plugin._WRITES_VALIDATORS, "tid" + ) + + assert config["influxdb3_auth_token"] == "env-token" + + +# --- flip suppression ------------------------------------------------------- + + +@pytest.mark.parametrize( + "flags, allowed, can_send", + [ + ([False, True, True, True], 2, True), # one sustained anomaly + ([False, True, False, True], 2, False), # flapping + ([False, True, False, True], 0, True), # suppression disabled + ([True], 2, True), # not enough history + ], +) +def test_check_state_changes(flags, allowed, can_send): + assert plugin.check_state_changes(deque(flags), allowed) is can_send + + +def test_inert_suppression_warns_for_narrow_count_windows_only(): + influxdb3_local = FakeInfluxdb3Local() + thresholds = [ + ("temp", 2.0, 5, 5), # no room for a transition + ("load", 2.0, 10, 2), # eight transitions fit + ("rate", 2.0, 5, timedelta(minutes=2)), # durations are not limited + ] + + plugin.warn_on_inert_suppression(influxdb3_local, thresholds, 2, "tid") + + warns = influxdb3_local.messages("warn") + assert len(warns) == 1 + assert "'temp'" in warns[0] and "window_count to 7" in warns[0] + + +@pytest.mark.parametrize("stored, expected", [(None, 0), ("", 0), ("x", 0), ("4", 4)]) +def test_read_counter_tolerates_unusable_values(stored, expected): + influxdb3_local = FakeInfluxdb3Local() + influxdb3_local.cache.put("key", stored) + + assert plugin.read_counter(influxdb3_local, "key") == expected + + +# --- process_writes --------------------------------------------------------- + + +def test_writes_waits_until_the_window_is_full(plugin_dir, sent): + influxdb3_local = FakeInfluxdb3Local() + + plugin.process_writes(influxdb3_local, batch(rows(20.0, 40.0)), dict(WRITES_ARGS)) + + assert sent == [] + assert any("Waiting for 5 points" in m for m in influxdb3_local.messages("info")) + + +def test_writes_count_threshold_alerts_after_consecutive_outliers(plugin_dir, sent): + influxdb3_local = FakeInfluxdb3Local() + seed_window(influxdb3_local) + + plugin.process_writes(influxdb3_local, batch(rows(40.0)), dict(WRITES_ARGS)) + assert sent == [] + assert influxdb3_local.cache.get(state_key("count-count")) == "1" + + plugin.process_writes(influxdb3_local, batch(rows(41.0)), dict(WRITES_ARGS)) + assert len(sent) == 1 + assert "outlier for 2 consecutive points" in sent[0]["payload"]["notification_text"] + assert sent[0]["payload"]["senders_config"] == { + "http": {"http_webhook_url": WEBHOOK} + } + assert influxdb3_local.cache.get(state_key("count-count")) == "0" + + +def test_writes_count_resets_when_value_returns_to_normal(plugin_dir, sent): + influxdb3_local = FakeInfluxdb3Local() + seed_window(influxdb3_local) + + plugin.process_writes(influxdb3_local, batch(rows(40.0, 20.5)), dict(WRITES_ARGS)) + + assert sent == [] + assert influxdb3_local.cache.get(state_key("count-count")) == "0" + + +def test_writes_duration_threshold_alerts_once_elapsed(plugin_dir, sent): + influxdb3_local = FakeInfluxdb3Local() + seed_window(influxdb3_local) + started = datetime.now(timezone.utc) - timedelta(hours=2) + influxdb3_local.cache.put( + state_key("time-time", threshold="3600.0s"), started.isoformat() + ) + args = {**WRITES_ARGS, "mad_thresholds": "temp:2:5:1h"} + + plugin.process_writes(influxdb3_local, batch(rows(40.0)), args) + + assert len(sent) == 1 + assert "outlier for 1:00:00" in sent[0]["payload"]["notification_text"] + assert influxdb3_local.cache.get(state_key("time-time", threshold="3600.0s")) == "" + + +def test_writes_duration_threshold_keeps_the_start_while_waiting(plugin_dir, sent): + influxdb3_local = FakeInfluxdb3Local() + seed_window(influxdb3_local) + started = (datetime.now(timezone.utc) - timedelta(minutes=5)).isoformat() + influxdb3_local.cache.put(state_key("time-time", threshold="3600.0s"), started) + args = {**WRITES_ARGS, "mad_thresholds": "temp:2:5:1h"} + + plugin.process_writes(influxdb3_local, batch(rows(40.0)), args) + + assert sent == [] + assert ( + influxdb3_local.cache.get(state_key("time-time", threshold="3600.0s")) + == started + ) + assert any("outlier ongoing" in m for m in influxdb3_local.messages("info")) + + +def test_writes_flapping_outlier_state_suppresses_notification(plugin_dir, sent): + """Regression: flips were counted over raw values, which suppressed every alert.""" + influxdb3_local = FakeInfluxdb3Local() + seed_window(influxdb3_local) + args = {**WRITES_ARGS, "state_change_count": "2"} + + # outlier, normal, outlier, outlier: two transitions when the threshold is reached + plugin.process_writes(influxdb3_local, batch(rows(40.0, 20.5, 40.0, 100.0)), args) + + assert sent == [] + assert any("outlier state flipped" in m for m in influxdb3_local.messages("warn")) + + +def test_writes_sustained_outlier_is_not_suppressed(plugin_dir, sent): + influxdb3_local = FakeInfluxdb3Local() + seed_window(influxdb3_local) + args = {**WRITES_ARGS, "state_change_count": "2"} + + plugin.process_writes(influxdb3_local, batch(rows(40.0, 41.0)), args) + + assert len(sent) == 1 + + +def test_writes_treats_state_change_count_of_one_as_disabled(plugin_dir, sent): + """A sustained anomaly records one transition, so 1 would suppress every alert.""" + influxdb3_local = FakeInfluxdb3Local() + seed_window(influxdb3_local) + args = {**WRITES_ARGS, "state_change_count": "1"} + + plugin.process_writes(influxdb3_local, batch(rows(40.0, 20.5, 40.0, 100.0)), args) + + assert len(sent) == 1 + assert any("treating it as 0" in m for m in influxdb3_local.messages("warn")) + assert not any("Suppressed" in m for m in influxdb3_local.messages("warn")) + + +def test_writes_shares_one_window_per_field_and_size(plugin_dir, sent): + """Regression: thresholds on one field used to reset each other's window.""" + influxdb3_local = FakeInfluxdb3Local() + seed_window(influxdb3_local) + seed_window(influxdb3_local, values=WARMUP[1:], window_count=4) + args = {**WRITES_ARGS, "mad_thresholds": "temp:2:5:2@temp:2:4:2"} + + plugin.process_writes(influxdb3_local, batch(rows(40.0)), args) + + assert len(influxdb3_local.cache.get(window_key(window_count=5))) == 5 + assert len(influxdb3_local.cache.get(window_key(window_count=4))) == 4 + assert not any("Waiting for" in m for m in influxdb3_local.messages("info")) + assert influxdb3_local.cache.get(state_key("count-count", window_count=5)) == "1" + assert influxdb3_local.cache.get(state_key("count-count", window_count=4)) == "1" + + +def test_writes_keeps_counters_of_thresholds_apart(plugin_dir, sent): + """Regression: thresholds differing only in the count shared one counter.""" + influxdb3_local = FakeInfluxdb3Local() + seed_window(influxdb3_local) + args = {**WRITES_ARGS, "mad_thresholds": "temp:2:5:2@temp:2:5:10"} + + plugin.process_writes(influxdb3_local, batch(rows(40.0)), args) + + assert sent == [] + assert influxdb3_local.cache.get(state_key("count-count", threshold="2")) == "1" + assert influxdb3_local.cache.get(state_key("count-count", threshold="10")) == "1" + + +def test_writes_sends_one_attempt_without_retrying(plugin_dir, monkeypatch): + attempts = [] + + def failing_post(url, headers=None, data=None, timeout=None): + attempts.append(url) + raise plugin.requests.ConnectionError("refused") + + monkeypatch.setattr(plugin.requests, "post", failing_post) + influxdb3_local = FakeInfluxdb3Local() + seed_window(influxdb3_local) + + plugin.process_writes(influxdb3_local, batch(rows(40.0, 41.0)), dict(WRITES_ARGS)) + + assert len(attempts) == 1 + assert any("Failed to send alert" in m for m in influxdb3_local.messages("error")) + + +@pytest.mark.parametrize( + "row", [{"host": "a", "hum": 5}, {"host": "a", "temp": "warm"}] +) +def test_writes_resets_state_when_the_field_is_unusable(plugin_dir, sent, row): + influxdb3_local = FakeInfluxdb3Local() + seed_window(influxdb3_local) + influxdb3_local.cache.put(state_key("count-count"), "1") + + plugin.process_writes(influxdb3_local, batch([row]), dict(WRITES_ARGS)) + + assert influxdb3_local.messages("error") == [] + assert influxdb3_local.cache.get(state_key("count-count")) == "0" + + +def test_writes_ignores_batches_of_other_tables(plugin_dir, sent): + influxdb3_local = FakeInfluxdb3Local() + + plugin.process_writes( + influxdb3_local, batch(rows(40.0), table="cpu"), dict(WRITES_ARGS) + ) + + assert sent == [] + assert not any("Starting writes process" in m for m in influxdb3_local.messages()) + assert not any("information_schema" in q for q in influxdb3_local.queries) + + +def test_writes_reports_unknown_measurement(plugin_dir, sent): + influxdb3_local = FakeInfluxdb3Local(tables=("cpu",)) + + plugin.process_writes(influxdb3_local, batch(rows(40.0)), dict(WRITES_ARGS)) + + assert any("not found in database" in m for m in influxdb3_local.messages("error")) + + +def test_writes_caches_configuration(plugin_dir, sent): + influxdb3_local = FakeInfluxdb3Local() + + plugin.process_writes(influxdb3_local, batch(rows(20.0)), dict(WRITES_ARGS)) + + assert influxdb3_local.cache.get(plugin._WRITES_CONFIG_CACHE_KEY) is not None + assert ( + influxdb3_local.cache.ttls[plugin._WRITES_CONFIG_CACHE_KEY] + == plugin._WRITES_CONFIG_TTL_SECONDS + ) + + +def test_writes_never_logs_credentials(plugin_dir, sent): + influxdb3_local = FakeInfluxdb3Local() + seed_window(influxdb3_local) + + plugin.process_writes(influxdb3_local, batch(rows(40.0, 41.0)), dict(WRITES_ARGS)) + + logged = " ".join(influxdb3_local.messages()) + assert TOKEN not in logged + assert WEBHOOK not in logged diff --git a/influxdata/state_change/README.md b/influxdata/state_change/README.md index be507e4..125f484 100644 --- a/influxdata/state_change/README.md +++ b/influxdata/state_change/README.md @@ -19,20 +19,20 @@ This plugin includes a JSON metadata schema in its docstring that defines suppor ### Required parameters -| Parameter | Type | Default | Description | -|----------------------|--------|----------|----------------------------------------------------------------------------------------------| -| `measurement` | string | required | Measurement to monitor for field changes | -| `field_change_count` | string | required | Dot-separated field thresholds (for example, "temp:3.load:2"). Supports count-based conditions | -| `senders` | string | required | Dot-separated notification channels with multi-channel alert support (Slack, Discord, etc.) | -| `window` | string | required | Time window for analysis. Format: `` (for example, "10m", "1h") | +| Parameter | Type | Default | Description | +|----------------------|--------|----------|------------------------------------------------------------------------------------------------------------------------| +| `measurement` | string | required | Measurement to monitor for field changes | +| `field_change_count` | string | required | Dot-separated field thresholds (for example, "temp:3.load:2" or "temp:3.disk.used:2"). Each count must be 1 or greater | +| `senders` | string | required | Dot-separated notification channels with multi-channel alert support (Slack, Discord, etc.) | +| `window` | string | required | Time window for analysis. Format: ``, units: `us`, `ms`, `s`, `min`, `h`, `d`, `w`. Must be positive | ### Data write trigger parameters -| Parameter | Type | Default | Description | -|--------------------|--------|----------|----------------------------------------------------------------------------------------------------------------| -| `measurement` | string | required | Measurement to monitor for threshold conditions | -| `field_thresholds` | string | required | Flexible threshold conditions with count-based and duration-based support (e.g., "temp:30:10@status:ok:1h") | -| `senders` | string | required | Dot-separated notification channels with multi-channel alert support (Slack, Discord, HTTP, SMS, WhatsApp) | +| Parameter | Type | Default | Description | +|--------------------|--------|----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `measurement` | string | required | Measurement to monitor for threshold conditions | +| `field_thresholds` | string | required | Threshold conditions with count-based and duration-based support (e.g., "temp:30:10@status:ok:1h"). Counts must be 1 or greater; durations must be positive | +| `senders` | string | required | Dot-separated notification channels with multi-channel alert support (Slack, Discord, HTTP, SMS, WhatsApp) | ### Notification parameters @@ -49,8 +49,10 @@ This plugin includes a JSON metadata schema in its docstring that defines suppor | Parameter | Type | Default | Description | |-----------------------|--------|---------|-------------------------------------------------------------------------------------------| -| `state_change_window` | number | 1 | Recent values to check for stability (configurable state change detection to reduce noise) | -| `state_change_count` | number | 1 | Max changes allowed within stability window (configurable state change detection) | +| `state_change_window` | number | 1 | Recent values to check for stability (reduces noise from flapping fields) | +| `state_change_count` | number | 1 | Changes within the stability window at which notifications start being suppressed | + +The stability check applies only when `state_change_window` is 2 or greater; the default of 1 leaves it off. Notifications are suppressed once the window contains `state_change_count` changes, so `state_change_count=3` is the setting that tolerates two flips. ### TOML configuration @@ -58,7 +60,11 @@ This plugin includes a JSON metadata schema in its docstring that defines suppor |--------------------|--------|---------|----------------------------------------------------------------------------------| | `config_file_path` | string | none | TOML config file path relative to `PLUGIN_DIR` (required for TOML configuration) | -*To use a TOML configuration file, set the `PLUGIN_DIR` environment variable and specify the `config_file_path` in the trigger arguments.* This is in addition to the `--plugin-dir` flag when starting InfluxDB 3. +*To use a TOML configuration file, set the `PLUGIN_DIR` environment variable and specify the `config_file_path` in the trigger arguments.* This is in addition to the `--plugin-dir` flag when starting InfluxDB 3. Relative paths are resolved against the first directory that is set: `PLUGIN_DIR`, then `INFLUXDB3_PLUGIN_DIR`, then the parent of `VIRTUAL_ENV`. Only that directory is used — the file is not looked up in the remaining ones. + +When `config_file_path` is set, the TOML file provides the whole configuration and inline trigger arguments are ignored. `INFLUXDB3_AUTH_TOKEN` from the environment still applies when `influxdb3_auth_token` is not set in the file. In TOML, `senders`, `field_thresholds`, and `field_change_count` use native structures (list, list of entries, table) instead of the inline string formats, though the inline strings are also accepted. + +Data write triggers cache the loaded configuration for 10 minutes to keep the write path fast, so configuration changes take effect within that window. Example TOML configuration files provided: @@ -80,6 +86,7 @@ The plugin assumes that the table schema is already defined in the database, as - **InfluxDB 3 Core/Enterprise**: with the Processing Engine enabled. - **Notification Sender Plugin for InfluxDB 3**: Required for sending notifications. See the [influxdata/notifier plugin](../notifier/README.md). - **Python packages**: + - `influxdata-plugin-utils>=0.3.0` (configuration loading, parsing, and schema introspection) - `requests` (for HTTP notifications) ### Installation steps @@ -97,6 +104,7 @@ The plugin assumes that the table schema is already defined in the database, as 2. Install required Python packages: ```bash + influxdb3 install package "influxdata-plugin-utils>=0.3.0" influxdb3 install package requests ``` @@ -113,7 +121,7 @@ influxdb3 create trigger \ --database mydb \ --path "gh:influxdata/state_change/state_change_check_plugin.py" \ --trigger-spec "every:10m" \ - --trigger-arguments "measurement=cpu,field_change_count=temp:3.load:2,window=10m,senders=slack,slack_webhook_url=$SLACK_WEBHOOK_URL" \ + --trigger-arguments "measurement=cpu,field_change_count=temp:3.load:2,window=10min,senders=slack,slack_webhook_url=$SLACK_WEBHOOK_URL" \ state_change_scheduler ``` @@ -186,7 +194,7 @@ Set `SLACK_WEBHOOK_URL` to your Slack incoming webhook URL. **Expected output** -When the field changes more than 5 times within 1 hour, a notification is sent: "Temperature sensor value changed 6 times in 1h for tags location=office" +When the field changes 5 or more times within 1 hour, a notification is sent: "Field value in table temperature changed 6 times in window 1:00:00 for tags location=office" ### Example 2: Advanced scheduled field change monitoring @@ -240,6 +248,9 @@ Set `SLACK_WEBHOOK_URL` to your Slack incoming webhook URL. - `state_change_check_plugin.py`: The main plugin code containing handlers for scheduled and data write triggers - `state_change_config_scheduler.toml`: Example TOML configuration for scheduled triggers - `state_change_config_data_writes.toml`: Example TOML configuration for data write triggers +- `test_state_change.py`: Pytest suite, runs without a live InfluxDB 3 server +- `requirements.txt`: Runtime dependencies (`influxdata-plugin-utils>=0.3.0`, `requests`) +- `requirements-dev.txt`: Development dependencies (`pytest`) ### Logging @@ -280,17 +291,21 @@ Handles real-time threshold monitoring on data writes. Evaluates incoming data a **Count-based thresholds** - Format: `field_name:"value":count` -- Example: `temp:"30.5":10` (10 occurrences of temperature = 30.5) +- Example: `temp:"30.5":10` (10 consecutive occurrences of temperature = 30.5) +- The count must be an integer of 1 or greater **Time-based thresholds** - Format: `field_name:"value":duration` - Example: `status:"error":5min` (status = error for 5 minutes) -- Supported units: `s`, `min`, `h`, `d`, `w` +- Supported units: `us`, `ms`, `s`, `min`, `h`, `d`, `w`; the duration must be positive **Multiple conditions** - Separate with `@`: `temp:"30":5@humidity:"high":10min` +- Segments that fail to parse are skipped with a warning; if none remain, the run stops with an error + +In TOML, the same thresholds are written as entries: `field_thresholds = [["temp", 30.5, 10], ["status", "error", "5min"]]`. ### Message template variables diff --git a/influxdata/state_change/manifest.toml b/influxdata/state_change/manifest.toml index b2a5b86..3de3969 100644 --- a/influxdata/state_change/manifest.toml +++ b/influxdata/state_change/manifest.toml @@ -2,7 +2,7 @@ manifest_schema_version = "1.3" [plugin] name = "state_change" -version = "1.2.0" +version = "1.3.0" description = "Provides field change and threshold monitoring capabilities through scheduler and data writes plugins. Detects changes in field values or threshold conditions with customizable notification templates." triggers = ["process_writes", "process_scheduled_call"] homepage = "https://www.influxdata.com/" @@ -16,7 +16,7 @@ exclude = [ [dependencies] database_version = ">=3.0.0" -python = ["requests"] +python = ["influxdata-plugin-utils>=0.3.0", "requests"] [[dependencies.plugins]] index_url = "https://github.com/influxdata/influxdb3_plugins/releases/download/registry/index.json" diff --git a/influxdata/state_change/requirements-dev.txt b/influxdata/state_change/requirements-dev.txt new file mode 100644 index 0000000..57bfbef --- /dev/null +++ b/influxdata/state_change/requirements-dev.txt @@ -0,0 +1,3 @@ +pytest +influxdata-plugin-utils>=0.3.0 +requests \ No newline at end of file diff --git a/influxdata/state_change/requirements.txt b/influxdata/state_change/requirements.txt index 663bd1f..3349a3f 100644 --- a/influxdata/state_change/requirements.txt +++ b/influxdata/state_change/requirements.txt @@ -1 +1,2 @@ +influxdata-plugin-utils>=0.3.0 requests \ No newline at end of file diff --git a/influxdata/state_change/state_change_check_plugin.py b/influxdata/state_change/state_change_check_plugin.py index 88dbf35..9fc7518 100644 --- a/influxdata/state_change/state_change_check_plugin.py +++ b/influxdata/state_change/state_change_check_plugin.py @@ -10,8 +10,8 @@ }, { "name": "field_change_count", - "example": "temp:3.load:2", - "description": "Dot-separated list of field thresholds (e.g., field:count).", + "example": "temp:3.disk.used:2", + "description": "Dot-separated list of field thresholds (e.g., field:count). Each count must be 1 or greater.", "required": true }, { @@ -23,7 +23,7 @@ { "name": "window", "example": "1h", - "description": "Time window for data analysis (e.g., '1h' for 1 hour). Units: 's', 'min', 'h', 'd', 'w'.", + "description": "Time window for data analysis (e.g., '1h' for 1 hour). Units: 'us', 'ms', 's', 'min', 'h', 'd', 'w'. Must be a positive duration.", "required": true }, { @@ -113,7 +113,7 @@ { "name": "config_file_path", "example": "config.toml", - "description": "Path to config file to override args. Format: 'config.toml'.", + "description": "Path to a TOML config file that replaces the trigger arguments entirely. Format: 'config.toml'.", "required": false } ], @@ -127,7 +127,7 @@ { "name": "field_thresholds", "example": "temp:'30.1':10@humidity:'true':2h", - "description": "Threshold conditions (e.g., field:value:count or field:value:time). Multiple conditions separated by '@'.", + "description": "Threshold conditions (e.g., field:value:count or field:value:time). Multiple conditions separated by '@'. Count must be 1 or greater; duration units: 'us', 'ms', 's', 'min', 'h', 'd', 'w'.", "required": true }, { @@ -145,13 +145,13 @@ { "name": "state_change_window", "example": "5", - "description": "Number of recent values to check for stability. Default: 1.", + "description": "Number of recent values to check for stability. The stability check applies only when this is 2 or greater. Default: 1.", "required": false }, { "name": "state_change_count", "example": "2", - "description": "Maximum allowed changes within state_change_window to allow notifications. Default: 1.", + "description": "Number of changes within state_change_window at which notifications start being suppressed. Default: 1.", "required": false }, { @@ -241,7 +241,7 @@ { "name": "config_file_path", "example": "config.toml", - "description": "Path to config file to override args. Format: 'config.toml'.", + "description": "Path to a TOML config file that replaces the trigger arguments entirely. Format: 'config.toml'.", "required": false } ] @@ -253,15 +253,24 @@ import random import re import time -import tomllib import uuid from collections import defaultdict, deque from datetime import datetime, timedelta, timezone -from pathlib import Path from string import Template from urllib.parse import urlparse import requests +from influxdata_plugin_utils.config import Validator, load_plugin_config +from influxdata_plugin_utils.introspection import ( + get_table_names, + get_tag_names, + query_window, +) +from influxdata_plugin_utils.parsing import ( + parse_delimited_list, + parse_int, + parse_timedelta, +) # Supported sender types with their required arguments AVAILABLE_SENDERS = { @@ -280,72 +289,109 @@ # List of keywords to exclude from argument validation in AVAILABLE_SENDERS EXCLUDED_KEYWORDS = ["headers", "token", "sid"] - -def get_all_measurements(influxdb3_local) -> list[str]: +_DEFAULT_NOTIFICATION_TEXT = ( + "Field $field in table $table changed $changes times in window $window " + "for tags $tags" +) +_DEFAULT_COUNT_TEXT = ( + "State change detected: Field $field in table $table changed to $value " + "during last $duration times. Row: $row" +) +_DEFAULT_TIME_TEXT = ( + "State change detected: Field $field in table $table changed to $value " + "during $duration. Row: $row" +) + +_CHANGE_COUNT_PAIR_RE = re.compile(r"(?P[^:]+):\s*(?P-?\d+)\s*(?:\.|$)") + + +def parse_window(raw) -> timedelta: + """Parse the analysis window, rejecting non-positive durations.""" + window: timedelta = parse_timedelta(raw) + if window <= timedelta(0): + raise ValueError(f"Invalid window: {raw!r} (must be a positive duration)") + return window + + +_COMMON_VALIDATORS = [ + Validator("measurement", required=True, cast=str), + Validator("senders", required=True), + Validator( + "port_override", + default=8181, + cast=lambda raw: parse_int(raw, minimum=1, maximum=65535), + ), + Validator("notification_path", default="notify", cast=str), +] + +_WRITES_VALIDATORS = _COMMON_VALIDATORS + [ + Validator("field_thresholds", required=True), + Validator( + "state_change_window", default=1, cast=lambda raw: parse_int(raw, minimum=0) + ), + Validator( + "state_change_count", default=1, cast=lambda raw: parse_int(raw, minimum=0) + ), + Validator("notification_count_text", default=_DEFAULT_COUNT_TEXT, cast=str), + Validator("notification_time_text", default=_DEFAULT_TIME_TEXT, cast=str), +] + +_SCHEDULED_VALIDATORS = _COMMON_VALIDATORS + [ + Validator("field_change_count", required=True), + Validator("window", required=True, cast=parse_window), + Validator("notification_text", default=_DEFAULT_NOTIFICATION_TEXT, cast=str), +] + +_WRITES_CONFIG_CACHE_KEY = "state_change:writes_config" +_WRITES_CONFIG_TTL_SECONDS = 10 * 60 + + +def _load_config( + influxdb3_local, args: dict, validators: list, task_id: str +) -> dict | None: """ - Retrieves a list of all tables of type 'BASE TABLE' from cache or the current InfluxDB database. + Load the plugin configuration, applying defaults and type casts. Args: influxdb3_local: InfluxDB client instance. + args (dict): Runtime arguments of the trigger. + validators (list): Validators providing defaults and casts for the mode. + task_id (str): Unique task identifier. Returns: - list[str]: List of table names (e.g., ["cpu", "memory", "disk"]). + dict | None: Config values keyed by lower-case name, or None if loading failed. """ - # check cache first - measurements: list = influxdb3_local.cache.get("measurements") - if measurements: - return measurements - - # if not in cache, query the database - result: list = influxdb3_local.query("SHOW TABLES") - measurements = [ - row["table_name"] for row in result if row.get("table_type") == "BASE TABLE" - ] - - # cache the result for 1 hour - influxdb3_local.cache.put(f"measurements", measurements, 60 * 60) - - return measurements + config_file_path = (args or {}).get("config_file_path") + if config_file_path and not str(config_file_path).endswith(".toml"): + influxdb3_local.error( + f"[{task_id}] Invalid config file format: expected a .toml file" + ) + return None + try: + loaded = load_plugin_config( + args, + validators=validators, + env_keys=["INFLUXDB3_AUTH_TOKEN"], + source="toml" if config_file_path else "args", + ) + except Exception as e: + influxdb3_local.error(f"[{task_id}] Failed to load configuration: {e}") + return None -def get_tag_names(influxdb3_local, measurement: str, task_id: str) -> list[str]: - """ - Retrieves the list of tag names for a measurement from cache or the database. + return {key.lower(): value for key, value in loaded.as_dict().items()} - Args: - influxdb3_local: InfluxDB client instance. - measurement (str): Name of the measurement to query. - task_id (str): The task ID. - Returns: - list[str]: List of tag names with 'Dictionary(Int32, Utf8)' data type. - """ - # check cache first - tags: list = influxdb3_local.cache.get(f"{measurement}_tags") - if tags: - return tags - - # if not in cache, query the database - query = """ - SELECT column_name - FROM information_schema.columns - WHERE table_name = $measurement - AND data_type = 'Dictionary(Int32, Utf8)' - """ - res: list[dict] = influxdb3_local.query(query, {"measurement": measurement}) - - if not res: +def get_measurement_tags(influxdb3_local, measurement: str, task_id: str) -> list[str]: + """Return the cached tag names of a measurement, logging when it has none.""" + tags: list[str] = get_tag_names(influxdb3_local, measurement) + if not tags: + tags = get_tag_names(influxdb3_local, measurement, use_cache=False) + if not tags: influxdb3_local.info( f"[{task_id}] No tags found for measurement '{measurement}'." ) - return [] - - tag_names: list[str] = [tag["column_name"] for tag in res] - - # cache the result for 1 hour - influxdb3_local.cache.put(f"{measurement}_tags", tag_names, 60 * 60) - - return tag_names + return tags def generate_cache_key( @@ -366,65 +412,55 @@ def generate_cache_key( return cache_key -def parse_senders(influxdb3_local, args: dict, task_id: str) -> dict: +def read_counter(influxdb3_local, cache_key: str) -> int: + """Read a breach counter, treating a missing or non-numeric entry as zero.""" + try: + return int(influxdb3_local.cache.get(cache_key)) + except (TypeError, ValueError): + return 0 + + +def parse_senders(influxdb3_local, config: dict, task_id: str) -> dict: """ - Parse and validate sender configurations from input arguments. + Parse and validate sender configurations from the loaded config. Args: influxdb3_local: InfluxDB client instance. - args (dict): Input arguments containing: - - "senders": dot-separated list of sender types (e.g., "slack.http"). - - For each sender, its own required keys (see AVAILABLE_SENDERS). - task_id (str): Unique task identifier used for logging context. + config (dict): Loaded config containing "senders" and related settings. + task_id (str): Unique task identifier. Returns: dict: A mapping `{sender_type: {key: value}}` for each valid sender. - For example: - { - "slack": { - "slack_webhook_url": "https://hooks.slack.com/...", - "slack_headers": "..." - }, - "sms": { ... } - } Raises: Exception: If no valid senders are found after parsing. """ senders_config: defaultdict = defaultdict(dict) - - senders: str | list = args.get("senders") - if args["use_config_file"]: - if not isinstance(senders, list): - raise Exception( - f"[{task_id}] 'senders' must be a list when using config file" - ) - else: - senders = senders.split(".") + senders: list = parse_delimited_list(config["senders"], sep=".") for sender in senders: if sender not in AVAILABLE_SENDERS: influxdb3_local.warn(f"[{task_id}] Invalid sender type: {sender}") continue for key in AVAILABLE_SENDERS[sender]: - if key not in args and not any(ex in key for ex in EXCLUDED_KEYWORDS): + if key not in config and not any(ex in key for ex in EXCLUDED_KEYWORDS): influxdb3_local.warn( f"[{task_id}] Required key '{key}' missing for sender '{sender}'" ) senders_config.pop(sender, None) break if "url" in key and not validate_webhook_url( - influxdb3_local, sender, args[key], task_id + influxdb3_local, sender, config[key], task_id ): senders_config.pop(sender, None) break - if key not in args: + if key not in config: continue - senders_config[sender][key] = args[key] + senders_config[sender][key] = config[key] if not senders_config: - raise Exception(f"[{task_id}] No valid senders configured") + raise Exception("No valid senders configured") return senders_config @@ -443,8 +479,8 @@ def send_notification( payload (dict): Dict to serialize as JSON in the POST body. task_id (str): Unique task identifier. - Raises: - requests.RequestException: If all retries fail or a non-2xx response is received. + Request failures and non-2xx responses are retried; after the final attempt + the error is logged and the alert is dropped. """ url: str = f"http://localhost:{port}/api/v3/engine/{path}" headers: dict = { @@ -480,36 +516,6 @@ def send_notification( ) -def parse_port_override(args: dict, task_id: str) -> int: - """ - Parse and validate the 'port_override' argument, converting it from string to int. - - Args: - args (dict): Runtime arguments containing 'port_override'. - task_id (str): Unique task identifier for logging context. - - Returns: - int: Parsed port number (1–65535), or 8181 if not provided. - - Raises: - Exception: If 'port_override' is provided but is not a valid integer in the range 1–65535. - """ - raw: str | int = args.get("port_override", 8181) - - try: - port = int(raw) - except (TypeError, ValueError): - raise Exception(f"[{task_id}] Invalid port_override, not an integer: {raw!r}") - - # Validate port range - if not (1 <= port <= 65535): - raise Exception( - f"[{task_id}] Invalid port_override, must be between 1 and 65535: {port}" - ) - - return port - - def validate_webhook_url(influxdb3_local, service: str, url: str, task_id: str) -> bool: """ Validate webhook URL format. @@ -575,115 +581,68 @@ def _coerce_value(raw: str) -> str | int | float | bool: return raw -def parse_field_thresholds( - influxdb3_local, - args: dict, - task_id: str, -) -> list: +def _parse_threshold_param( + influxdb3_local, raw, task_id: str +) -> int | timedelta | None: """ - Extracts and parses field threshold definitions from args or use values from config file. + Parse the third part of a threshold into a consecutive count or a duration. - Args: - influxdb3_local: InfluxDB client instance. - args (dict): Dictionary with the 'field_thresholds' key. - task_id (str): Unique identifier for the current task, used for logging. + A bare integer is a count of consecutive matching points; anything else is a + duration such as '10s' or '2h'. - The args dict must contain the key "field_thresholds" with a value like: - 'field_name-1:"value 2":10@field_name-2:"value 1":1s' + Returns: + int | timedelta | None: The parsed threshold, or None when it is invalid. + """ + if isinstance(raw, bool): + influxdb3_local.warn(f"[{task_id}] Invalid threshold parameter: {raw!r}") + return None - Each '@'-separated segment must contain exactly two ':' characters, - producing three parts: field_name, raw_value, and raw_third. + if isinstance(raw, int) or re.fullmatch(r"-?\d+", str(raw).strip()): + count = int(raw) + if count < 1: + influxdb3_local.warn( + f"[{task_id}] Invalid threshold count {count}, must be 1 or greater" + ) + return None + return count - - raw_value is coerced into int, float, bool, or str. - - raw_third is either: - • A plain integer (e.g., "10") - • A duration with unit suffix: , where unit ∈ {s, min, h, d, w} + try: + duration: timedelta = parse_timedelta(raw) + except ValueError as e: + influxdb3_local.warn(f"[{task_id}] Invalid threshold duration {raw!r}: {e}") + return None + + if duration <= timedelta(0): + influxdb3_local.warn( + f"[{task_id}] Invalid threshold duration {raw!r}, must be positive" + ) + return None + return duration - Valid units and their corresponding timedelta keyword: - "s" → "seconds" - "min" → "minutes" - "h" → "hours" - "d" → "days" - "w" → "weeks" - Returns: - A list of lists [field_name, coerced_value, converted_third]: - - coerced_value is int, float, bool, or str - - converted_third is int or timedelta +def _thresholds_from_entries(influxdb3_local, entries: list, task_id: str) -> list: + """Parse thresholds given as [field, value, count_or_duration] entries.""" + thresholds: list = [] - Example: - args = { - ... "field_thresholds": 'temp:"30":60@humidity:"true":2h' - ... } - parse_field_thresholds(args) - [ - ["temp", 30, 60], - ["humidity", True, datetime.timedelta(hours=2)] - ] - """ - valid_units: dict[str, str] = { - "s": "seconds", - "min": "minutes", - "h": "hours", - "d": "days", - "w": "weeks", - } + for entry in entries: + if not isinstance(entry, (list, tuple)) or len(entry) != 3: + influxdb3_local.warn( + f"[{task_id}] Invalid threshold '{entry}', expected [field, value, count_or_duration]" + ) + continue + threshold_param = _parse_threshold_param(influxdb3_local, entry[2], task_id) + if threshold_param is None: + continue + thresholds.append((str(entry[0]), entry[1], threshold_param)) - raw_input: str | list = args.get("field_thresholds") - results: list = [] + return thresholds - if args["use_config_file"]: - if not isinstance(raw_input, list): - raise Exception( - "[{task_id}] field_thresholds must be a list while using config file" - ) - for threshold in raw_input: - try: - field_name = str(threshold[0]) - value = threshold[1] - duration: str | int = threshold[2] - if isinstance(duration, str): - num_part, unit_part = "", "" - for unit in sorted(valid_units.keys(), key=len, reverse=True): - if duration.endswith(unit): - num_part = duration[: -len(unit)] - unit_part = unit - break - if not num_part or unit_part not in valid_units: - influxdb3_local.warn( - f"[{task_id}] Invalid duration format '{duration}'" - ) - continue - try: - num = int(num_part) - except ValueError: - influxdb3_local.warn( - f"[{task_id}] Invalid number in duration '{duration}'" - ) - continue - threshold_param: timedelta | int = timedelta( - **{valid_units[unit_part]: num} - ) - elif isinstance(duration, int): - threshold_param = duration - else: - influxdb3_local.warn( - f"[{task_id}] Invalid duration format '{duration}'" - ) - continue - results.append([field_name, value, threshold_param]) - except Exception: - influxdb3_local.warn( - f"[{task_id}] Invalid duration definition: {threshold}, skipping" - ) - if not results: - raise Exception( - f"[{task_id}] No valid field threshold segments found in {raw_input}" - ) - return results - segments: list = [seg.strip() for seg in raw_input.split("@") if seg.strip()] - for segment in segments: +def _thresholds_from_string(influxdb3_local, raw: str, task_id: str) -> list: + """Parse thresholds given as '::' joined by '@'.""" + thresholds: list = [] + + for segment in parse_delimited_list(raw, sep="@"): # Each segment must contain exactly two ':' characters if segment.count(":") != 2: influxdb3_local.warn( @@ -691,51 +650,132 @@ def parse_field_thresholds( ) continue - # Split into three parts: field_name, raw_value, raw_third - field_name, raw_value, raw_third = segment.split(":", 2) - field_name = field_name.strip() - raw_value = raw_value.strip() - raw_third = raw_third.strip() - - # Coerce raw_value into int, float, bool, or str - value = _coerce_value(raw_value) - - # Parse raw_third: integer or duration - if re.fullmatch(r"-?\d+", raw_third): - third_converted: int | timedelta = int(raw_third) - else: - # Attempt duration parsing: - num_part: str = "" - unit_part: str = "" - for unit in sorted(valid_units.keys(), key=len, reverse=True): - if raw_third.endswith(unit): - num_part = raw_third[: -len(unit)] - unit_part = unit - break - - if not num_part or unit_part not in valid_units: - influxdb3_local.warn( - f"[{task_id}] Invalid duration format: {raw_third}" - ) - continue + field_name, raw_value, raw_param = segment.split(":", 2) + threshold_param = _parse_threshold_param( + influxdb3_local, raw_param.strip(), task_id + ) + if threshold_param is None: + continue + thresholds.append( + (field_name.strip(), _coerce_value(raw_value), threshold_param) + ) - try: - num = int(num_part) - except ValueError: - influxdb3_local.warn(f"[{task_id}] Invalid duration number: {num_part}") - continue + return thresholds - kw: str = valid_units[unit_part] - third_converted = timedelta(**{kw: num}) - results.append((field_name, value, third_converted)) +def parse_field_thresholds(influxdb3_local, config: dict, task_id: str) -> list: + """ + Parse the field thresholds used by the data write trigger. - if not results: + Thresholds come either as entries of [field, value, count_or_duration] (TOML) or + as a string of '::' expressions separated by '@'. + + Args: + influxdb3_local: InfluxDB client instance. + config (dict): Loaded config containing "field_thresholds". + task_id (str): Unique task identifier. + + Returns: + list[tuple]: Tuples of (field_name, target_value, count_or_duration). + + Example: + 'temp:"30":60@humidity:"true":2h' + [ + ("temp", 30, 60), + ("humidity", True, datetime.timedelta(hours=2)), + ] + """ + raw: str | list = config["field_thresholds"] + + if isinstance(raw, (list, tuple)): + thresholds = _thresholds_from_entries(influxdb3_local, raw, task_id) + elif isinstance(raw, str): + thresholds = _thresholds_from_string(influxdb3_local, raw, task_id) + else: raise Exception( - f"[{task_id}] No valid field threshold segments found in {raw_input}" + "'field_thresholds' must be a list of entries or a string, " + f"got {type(raw).__name__}" + ) + + if not thresholds: + raise Exception("No valid field thresholds provided.") + return thresholds + + +def _change_counts_from_string(influxdb3_local, raw: str, task_id: str) -> list: + """ + Split a string of 'field:count' pairs joined by '.' into (field, count) tuples. + + Example: + 'temp:3.disk.used:2' -> [('temp', '3'), ('disk.used', '2')] + """ + pairs: list = [] + text: str = raw.strip() + position: int = 0 + + while position < len(text): + match = _CHANGE_COUNT_PAIR_RE.match(text, position) + if match: + pairs.append((match.group("field"), match.group("count"))) + position = match.end() + continue + + next_dot: int = text.find(".", position) + skipped: str = text[position:] if next_dot == -1 else text[position:next_dot] + influxdb3_local.warn( + f"[{task_id}] Invalid format of field_change_count, expected 'field:count' in: {skipped}" ) + if next_dot == -1: + break + position = next_dot + 1 - return results + return pairs + + +def parse_field_change_count( + influxdb3_local, config: dict, task_id: str +) -> dict[str, int]: + """ + Parse the per-field change thresholds used by the scheduled trigger. + + Thresholds come either as a mapping of {field: count} (TOML) or as a string of + 'field:count' pairs separated by '.'. + + Args: + influxdb3_local: InfluxDB client instance. + config (dict): Loaded config containing "field_change_count". + task_id (str): Unique task identifier. + + Returns: + dict[str, int]: Field names mapped to their change count thresholds. + + Raises: + Exception: If the value has an unsupported type or no valid fields are found. + """ + raw: str | dict = config["field_change_count"] + + if isinstance(raw, dict): + pairs: list = list(raw.items()) + elif isinstance(raw, str): + pairs = _change_counts_from_string(influxdb3_local, raw, task_id) + else: + raise Exception( + "'field_change_count' must be a mapping or a string, " + f"got {type(raw).__name__}" + ) + + field_counts: dict = {} + for field, raw_count in pairs: + try: + field_counts[str(field).strip()] = parse_int(raw_count, minimum=1) + except ValueError as e: + influxdb3_local.warn( + f"[{task_id}] Invalid change count for field '{field}': {e}" + ) + + if not field_counts: + raise Exception("No valid entries found in field_change_count.") + return field_counts def check_state_changes(cached_values: deque, state_change_count: int) -> bool: @@ -744,12 +784,12 @@ def check_state_changes(cached_values: deque, state_change_count: int) -> bool: Args: cached_values (deque): A deque of recent field values (size = state_change_window). - state_change_count (int): Maximum allowed number of changes within the window. + state_change_count (int): Number of changes at which notifications are suppressed. Returns: bool: - True if the number of value changes in cached_values is <= state_change_count, - False if it exceeds state_change_count. + True while the number of value changes in cached_values stays below + state_change_count, False once it reaches it. """ # If fewer than 2 values, there can be no change if len(cached_values) < 2: @@ -774,154 +814,86 @@ def check_state_changes(cached_values: deque, state_change_count: int) -> bool: return True -def process_writes(influxdb3_local, table_batches: list, args: dict | None = None): +def process_writes(influxdb3_local, table_batches: list, args: dict): """ - Data write trigger entry point implementing field‐level thresholds with “count” and “duration” logic, - while also suppressing notifications if the field value has flipped too many times recently. + Data write trigger entry point implementing field-level thresholds with "count" and + "duration" logic, while also suppressing notifications if the field value has flipped + too many times recently. - When you create a Data Write trigger, point to this file and the function name must be `process_writes`. - Other names are not supported. + When you create a Data Write trigger, point to this file and the function name must be + `process_writes`. Other names are not supported. - The trigger fires on each WAL flush. All newly written rows within that flush—optionally filtered by - a configured measurement—are grouped into `table_batches`. + The trigger fires on each WAL flush. All newly written rows within that flush are grouped + into `table_batches`; only batches of the configured measurement are processed. Args: - influxdb3_local: - InfluxDB client instance (for logging, SQL queries, writing, and cache). - table_batches (list): - A list of dicts, each with: - - "table_name": str - - "rows": list[dict] # Each dict is one row of data, containing fields and tags. - args (dict, optional): - Must include: - - "measurement": measurement (table) name to monitor (str). - - "field_thresholds": string defining thresholds, parsed by `parse_field_thresholds`. - - "senders": dot-separated list of notification channels (e.g., "slack.sms"). - May also include: - - "config_file_path": path to config file to override args (str). - - "state_change_window": integer count of last values to consider for flip detection. - - "state_change_count": integer threshold of flips to suppress notifications. - - "port_override": HTTP port for notification plugin (default 8181). - - "influxdb3_auth_token": API v3 token (or provided via ENV var INFLUXDB3_AUTH_TOKEN). - - "notification_path": path on engine (default "notify"). - - "notification_text": template for alert text, with placeholders: - $table, $field, $value, $duration, $row. - - Raises: - Exception: Captures and logs any unexpected error (with `influxdb3_local.error`). + influxdb3_local: InfluxDB client instance (for logging, SQL queries, and cache). + table_batches (list): Dicts with "table_name" (str) and "rows" (list[dict]). + args (dict): Runtime arguments of the trigger. """ - task_id: str = str(uuid.uuid4()) - influxdb3_local.info(f"[{task_id}] Starting writes process with args: {args}") + if not table_batches: + return - # Override args with config file if specified - if args: - if path := args.get("config_file_path", None): - if not path.endswith(".toml"): - influxdb3_local.error( - f"[{task_id}] Invalid config file format: expected a .toml file" - ) - return - try: - plugin_dir_var: str | None = os.getenv("PLUGIN_DIR", None) - if plugin_dir_var: - file_path = Path(plugin_dir_var) / path - else: - # Fallbacks for servers where the operator has not exported PLUGIN_DIR: - # - INFLUXDB3_PLUGIN_DIR: set when the server is configured via env var - # - VIRTUAL_ENV: exported by the processing engine; default venv is /.venv - candidates: list[str] = [] - if influxdb3_plugin_dir := os.environ.get("INFLUXDB3_PLUGIN_DIR"): - candidates.append(influxdb3_plugin_dir) - if virtual_env := os.environ.get("VIRTUAL_ENV"): - candidates.append(str(Path(virtual_env).parent)) - - resolved = None - for base in candidates: - candidate = Path(base) / path - if candidate.exists(): - resolved = candidate - break - - if resolved is None: - candidates_str = ", ".join(candidates) if candidates else "none available" - influxdb3_local.error( - f"[{task_id}] PLUGIN_DIR env var not set and config file path " - f"'{path}' was not found via fallbacks (tried: {candidates_str})" - ) - return - file_path = resolved - influxdb3_local.info(f"[{task_id}] Reading config file {file_path}") - with open(file_path, "rb") as f: - args = tomllib.load(f) - args["use_config_file"] = True - influxdb3_local.info(f"[{task_id}] New args content: {args}") - except Exception: - influxdb3_local.error(f"[{task_id}] Failed to read config file") - return - else: - args["use_config_file"] = False - - if ( - not args - or "measurement" not in args - or "field_thresholds" not in args - or "senders" not in args - ): - influxdb3_local.error( - f"[{task_id}] Missing required arguments: measurement, field_thresholds, or senders" + task_id: str = str(uuid.uuid4()) + config: dict | None = influxdb3_local.cache.get(_WRITES_CONFIG_CACHE_KEY) + if config is None: + config = _load_config(influxdb3_local, args, _WRITES_VALIDATORS, task_id) + if config is None: + return + influxdb3_local.cache.put( + _WRITES_CONFIG_CACHE_KEY, config, _WRITES_CONFIG_TTL_SECONDS ) - return - # Parse configuration - measurement: str = args["measurement"] - all_measurements: list = get_all_measurements(influxdb3_local) + measurement: str = config["measurement"] + all_measurements: list = get_table_names(influxdb3_local) if measurement not in all_measurements: influxdb3_local.error( f"[{task_id}] Measurement '{measurement}' not found in database" ) return + monitored_batches: list = [ + table_batch + for table_batch in table_batches + if table_batch["table_name"] == measurement + ] + if not monitored_batches: + return + + influxdb3_local.info(f"[{task_id}] Starting writes process") + try: - field_thresholds: list = parse_field_thresholds(influxdb3_local, args, task_id) + field_thresholds: list = parse_field_thresholds( + influxdb3_local, config, task_id + ) influxdb3_local.info(f"[{task_id}] Field thresholds: {field_thresholds}") - senders_config: dict = parse_senders(influxdb3_local, args, task_id) - tags: list = get_tag_names(influxdb3_local, measurement, task_id) - port_override: int = parse_port_override(args, task_id) - state_change_window: int = int(args.get("state_change_window", 1)) - state_change_count: int = int(args.get("state_change_count", 1)) - notification_path: str = args.get("notification_path", "notify") - influxdb3_auth_token: str = args.get("influxdb3_auth_token") or os.getenv( - "INFLUXDB3_AUTH_TOKEN" + senders_config: dict = parse_senders(influxdb3_local, config, task_id) + port_override: int = config["port_override"] + state_change_window: int = config["state_change_window"] + state_change_count: int = config["state_change_count"] + notification_path: str = config["notification_path"] + influxdb3_auth_token: str = ( + config.get("influxdb3_auth_token") + or os.getenv("INFLUXDB3_AUTH_TOKEN") + or "" ) - if influxdb3_auth_token is None: + if not influxdb3_auth_token: influxdb3_local.error( f"[{task_id}] Missing required argument: influxdb3_auth_token" ) return - notification_count_tpl = args.get( - "notification_count_text", - "State change detected: Field $field in table $table changed to $value during last $duration times. Row: $row", - ) - notification_time_tpl = args.get( - "notification_time_text", - "State change detected: Field $field in table $table changed to $value during $duration. Row: $row", - ) + notification_count_tpl: str = config["notification_count_text"] + notification_time_tpl: str = config["notification_time_text"] - # Process incoming data - for table_batch in table_batches: - # Skip non-matching tables - if table_batch["table_name"] != measurement: - continue + tags: list = get_measurement_tags(influxdb3_local, measurement, task_id) - # Process rows in this batch + for table_batch in monitored_batches: for row in table_batch["rows"]: for field_name, target_value, threshold_param in field_thresholds: - # Get cache keys - if isinstance(threshold_param, timedelta): - duration_suffix: str = "time" - else: - duration_suffix = "count" + is_duration: bool = isinstance(threshold_param, timedelta) + duration_suffix: str = "time" if is_duration else "count" + reset_value: str = "" if is_duration else "0" duration_cache_key: str = generate_cache_key( measurement=measurement, @@ -940,13 +912,12 @@ def process_writes(influxdb3_local, table_batches: list, args: dict | None = Non influxdb3_local.info( f"[{task_id}] Field '{field_name}' not present in row. Cache key: {duration_cache_key}. Resetting state." ) - influxdb3_local.cache.put(duration_cache_key, "") + influxdb3_local.cache.put(duration_cache_key, reset_value) continue # Check if the condition is satisfied: row[field_name] == target_value condition_met: bool = current_val == target_value - # Get cache keys values_cache_key: str = generate_cache_key( measurement=measurement, field=field_name, @@ -955,7 +926,6 @@ def process_writes(influxdb3_local, table_batches: list, args: dict | None = Non tags=tags, row=row, ) - # Get cached values cached_values = influxdb3_local.cache.get( values_cache_key, default=deque(maxlen=state_change_window) ) @@ -971,9 +941,9 @@ def process_writes(influxdb3_local, table_batches: list, args: dict | None = Non ) cached_values.append(current_val) - if duration_suffix == "count": - cached_state: int = int( - influxdb3_local.cache.get(duration_cache_key, default=0) + if not is_duration: + cached_state: int = read_counter( + influxdb3_local, duration_cache_key ) if condition_met: @@ -983,7 +953,6 @@ def process_writes(influxdb3_local, table_batches: list, args: dict | None = Non influxdb3_local.error( f"[{task_id}] State change detected: {field_name} in table {measurement} changed to {target_value} during last {threshold_param} values. Row: {duration_cache_key}, sending alert" ) - # Send notification payload: dict = { "notification_text": interpolate_notification_text( notification_count_tpl, @@ -1026,22 +995,19 @@ def process_writes(influxdb3_local, table_batches: list, args: dict | None = Non # Condition failed → reset count influxdb3_local.cache.put(duration_cache_key, "0") - else: # duration_suffix == "time" + else: required_duration: timedelta = threshold_param - cached_state: str = influxdb3_local.cache.get( + prev_start_iso: str = influxdb3_local.cache.get( duration_cache_key, default="" ) if condition_met: - # Parse cached start time, if any - prev_start_iso: str = cached_state + start_time = None if prev_start_iso: try: start_time = datetime.fromisoformat(prev_start_iso) except Exception: start_time = None - else: - start_time = None # Use current UTC time rather than row's "time" field now = datetime.now(timezone.utc) @@ -1060,7 +1026,6 @@ def process_writes(influxdb3_local, table_batches: list, args: dict | None = Non influxdb3_local.error( f"[{task_id}] Threshold duration reached for row: {duration_cache_key}, target_value={target_value} (required {required_duration})" ) - # Send notification payload: dict = { "notification_text": interpolate_notification_text( notification_time_tpl, @@ -1093,15 +1058,15 @@ def process_writes(influxdb3_local, table_batches: list, args: dict | None = Non influxdb3_local.cache.put(duration_cache_key, "") else: - # Update elapsed (keep original start in cache) + # Keep the original start in cache and wait influxdb3_local.warn( - f"[{task_id}] Threshold duration reached for row: {row}, target_value={target_value} with elapsed={elapsed} (required {required_duration})" + f"[{task_id}] Condition still holding for row: {duration_cache_key}, target_value={target_value} with elapsed={elapsed} (required {required_duration})" ) else: # Condition failed → reset any stored start time - if cached_state: + if prev_start_iso: influxdb3_local.info( - f"[{task_id}] Condition failed for row: {row}, clearing duration cache" + f"[{task_id}] Condition failed for row: {duration_cache_key}, clearing duration cache" ) influxdb3_local.cache.put(duration_cache_key, "") @@ -1111,211 +1076,29 @@ def process_writes(influxdb3_local, table_batches: list, args: dict | None = Non influxdb3_local.error(f"[{task_id}] Error: {str(e)}") -def parse_field_change_count( - influxdb3_local, args: dict, task_id: str -) -> dict[str, int]: - """ - Parses the 'field_change_count' parameter into a dictionary of field names and their change thresholds or use values from config file. - - Args: - influxdb3_local: InfluxDB client instance. - args (dict): Dictionary with the 'field_change_count' key. - task_id (str): Unique task identifier for logging. - - Returns: - dict[str, int]: Dictionary mapping field names to change counts. - - Raises: - Exception: If the format is invalid or no valid fields are found. - """ - raw_input: str | dict = args.get("field_change_count") - field_counts: dict = {} - - if args["use_config_file"]: - if not isinstance(raw_input, dict): - raise Exception( - f"[{task_id}] field_change_count must be a dictionary when using config file" - ) - return raw_input - - pairs: list = raw_input.split(".") - for pair in pairs: - if ":" not in pair: - influxdb3_local.warn( - f"[{task_id}] Invalid format of field_change_count, missing ':' in pair: {pair}" - ) - continue - field, count_str = pair.split(":", 1) - try: - count: int = int(count_str) - field_counts[field.strip()] = count - except ValueError: - influxdb3_local.warn( - f"[{task_id}] Invalid format of field_change_count, invalid count: {count_str} in pair: {pair}" - ) - continue - - if not field_counts: - raise Exception(f"[{task_id}] No valid entries found in field_change_count.") - - return field_counts - - -def parse_window(args: dict, task_id: str) -> timedelta: +def process_scheduled_call(influxdb3_local, call_time: datetime, args: dict) -> None: """ - Parses the 'window' argument from args and converts it into a timedelta object. + Scheduled trigger entry point that counts how often fields change within a time window + and sends a notification when a field exceeds its configured change threshold. Args: - args (dict): Dictionary with the 'window' key (e.g., {"window": "2h"}). - task_id (str): Unique task identifier. - - Returns: - timedelta: Parsed time interval. - - Raises: - Exception: If window is missing or has an invalid format. - """ - valid_units: dict = { - "s": "seconds", - "min": "minutes", - "h": "hours", - "d": "days", - "w": "weeks", - } - - window: str | None = args.get("window") - - match = re.fullmatch(r"(\d+)([a-zA-Z]+)", window) - if match: - number, unit = match.groups() - number = int(number) - if number >= 1 and unit in valid_units: - return timedelta(**{valid_units[unit]: number}) - - raise Exception(f"[{task_id}] Invalid interval format: {window}.") - - -def build_query(measurement: str, start_time: datetime, end_time: datetime) -> str: - """ - Builds an SQL query to select all data from a measurement within a time range. - - Args: - measurement (str): Name of the measurement/table. - start_time (datetime): Start time (inclusive). - end_time (datetime): End time (exclusive). - - Returns: - str: SQL query string selecting all data between start_time and end_time. - """ - start_iso: str = start_time.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - end_iso: str = end_time.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - - query = f""" - SELECT * - FROM '{measurement}' - WHERE time >= '{start_iso}' - AND time < '{end_iso}' - ORDER BY time ASC - """ - return query - - -def process_scheduled_call( - influxdb3_local, call_time: datetime, args: dict | None = None -) -> None: - """ - Entry point for the InfluxDB scheduler plugin that monitors field changes - in a measurement and triggers notifications if a threshold is exceeded. - - It queries a specified measurement within a time window, detects how many times certain - fields have changed, and sends notifications if those changes exceed - predefined thresholds. - - Args: - influxdb3_local: Instance of the InfluxDB client used for querying - and logging. - call_time (datetime): The UTC timestamp at which the scheduler triggers - this function. This defines the end of the time window. - args (dict, optional): Dictionary containing the following required keys: - - config_file_path (str): path to config file to override args. - - measurement (str): The name of the measurement to query. - - field_change_count (dict): Mapping of field names to change thresholds. - - senders (dict): Configuration for notification senders. - - window (str | int): Duration to look back from `call_time`. - - influxdb3_auth_token (str, optional): Token for authentication. - - notification_path (str, optional): Endpoint path for notifications. - - notification_text (str, optional): Template for the alert message. - - port_override (int, optional): Custom port for notification endpoint. - - Raises: - No exceptions are raised directly; all errors are caught and logged. + influxdb3_local: InfluxDB client instance used for querying and logging. + call_time (datetime): UTC timestamp of the scheduled run; the end of the window. + args (dict): Runtime arguments of the trigger. """ task_id: str = str(uuid.uuid4()) - influxdb3_local.info(f"[{task_id}] Starting scheduled field change check at {call_time} with args: {args}.") - - # Override args with config file if specified - if args: - if path := args.get("config_file_path", None): - if not path.endswith(".toml"): - influxdb3_local.error( - f"[{task_id}] Invalid config file format: expected a .toml file" - ) - return - try: - plugin_dir_var: str | None = os.getenv("PLUGIN_DIR", None) - if plugin_dir_var: - file_path = Path(plugin_dir_var) / path - else: - # Fallbacks for servers where the operator has not exported PLUGIN_DIR: - # - INFLUXDB3_PLUGIN_DIR: set when the server is configured via env var - # - VIRTUAL_ENV: exported by the processing engine; default venv is /.venv - candidates: list[str] = [] - if influxdb3_plugin_dir := os.environ.get("INFLUXDB3_PLUGIN_DIR"): - candidates.append(influxdb3_plugin_dir) - if virtual_env := os.environ.get("VIRTUAL_ENV"): - candidates.append(str(Path(virtual_env).parent)) - - resolved = None - for base in candidates: - candidate = Path(base) / path - if candidate.exists(): - resolved = candidate - break - - if resolved is None: - candidates_str = ", ".join(candidates) if candidates else "none available" - influxdb3_local.error( - f"[{task_id}] PLUGIN_DIR env var not set and config file path " - f"'{path}' was not found via fallbacks (tried: {candidates_str})" - ) - return - file_path = resolved - influxdb3_local.info(f"[{task_id}] Reading config file {file_path}") - with open(file_path, "rb") as f: - args = tomllib.load(f) - args["use_config_file"] = True - influxdb3_local.info(f"[{task_id}] New args content: {args}") - except Exception: - influxdb3_local.error(f"[{task_id}] Failed to read config file") - return - else: - args["use_config_file"] = False - - # Check for required arguments - if ( - not args - or "measurement" not in args - or "field_change_count" not in args - or "senders" not in args - or "window" not in args - ): - influxdb3_local.error( - f"[{task_id}] Missing required arguments: measurement, field_change_count, senders, or window" - ) + influxdb3_local.info( + f"[{task_id}] Starting scheduled field change check at {call_time}" + ) + + config: dict | None = _load_config( + influxdb3_local, args, _SCHEDULED_VALIDATORS, task_id + ) + if config is None: return - measurement: str = args["measurement"] - all_measurements: list = get_all_measurements(influxdb3_local) + measurement: str = config["measurement"] + all_measurements: list = get_table_names(influxdb3_local) if measurement not in all_measurements: influxdb3_local.error( f"[{task_id}] Measurement '{measurement}' not found in database" @@ -1323,53 +1106,55 @@ def process_scheduled_call( return try: - # Extract and validate parameters - field_counts: dict = parse_field_change_count(influxdb3_local, args, task_id) + field_counts: dict = parse_field_change_count(influxdb3_local, config, task_id) influxdb3_local.info(f"[{task_id}] Field change counts: {field_counts}") - senders_config: dict = parse_senders(influxdb3_local, args, task_id) - tags: list = get_tag_names(influxdb3_local, measurement, task_id) - window: timedelta = parse_window(args, task_id) - notification_path: str = args.get("notification_path", "notify") - port_override: int = parse_port_override(args, task_id) - influxdb3_auth_token: str = args.get("influxdb3_auth_token") or os.getenv( - "INFLUXDB3_AUTH_TOKEN" + senders_config: dict = parse_senders(influxdb3_local, config, task_id) + port_override: int = config["port_override"] + notification_path: str = config["notification_path"] + influxdb3_auth_token: str = ( + config.get("influxdb3_auth_token") + or os.getenv("INFLUXDB3_AUTH_TOKEN") + or "" ) - if influxdb3_auth_token is None: + if not influxdb3_auth_token: influxdb3_local.error( f"[{task_id}] Missing required argument: influxdb3_auth_token" ) return - notification_tpl: str = args.get( - "notification_text", - "Field $field in table $table changed $changes times in window $window for tags $tags", - ) + notification_tpl: str = config["notification_text"] - # Calculate time range + tags: list = get_measurement_tags(influxdb3_local, measurement, task_id) + window: timedelta = config["window"] end_time: datetime = call_time.replace(tzinfo=timezone.utc) start_time: datetime = end_time - window - influxdb3_local.info(f"[{task_id}] Querying '{measurement}' from {start_time} to {end_time}") + influxdb3_local.info( + f"[{task_id}] Querying '{measurement}' from {start_time} to {end_time}" + ) - # Build query to get data - query: str = build_query(measurement, start_time, end_time) - results: list = influxdb3_local.query(query) + results: list = query_window( + influxdb3_local, + measurement, + start=start_time.strftime("%Y-%m-%dT%H:%M:%SZ"), + end=end_time.strftime("%Y-%m-%dT%H:%M:%SZ"), + ) if not results: influxdb3_local.info( f"[{task_id}] No data found in '{measurement}' from {start_time} to {end_time}." ) return - influxdb3_local.info(f"[{task_id}] Retrieved {len(results)} records from {measurement}") + influxdb3_local.info( + f"[{task_id}] Retrieved {len(results)} records from {measurement}" + ) # Group data by unique tag combinations tag_combinations = defaultdict(list) for row in results: - tag_values = tuple(row[tag] if tag in row else "None" for tag in tags) + tag_values = tuple(row.get(tag, "None") for tag in tags) tag_combinations[tag_values].append(row) - # Process each tag combination for tag_values, rows in tag_combinations.items(): for field, count_threshold in field_counts.items(): - # Count changes changes: int = 0 prev_value = None for row in rows: @@ -1382,9 +1167,8 @@ def process_scheduled_call( if changes >= count_threshold: influxdb3_local.error( - f"[{task_id}] Found {count_threshold} changes in field '{field}' for tags {tag_values}, sending alert..." + f"[{task_id}] Found {changes} changes (threshold {count_threshold}) in field '{field}' for tags {tag_values}, sending alert..." ) - # Send notification tag_str = ", ".join( f"{tag}={value}" for tag, value in zip(tags, tag_values) ) @@ -1411,4 +1195,4 @@ def process_scheduled_call( ) except Exception as e: - influxdb3_local.error(f"[{task_id}] Error: {str(e)}") + influxdb3_local.error(f"[{task_id}] Error: {str(e)}") \ No newline at end of file diff --git a/influxdata/state_change/state_change_config_data_writes.toml b/influxdata/state_change/state_change_config_data_writes.toml index 5162293..dcde7f7 100644 --- a/influxdata/state_change/state_change_config_data_writes.toml +++ b/influxdata/state_change/state_change_config_data_writes.toml @@ -14,8 +14,9 @@ measurement = "your_measurement" # e.g., "cpu", "temperature", "home" # Format: [[field, value, count_or_duration], ...] # - field: field name (string) # - value: threshold value (number or boolean) -# - count_or_duration: integer (count-based) or duration string (e.g., "10s") for duration-based triggering -# Supported units for duration: s (seconds), min (minutes), h (hours), d (days), w (weeks) +# - count_or_duration: integer >= 1 (count-based) or duration string (e.g., "10s") for duration-based triggering +# Supported units for duration: us (microseconds), ms (milliseconds), s (seconds), +# min (minutes), h (hours), d (days), w (weeks) field_thresholds = [["field1", 0.0, "your_count_or_duration"]] # e.g., [["temp", 30, 1], ["temp", 15, "10s"]] # Notification channels @@ -29,10 +30,10 @@ senders = ["your_channel"] # e.g., ["slack"], ["sms", "http"] # Stability check parameters # Number of recent values to assess stability -# Specify an integer ≥ 1; default is 1 +# The check applies only when this is 2 or greater; default is 1 #state_change_window = 1 # e.g., 5 -# Maximum allowed changes in that window +# Number of changes in that window at which notifications start being suppressed # Specify an integer ≥ 1; default is 1 #state_change_count = 1 # e.g., 2 diff --git a/influxdata/state_change/state_change_config_scheduler.toml b/influxdata/state_change/state_change_config_scheduler.toml index 6d7c90d..cb607c6 100644 --- a/influxdata/state_change/state_change_config_scheduler.toml +++ b/influxdata/state_change/state_change_config_scheduler.toml @@ -12,16 +12,17 @@ measurement = "your_measurement" # e.g., "cpu", "temperature", "home" # Field change thresholds per field # Format: {field = count, ...} # - field: field name (string) -# - count: number of changes to trigger notification (integer) -field_change_count = {field1 = 0} # e.g., {temp = 3, hum = 2} +# - count: number of changes to trigger notification (integer >= 1) +field_change_count = {field1 = 1} # e.g., {temp = 3, hum = 2, "disk.used" = 1} # Notification channels # Specify a list of notification channels (strings) senders = ["your_channel"] # e.g., ["slack"], ["slack", "http"] # Analysis window duration -# Format: , where unit is s (seconds), min (minutes), h (hours), d (days), w (weeks) -window = "your_window" # e.g., "10m", "24h" +# Format: , where unit is us (microseconds), ms (milliseconds), s (seconds), +# min (minutes), h (hours), d (days), w (weeks). Must be a positive duration. +window = "your_window" # e.g., "10min", "24h" ########## Optional Parameters ########## # InfluxDB 3 API token for notifications diff --git a/influxdata/state_change/test_state_change.py b/influxdata/state_change/test_state_change.py new file mode 100644 index 0000000..c4a9398 --- /dev/null +++ b/influxdata/state_change/test_state_change.py @@ -0,0 +1,665 @@ +import json +from collections import deque +from datetime import datetime, timedelta, timezone + +import pytest + +import state_change_check_plugin as plugin + +TOKEN = "apiv3_secret_token_value" +WEBHOOK = "https://example.com/hook" + + +class FakeCache: + def __init__(self): + self.store = {} + self.ttls = {} + + def get(self, key, default=None, use_global=None): + return self.store.get(key, default) + + def put(self, key, value, ttl=None, use_global=None): + self.store[key] = value + self.ttls[key] = ttl + + def delete(self, key, use_global=None): + return self.store.pop(key, None) is not None + + +class FakeInfluxdb3Local: + """Stub of the runtime client: logging, trigger-local cache and queries.""" + + def __init__(self, tables=("home",), tags=("host",), rows=None): + self.cache = FakeCache() + self.logs = [] + self.tables = list(tables) + self.tags = list(tags) + self.rows = rows or [] + self.queries = [] + + def info(self, message): + self.logs.append(("info", message)) + + def warn(self, message): + self.logs.append(("warn", message)) + + def error(self, message): + self.logs.append(("error", message)) + + def query(self, query, params=None): + self.queries.append(query) + if "SHOW TABLES" in query: + return [{"table_name": t, "table_type": "BASE TABLE"} for t in self.tables] + if "information_schema" in query: + return [{"column_name": tag} for tag in self.tags] + return self.rows + + def messages(self, level=None): + return [m for lvl, m in self.logs if level is None or lvl == level] + + +class FakeResponse: + def raise_for_status(self): + return None + + def json(self): + return {"results": "recorded"} + + +@pytest.fixture +def sent(monkeypatch): + """Collect notification payloads instead of posting them.""" + posts = [] + + def fake_post(url, headers=None, data=None, timeout=None): + posts.append({"url": url, "headers": headers, "payload": json.loads(data)}) + return FakeResponse() + + monkeypatch.setattr(plugin.requests, "post", fake_post) + monkeypatch.setattr(plugin.time, "sleep", lambda seconds: None) + return posts + + +@pytest.fixture +def plugin_dir(monkeypatch, tmp_path): + monkeypatch.setenv("PLUGIN_DIR", str(tmp_path)) + monkeypatch.delenv("INFLUXDB3_AUTH_TOKEN", raising=False) + return tmp_path + + +WRITES_ARGS = { + "measurement": "home", + "field_thresholds": "temp:30:2", + "senders": "http", + "http_webhook_url": WEBHOOK, + "influxdb3_auth_token": TOKEN, +} +SCHEDULED_ARGS = { + "measurement": "home", + "field_change_count": "value:2", + "senders": "http", + "http_webhook_url": WEBHOOK, + "influxdb3_auth_token": TOKEN, + "window": "1h", +} + + +def batch(rows, table="home"): + return [{"table_name": table, "rows": rows}] + + +def count_key(field="temp", value=30, host="a"): + return plugin.generate_cache_key( + "home", field, value, "count", ["host"], {"host": host} + ) + + +def time_key(field="temp", value=30, host="a"): + return plugin.generate_cache_key( + "home", field, value, "time", ["host"], {"host": host} + ) + + +# --- parsing ---------------------------------------------------------------- + + +@pytest.mark.parametrize( + "raw, expected", + [ + ("10", 10), + (10, 10), + ("2h", timedelta(hours=2)), + ("500ms", timedelta(milliseconds=500)), + ("5min", timedelta(minutes=5)), + ], +) +def test_threshold_param_accepts_counts_and_durations(raw, expected): + assert plugin._parse_threshold_param(FakeInfluxdb3Local(), raw, "tid") == expected + + +@pytest.mark.parametrize("raw", ["0", "-5", "0s", "abc", True, "2x"]) +def test_threshold_param_rejects_invalid_values(raw): + influxdb3_local = FakeInfluxdb3Local() + + assert plugin._parse_threshold_param(influxdb3_local, raw, "tid") is None + assert influxdb3_local.messages("warn") + + +def test_field_thresholds_from_string(): + thresholds = plugin.parse_field_thresholds( + FakeInfluxdb3Local(), + {"field_thresholds": "temp:'30.1':10@humidity:'true':2h"}, + "tid", + ) + + assert thresholds == [ + ("temp", 30.1, 10), + ("humidity", True, timedelta(hours=2)), + ] + + +def test_field_thresholds_from_toml_entries(): + thresholds = plugin.parse_field_thresholds( + FakeInfluxdb3Local(), + {"field_thresholds": [["temp", 30, 1], ["status", "error", "10s"]]}, + "tid", + ) + + assert thresholds == [ + ("temp", 30, 1), + ("status", "error", timedelta(seconds=10)), + ] + + +def test_field_thresholds_skips_malformed_segments(): + influxdb3_local = FakeInfluxdb3Local() + + thresholds = plugin.parse_field_thresholds( + influxdb3_local, {"field_thresholds": "temp:30@humidity:5:1"}, "tid" + ) + + assert thresholds == [("humidity", 5, 1)] + assert any("must have exactly 2 colons" in m for m in influxdb3_local.messages("warn")) + + +def test_field_thresholds_without_valid_entries_raises(): + with pytest.raises(Exception, match="No valid field thresholds"): + plugin.parse_field_thresholds( + FakeInfluxdb3Local(), {"field_thresholds": "temp:30"}, "tid" + ) + + +def test_field_thresholds_rejects_unsupported_type(): + with pytest.raises(Exception, match="must be a list of entries or a string"): + plugin.parse_field_thresholds( + FakeInfluxdb3Local(), {"field_thresholds": 42}, "tid" + ) + + +def test_field_change_count_from_string(): + assert plugin.parse_field_change_count( + FakeInfluxdb3Local(), {"field_change_count": "temp:3.load:2"}, "tid" + ) == {"temp": 3, "load": 2} + + +def test_field_change_count_from_toml_mapping(): + assert plugin.parse_field_change_count( + FakeInfluxdb3Local(), {"field_change_count": {"temp": 3}}, "tid" + ) == {"temp": 3} + + +@pytest.mark.parametrize( + "raw, expected", + [ + ("disk.used:2", {"disk.used": 2}), + ("temp:3.disk.used:2.a.b.c:10", {"temp": 3, "disk.used": 2, "a.b.c": 10}), + ("sensor.1.value:5", {"sensor.1.value": 5}), + ("temp : 3", {"temp": 3}), + ], +) +def test_field_change_count_accepts_dotted_field_names(raw, expected): + """A pair ends at the dot after the count, so field names may contain dots.""" + influxdb3_local = FakeInfluxdb3Local() + + assert ( + plugin.parse_field_change_count( + influxdb3_local, {"field_change_count": raw}, "tid" + ) + == expected + ) + assert influxdb3_local.messages("warn") == [] + + +@pytest.mark.parametrize( + "raw, expected", + [("temp:abc.load:2", {"load": 2}), ("temp:0.load:2", {"load": 2})], +) +def test_field_change_count_skips_invalid_pairs(raw, expected): + influxdb3_local = FakeInfluxdb3Local() + + assert ( + plugin.parse_field_change_count( + influxdb3_local, {"field_change_count": raw}, "tid" + ) + == expected + ) + assert influxdb3_local.messages("warn") + + +def test_field_change_count_without_valid_entries_raises(): + with pytest.raises(Exception, match="No valid entries"): + plugin.parse_field_change_count( + FakeInfluxdb3Local(), {"field_change_count": "temp:0"}, "tid" + ) + + +@pytest.mark.parametrize( + "raw, expected", + [("1h", timedelta(hours=1)), ("30s", timedelta(seconds=30))], +) +def test_parse_window(raw, expected): + assert plugin.parse_window(raw) == expected + + +@pytest.mark.parametrize("raw", ["0s", "10m", "abc"]) +def test_parse_window_rejects_invalid(raw): + with pytest.raises(ValueError): + plugin.parse_window(raw) + + +# --- senders ---------------------------------------------------------------- + + +def test_senders_from_string_and_list(): + config = {"senders": "http.slack", "http_webhook_url": WEBHOOK, "slack_webhook_url": WEBHOOK} + + from_string = plugin.parse_senders(FakeInfluxdb3Local(), config, "tid") + from_list = plugin.parse_senders( + FakeInfluxdb3Local(), {**config, "senders": ["http", "slack"]}, "tid" + ) + + assert set(from_string) == {"http", "slack"} == set(from_list) + + +def test_senders_skips_unknown_channel(): + influxdb3_local = FakeInfluxdb3Local() + + senders = plugin.parse_senders( + influxdb3_local, + {"senders": "telegram.http", "http_webhook_url": WEBHOOK}, + "tid", + ) + + assert set(senders) == {"http"} + assert any("Invalid sender type: telegram" in m for m in influxdb3_local.messages("warn")) + + +def test_senders_requires_webhook_url(): + with pytest.raises(Exception, match="No valid senders"): + plugin.parse_senders(FakeInfluxdb3Local(), {"senders": "http"}, "tid") + + +def test_senders_rejects_non_http_scheme(): + influxdb3_local = FakeInfluxdb3Local() + + with pytest.raises(Exception, match="No valid senders"): + plugin.parse_senders( + influxdb3_local, + {"senders": "http", "http_webhook_url": "ftp://example.com/hook"}, + "tid", + ) + assert any("must start with" in m for m in influxdb3_local.messages("error")) + + +# --- configuration ---------------------------------------------------------- + + +def test_load_config_applies_defaults(plugin_dir): + config = plugin._load_config( + FakeInfluxdb3Local(), dict(WRITES_ARGS), plugin._WRITES_VALIDATORS, "tid" + ) + + assert config["port_override"] == 8181 + assert config["notification_path"] == "notify" + assert config["state_change_window"] == 1 + assert config["state_change_count"] == 1 + + +def test_load_config_reads_toml(plugin_dir): + (plugin_dir / "writes.toml").write_text( + "measurement = 'home'\n" + "senders = ['http']\n" + "http_webhook_url = 'https://example.com/hook'\n" + "influxdb3_auth_token = 'tok'\n" + "field_thresholds = [['temp', 30, 1]]\n" + "state_change_window = 4\n" + ) + + config = plugin._load_config( + FakeInfluxdb3Local(), + {"config_file_path": "writes.toml"}, + plugin._WRITES_VALIDATORS, + "tid", + ) + + assert config["senders"] == ["http"] + assert config["field_thresholds"] == [["temp", 30, 1]] + assert config["state_change_window"] == 4 + + +def test_load_config_rejects_non_toml_path(plugin_dir): + influxdb3_local = FakeInfluxdb3Local() + + assert ( + plugin._load_config( + influxdb3_local, + {"config_file_path": "config.yaml"}, + plugin._WRITES_VALIDATORS, + "tid", + ) + is None + ) + assert any("expected a .toml file" in m for m in influxdb3_local.messages("error")) + + +@pytest.mark.parametrize( + "override", + [{"measurement": None}, {"state_change_window": "-1"}, {"port_override": "70000"}], +) +def test_load_config_reports_validation_failures(plugin_dir, override): + args = {**WRITES_ARGS, **override} + args = {key: value for key, value in args.items() if value is not None} + influxdb3_local = FakeInfluxdb3Local() + + assert ( + plugin._load_config( + influxdb3_local, args, plugin._WRITES_VALIDATORS, "tid" + ) + is None + ) + assert any("Failed to load configuration" in m for m in influxdb3_local.messages("error")) + + +def test_load_config_accepts_zero_stability_settings(plugin_dir): + """Existing triggers may pass 0; it behaves like the default of 1.""" + args = {**WRITES_ARGS, "state_change_window": "0", "state_change_count": "0"} + + config = plugin._load_config( + FakeInfluxdb3Local(), args, plugin._WRITES_VALIDATORS, "tid" + ) + + assert config["state_change_window"] == 0 + assert config["state_change_count"] == 0 + + +def test_writes_with_zero_window_keeps_alerting(plugin_dir, sent): + influxdb3_local = FakeInfluxdb3Local() + args = {**WRITES_ARGS, "field_thresholds": "temp:30:1", "state_change_window": "0"} + + plugin.process_writes(influxdb3_local, batch([{"host": "a", "temp": 30}]), args) + + assert len(sent) == 1 + + +def test_load_config_uses_token_from_environment(plugin_dir, monkeypatch): + monkeypatch.setenv("INFLUXDB3_AUTH_TOKEN", "env-token") + args = {key: value for key, value in WRITES_ARGS.items() if key != "influxdb3_auth_token"} + + config = plugin._load_config( + FakeInfluxdb3Local(), args, plugin._WRITES_VALIDATORS, "tid" + ) + + assert config["influxdb3_auth_token"] == "env-token" + + +# --- stability and counters ------------------------------------------------- + + +@pytest.mark.parametrize( + "values, allowed, stable", + [ + ([], 1, True), + ([1], 1, True), + ([1, 1, 1], 1, True), + ([1, 2, 1], 2, False), + ([1, 2, 2], 2, True), + ], +) +def test_check_state_changes(values, allowed, stable): + assert plugin.check_state_changes(deque(values), allowed) is stable + + +@pytest.mark.parametrize("stored, expected", [("3", 3), ("", 0), (None, 0), ("x", 0)]) +def test_read_counter_tolerates_unusable_values(stored, expected): + influxdb3_local = FakeInfluxdb3Local() + if stored is not None: + influxdb3_local.cache.put("key", stored) + + assert plugin.read_counter(influxdb3_local, "key") == expected + + +# --- process_writes --------------------------------------------------------- + + +def test_writes_count_threshold_alerts_after_enough_matches(plugin_dir, sent): + influxdb3_local = FakeInfluxdb3Local() + + plugin.process_writes( + influxdb3_local, batch([{"host": "a", "temp": 30}]), dict(WRITES_ARGS) + ) + assert sent == [] + assert influxdb3_local.cache.get(count_key()) == "1" + + plugin.process_writes( + influxdb3_local, batch([{"host": "a", "temp": 30}]), dict(WRITES_ARGS) + ) + assert len(sent) == 1 + assert "changed to 30" in sent[0]["payload"]["notification_text"] + assert sent[0]["payload"]["senders_config"] == {"http": {"http_webhook_url": WEBHOOK}} + assert influxdb3_local.cache.get(count_key()) == "0" + + +def test_writes_count_resets_when_condition_fails(plugin_dir, sent): + influxdb3_local = FakeInfluxdb3Local() + + plugin.process_writes( + influxdb3_local, + batch([{"host": "a", "temp": 30}, {"host": "a", "temp": 25}]), + dict(WRITES_ARGS), + ) + + assert sent == [] + assert influxdb3_local.cache.get(count_key()) == "0" + + +def test_writes_missing_field_does_not_break_the_batch(plugin_dir, sent): + """Regression: an absent field used to store '' and crash the next int() read.""" + influxdb3_local = FakeInfluxdb3Local() + + plugin.process_writes( + influxdb3_local, + batch([{"host": "a", "hum": 5}, {"host": "a", "temp": 30}]), + dict(WRITES_ARGS), + ) + + assert influxdb3_local.messages("error") == [] + assert influxdb3_local.cache.get(count_key()) == "1" + + +def test_writes_duration_threshold_alerts_once_elapsed(plugin_dir, sent): + influxdb3_local = FakeInfluxdb3Local() + args = {**WRITES_ARGS, "field_thresholds": "temp:30:1h"} + started = datetime.now(timezone.utc) - timedelta(hours=2) + influxdb3_local.cache.put(time_key(), started.isoformat()) + + plugin.process_writes(influxdb3_local, batch([{"host": "a", "temp": 30}]), args) + + assert len(sent) == 1 + assert influxdb3_local.cache.get(time_key()) == "" + + +def test_writes_duration_threshold_waits_and_keeps_start(plugin_dir, sent): + influxdb3_local = FakeInfluxdb3Local() + args = {**WRITES_ARGS, "field_thresholds": "temp:30:1h"} + started = (datetime.now(timezone.utc) - timedelta(minutes=5)).isoformat() + influxdb3_local.cache.put(time_key(), started) + + plugin.process_writes(influxdb3_local, batch([{"host": "a", "temp": 30}]), args) + + assert sent == [] + assert influxdb3_local.cache.get(time_key()) == started + assert any("Condition still holding" in m for m in influxdb3_local.messages("warn")) + + +def test_writes_unstable_data_suppresses_notification(plugin_dir, sent): + influxdb3_local = FakeInfluxdb3Local() + args = { + **WRITES_ARGS, + "field_thresholds": "temp:30:1", + "state_change_window": "3", + "state_change_count": "2", + } + values_key = plugin.generate_cache_key( + "home", "temp", 30, "values", ["host"], {"host": "a"} + ) + influxdb3_local.cache.put(values_key, deque([25, 30, 25], maxlen=3)) + + plugin.process_writes(influxdb3_local, batch([{"host": "a", "temp": 30}]), args) + + assert sent == [] + assert any("unstable data state" in m for m in influxdb3_local.messages("warn")) + + +def test_writes_ignores_batches_of_other_tables(plugin_dir, sent): + influxdb3_local = FakeInfluxdb3Local() + + plugin.process_writes( + influxdb3_local, batch([{"host": "a", "temp": 30}], table="cpu"), dict(WRITES_ARGS) + ) + + assert sent == [] + assert not any("Starting writes process" in m for m in influxdb3_local.messages()) + assert not any("information_schema" in q for q in influxdb3_local.queries) + + +def test_writes_reports_unknown_measurement(plugin_dir, sent): + influxdb3_local = FakeInfluxdb3Local(tables=("cpu",)) + + plugin.process_writes( + influxdb3_local, batch([{"host": "a", "temp": 30}]), dict(WRITES_ARGS) + ) + + assert any("not found in database" in m for m in influxdb3_local.messages("error")) + + +def test_writes_caches_configuration(plugin_dir, sent): + influxdb3_local = FakeInfluxdb3Local() + + plugin.process_writes( + influxdb3_local, batch([{"host": "a", "temp": 30}]), dict(WRITES_ARGS) + ) + + assert influxdb3_local.cache.get(plugin._WRITES_CONFIG_CACHE_KEY) is not None + assert ( + influxdb3_local.cache.ttls[plugin._WRITES_CONFIG_CACHE_KEY] + == plugin._WRITES_CONFIG_TTL_SECONDS + ) + + +def test_writes_never_logs_credentials(plugin_dir, sent): + influxdb3_local = FakeInfluxdb3Local() + + plugin.process_writes( + influxdb3_local, batch([{"host": "a", "temp": 30}]), dict(WRITES_ARGS) + ) + + logged = " ".join(influxdb3_local.messages()) + assert TOKEN not in logged + assert WEBHOOK not in logged + + +# --- process_scheduled_call ------------------------------------------------- + + +def test_scheduled_alerts_when_changes_reach_threshold(plugin_dir, sent): + rows = [ + {"host": "a", "value": 1}, + {"host": "a", "value": 2}, + {"host": "a", "value": 1}, + ] + influxdb3_local = FakeInfluxdb3Local(rows=rows) + + plugin.process_scheduled_call( + influxdb3_local, datetime(2026, 8, 16, 12, 0, 0), dict(SCHEDULED_ARGS) + ) + + assert len(sent) == 1 + assert "changed 2 times" in sent[0]["payload"]["notification_text"] + assert any( + "Found 2 changes (threshold 2)" in m for m in influxdb3_local.messages("error") + ) + + +def test_scheduled_stays_quiet_below_threshold(plugin_dir, sent): + rows = [{"host": "a", "value": 1}, {"host": "a", "value": 2}] + influxdb3_local = FakeInfluxdb3Local(rows=rows) + args = {**SCHEDULED_ARGS, "field_change_count": "value:5"} + + plugin.process_scheduled_call( + influxdb3_local, datetime(2026, 8, 16, 12, 0, 0), args + ) + + assert sent == [] + + +def test_scheduled_counts_changes_per_tag_combination(plugin_dir, sent): + rows = [ + {"host": "a", "value": 1}, + {"host": "a", "value": 2}, + {"host": "a", "value": 3}, + {"host": "b", "value": 7}, + ] + influxdb3_local = FakeInfluxdb3Local(rows=rows) + + plugin.process_scheduled_call( + influxdb3_local, datetime(2026, 8, 16, 12, 0, 0), dict(SCHEDULED_ARGS) + ) + + assert len(sent) == 1 + assert "host=a" in sent[0]["payload"]["notification_text"] + + +def test_scheduled_handles_empty_window(plugin_dir, sent): + influxdb3_local = FakeInfluxdb3Local(rows=[]) + + plugin.process_scheduled_call( + influxdb3_local, datetime(2026, 8, 16, 12, 0, 0), dict(SCHEDULED_ARGS) + ) + + assert sent == [] + assert any("No data found" in m for m in influxdb3_local.messages("info")) + + +def test_scheduled_queries_the_configured_window(plugin_dir, sent): + influxdb3_local = FakeInfluxdb3Local(rows=[]) + + plugin.process_scheduled_call( + influxdb3_local, datetime(2026, 8, 16, 12, 0, 0), dict(SCHEDULED_ARGS) + ) + + data_query = influxdb3_local.queries[-1] + assert '"home"' in data_query + assert "time >= $start AND time < $end" in data_query + assert any("from 2026-08-16 11:00:00+00:00" in m for m in influxdb3_local.messages("info")) + + +def test_scheduled_never_logs_credentials(plugin_dir, sent): + influxdb3_local = FakeInfluxdb3Local(rows=[{"host": "a", "value": 1}]) + + plugin.process_scheduled_call( + influxdb3_local, datetime(2026, 8, 16, 12, 0, 0), dict(SCHEDULED_ARGS) + ) + + logged = " ".join(influxdb3_local.messages()) + assert TOKEN not in logged + assert WEBHOOK not in logged diff --git a/influxdata/stateless_adtk_detector/README.md b/influxdata/stateless_adtk_detector/README.md index 2318482..4cbaeb7 100644 --- a/influxdata/stateless_adtk_detector/README.md +++ b/influxdata/stateless_adtk_detector/README.md @@ -25,15 +25,33 @@ This plugin includes a JSON metadata schema in its docstring that defines suppor | `field` | string | required | Numeric field to evaluate | | `detectors` | string | required | Dot-separated list of advanced ADTK detectors for different anomaly types | | `detector_params` | string | required | Base64-encoded JSON parameters for each detector | -| `window` | string | required | Data analysis window with flexible scheduling. Format: `` (e.g., "1h", "30m") | +| `window` | string | required | Data analysis window. Format: `` (e.g., "1h", "30min"). Must be positive | | `senders` | string | required | Dot-separated notification channels with multi-channel notification support | +Duration units: `us`, `ms`, `s`, `min`, `h`, `d`, `w`. + ### Advanced parameters -| Parameter | Type | Default | Description | -|--------------------------|--------|---------|----------------------------------------------------------------------------------------------| -| `min_consensus` | number | 1 | Minimum detectors required to agree for consensus-based filtering to reduce false positives | -| `min_condition_duration` | string | "0s" | Minimum duration for configurable anomaly persistence before alerting | +| Parameter | Type | Default | Description | +|-----------------------------|---------|----------|------------------------------------------------------------------------------------------------------------| +| `min_consensus` | number | 1 | Minimum detectors required to agree for consensus-based filtering to reduce false positives (1 or greater) | +| `min_condition_duration` | string | "0s" | Minimum duration for configurable anomaly persistence before alerting | +| `group_by_tags` | bool | false | Analyze every tag combination as its own time series | +| `max_notifications_per_run` | number | 20 | Maximum number of notifications a single run may send | + +#### Analyzing tagged measurements + +By default the whole window forms a single time series. When a measurement holds several tag combinations (for example `host=server1` and `host=server2`), those rows share timestamps, and ADTK keeps only the first value of each timestamp — so one arbitrary series is analyzed and the rest of the data is ignored. + +Set `group_by_tags=true` to analyze each tag combination separately. Every series then gets its own detector run, its own consensus evaluation, and its own debounce state, so a measurement with N tag combinations can produce up to N notifications per run. + +#### Notification behavior + +The timestamp of the last alerted point is remembered per series, so a `window` longer than the trigger interval does not resend anomalies that earlier runs already reported. + +A single run sends at most `max_notifications_per_run` notifications. Anomalies beyond the limit are counted in a warning and are not resent by later runs — raise the limit if a run legitimately produces more alerts. + +Points that have no value for `field` are dropped before detection and reported in the log; without this a single NULL makes every detector fail. ### Notification parameters @@ -52,6 +70,8 @@ This plugin includes a JSON metadata schema in its docstring that defines suppor *To use a TOML configuration file, set the `PLUGIN_DIR` environment variable and specify the `config_file_path` in the trigger arguments.* This is in addition to the `--plugin-dir` flag when starting InfluxDB 3. +When a config file is given, it replaces the inline trigger arguments entirely. In TOML, `detectors` and `senders` accept either a list (`["QuantileAD", "PersistAD"]`) or a dot-separated string, and detector parameters may be given as a `[detector_params]` table or as a base64-encoded JSON string. + #### Example TOML configuration [adtk_anomaly_config_scheduler.toml](adtk_anomaly_config_scheduler.toml) @@ -74,10 +94,17 @@ For more information on using TOML configuration files, see the Using TOML Confi ## Software Requirements - **InfluxDB 3 Core/Enterprise**: with the Processing Engine enabled. +- **Python 3.11+** - **Python packages**: + - `influxdata-plugin-utils>=0.3.0` (for configuration loading, parsing, and schema introspection) - `adtk` (for anomaly detection) - - `pandas` (for data manipulation) + - `pandas<3` (for data manipulation) - `requests` (for HTTP notifications) + +`pandas` must stay below 3.0. Window-based detectors (`LevelShiftAD`, `VolatilityShiftAD`, `PersistAD`) +return `NaN` for the first `window` points, which pandas 3 refuses to store in a boolean result. With +pandas 3 installed those three detectors fail with `Invalid value 'nan' for dtype 'bool'`, are skipped +with a warning, and stop contributing to the consensus. - **Notification Sender Plugin** *(optional)*: Required if using the `senders` parameter. See the [influxdata/notifier plugin](../notifier/README.md). ### Installation steps @@ -95,9 +122,10 @@ For more information on using TOML configuration files, see the Using TOML Confi 2. Install required Python packages: ```bash + influxdb3 install package "influxdata-plugin-utils>=0.3.0" influxdb3 install package requests influxdb3 install package adtk - influxdb3 install package pandas + influxdb3 install package "pandas<3" ``` 3. *(Optional)* For notifications, install the [influxdata/notifier plugin](../notifier/README.md) and create an HTTP trigger for it. @@ -113,7 +141,7 @@ influxdb3 create trigger \ --database mydb \ --path "gh:influxdata/stateless_adtk_detector/adtk_anomaly_detection_plugin.py" \ --trigger-spec "every:10m" \ - --trigger-arguments "measurement=cpu,field=usage,detectors=QuantileAD.LevelShiftAD,detector_params=eyJRdWFudGlsZUFKIjogeyJsb3ciOiAwLjA1LCAiaGlnaCI6IDAuOTV9LCAiTGV2ZWxTaGlmdEFKIjogeyJ3aW5kb3ciOiA1fX0=,window=10m,senders=slack,slack_webhook_url=$SLACK_WEBHOOK_URL" \ + --trigger-arguments "measurement=cpu,field=usage,detectors=QuantileAD.LevelShiftAD,detector_params=eyJRdWFudGlsZUFKIjogeyJsb3ciOiAwLjA1LCAiaGlnaCI6IDAuOTV9LCAiTGV2ZWxTaGlmdEFKIjogeyJ3aW5kb3ciOiA1fX0=,window=10min,senders=slack,slack_webhook_url=$SLACK_WEBHOOK_URL" \ anomaly_detector ``` @@ -157,7 +185,7 @@ influxdb3 create trigger \ --database monitoring \ --path "gh:influxdata/stateless_adtk_detector/adtk_anomaly_detection_plugin.py" \ --trigger-spec "every:15m" \ - --trigger-arguments "measurement=cpu_metrics,field=utilization,detectors=QuantileAD.LevelShiftAD,detector_params=eyJRdWFudGlsZUFEIjogeyJsb3ciOiAwLjEsICJoaWdoIjogMC45fSwgIkxldmVsU2hpZnRBRCI6IHsid2luZG93IjogMTB9fQ==,min_consensus=2,window=30m,senders=discord,discord_webhook_url=$DISCORD_WEBHOOK_URL" \ + --trigger-arguments "measurement=cpu_metrics,field=utilization,detectors=QuantileAD.LevelShiftAD,detector_params=eyJRdWFudGlsZUFEIjogeyJsb3ciOiAwLjEsICJoaWdoIjogMC45fSwgIkxldmVsU2hpZnRBRCI6IHsid2luZG93IjogMTB9fQ==,min_consensus=2,window=30min,senders=discord,discord_webhook_url=$DISCORD_WEBHOOK_URL" \ cpu_consensus_detector ``` @@ -175,7 +203,7 @@ influxdb3 create trigger \ --database trading \ --path "gh:influxdata/stateless_adtk_detector/adtk_anomaly_detection_plugin.py" \ --trigger-spec "every:1m" \ - --trigger-arguments "measurement=stock_prices,field=price,detectors=VolatilityShiftAD,detector_params=eyJWb2xhdGlsaXR5U2hpZnRBRCI6IHsid2luZG93IjogMjB9fQ==,window=1h,min_condition_duration=5m,senders=sms,twilio_from_number=+1234567890,twilio_to_number=+0987654321" \ + --trigger-arguments "measurement=stock_prices,field=price,detectors=VolatilityShiftAD,detector_params=eyJWb2xhdGlsaXR5U2hpZnRBRCI6IHsid2luZG93IjogMjB9fQ==,window=1h,min_condition_duration=5min,senders=sms,twilio_from_number=+1234567890,twilio_to_number=+0987654321" \ volatility_detector ``` @@ -186,6 +214,9 @@ influxdb3 create trigger \ - `adtk_anomaly_detection_plugin.py`: The main plugin code containing the scheduled handler for anomaly detection - `adtk_anomaly_config_scheduler.toml`: Example TOML configuration file +- `test_adtk_anomaly_detection.py`: Pytest suite (49 tests, runs without a live InfluxDB 3 server) +- `requirements.txt`: Runtime dependencies (`influxdata-plugin-utils>=0.3.0`, `requests`, `adtk`, `pandas<3`) +- `requirements-dev.txt`: Development dependencies (`pytest`) ### Logging @@ -209,6 +240,14 @@ Key operations: 4. Evaluates consensus across detectors 5. Sends notifications when anomalies are confirmed +#### `parse_detectors(influxdb3_local, config, task_id)` + +Resolves the detectors to apply together with their parameters. Detectors that are unknown, have no entry in `detector_params`, or miss a parameter required to construct them are skipped with a warning and do not count toward `min_consensus`. + +#### `split_by_tags(df, tags, group_by_tags)` + +Splits query results into one frame per tag combination when `group_by_tags` is enabled, so detectors never mix values written under different tag sets. + ## Troubleshooting ### Common issues @@ -221,9 +260,25 @@ Key operations: **Solution**: Increase `min_consensus` to require more detectors to agree. Add `min_condition_duration` to require anomalies to persist. Adjust detector-specific thresholds in `detector_params`. +#### Issue: A newly created measurement is reported as not found + +**Solution**: Table and tag names are cached for one hour per trigger. Wait for the cache to expire, or recreate the trigger to clear it. + +#### Issue: Anomalies of some tag combinations are never detected + +**Solution**: Set `group_by_tags=true`. Without it, rows of different tag combinations share timestamps and only the first series survives. + +#### Issue: A detector is skipped with a warning + +**Solution**: The warning names the reason: the detector is not in the supported list (check the spelling), has no entry in `detector_params`, or misses a required parameter (`window` for `LevelShiftAD` and `VolatilityShiftAD`). `Invalid value 'nan' for dtype 'bool'` means pandas 3 is installed — downgrade to `pandas<3`. + +#### Issue: No anomalies are ever reported + +**Solution**: Check the warnings. `min_consensus` must not exceed the number of detectors that were actually applied — skipped detectors reduce that count. `min_condition_duration` must be shorter than `window`, otherwise no anomaly can persist long enough within a single query window. + #### Issue: Missing dependencies -**Solution**: Install required packages: `adtk`, `pandas`, `requests`. Ensure the Notifier Plugin is installed for notifications. +**Solution**: Install required packages: `influxdata-plugin-utils`, `adtk`, `pandas`, `requests`. Ensure the Notifier Plugin is installed for notifications. #### Issue: Data quality issues diff --git a/influxdata/stateless_adtk_detector/adtk_anomaly_config_scheduler.toml b/influxdata/stateless_adtk_detector/adtk_anomaly_config_scheduler.toml index be16b92..7c789b1 100644 --- a/influxdata/stateless_adtk_detector/adtk_anomaly_config_scheduler.toml +++ b/influxdata/stateless_adtk_detector/adtk_anomaly_config_scheduler.toml @@ -16,15 +16,16 @@ field = "your_field" # e.g., "usage", "value", "temp" # ADTK detectors to use # Supported detectors: GeneralizedESDTestAD, InterQuartileRangeAD, ThresholdAD, QuantileAD, LevelShiftAD, VolatilityShiftAD, PersistAD, SeasonalAD -# Specify a list of detector names (strings) +# Specify a list of detector names (strings), or a dot-separated string detectors = ["your_detector"] # e.g., ["QuantileAD"], ["GeneralizedESDTestAD", "InterQuartileRangeAD"] # Time window for analysis -# Format: , where unit is s (seconds), min (minutes), h (hours), d (days), w (weeks) -window = "your_window" # e.g., "15m", "24h" +# Format: , where unit is us, ms, s (seconds), min (minutes), h (hours), d (days), w (weeks) +# Must be a positive duration +window = "your_window" # e.g., "15min", "24h" # Notification channels -# Specify a list of notification channels (strings) +# Specify a list of notification channels (strings), or a dot-separated string senders = ["your_channel"] # e.g., ["slack"], ["http", "sms"] ########## Optional Parameters ########## @@ -32,9 +33,17 @@ senders = ["your_channel"] # e.g., ["slack"], ["http", "sms"] # Specify an integer ≥ 1; default is 1 #min_consensus = 1 # e.g., 2 +# Analyze every tag combination as its own time series +# Without it the whole window is one series and rows sharing a timestamp are collapsed +#group_by_tags = true # default is false + +# Maximum number of notifications a single run may send +# Anomalies beyond the limit are reported in a warning and not resent later +#max_notifications_per_run = 20 # e.g., 50 + # Minimum duration an anomaly condition must persist before notifying -# Format: , where unit is s (seconds), min (minutes), h (hours), d (days), w (weeks) -#min_condition_duration = "your_duration" # e.g., "1m", "10m" +# Format: , where unit is us, ms, s (seconds), min (minutes), h (hours), d (days), w (weeks) +#min_condition_duration = "your_duration" # e.g., "1min", "10min" # InfluxDB 3 API token # Specify the token (string); can also be provided via INFLUXDB3_AUTH_TOKEN environment variable @@ -84,14 +93,12 @@ senders = ["your_channel"] # e.g., ["slack"], ["http", "sms"] #twilio_to_number = "your_twilio_to_number" # e.g., "+0987654321" # --- WhatsApp (via Twilio) --- -# WhatsApp sender number (required for WhatsApp, format: +1234567890) -#whatsapp_from_number = "your_whatsapp_from_number" # e.g., "+1234567890" -# WhatsApp recipient number (required for WhatsApp, format: +0987654321) -#whatsapp_to_number = "your_whatsapp_to_number" # e.g., "+0987654321" +# WhatsApp uses the same twilio_* settings as SMS above # Detector parameters (Required) # Format: {"DetectorName": {param1: value, ...}, ...} # Specify parameters for each detector listed in detectors +# A base64-encoded JSON string is also accepted: detector_params = "eyJRdWFudGlsZUFEIjogey4uLn19" [detector_params] your_detector = { param1 = "value1", param2 = "value2" } # e.g., QuantileAD = { low = 0.05, high = 0.95 } diff --git a/influxdata/stateless_adtk_detector/adtk_anomaly_detection_plugin.py b/influxdata/stateless_adtk_detector/adtk_anomaly_detection_plugin.py index 03548e7..ef1a050 100644 --- a/influxdata/stateless_adtk_detector/adtk_anomaly_detection_plugin.py +++ b/influxdata/stateless_adtk_detector/adtk_anomaly_detection_plugin.py @@ -17,31 +17,37 @@ { "name": "detectors", "example": "QuantileAD.LevelShiftAD", - "description": "Dot-separated list of ADTK detectors. Supported: GeneralizedESDTestAD, InterQuartileRangeAD, ThresholdAD, QuantileAD, LevelShiftAD, VolatilityShiftAD, PersistAD, SeasonalAD.", + "description": "Dot-separated list of ADTK detectors (a TOML config may use a list instead). Supported: GeneralizedESDTestAD, InterQuartileRangeAD, ThresholdAD, QuantileAD, LevelShiftAD, VolatilityShiftAD, PersistAD, SeasonalAD.", "required": true }, { "name": "detector_params", "example": "eyJRdWFudGlsZUFKIjogeyJsb3dfcXVhbnRpbGUiOiA...", - "description": "Base64-encoded JSON string specifying parameters for each detector.", + "description": "Base64-encoded JSON string specifying parameters for each detector (a TOML config may use a [detector_params] table instead).", "required": true }, { "name": "min_consensus", "example": "2", - "description": "Minimum number of detectors that must agree to flag a point as anomalous. Default: 1.", + "description": "Minimum number of detectors that must agree to flag a point as anomalous. Must be 1 or greater. Default: 1.", + "required": false + }, + { + "name": "group_by_tags", + "example": "true", + "description": "Analyze every tag combination as its own time series. When disabled (default), all rows of the window form a single series and rows sharing a timestamp are collapsed to the first one. Default: false.", "required": false }, { "name": "window", "example": "1h", - "description": "Time window for data analysis (e.g., `1h` for 1 hour). Units: `s`, `min`, `h`, `d`, `w`.", + "description": "Time window for data analysis (e.g., `1h` for 1 hour). Must be a positive duration. Units: `us`, `ms`, `s`, `min`, `h`, `d`, `w`.", "required": true }, { "name": "senders", "example": "slack.discord", - "description": "Dot-separated list of notification channels. Supported channels: slack, discord, http, sms, whatsapp.", + "description": "Dot-separated list of notification channels (a TOML config may use a list instead). Supported channels: slack, discord, http, sms, whatsapp.", "required": true }, { @@ -52,14 +58,20 @@ }, { "name": "min_condition_duration", - "example": "5m", - "description": "Minimum duration for an anomaly condition to persist before triggering a notification (e.g., `5m`). Units: `s`, `min`, `h`, `d`, `w`. Default: `0s`.", + "example": "5min", + "description": "Minimum duration for an anomaly condition to persist before triggering a notification (e.g., `5min`). Units: `us`, `ms`, `s`, `min`, `h`, `d`, `w`. Default: `0s`.", + "required": false + }, + { + "name": "max_notifications_per_run", + "example": "20", + "description": "Maximum number of notifications sent by a single run. Anomalies beyond the limit are counted in a warning and not resent later. Default: 20.", "required": false }, { "name": "notification_text", "example": "Anomaly detected in $table.$field with value $value by $detectors. Tags: $tags", - "description": "Template for notification message with variables `$table`, `$field`, `$value`, `$detectors`, `$tags`.", + "description": "Template for notification message with variables `$table`, `$field`, `$value`, `$detectors`, `$tags`, `$timestamp`.", "required": false }, { @@ -149,11 +161,9 @@ import os import random import time -import tomllib import uuid from collections import defaultdict -from datetime import datetime, timedelta, timezone -from pathlib import Path +from datetime import datetime, timedelta from string import Template from urllib.parse import urlparse @@ -170,6 +180,18 @@ ThresholdAD, VolatilityShiftAD, ) +from influxdata_plugin_utils.config import Validator, load_plugin_config +from influxdata_plugin_utils.introspection import ( + get_table_names, + get_tag_names, + query_window, +) +from influxdata_plugin_utils.parsing import ( + parse_bool, + parse_delimited_list, + parse_int, + parse_timedelta, +) # Supported sender types with their required arguments AVAILABLE_SENDERS = { @@ -200,51 +222,105 @@ "SeasonalAD": SeasonalAD, } +# Detectors that cannot be constructed without these parameters +REQUIRED_DETECTOR_PARAMS = { + "LevelShiftAD": ["window"], + "VolatilityShiftAD": ["window"], +} -def get_all_measurements(influxdb3_local) -> list[str]: - """ - Retrieves a list of all tables of type 'BASE TABLE' from the current InfluxDB database. +# Detectors that classify points on their own and must not be fitted +UNFITTED_DETECTORS = ("ThresholdAD",) - Args: - influxdb3_local: InfluxDB client instance. - - Returns: - list[str]: List of table names (e.g., ["cpu", "memory", "disk"]). - """ - result: list = influxdb3_local.query("SHOW TABLES") - return [ - row["table_name"] for row in result if row.get("table_type") == "BASE TABLE" - ] +_DEFAULT_NOTIFICATION_TEXT = ( + "Anomaly detected in $table.$field with value $value by $detectors. Tags: $tags" +) -def get_tag_names(influxdb3_local, measurement: str, task_id: str) -> list[str]: +def parse_window(raw) -> timedelta: + """Parse the analysis window, rejecting non-positive durations.""" + window: timedelta = parse_timedelta(raw) + if window <= timedelta(0): + raise ValueError(f"Invalid window: {raw!r} (must be a positive duration)") + return window + + +_VALIDATORS = [ + Validator("measurement", required=True, cast=str), + Validator("field", required=True, cast=str), + Validator( + "detectors", + required=True, + cast=lambda raw: parse_delimited_list(raw, sep="."), + ), + Validator("detector_params", required=True), + Validator("senders", required=True), + Validator("window", required=True, cast=parse_window), + Validator("min_consensus", default=1, cast=lambda raw: parse_int(raw, minimum=1)), + Validator("group_by_tags", default=False, cast=parse_bool), + Validator("min_condition_duration", default="0s", cast=parse_timedelta), + Validator( + "max_notifications_per_run", + default=20, + cast=lambda raw: parse_int(raw, minimum=1), + ), + Validator("notification_text", default=_DEFAULT_NOTIFICATION_TEXT, cast=str), + Validator("notification_path", default="notify", cast=str), + Validator( + "port_override", + default=8181, + cast=lambda raw: parse_int(raw, minimum=1, maximum=65535), + ), +] + + +def _load_config(influxdb3_local, args: dict, task_id: str) -> dict | None: """ - Retrieves the list of tag names for a measurement. + Load the plugin configuration, applying defaults and type casts. + + A TOML file referenced by 'config_file_path' replaces the inline arguments; + INFLUXDB3_AUTH_TOKEN from the environment is used when the token is not + configured explicitly. Args: influxdb3_local: InfluxDB client instance. - measurement (str): Name of the measurement to query. - task_id (str): The task ID. + args (dict): Runtime arguments of the trigger. + task_id (str): Unique task identifier. Returns: - list[str]: List of tag names with 'Dictionary(Int32, Utf8)' data type. + dict | None: Config values keyed by lower-case name, or None if loading failed. """ - query: str = """ - SELECT column_name - FROM information_schema.columns - WHERE table_name = $measurement - AND data_type = 'Dictionary(Int32, Utf8)' - """ - res: list[dict] = influxdb3_local.query(query, {"measurement": measurement}) + config_file_path = (args or {}).get("config_file_path") + if config_file_path and not str(config_file_path).endswith(".toml"): + influxdb3_local.error( + f"[{task_id}] Invalid config file format: expected a .toml file" + ) + return None + + try: + loaded = load_plugin_config( + args, + validators=_VALIDATORS, + env_keys=["INFLUXDB3_AUTH_TOKEN"], + source="toml" if config_file_path else "args", + ) + except Exception as e: + influxdb3_local.error(f"[{task_id}] Failed to load configuration: {e}") + return None + + return {key.lower(): value for key, value in loaded.as_dict().items()} - if not res: + +def get_measurement_tags(influxdb3_local, measurement: str, task_id: str) -> list[str]: + """Return the cached tag names of a measurement, logging when it has none.""" + tags: list[str] = get_tag_names(influxdb3_local, measurement) + if not tags: + # an empty list stays cached for an hour and would hide tags added later + tags = get_tag_names(influxdb3_local, measurement, use_cache=False) + if not tags: influxdb3_local.info( f"[{task_id}] No tags found for measurement '{measurement}'." ) - return [] - - tag_names: list[str] = [tag["column_name"] for tag in res] - return tag_names + return tags def generate_cache_key( @@ -269,15 +345,14 @@ def generate_cache_key( return base -def parse_senders(influxdb3_local, args: dict, task_id: str) -> dict: +def parse_senders(influxdb3_local, config: dict, task_id: str) -> dict: """ - Parse and validate sender configurations from input arguments. + Parse and validate sender configurations from the loaded config. Args: influxdb3_local: InfluxDB client instance. - args (dict): Input arguments containing: - - "senders": dot-separated list of sender types (e.g., "slack.http"). - - For each sender, its own required keys (see AVAILABLE_SENDERS). + config (dict): Loaded config containing "senders" and the sender-specific + keys listed in AVAILABLE_SENDERS. task_id (str): Unique task identifier used for logging context. Returns: @@ -295,45 +370,37 @@ def parse_senders(influxdb3_local, args: dict, task_id: str) -> dict: Exception: If no valid senders are found after parsing. """ senders_config: defaultdict = defaultdict(dict) - - senders: str | list = args.get("senders") - if args["use_config_file"]: - if not isinstance(senders, list): - raise Exception( - f"[{task_id}] 'senders' must be a list when using config file" - ) - else: - senders = senders.split(".") + senders: list = parse_delimited_list(config["senders"], sep=".") for sender in senders: if sender not in AVAILABLE_SENDERS: influxdb3_local.warn(f"[{task_id}] Invalid sender type: {sender}") continue for key in AVAILABLE_SENDERS[sender]: - if key not in args and not any(ex in key for ex in EXCLUDED_KEYWORDS): + if key not in config and not any(ex in key for ex in EXCLUDED_KEYWORDS): influxdb3_local.warn( f"[{task_id}] Required key '{key}' missing for sender '{sender}'" ) senders_config.pop(sender, None) break if "url" in key and not validate_webhook_url( - influxdb3_local, sender, args[key], task_id + influxdb3_local, sender, config[key], task_id ): senders_config.pop(sender, None) break - if key not in args: + if key not in config: continue - senders_config[sender][key] = args[key] + senders_config[sender][key] = config[key] if not senders_config: - raise Exception(f"[{task_id}] No valid senders configured") + raise Exception("No valid senders configured") return senders_config def send_notification( influxdb3_local, port: int, path: str, token: str, payload: dict, task_id: str -) -> None: +) -> bool: """ Send a JSON POST to the given InfluxDB 3 webhook endpoint, with up to 3 retry attempts and randomized backoff delays between attempts. @@ -346,8 +413,8 @@ def send_notification( payload (dict): Dict to serialize as JSON in the POST body. task_id (str): Unique task identifier. - Raises: - requests.RequestException: If all retries fail or a non-2xx response is received. + Returns: + bool: True when the alert was accepted, False when every attempt failed. """ url: str = f"http://localhost:{port}/api/v3/engine/{path}" headers: dict = { @@ -366,7 +433,7 @@ def send_notification( influxdb3_local.info( f"[{task_id}] Alert sent to notification plugin with results: {resp.json()['results']}" ) - break + return True except requests.RequestException as e: influxdb3_local.warn( f"[{task_id}] [Attempt {attempt}/{max_retries}] Error sending alert to notification plugin: {e}" @@ -382,35 +449,7 @@ def send_notification( f"[{task_id}] Failed to send alert to notification plugin after {max_retries} attempts: {e}" ) - -def parse_port_override(args: dict, task_id: str) -> int: - """ - Parse and validate the 'port_override' argument, converting it from string to int. - - Args: - args (dict): Runtime arguments containing 'port_override'. - task_id (str): Unique task identifier for logging context. - - Returns: - int: Parsed port number (1–65535), or 8181 if not provided. - - Raises: - Exception: If 'port_override' is provided but is not a valid integer in the range 1–65535. - """ - raw: str | int = args.get("port_override", 8181) - - try: - port = int(raw) - except (TypeError, ValueError): - raise Exception(f"[{task_id}] Invalid port_override, not an integer: {raw!r}") - - # Validate port range - if not (1 <= port <= 65535): - raise Exception( - f"[{task_id}] Invalid port_override, must be between 1 and 65535: {port}" - ) - - return port + return False def validate_webhook_url(influxdb3_local, service: str, url: str, task_id: str) -> bool: @@ -455,130 +494,137 @@ def interpolate_notification_text(text: str, row_data: dict) -> str: return Template(text).safe_substitute(row_data) -def parse_time_duration(raw: str, task_id: str) -> timedelta: +def decode_detector_params(raw: str | dict) -> dict: """ - Parse a time duration string into a timedelta. - - Args: - raw (str): Duration string (e.g., "5m", "1h"). - task_id (str): Unique task identifier for logging. + Decode 'detector_params' from a mapping or a base64-encoded JSON string. Returns: - timedelta: Parsed duration. + dict: Mapping of detector names to their parameter dictionaries. Raises: - Exception: If the duration format is invalid. + Exception: If the value is not valid base64 or not a JSON object. """ - valid_units: dict = { - "s": "seconds", - "min": "minutes", - "h": "hours", - "d": "days", - "w": "weeks", - } - num_part, unit_part = "", "" - for unit in sorted(valid_units.keys(), key=len, reverse=True): - if raw.endswith(unit): - num_part = raw[: -len(unit)] - unit_part = unit - break - if not num_part or unit_part not in valid_units: - raise Exception(f"[{task_id}] Invalid duration format: {raw}") + if isinstance(raw, dict): + return raw + try: - num = int(num_part) - except ValueError: - raise Exception(f"[{task_id}] Invalid number in duration: {raw}") - return timedelta(**{valid_units[unit_part]: num}) + decoded: str = base64.b64decode(raw).decode("utf-8") + except Exception: + raise Exception("Invalid base64 encoding in detector_params") + try: + params = json.loads(decoded) + except json.JSONDecodeError: + raise Exception(f"Invalid JSON in decoded detector_params: {decoded}") -def parse_detector_params( - influxdb3_local, args: dict, detectors: list, task_id: str -) -> dict: - """ - Parse and validate detector parameters from args, expecting detector_params as a base64-encoded JSON string or use values from config file. + if not isinstance(params, dict): + raise Exception("detector_params must decode to a JSON object") + return params - Args: - influxdb3_local: InfluxDB client instance. - args (dict): Must include "detector_params" as a base64-encoded JSON string. - detectors (list): List of detector names. - task_id (str): Unique task identifier. + +def parse_detectors(influxdb3_local, config: dict, task_id: str) -> tuple[list, dict]: + """ + Resolve the detectors to apply together with their parameters. Returns: - dict: Mapping of detector names to their parameter dictionaries. + tuple[list, dict]: Applicable detector names and their parameters. Raises: - Exception: If detector_params is not valid base64, contains invalid JSON, or is missing required parameters. + Exception: If no detector is applicable. """ - input_params: str | dict = args["detector_params"] - - if args["use_config_file"]: - if isinstance(input_params, dict): - params: dict = input_params - else: - raise Exception( - f"[{task_id}] detector_params must be a dict when using config file" - ) - else: - try: - # Decode base64-encoded string - decoded_bytes: bytes = base64.b64decode(input_params) - decoded_str: str = decoded_bytes.decode("utf-8") - except Exception: - raise Exception( - f"[{task_id}] Invalid base64 encoding in detector_params: {input_params}" - ) - - try: - # Parse JSON from decoded string - params: dict = json.loads(decoded_str) - except json.JSONDecodeError: - raise Exception( - f"[{task_id}] Invalid JSON in decoded detector_params: {decoded_str}" - ) + params: dict = decode_detector_params(config["detector_params"]) - valid_params: dict = {} - for detector in detectors[:]: + detectors: list = [] + detector_params: dict = {} + for detector in config["detectors"]: + if detector not in AVAILABLE_DETECTORS: + influxdb3_local.warn(f"[{task_id}] Unknown detector: {detector}") + continue if detector not in params: influxdb3_local.warn( f"[{task_id}] Missing parameters for detector: {detector}" ) continue - valid_params[detector] = params[detector] + if not isinstance(params[detector], dict): + influxdb3_local.warn( + f"[{task_id}] Parameters for detector {detector} must be a mapping" + ) + continue - # Validate required parameters - if detector == "LevelShiftAD": - if "window" not in valid_params[detector]: - influxdb3_local.warn( - f"[{task_id}] LevelShiftAD requires 'window' parameter" - ) - del valid_params[detector] - detectors.remove(detector) - continue - elif detector == "VolatilityShiftAD": - if "window" not in valid_params[detector]: - influxdb3_local.warn( - f"[{task_id}] VolatilityShiftAD requires 'window' parameter" - ) - del valid_params[detector] - detectors.remove(detector) - continue + missing: list = [ + name + for name in REQUIRED_DETECTOR_PARAMS.get(detector, []) + if name not in params[detector] + ] + if missing: + influxdb3_local.warn( + f"[{task_id}] {detector} requires the '{', '.join(missing)}' parameter" + ) + continue - if not valid_params: - raise Exception( - f"[{task_id}] No valid detector parameters found in detector_params: {params}" - ) + detectors.append(detector) + detector_params[detector] = params[detector] - return valid_params + if not detectors: + raise Exception(f"No applicable detectors in {config['detectors']}") + return detectors, detector_params -def parse_min_consensus(min_consensus: str | int, task_id: str) -> int: - """Validate and convert min_consensus to an integer.""" - try: - return int(min_consensus) - except (TypeError, ValueError): - raise Exception( - f"[{task_id}] Invalid min_consensus, not an integer: {min_consensus!r}" - ) + +def format_tags(row: pd.Series, tags: list) -> str: + """Render the tag values of a row as 'tag=value' pairs.""" + return ", ".join(f"{tag}={row.get(tag, 'None')}" for tag in tags) + + +def split_by_tags(df: pd.DataFrame, tags: list, group_by_tags: bool) -> list: + """ + Split query results into one frame per tag combination. + """ + if not group_by_tags or not tags: + return [df] + return [group for _, group in df.groupby(tags, dropna=False, sort=False)] + + +def detect_anomalies( + influxdb3_local, + series: pd.Series, + detectors: list, + detector_params: dict, + min_consensus: int, + task_id: str, +) -> pd.Series | None: + """ + Apply every detector to the series and combine their verdicts by consensus. + + Returns: + pd.Series | None: True for every point flagged by at least 'min_consensus' + detectors, or None if no detector could be applied. + """ + anomaly_results: list = [] + for detector_name in detectors: + try: + params: dict = detector_params[detector_name] + influxdb3_local.info( + f"[{task_id}] Applying detector {detector_name} with params {params}" + ) + detector = AVAILABLE_DETECTORS[detector_name](**params) + if detector_name not in UNFITTED_DETECTORS: + detector.fit(series) + anomalies: pd.Series = detector.detect(series) + anomaly_results.append(anomalies) + influxdb3_local.info( + f"[{task_id}] Detector {detector_name} found {anomalies.sum()} anomalies" + ) + except Exception as e: + influxdb3_local.warn( + f"[{task_id}] Failed to apply detector {detector_name}: {e}" + ) + + if not anomaly_results: + return None + + anomaly_df = pd.concat(anomaly_results, axis=1).fillna(False) + return (anomaly_df.sum(axis=1) >= min_consensus).astype(bool) def process_scheduled_call( @@ -589,23 +635,24 @@ def process_scheduled_call( Queries a specified measurement and field within a time window, applies one or more ADTK detectors, and sends notifications for anomalies. Supports consensus-based detection - (all detectors must agree) and optional debounce logic. + (a configurable number of detectors must agree) and optional debounce logic. Args: influxdb3_local: InfluxDB client for querying, caching, and logging. call_time (datetime): UTC timestamp at which the scheduler triggers this function. args (dict): Required: - - table (str): Measurement name to query. + - measurement (str): Measurement name to query. - field (str): Numeric field to evaluate. - detectors (str): Dot-separated list of ADTK detectors. - - detector_params (str): JSON string of detector parameters. - - min_consensus (int): Minimum number of detectors required to flag anomaly. + - detector_params (str): Base64-encoded JSON of detector parameters. - window (str): Time window for data query (e.g., "1h"). - senders (str): Dot-separated notification channels. Optional: - config_file_path (str): path to config file to override args. - - min_condition_duration (str): Minimum anomaly duration (e.g., "5m"). + - min_consensus (int): Detectors required to flag an anomaly (default: 1). + - min_condition_duration (str): Minimum anomaly duration (e.g., "5min"). + - max_notifications_per_run (int): Notification cap per run (default: 20). - notification_text (str): Message template. - notification_path (str): Path for notification plugin (default: "notify"). - port_override (int): HTTP port (default: 8181). @@ -615,135 +662,79 @@ def process_scheduled_call( All exceptions are caught and logged via influxdb3_local.error. """ task_id: str = str(uuid.uuid4()) - influxdb3_local.info(f"[{task_id}] Starting anomaly detection scheduled call at {call_time} with args: {args}") + influxdb3_local.info( + f"[{task_id}] Starting anomaly detection scheduled call at {call_time}" + ) - # Override args with config file if specified - if args: - if path := args.get("config_file_path", None): - if not path.endswith(".toml"): - influxdb3_local.error( - f"[{task_id}] Invalid config file format: expected a .toml file" - ) - return - try: - plugin_dir_var: str | None = os.getenv("PLUGIN_DIR", None) - if plugin_dir_var: - file_path = Path(plugin_dir_var) / path - else: - # Fallbacks for servers where the operator has not exported PLUGIN_DIR: - # - INFLUXDB3_PLUGIN_DIR: set when the server is configured via env var - # - VIRTUAL_ENV: exported by the processing engine; default venv is /.venv - candidates: list[str] = [] - if influxdb3_plugin_dir := os.environ.get("INFLUXDB3_PLUGIN_DIR"): - candidates.append(influxdb3_plugin_dir) - if virtual_env := os.environ.get("VIRTUAL_ENV"): - candidates.append(str(Path(virtual_env).parent)) - - resolved = None - for base in candidates: - candidate = Path(base) / path - if candidate.exists(): - resolved = candidate - break - - if resolved is None: - candidates_str = ", ".join(candidates) if candidates else "none available" - influxdb3_local.error( - f"[{task_id}] PLUGIN_DIR env var not set and config file path " - f"'{path}' was not found via fallbacks (tried: {candidates_str})" - ) - return - file_path = resolved - influxdb3_local.info(f"[{task_id}] Reading config file {file_path}") - with open(file_path, "rb") as f: - args = tomllib.load(f) - args["use_config_file"] = True - influxdb3_local.info(f"[{task_id}] New args content: {args}") - except Exception: - influxdb3_local.error(f"[{task_id}] Failed to read config file") - return - else: - args["use_config_file"] = False - - if ( - not args - or "measurement" not in args - or "field" not in args - or "detectors" not in args - or "detector_params" not in args - or "window" not in args - or "senders" not in args - ): - influxdb3_local.error( - f"[{task_id}] Missing required arguments: measurement, field, detectors, detector_params, window, or senders" - ) + config: dict | None = _load_config(influxdb3_local, args, task_id) + if config is None: return try: - # Parse configuration - influxdb3_local.info(f"[{task_id}] Starting configuration parsing") - measurement: str = args["measurement"] - # Validate measurement - all_measurements: list = get_all_measurements(influxdb3_local) - if measurement not in all_measurements: + measurement: str = config["measurement"] + if measurement not in get_table_names(influxdb3_local): influxdb3_local.error(f"[{task_id}] Measurement '{measurement}' not found") return - influxdb3_local.info(f"[{task_id}] Validated measurement: {measurement}") - field: str = args["field"] - detectors: list = ( - args["detectors"].split(".") - if not args.get("use_config_file") - else args["detectors"] - ) - detector_params: dict = parse_detector_params( - influxdb3_local, args, detectors, task_id - ) - influxdb3_local.info(f"[{task_id}] Retrieved detector_params: {detector_params}") - - min_consensus: int = parse_min_consensus(args.get("min_consensus", 1), task_id) - window: timedelta = parse_time_duration(args["window"], task_id) - senders_config: dict = parse_senders(influxdb3_local, args, task_id) - port_override: int = parse_port_override(args, task_id) - min_condition_duration: timedelta = parse_time_duration( - args.get("min_condition_duration", "0s"), task_id - ) - notification_path: str = args.get("notification_path", "notify") - notification_text: str = args.get( - "notification_text", - "Anomaly detected in $table.$field with value $value by $detectors. Tags: $tags", + field: str = config["field"] + detectors, detector_params = parse_detectors(influxdb3_local, config, task_id) + influxdb3_local.info( + f"[{task_id}] Retrieved detector_params: {detector_params}" ) - influxdb3_auth_token: str = args.get("influxdb3_auth_token") or os.getenv( - "INFLUXDB3_AUTH_TOKEN" + + min_consensus: int = config["min_consensus"] + if min_consensus > len(detectors): + influxdb3_local.warn( + f"[{task_id}] min_consensus={min_consensus} exceeds the {len(detectors)} applicable detectors, no point can reach consensus" + ) + + group_by_tags: bool = config["group_by_tags"] + max_notifications_per_run: int = config["max_notifications_per_run"] + window: timedelta = config["window"] + senders_config: dict = parse_senders(influxdb3_local, config, task_id) + port_override: int = config["port_override"] + min_condition_duration: timedelta = config["min_condition_duration"] + if min_condition_duration >= window: + influxdb3_local.warn( + f"[{task_id}] min_condition_duration={min_condition_duration} is not shorter than window={window}, an anomaly can never persist long enough to alert" + ) + notification_path: str = config["notification_path"] + notification_text: str = config["notification_text"] + influxdb3_auth_token: str = ( + config.get("influxdb3_auth_token") + or os.getenv("INFLUXDB3_AUTH_TOKEN") + or "" ) if not influxdb3_auth_token: influxdb3_local.error(f"[{task_id}] Missing influxdb3_auth_token") return - influxdb3_local.info(f"[{task_id}] Configuration completed: field={field}, detectors={len(detectors)}, min_consensus={min_consensus}, window={window}") + influxdb3_local.info( + f"[{task_id}] Configuration completed: field={field}, detectors={len(detectors)}, min_consensus={min_consensus}, window={window}" + ) # Query data - tags: list = get_tag_names(influxdb3_local, measurement, task_id) - tags_clause: str = ", ".join([f'"{tag}"' for tag in tags]) + tags: list = get_measurement_tags(influxdb3_local, measurement, task_id) end_time: datetime = call_time start_time: datetime = end_time - window - influxdb3_local.info(f"[{task_id}] Querying {measurement}.{field} from {start_time} to {end_time}") - query: str = f""" - SELECT "{field}", "time", {tags_clause} - FROM "{measurement}" - WHERE time >= $start_time AND time < $end_time - ORDER BY time - """ - result: list = influxdb3_local.query( - query, - {"start_time": start_time.isoformat(), "end_time": end_time.isoformat()}, + influxdb3_local.info( + f"[{task_id}] Querying {measurement}.{field} from {start_time} to {end_time}" + ) + result: list = query_window( + influxdb3_local, + measurement, + start=start_time.isoformat(), + end=end_time.isoformat(), + columns=[field, "time", *tags], ) if not result: influxdb3_local.info( f"[{task_id}] No data found for {measurement}.{field} from {start_time} to {end_time}" ) return - influxdb3_local.info(f"[{task_id}] Retrieved {len(result)} records from {measurement}") + influxdb3_local.info( + f"[{task_id}] Retrieved {len(result)} records from {measurement}" + ) # Convert to pandas Series df: pd.DataFrame = pd.DataFrame(result) @@ -752,72 +743,114 @@ def process_scheduled_call( f"[{task_id}] Field '{field}' or 'time' not found in query results" ) return - series: pd.Series = pd.Series( - df[field].values, index=pd.to_datetime(df["time"], unit="ns") - ) - series = validate_series(series) # Ensure regular sampling and time order - influxdb3_local.info(f"[{task_id}] Prepared time series data with {len(series)} points") + groups: list = split_by_tags(df, tags, group_by_tags) # Apply detectors - influxdb3_local.info(f"[{task_id}] Starting anomaly detection with {len(detectors)} detectors") - anomaly_results: list = [] - for detector_name in detectors: - try: - params: dict = detector_params[detector_name] + influxdb3_local.info( + f"[{task_id}] Starting anomaly detection with {len(detectors)} detectors on {len(groups)} series" + ) + # Process anomalies with debounce logic + influxdb3_local.info( + f"[{task_id}] Processing anomalies with debounce logic (min_condition_duration={min_condition_duration})" + ) + processed_anomalies = 0 + sent_notifications = 0 + failed_notifications = 0 + suppressed_notifications = 0 + for group in groups: + series_label: str = ( + f" (tags: {format_tags(group.iloc[0], tags)})" + if group_by_tags and tags + else "" + ) + # a time index keeps the per-point row lookup below out of a full scan + rows: pd.DataFrame = group.drop_duplicates(subset="time") + rows.index = pd.to_datetime(rows["time"], unit="ns") + + series: pd.Series = rows[field].dropna() + missing_values: int = len(rows) - len(series) + if missing_values: + # detectors raise on NaN, which would drop the whole series influxdb3_local.info( - f"[{task_id}] Applying detector {detector_name} with params {params}" + f"[{task_id}] Skipped {missing_values} points without a '{field}' value{series_label}" ) - detector_class = AVAILABLE_DETECTORS[detector_name] - detector = detector_class(**params) - if detector_name not in ("ThresholdAD",): - detector.fit(series) - anomalies: pd.Series = detector.detect(series) - anomaly_results.append(anomalies) - anomaly_count = anomalies.sum() - influxdb3_local.info(f"[{task_id}] Detector {detector_name} found {anomaly_count} anomalies") - except Exception as e: - influxdb3_local.warn( - f"[{task_id}] Failed to apply detector {detector_name}: {e}" + if series.empty: + influxdb3_local.info( + f"[{task_id}] No values to analyze for {measurement}.{field}{series_label}" ) + continue - if not anomaly_results: - influxdb3_local.error( - f"[{task_id}] No valid detectors applied to {measurement}.{field}" + series = validate_series(series) # Ensure regular sampling and time order + influxdb3_local.info( + f"[{task_id}] Prepared time series data with {len(series)} points{series_label}" ) - return - # Consensus: point is anomaly if >= min_consensus detectors agree - anomaly_df = pd.concat(anomaly_results, axis=1).fillna(False) - anomaly_count = anomaly_df.sum(axis=1) - consensus_anomalies = anomaly_count >= min_consensus - consensus_anomalies = consensus_anomalies.astype(bool) - total_consensus_anomalies = consensus_anomalies.sum() - influxdb3_local.info(f"[{task_id}] Consensus analysis: {total_consensus_anomalies} anomalies detected with min_consensus={min_consensus}") + consensus_anomalies = detect_anomalies( + influxdb3_local, + series, + detectors, + detector_params, + min_consensus, + task_id, + ) + if consensus_anomalies is None: + influxdb3_local.error( + f"[{task_id}] No valid detectors applied to {measurement}.{field}{series_label}" + ) + continue + influxdb3_local.info( + f"[{task_id}] Consensus analysis: {consensus_anomalies.sum()} anomalies detected with min_consensus={min_consensus}{series_label}" + ) - # Process anomalies with debounce logic - influxdb3_local.info(f"[{task_id}] Processing anomalies with debounce logic (min_condition_duration={min_condition_duration})") - processed_anomalies = 0 - sent_notifications = 0 - for idx, is_anomaly in consensus_anomalies.items(): - row: pd.Series = df[df["time"] == pd.Timestamp(idx.isoformat()).value].iloc[ - 0 - ] - time_datetime: datetime = pd.to_datetime(row["time"], unit="ns") - cache_key: str = generate_cache_key(measurement, field, tags, row) - tag_str: str = ", ".join(f"{t}={row.get(t, 'None')}" for t in tags) - start_time_str: str = influxdb3_local.cache.get(cache_key, default="") - - if is_anomaly: - if not start_time_str: - if min_condition_duration > timedelta(0): - # Start of a new anomaly - influxdb3_local.cache.put(cache_key, time_datetime.isoformat()) - influxdb3_local.info( - f"[{task_id}] Anomaly started for {measurement}.{field} (tags: {tag_str}), waiting for duration {min_condition_duration}" - ) - continue + for timestamp, is_anomaly in consensus_anomalies.items(): + row: pd.Series = rows.loc[timestamp] + cache_key: str = generate_cache_key(measurement, field, tags, row) + alert_key: str = f"{cache_key}:last_alert" + tag_str: str = format_tags(row, tags) + + last_alert_str: str = influxdb3_local.cache.get(alert_key, default="") + if last_alert_str and timestamp <= pd.Timestamp(last_alert_str): + continue + + processed_anomalies += 1 + start_time_str: str = influxdb3_local.cache.get(cache_key, default="") + alert_reason: str | None = None + + if is_anomaly: + if not start_time_str: + if min_condition_duration > timedelta(0): + # Start of a new anomaly + influxdb3_local.cache.put(cache_key, timestamp.isoformat()) + influxdb3_local.info( + f"[{task_id}] Anomaly started for {measurement}.{field} (tags: {tag_str}), waiting for duration {min_condition_duration}" + ) + continue + alert_reason = f"Anomaly detected for {measurement}.{field} (tags: {tag_str}), sending alert" + else: + # Check duration + elapsed: timedelta = timestamp - pd.Timestamp(start_time_str) + if elapsed < min_condition_duration: + influxdb3_local.info( + f"[{task_id}] Anomaly ongoing for {elapsed} < {min_condition_duration} for {measurement}.{field} (tags: {tag_str})" + ) + continue + alert_reason = f"Anomaly persisted for {elapsed} for {measurement}.{field} (tags: {tag_str}), sending alert" + elif start_time_str: + # Reset cache if anomaly stops + influxdb3_local.cache.delete(cache_key) + influxdb3_local.info( + f"[{task_id}] Anomaly cleared for {measurement}.{field} (tags: {tag_str})" + ) + + if alert_reason is None: + continue - # Send notification + if ( + sent_notifications + failed_notifications + >= max_notifications_per_run + ): + suppressed_notifications += 1 + else: payload: dict = { "notification_text": interpolate_notification_text( notification_text, @@ -827,14 +860,13 @@ def process_scheduled_call( "value": row[field], "detectors": ".".join(detectors), "tags": tag_str, + "timestamp": timestamp.isoformat(), }, ), "senders_config": senders_config, } - influxdb3_local.error( - f"[{task_id}] Anomaly detected for {measurement}.{field} (tags: {tag_str}), sending alert" - ) - send_notification( + influxdb3_local.error(f"[{task_id}] {alert_reason}") + delivered: bool = send_notification( influxdb3_local, port_override, notification_path, @@ -842,57 +874,26 @@ def process_scheduled_call( payload, task_id, ) + if not delivered: + # leave the state untouched so a later run can alert again + failed_notifications += 1 + continue sent_notifications += 1 - else: - # Check duration - duration_start_time: datetime = datetime.fromisoformat( - start_time_str - ) - elapsed: timedelta = time_datetime - duration_start_time - if elapsed >= min_condition_duration: - # Send notification - payload: dict = { - "notification_text": interpolate_notification_text( - notification_text, - { - "table": measurement, - "field": field, - "value": row[field], - "detectors": ".".join(detectors), - "tags": tag_str, - }, - ), - "senders_config": senders_config, - } - influxdb3_local.error( - f"[{task_id}] Anomaly persisted for {elapsed} for {measurement}.{field} (tags: {tag_str}), sending alert" - ) - send_notification( - influxdb3_local, - port_override, - notification_path, - influxdb3_auth_token, - payload, - task_id, - ) - influxdb3_local.cache.put( - cache_key, "" - ) # Reset cache after sending - sent_notifications += 1 - else: - influxdb3_local.info( - f"[{task_id}] Anomaly ongoing for {elapsed} < {min_condition_duration} for {measurement}.{field} (tags: {tag_str})" - ) - else: - # Reset cache if anomaly stops - if start_time_str: - influxdb3_local.cache.put(cache_key, "") - influxdb3_local.info( - f"[{task_id}] Anomaly cleared for {measurement}.{field} (tags: {tag_str})" - ) - processed_anomalies += 1 - influxdb3_local.info(f"[{task_id}] Anomaly processing completed: {processed_anomalies} points processed, {sent_notifications} notifications sent") + influxdb3_local.cache.delete(cache_key) + influxdb3_local.cache.put(alert_key, timestamp.isoformat()) + + if failed_notifications: + influxdb3_local.warn( + f"[{task_id}] {failed_notifications} notifications could not be delivered, the next run will alert on them again" + ) + if suppressed_notifications: + influxdb3_local.warn( + f"[{task_id}] Suppressed {suppressed_notifications} notifications after reaching max_notifications_per_run={max_notifications_per_run}" + ) + influxdb3_local.info( + f"[{task_id}] Anomaly processing completed: {processed_anomalies} points processed, {sent_notifications} notifications sent" + ) except Exception as e: influxdb3_local.error(f"[{task_id}] Error: {e}") diff --git a/influxdata/stateless_adtk_detector/manifest.toml b/influxdata/stateless_adtk_detector/manifest.toml index 210f5ef..d563f9e 100644 --- a/influxdata/stateless_adtk_detector/manifest.toml +++ b/influxdata/stateless_adtk_detector/manifest.toml @@ -2,7 +2,7 @@ manifest_schema_version = "1.3" [plugin] name = "stateless_adtk_detector" -version = "1.2.0" +version = "1.3.0" description = "Provides anomaly detection capabilities for time series data using the ADTK library. Supports multiple stateless detectors with consensus-based detection and customizable notification messages." triggers = ["process_scheduled_call"] homepage = "https://www.influxdata.com/" @@ -16,7 +16,7 @@ exclude = [ [dependencies] database_version = ">=3.0.0" -python = ["requests", "adtk", "pandas"] +python = ["influxdata-plugin-utils>=0.3.0", "requests", "adtk", "pandas<3"] [[dependencies.plugins]] index_url = "https://github.com/influxdata/influxdb3_plugins/releases/download/registry/index.json" diff --git a/influxdata/stateless_adtk_detector/requirements-dev.txt b/influxdata/stateless_adtk_detector/requirements-dev.txt new file mode 100644 index 0000000..f145028 --- /dev/null +++ b/influxdata/stateless_adtk_detector/requirements-dev.txt @@ -0,0 +1,5 @@ +pytest +influxdata-plugin-utils>=0.3.0 +requests +adtk +pandas<3 diff --git a/influxdata/stateless_adtk_detector/requirements.txt b/influxdata/stateless_adtk_detector/requirements.txt index 4592bd5..559c4f4 100644 --- a/influxdata/stateless_adtk_detector/requirements.txt +++ b/influxdata/stateless_adtk_detector/requirements.txt @@ -1,3 +1,4 @@ +influxdata-plugin-utils>=0.3.0 requests adtk -pandas \ No newline at end of file +pandas<3 diff --git a/influxdata/stateless_adtk_detector/test_adtk_anomaly_detection.py b/influxdata/stateless_adtk_detector/test_adtk_anomaly_detection.py new file mode 100644 index 0000000..257da81 --- /dev/null +++ b/influxdata/stateless_adtk_detector/test_adtk_anomaly_detection.py @@ -0,0 +1,738 @@ +import base64 +import json +from datetime import datetime, timedelta, timezone +from itertools import chain + +import pandas as pd +import pytest + +import adtk_anomaly_detection_plugin as plugin + + +class FakeCache: + def __init__(self): + self.store = {} + self.ttls = {} + + def get(self, key, default=None, use_global=None): + return self.store.get(key, default) + + def put(self, key, value, ttl=None, use_global=None): + self.store[key] = value + self.ttls[key] = ttl + + def delete(self, key, use_global=None): + return self.store.pop(key, None) is not None + + +class FakeInfluxdb3Local: + """Stub of the runtime client: logging, trigger-local cache and queries.""" + + def __init__(self, tables=("cpu",), tags=("host",), rows=None): + self.cache = FakeCache() + self.logs = [] + self.tables = list(tables) + self.tags = list(tags) + self.rows = rows or [] + self.queries = [] + + def info(self, message): + self.logs.append(("info", message)) + + def warn(self, message): + self.logs.append(("warn", message)) + + def error(self, message): + self.logs.append(("error", message)) + + def query(self, query, params=None): + self.queries.append((query, params)) + if "SHOW TABLES" in query: + return [{"table_name": t, "table_type": "BASE TABLE"} for t in self.tables] + if "information_schema" in query: + return [{"column_name": tag} for tag in self.tags] + return self.rows + + def messages(self, level=None): + return [m for lvl, m in self.logs if level is None or lvl == level] + + def logged(self, fragment, level=None): + return any(fragment in message for message in self.messages(level)) + + def window_query(self): + return next( + (q, p) + for q, p in self.queries + if "information_schema" not in q and "SHOW" not in q + ) + + +class FakeResponse: + def __init__(self, status_code=200): + self.status_code = status_code + + def raise_for_status(self): + if self.status_code >= 400: + raise plugin.requests.HTTPError(f"{self.status_code} Server Error") + + def json(self): + return {"results": "recorded"} + + +@pytest.fixture +def sent(monkeypatch): + """Collect notification payloads instead of posting them.""" + posts = [] + + def fake_post(url, headers=None, data=None, timeout=None): + posts.append({"url": url, "headers": headers, "payload": json.loads(data)}) + return FakeResponse() + + monkeypatch.setattr(plugin.requests, "post", fake_post) + monkeypatch.setattr(plugin.time, "sleep", lambda seconds: None) + return posts + + +@pytest.fixture +def plugin_dir(monkeypatch, tmp_path): + monkeypatch.setenv("PLUGIN_DIR", str(tmp_path)) + monkeypatch.delenv("INFLUXDB3_PLUGIN_DIR", raising=False) + monkeypatch.delenv("INFLUXDB3_AUTH_TOKEN", raising=False) + return tmp_path + + +START = datetime(2026, 8, 13, 12, 0, tzinfo=timezone.utc) +CALL_TIME = START + timedelta(hours=1) + + +def encode(params): + return base64.b64encode(json.dumps(params).encode()).decode() + + +ARGS = { + "measurement": "cpu", + "field": "usage", + "detectors": "ThresholdAD", + "detector_params": encode({"ThresholdAD": {"high": 200}}), + "window": "2h", + "senders": "http", + "http_webhook_url": "https://example.com/hook", + "influxdb3_auth_token": "tok", +} + + +def rows(values, host="server1", start=START, step=timedelta(minutes=1)): + """Build query results: one row per value, one step apart.""" + return [ + { + "usage": value, + "time": int((start + step * index).timestamp() * 1_000_000_000), + "host": host, + } + for index, value in enumerate(values) + ] + + +def merged(*row_lists): + """Interleave several series the way a time-ordered query returns them.""" + return sorted(chain(*row_lists), key=lambda row: row["time"]) + + +def run(args=None, local=None, call_time=CALL_TIME): + local = local or FakeInfluxdb3Local(rows=rows([10.0, 999.0, 10.0])) + plugin.process_scheduled_call(local, call_time, {**ARGS, **(args or {})}) + return local + + +def series_of(values, start=START): + index = pd.to_datetime( + [ + int((start + timedelta(minutes=i)).timestamp() * 1e9) + for i in range(len(values)) + ], + unit="ns", + ) + return pd.Series(values, index=index) + + +# --- configuration ---------------------------------------------------------- + + +def test_config_applies_defaults(plugin_dir): + config = plugin._load_config(FakeInfluxdb3Local(), dict(ARGS), "tid") + + assert config["min_consensus"] == 1 + assert config["group_by_tags"] is False + assert config["max_notifications_per_run"] == 20 + assert config["min_condition_duration"] == timedelta(0) + assert config["port_override"] == 8181 + assert config["notification_path"] == "notify" + assert config["notification_text"] == plugin._DEFAULT_NOTIFICATION_TEXT + assert config["window"] == timedelta(hours=2) + assert config["detectors"] == ["ThresholdAD"] + + +def test_config_reports_missing_required_argument(plugin_dir): + local = FakeInfluxdb3Local() + args = {key: value for key, value in ARGS.items() if key != "field"} + + assert plugin._load_config(local, args, "tid") is None + assert local.logged("field is required", "error") + + +@pytest.mark.parametrize( + "override", + [ + {"min_consensus": "0"}, + {"window": "0s"}, + {"window": "10m"}, + {"port_override": "70000"}, + {"group_by_tags": "maybe"}, + {"max_notifications_per_run": "0"}, + {"min_condition_duration": "-5min"}, + ], +) +def test_config_rejects_invalid_values(plugin_dir, override): + local = FakeInfluxdb3Local() + + assert plugin._load_config(local, {**ARGS, **override}, "tid") is None + assert local.logged("Failed to load configuration", "error") + + +def test_config_rejects_non_toml_path(plugin_dir): + local = FakeInfluxdb3Local() + + assert ( + plugin._load_config(local, {"config_file_path": "config.yaml"}, "tid") is None + ) + assert local.logged("expected a .toml file", "error") + + +def test_config_reports_missing_toml_file(plugin_dir): + local = FakeInfluxdb3Local() + + assert ( + plugin._load_config(local, {"config_file_path": "absent.toml"}, "tid") is None + ) + assert local.logged("Failed to load configuration", "error") + + +def test_config_from_toml_uses_native_structures(plugin_dir, sent): + (plugin_dir / "adtk.toml").write_text( + 'measurement = "cpu"\n' + 'field = "usage"\n' + 'detectors = ["ThresholdAD"]\n' + 'window = "2h"\n' + 'senders = ["http"]\n' + 'http_webhook_url = "https://example.com/hook"\n' + 'influxdb3_auth_token = "from-toml"\n' + "port_override = 8182\n" + 'notification_path = "custom/notify"\n' + "\n[detector_params]\n" + "ThresholdAD = { high = 200 }\n" + ) + local = FakeInfluxdb3Local(rows=rows([10.0, 999.0])) + + plugin.process_scheduled_call(local, CALL_TIME, {"config_file_path": "adtk.toml"}) + + assert len(sent) == 1 + assert sent[0]["url"] == "http://localhost:8182/api/v3/engine/custom/notify" + assert sent[0]["headers"]["Authorization"] == "Bearer from-toml" + + +def test_config_from_toml_accepts_inline_spellings(plugin_dir, sent): + (plugin_dir / "adtk.toml").write_text( + 'measurement = "cpu"\n' + 'field = "usage"\n' + 'detectors = "ThresholdAD"\n' + f'detector_params = "{encode({"ThresholdAD": {"high": 200}})}"\n' + 'window = "2h"\n' + 'senders = "http.slack"\n' + 'http_webhook_url = "https://example.com/hook"\n' + 'slack_webhook_url = "https://hooks.slack.com/services/T"\n' + 'influxdb3_auth_token = "tok"\n' + ) + local = FakeInfluxdb3Local(rows=rows([10.0, 999.0])) + + plugin.process_scheduled_call(local, CALL_TIME, {"config_file_path": "adtk.toml"}) + + assert set(sent[0]["payload"]["senders_config"]) == {"http", "slack"} + + +def test_token_falls_back_to_environment(monkeypatch, plugin_dir, sent): + monkeypatch.setenv("INFLUXDB3_AUTH_TOKEN", "from-env") + args = {key: value for key, value in ARGS.items() if key != "influxdb3_auth_token"} + local = FakeInfluxdb3Local(rows=rows([10.0, 999.0])) + + plugin.process_scheduled_call(local, CALL_TIME, args) + + assert sent[0]["headers"]["Authorization"] == "Bearer from-env" + + +def test_missing_token_stops_the_run(plugin_dir, sent): + local = FakeInfluxdb3Local(rows=rows([999.0])) + args = dict(ARGS) + args["influxdb3_auth_token"] = "" + + plugin.process_scheduled_call(local, CALL_TIME, args) + + assert local.logged("Missing influxdb3_auth_token", "error") + assert sent == [] + + +# --- detector parameters ---------------------------------------------------- + + +def test_decode_detector_params_accepts_mapping_and_base64(): + params = {"ThresholdAD": {"high": 200}} + + assert plugin.decode_detector_params(params) == params + assert plugin.decode_detector_params(encode(params)) == params + + +@pytest.mark.parametrize( + "raw, message", + [ + ("!!!not base64!!!", "Invalid base64 encoding"), + (base64.b64encode(b"{broken").decode(), "Invalid JSON"), + (encode(["ThresholdAD"]), "must decode to a JSON object"), + ], +) +def test_decode_detector_params_rejects_invalid(raw, message): + with pytest.raises(Exception, match=message): + plugin.decode_detector_params(raw) + + +def test_parse_detectors_skips_detectors_it_cannot_apply(): + local = FakeInfluxdb3Local() + config = { + "detectors": ["ThresholdAD", "Nonsense", "PersistAD", "LevelShiftAD"], + "detector_params": { + "ThresholdAD": {"high": 200}, + "LevelShiftAD": {}, + "Nonsense": {}, + }, + } + + detectors, params = plugin.parse_detectors(local, config, "tid") + + assert detectors == ["ThresholdAD"] + assert set(params) == {"ThresholdAD"} + assert local.logged("Unknown detector: Nonsense", "warn") + assert local.logged("Missing parameters for detector: PersistAD", "warn") + assert local.logged("LevelShiftAD requires the 'window' parameter", "warn") + + +def test_parse_detectors_rejects_non_mapping_parameters(): + local = FakeInfluxdb3Local() + config = {"detectors": ["ThresholdAD"], "detector_params": {"ThresholdAD": [200]}} + + with pytest.raises(Exception, match="No applicable detectors"): + plugin.parse_detectors(local, config, "tid") + assert local.logged("must be a mapping", "warn") + + +# --- detection and consensus ------------------------------------------------ + + +def test_detect_anomalies_requires_min_consensus_agreement(): + local = FakeInfluxdb3Local() + series = plugin.validate_series(series_of([10.0, 11.0, 12.0, 300.0, 999.0])) + detectors = ["QuantileAD", "ThresholdAD"] + params = {"QuantileAD": {"high": 0.6}, "ThresholdAD": {"high": 500}} + + lenient = plugin.detect_anomalies(local, series, detectors, params, 1, "tid") + strict = plugin.detect_anomalies(local, series, detectors, params, 2, "tid") + + assert lenient.sum() == 2 + assert strict.sum() == 1 + assert strict[strict].index == series.index[-1:] + + +def test_detect_anomalies_returns_none_when_every_detector_fails(): + local = FakeInfluxdb3Local() + series = plugin.validate_series(series_of([1.0, 2.0, 3.0])) + + result = plugin.detect_anomalies( + local, series, ["ThresholdAD"], {"ThresholdAD": {"nonsense": 1}}, 1, "tid" + ) + + assert result is None + assert local.logged("Failed to apply detector ThresholdAD", "warn") + + +def test_only_trainable_detectors_are_fitted(monkeypatch): + fitted = [] + + class Recording: + def __init__(self, **params): + self.name = params["name"] + + def fit(self, series): + fitted.append(self.name) + + def detect(self, series): + return pd.Series(False, index=series.index) + + monkeypatch.setitem(plugin.AVAILABLE_DETECTORS, "ThresholdAD", Recording) + monkeypatch.setitem(plugin.AVAILABLE_DETECTORS, "PersistAD", Recording) + series = series_of([1.0, 2.0]) + + plugin.detect_anomalies( + FakeInfluxdb3Local(), + series, + ["ThresholdAD", "PersistAD"], + {"ThresholdAD": {"name": "threshold"}, "PersistAD": {"name": "persist"}}, + 1, + "tid", + ) + + assert fitted == ["persist"] + + +def test_unreachable_min_consensus_warns(plugin_dir, sent): + local = run({"min_consensus": "3"}) + + assert local.logged("min_consensus=3 exceeds the 1 applicable detectors", "warn") + assert sent == [] + + +# --- series preparation ----------------------------------------------------- + + +def test_split_by_tags_groups_only_when_enabled(): + frame = pd.DataFrame(merged(rows([1.0, 2.0]), rows([3.0, 4.0], host="server2"))) + + assert len(plugin.split_by_tags(frame, ["host"], False)) == 1 + assert len(plugin.split_by_tags(frame, ["host"], True)) == 2 + assert len(plugin.split_by_tags(frame, [], True)) == 1 + + +def test_null_values_are_dropped_before_detection(plugin_dir, sent): + local = run(local=FakeInfluxdb3Local(rows=rows([10.0, None, 999.0, 10.0]))) + + assert len(sent) == 1 + assert local.logged("Skipped 1 points without a 'usage' value", "info") + assert local.messages("warn") == [] + + +def test_series_without_any_value_is_skipped(plugin_dir, sent): + local = run(local=FakeInfluxdb3Local(rows=rows([None, None]))) + + assert sent == [] + assert local.logged("No values to analyze", "info") + assert local.messages("error") == [] + + +def test_duplicate_timestamps_keep_the_first_row(plugin_dir, sent): + duplicated = rows([10.0, 999.0]) + duplicated.append({**duplicated[1], "usage": 10.0}) + + local = run(local=FakeInfluxdb3Local(rows=duplicated)) + + assert local.logged("Prepared time series data with 2 points", "info") + assert len(sent) == 1 + assert ( + sent[0]["payload"]["notification_text"] + == "Anomaly detected in cpu.usage with value 999.0 by ThresholdAD. Tags: host=server1" + ) + + +def test_cache_key_sorts_tags_and_marks_missing_ones(): + row = pd.Series({"region": "eu", "host": "server1"}) + + key = plugin.generate_cache_key("cpu", "usage", ["region", "host", "rack"], row) + + assert key == "cpu:usage:host=server1:rack=None:region=eu" + assert plugin.format_tags(row, ["host", "rack"]) == "host=server1, rack=None" + + +# --- senders ---------------------------------------------------------------- + + +def test_senders_collect_channel_arguments(): + config = { + "senders": "http.slack", + "http_webhook_url": "https://example.com/hook", + "slack_webhook_url": "https://hooks.slack.com/services/T", + "slack_headers": "eyJhIjogMX0=", + } + + senders = plugin.parse_senders(FakeInfluxdb3Local(), config, "tid") + + assert senders["http"] == {"http_webhook_url": "https://example.com/hook"} + assert senders["slack"] == { + "slack_webhook_url": "https://hooks.slack.com/services/T", + "slack_headers": "eyJhIjogMX0=", + } + + +def test_senders_drop_channel_without_required_argument(): + local = FakeInfluxdb3Local() + config = {"senders": "http.sms", "http_webhook_url": "https://example.com/hook"} + + senders = plugin.parse_senders(local, config, "tid") + + assert set(senders) == {"http"} + assert local.logged( + "Required key 'twilio_to_number' missing for sender 'sms'", "warn" + ) + + +def test_senders_reject_unusable_configuration(): + local = FakeInfluxdb3Local() + + with pytest.raises(Exception, match="No valid senders configured"): + plugin.parse_senders(local, {"senders": "carrier_pigeon"}, "tid") + assert local.logged("Invalid sender type: carrier_pigeon", "warn") + + with pytest.raises(Exception, match="No valid senders configured"): + plugin.parse_senders( + FakeInfluxdb3Local(), + {"senders": "http", "http_webhook_url": "ftp://x"}, + "tid", + ) + + +# --- notifications ---------------------------------------------------------- + + +def test_notification_payload_carries_template_variables(plugin_dir, sent): + run( + { + "notification_text": "$table.$field=$value at $timestamp by $detectors ($tags) $unknown" + }, + local=FakeInfluxdb3Local(rows=rows([10.0, 999.0])), + ) + + text = sent[0]["payload"]["notification_text"] + assert text == ( + "cpu.usage=999.0 at 2026-08-13T12:01:00 by ThresholdAD (host=server1) $unknown" + ) + assert sent[0]["payload"]["senders_config"] == { + "http": {"http_webhook_url": "https://example.com/hook"} + } + + +def test_notification_cap_suppresses_the_rest(plugin_dir, sent): + local = run( + {"max_notifications_per_run": "2"}, + local=FakeInfluxdb3Local(rows=rows([999.0] * 5)), + ) + + assert len(sent) == 2 + assert local.logged("Suppressed 3 notifications", "warn") + + sent.clear() + plugin.process_scheduled_call( + local, CALL_TIME, {**ARGS, "max_notifications_per_run": "2"} + ) + assert sent == [] + + +@pytest.fixture +def failing_delivery(monkeypatch): + """Make every notification attempt fail and count the attempts.""" + attempts = [] + + def failing_post(url, headers=None, data=None, timeout=None): + attempts.append(url) + raise plugin.requests.ConnectionError("refused") + + monkeypatch.setattr(plugin.requests, "post", failing_post) + monkeypatch.setattr(plugin.time, "sleep", lambda seconds: None) + return attempts + + +def test_failed_delivery_is_retried_by_the_next_run(plugin_dir, failing_delivery): + local = run(local=FakeInfluxdb3Local(rows=rows([999.0]))) + + assert len(failing_delivery) == 3 + assert local.logged( + "Failed to send alert to notification plugin after 3 attempts", "error" + ) + assert local.logged("1 notifications could not be delivered", "warn") + assert local.logged("0 notifications sent", "info") + # the point stays unhandled, so a later run alerts on it again + assert not any(key.endswith(":last_alert") for key in local.cache.store) + + plugin.process_scheduled_call(local, CALL_TIME, dict(ARGS)) + assert len(failing_delivery) == 6 + + +def test_failed_deliveries_count_towards_the_cap(plugin_dir, failing_delivery): + run( + {"max_notifications_per_run": "2"}, + local=FakeInfluxdb3Local(rows=rows([999.0] * 6)), + ) + + assert len(failing_delivery) == 6 # two alerts, three attempts each + + +# --- debounce --------------------------------------------------------------- + + +def test_anomaly_alerts_immediately_without_debounce(plugin_dir, sent): + local = run(local=FakeInfluxdb3Local(rows=rows([10.0, 999.0, 999.0]))) + + assert len(sent) == 2 + assert local.logged("Anomaly detected for cpu.usage", "error") + + +def test_debounce_waits_for_the_configured_duration(plugin_dir, sent): + local = run( + {"min_condition_duration": "3min"}, + local=FakeInfluxdb3Local(rows=rows([10.0] + [999.0] * 5)), + ) + + assert len(sent) == 1 + assert local.logged("Anomaly started for cpu.usage", "info") + assert local.logged("Anomaly ongoing for 0 days 00:01:00", "info") + assert local.logged("Anomaly persisted for 0 days 00:03:00", "error") + + +def test_debounce_state_is_cleared_when_the_anomaly_stops(plugin_dir, sent): + local = run( + {"min_condition_duration": "1h"}, + local=FakeInfluxdb3Local(rows=rows([999.0, 999.0, 10.0])), + ) + + assert sent == [] + assert local.logged("Anomaly cleared for cpu.usage", "info") + assert "cpu:usage:host=server1" not in local.cache.store + + +def test_debounce_longer_than_the_window_warns(plugin_dir, sent): + local = run({"window": "10min", "min_condition_duration": "1h"}) + + assert local.logged("is not shorter than window", "warn") + + +# --- overlapping windows ---------------------------------------------------- + + +def test_same_anomaly_is_reported_once_across_runs(plugin_dir, sent): + local = FakeInfluxdb3Local(rows=rows([10.0, 999.0, 10.0])) + + for _ in range(3): + plugin.process_scheduled_call(local, CALL_TIME, dict(ARGS)) + + assert len(sent) == 1 + assert ( + local.cache.store["cpu:usage:host=server1:last_alert"] == "2026-08-13T12:01:00" + ) + + +def test_nanosecond_timestamps_are_not_realerted(plugin_dir, sent): + # datetime.fromisoformat truncates to microseconds, which would let a point + # with nanosecond precision pass the "already alerted" check on every run + nanos = [ + { + "usage": 999.0 if index else 10.0, + "time": int(START.timestamp() * 1_000_000_000) + + index * 60_000_000_000 + + 678_851_840, + "host": "server1", + } + for index in range(2) + ] + local = FakeInfluxdb3Local(rows=nanos) + + plugin.process_scheduled_call(local, CALL_TIME, dict(ARGS)) + assert len(sent) == 1 + + plugin.process_scheduled_call(local, CALL_TIME, dict(ARGS)) + assert len(sent) == 1 + + +def test_anomaly_after_the_last_alert_is_reported(plugin_dir, sent): + local = FakeInfluxdb3Local(rows=rows([10.0, 999.0])) + plugin.process_scheduled_call(local, CALL_TIME, dict(ARGS)) + + local.rows = rows([10.0, 999.0, 10.0, 999.0]) + sent.clear() + plugin.process_scheduled_call(local, CALL_TIME, dict(ARGS)) + + assert len(sent) == 1 + assert ( + local.cache.store["cpu:usage:host=server1:last_alert"] == "2026-08-13T12:03:00" + ) + + +# --- tag grouping ----------------------------------------------------------- + + +def test_without_grouping_only_the_first_series_is_analyzed(plugin_dir, sent): + local = run( + local=FakeInfluxdb3Local( + rows=merged(rows([10.0, 10.0]), rows([20.0, 999.0], host="server2")) + ) + ) + + assert sent == [] + assert local.logged("Prepared time series data with 2 points", "info") + + +def test_grouping_analyzes_every_tag_combination(plugin_dir, sent): + local = run( + {"group_by_tags": "true"}, + local=FakeInfluxdb3Local( + rows=merged(rows([10.0, 10.0]), rows([20.0, 999.0], host="server2")) + ), + ) + + assert len(sent) == 1 + assert "host=server2" in sent[0]["payload"]["notification_text"] + assert local.logged("on 2 series", "info") + assert local.logged("(tags: host=server1)", "info") + + +def test_grouping_keeps_debounce_state_per_series(plugin_dir, sent): + local = run( + {"group_by_tags": "true", "min_condition_duration": "2min"}, + local=FakeInfluxdb3Local( + rows=merged(rows([999.0] * 4), rows([999.0] * 4, host="server2")) + ), + ) + + assert len(sent) == 2 + assert sorted(k for k in local.cache.store if k.endswith(":last_alert")) == [ + "cpu:usage:host=server1:last_alert", + "cpu:usage:host=server2:last_alert", + ] + + +# --- scheduled flow guards -------------------------------------------------- + + +def test_unknown_measurement_is_reported(plugin_dir, sent): + local = run(local=FakeInfluxdb3Local(tables=("memory",), rows=rows([999.0]))) + + assert local.logged("Measurement 'cpu' not found", "error") + assert sent == [] + + +def test_empty_window_is_reported(plugin_dir, sent): + local = run(local=FakeInfluxdb3Local(rows=[])) + + assert local.logged("No data found for cpu.usage", "info") + assert sent == [] + + +def test_missing_field_column_is_reported(plugin_dir, sent): + local = run( + local=FakeInfluxdb3Local(rows=[{"other": 1.0, "time": 0, "host": "server1"}]) + ) + + assert local.logged("Field 'usage' or 'time' not found", "error") + assert sent == [] + + +def test_query_covers_the_window_and_quotes_identifiers(plugin_dir, sent): + local = run(local=FakeInfluxdb3Local(rows=rows([10.0]))) + query, params = local.window_query() + + assert '"usage", "time", "host"' in query + assert 'FROM "cpu"' in query + assert params["start"] == (CALL_TIME - timedelta(hours=2)).isoformat() + assert params["end"] == CALL_TIME.isoformat() diff --git a/influxdata/stock_plugin/README.md b/influxdata/stock_plugin/README.md index 8cb0fcb..1dc4b97 100644 --- a/influxdata/stock_plugin/README.md +++ b/influxdata/stock_plugin/README.md @@ -20,12 +20,16 @@ This plugin includes a JSON metadata schema in its docstring that defines the su ### Optional parameters -| Parameter | Type | Default | Description | -|---------------|--------|--------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `database` | string | `stocks` | Target database for writes. Overrides the TOML `database` key if both are set. | -| `portfolio` | string | `AAPL:1\|MSFT:1\|GOOG:1` | Inline holdings: pipe-separated `SYMBOL:QUANTITY[:PORTFOLIO_NAME]` entries (e.g. `AAPL:10:401k\|MSFT:5:401k\|GOOG:2.5:brokerage`). Portfolio defaults to `main`. When omitted and no TOML config is found, falls back to the default shown. | -| `categories` | string | none | Inline category map: pipe-separated `PORTFOLIO:CATEGORY` entries (e.g. `401k:Retirement\|brokerage:Investment`). | -| `config_path` | string | `stock_plugin.toml` | Path to the TOML config file, relative to the InfluxDB plugin directory (or absolute). | +| Parameter | Type | Default | Description | +|-----------------------------|---------|--------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `database` | string | `stocks` | Target database for writes. | +| `portfolio` | string | `AAPL:1\|MSFT:1\|GOOG:1` | Inline holdings: pipe-separated `SYMBOL:QUANTITY[:PORTFOLIO_NAME]` entries (e.g. `AAPL:10:401k\|MSFT:5:401k\|GOOG:2.5:brokerage`). Portfolio defaults to `main`. When omitted and no TOML config is found, falls back to the default shown. | +| `categories` | string | none | Inline category map: pipe-separated `PORTFOLIO:CATEGORY` entries (e.g. `401k:Retirement\|brokerage:Investment`). | +| `config_path` | string | `stock_plugin.toml` | Path to the TOML config file, relative to the InfluxDB plugin directory (or absolute). The default file is loaded when it exists; an explicit `config_path` that does not exist is an error. | +| `write_during_closed_hours` | boolean | `true` | See the TOML table below. Also settable as a trigger argument. | +| `mutual_fund_check_time` | string | `18:00` | See the TOML table below. Also settable as a trigger argument. | +| `market_calendar` | string | `NYSE` | See the TOML table below. Also settable as a trigger argument. | +| `market_timezone` | string | `America/New_York` | See the TOML table below. Also settable as a trigger argument. | *If neither `portfolio` nor a TOML file with `[holdings.]` is provided, the plugin runs with the default holdings `AAPL:1|MSFT:1|GOOG:1` in the `main` portfolio.* @@ -33,15 +37,26 @@ This plugin includes a JSON metadata schema in its docstring that defines the su The TOML file is the recommended way to configure anything more than a handful of holdings. The plugin reads it from `config_path` (default: `/stock_plugin.toml`). -| Key | Type | Default | Description | -|-----------------------------|---------|----------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `database` | string | `stocks` | Target database for writes. | -| `write_during_closed_hours` | boolean | `true` | When `false`, stocks/ETFs are skipped outside the configured exchange's regular session (the calendar handles holidays and early closes). Mutual funds always follow their own daily check schedule. | -| `mutual_fund_check_time` | string | `"18:00"` | Time of day in `market_timezone` after which the plugin fetches mutual fund NAV. Mutual funds are fetched at most once per local calendar day, at the first tick at or after this time. Bootstrap exception: a mutual fund with no cached asset type is fetched on its first tick regardless of time. | -| `market_calendar` | string | `"NYSE"` | Exchange calendar used for the market-hours check. Any name accepted by [pandas_market_calendars](https://pandas-market-calendars.readthedocs.io/) (e.g. `NYSE`, `LSE`, `TSX`, `JPX`, `XETR`, `ASX`, `HKEX`). | -| `market_timezone` | string | `"America/New_York"` | IANA timezone for the exchange's local time. Used for `mutual_fund_check_time` comparisons and for resolving the "today" date the calendar consults. | -| `[portfolio_categories]` | table | empty | Maps portfolio name to category name. Portfolios not listed are uncategorized (omitted from `category_totals`). | -| `[holdings.]` | table | default holdings | Holdings for each portfolio. Each entry is `SYMBOL = quantity`. Fractional quantities supported. Quote symbols containing dots, for example `"VOD.L" = 10`. Duplicate same-symbol entries in one portfolio are aggregated. The portfolio name `_total` is reserved. When no `[holdings.*]` section is present, the plugin falls back to `AAPL:1\|MSFT:1\|GOOG:1`. | +Trigger arguments take precedence over TOML keys of the same name, so a TOML file can hold the full portfolio shape while a trigger argument overrides a single setting. + +Holdings and categories are the exception, because each is spelled differently per source: + +| Setting | Trigger argument | TOML | +|------------|---------------------------------------|--------------------------------| +| Holdings | `portfolio=AAPL:10:401k\|MSFT:5:401k` | `[holdings.401k]` tables | +| Categories | `categories=401k:Retirement` | `[portfolio_categories]` table | + +Each spelling is read only from its own source: a top-level `portfolio` or `categories` key in the TOML file is ignored, as is a trigger argument named `holdings` or `portfolio_categories`. When both sources are present, the trigger argument replaces the TOML tables entirely rather than merging with them. + +| Key | Type | Default | Description | +|-----------------------------|---------|----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `database` | string | `stocks` | Target database for writes. | +| `write_during_closed_hours` | boolean | `true` | When `false`, stocks/ETFs are skipped outside the configured exchange's regular session (the calendar handles holidays and early closes). Mutual funds always follow their own daily check schedule. | +| `mutual_fund_check_time` | string | `"18:00"` | Time of day in `market_timezone` after which the plugin fetches mutual fund NAV. Mutual funds are fetched at most once per local calendar day, at the first tick at or after this time. Bootstrap exception: a mutual fund with no cached asset type is fetched on its first tick regardless of time. | +| `market_calendar` | string | `"NYSE"` | Exchange calendar used for the market-hours check. Any name accepted by [pandas_market_calendars](https://pandas-market-calendars.readthedocs.io/) (e.g. `NYSE`, `LSE`, `TSX`, `JPX`, `XETR`, `ASX`, `HKEX`). | +| `market_timezone` | string | `"America/New_York"` | IANA timezone for the exchange's local time. Used for `mutual_fund_check_time` comparisons and for resolving the "today" date the calendar consults. | +| `[portfolio_categories]` | table | empty | Maps portfolio name to category name. Portfolios not listed are uncategorized (omitted from `category_totals`). | +| `[holdings.]` | table | default holdings | Holdings for each portfolio. Each entry is `SYMBOL = quantity`. Fractional quantities supported; the quantity must be a finite number (`inf` and `nan` are rejected). Quote symbols containing dots, for example `"VOD.L" = 10`. Duplicate same-symbol entries in one portfolio are aggregated. The portfolio name `_total` is reserved. When no `[holdings.*]` section is present, the plugin falls back to `AAPL:1\|MSFT:1\|GOOG:1`. | The trigger spec is the source of truth for cadence. For example, `--trigger-spec "every:15m"` runs the plugin every 15 minutes. @@ -52,9 +67,11 @@ The trigger spec is the source of truth for cadence. For example, `--trigger-spe ## Software requirements - **InfluxDB 3 Core/Enterprise**: with the Processing Engine enabled. +- **Python 3.11 or higher** - **Python packages** (installed into the plugin venv): - `yfinance` — Yahoo Finance scraper for price data - - `pandas_market_calendars` — NYSE calendar for accurate market-hours and holiday gating + - `pandas_market_calendars` — exchange calendars for accurate market-hours and holiday gating + - `influxdata-plugin-utils>=0.3.0` — shared configuration, parsing, and write helpers ### Installation steps @@ -71,7 +88,7 @@ The trigger spec is the source of truth for cadence. For example, `--trigger-spe 2. Install required Python packages: ```bash - influxdb3 install package yfinance pandas_market_calendars + influxdb3 install package yfinance pandas_market_calendars influxdata-plugin-utils ``` 3. Copy `stock_plugin.toml.example` to `/stock_plugin.toml` and edit it with your holdings and categories. @@ -173,9 +190,9 @@ WHERE missing_symbols = 0; #### `process_scheduled_call(influxdb3_local, call_time, args)` -Entry point for the scheduled trigger. Resolves the plugin directory from `INFLUXDB3_PLUGIN_DIR`, normalizes `call_time` to UTC, and delegates to `_main` with the runtime-injected `LineBuilder` and the live `influxdb3_local`. All side-effecting work lives in `_main` so its logic can be reasoned about with injected dependencies. +Entry point for the scheduled trigger. Stamps one UTC timestamp for the whole run and delegates to `_main` with the runtime-injected `LineBuilder` and the live `influxdb3_local`. All side-effecting work lives in `_main` so its logic can be reasoned about with injected dependencies. -#### `_main(local, args, fetcher, line_builder_cls, plugin_dir, now_ns)` +#### `_main(local, args, fetcher, line_builder_cls, now_ns, task_id)` Drives the full plugin flow: @@ -185,9 +202,13 @@ Drives the full plugin flow: 4. Build carry-forward `HoldingRow`s for intentionally-skipped symbols whose last known price is cached. 5. Aggregate per-portfolio totals + a grand `_total` row. 6. Aggregate per-category totals across portfolios. -7. Emit line protocol via `LineBuilder` for `stock_holdings`, `portfolio_totals`, and `category_totals`. +7. Write `stock_holdings`, `portfolio_totals`, and `category_totals` as a single batched payload. 8. Log a single summary line. +#### `resolve_config(args)` + +Merges the TOML file with the trigger arguments and validates the result. The TOML path comes from `config_path`; relative paths resolve against the plugin directory (`PLUGIN_DIR`, `INFLUXDB3_PLUGIN_DIR`, or the `VIRTUAL_ENV` parent). Returns a `ResolvedConfig`, raising `ValueError` on any invalid value. + ### Measurements and fields #### `stock_holdings` @@ -241,7 +262,7 @@ When `write_during_closed_hours` is false, the plugin uses `pandas_market_calend **Solution:** Verify the ticker symbol and check whether Yahoo Finance exposes current price data for that instrument. -The plugin carries forward the last known price for intentionally skipped symbols, but it cannot value a new holding until the first successful fetch. +A symbol whose price is missing or not a finite number is counted as a fetch failure and reported in the summary log; the optional `previous_close`, `day_open`, `day_high`, and `day_low` fields are simply omitted when unusable. The plugin carries forward the last known price for intentionally skipped symbols, but it cannot value a new holding until the first successful fetch. ### Debugging tips diff --git a/influxdata/stock_plugin/manifest.toml b/influxdata/stock_plugin/manifest.toml index 559fe36..b973be4 100644 --- a/influxdata/stock_plugin/manifest.toml +++ b/influxdata/stock_plugin/manifest.toml @@ -2,7 +2,7 @@ manifest_schema_version = "1.2" [plugin] name = "stock_plugin" -version = "0.2.0" +version = "0.3.0" description = "Tracks stock, ETF, and mutual fund portfolio values from Yahoo Finance with market-hours gating and rollups." triggers = ["process_scheduled_call"] homepage = "https://www.influxdata.com/" @@ -16,4 +16,8 @@ exclude = [ [dependencies] database_version = ">=3.0.0" -python = ["yfinance>=0.2.40", "pandas_market_calendars>=4.4"] +python = [ + "yfinance>=0.2.40", + "pandas_market_calendars>=4.4", + "influxdata-plugin-utils>=0.3.0", +] diff --git a/influxdata/stock_plugin/requirements-dev.txt b/influxdata/stock_plugin/requirements-dev.txt new file mode 100644 index 0000000..cd2aedf --- /dev/null +++ b/influxdata/stock_plugin/requirements-dev.txt @@ -0,0 +1,4 @@ +pytest +yfinance>=0.2.40 +pandas_market_calendars>=4.4 +influxdata-plugin-utils>=0.3.0 diff --git a/influxdata/stock_plugin/requirements.txt b/influxdata/stock_plugin/requirements.txt index ac12b67..3b29187 100644 --- a/influxdata/stock_plugin/requirements.txt +++ b/influxdata/stock_plugin/requirements.txt @@ -1,2 +1,3 @@ yfinance>=0.2.40 pandas_market_calendars>=4.4 +influxdata-plugin-utils>=0.3.0 diff --git a/influxdata/stock_plugin/stock_plugin.py b/influxdata/stock_plugin/stock_plugin.py index 1973456..3c633f4 100644 --- a/influxdata/stock_plugin/stock_plugin.py +++ b/influxdata/stock_plugin/stock_plugin.py @@ -23,7 +23,31 @@ { "name": "config_path", "example": "stock_plugin.toml", - "description": "Path to TOML configuration file. Supports absolute paths or relative paths resolved from INFLUXDB3_PLUGIN_DIR, PLUGIN_DIR, VIRTUAL_ENV parent, or the plugin directory.", + "description": "Path to TOML configuration file. Absolute paths are used as-is; relative paths resolve against the plugin directory (PLUGIN_DIR, INFLUXDB3_PLUGIN_DIR, or the VIRTUAL_ENV parent). Defaults to stock_plugin.toml, loaded when present.", + "required": false + }, + { + "name": "write_during_closed_hours", + "example": "true", + "description": "Write rows even when the configured market is closed. When false, stocks and ETFs are skipped outside the exchange session. Defaults to true.", + "required": false + }, + { + "name": "mutual_fund_check_time", + "example": "18:00", + "description": "Time of day in market_timezone, as HH:MM, after which mutual fund NAV is fetched. Defaults to 18:00.", + "required": false + }, + { + "name": "market_calendar", + "example": "NYSE", + "description": "Exchange calendar governing the market-open check. Any name accepted by pandas_market_calendars. Defaults to NYSE.", + "required": false + }, + { + "name": "market_timezone", + "example": "America/New_York", + "description": "IANA timezone of the exchange, used for mutual_fund_check_time and for the local market day. Defaults to America/New_York.", "required": false } ] @@ -32,8 +56,7 @@ from __future__ import annotations -import os -import tomllib +import math import uuid from dataclasses import dataclass from datetime import datetime, timezone @@ -41,6 +64,27 @@ from typing import Optional from zoneinfo import ZoneInfo +from influxdata_plugin_utils.config import Validator, load_plugin_config, resolve_path +from influxdata_plugin_utils.parsing import ( + parse_bool, + parse_delimited_list, + parse_timestamp_ns, +) +from influxdata_plugin_utils.write import build_line, write_data + + +# Default holdings +DEFAULT_PORTFOLIO = "AAPL:1|MSFT:1|GOOG:1" + +DEFAULT_DATABASE = "stocks" + +# TOML config loaded when `config_path` is not set and the file exists +DEFAULT_CONFIG_FILE = "stock_plugin.toml" + +# Holdings and categories are spelled differently per medium, so each spelling is read only from its own medium. +INLINE_ARG_KEYS = ("portfolio", "categories") +TOML_TABLE_KEYS = ("holdings", "portfolio_categories") + def _parse_hhmm(s: str) -> tuple[int, int]: """Parse a HH:MM time string. Raises ValueError on bad input.""" @@ -57,6 +101,80 @@ def _parse_hhmm(s: str) -> tuple[int, int]: return hour, minute +def _parse_quantity(raw, where: str) -> float: + """Parse a holding quantity, rejecting non-finite values. + + `inf` and `nan` are valid TOML floats and valid input to float(), but they + cannot be written as a field, so they are rejected as configuration errors. + """ + try: + quantity = float(raw) + except (TypeError, ValueError) as e: + raise ValueError(f"invalid quantity {raw!r} {where}") from e + if not math.isfinite(quantity): + raise ValueError(f"invalid quantity {raw!r} {where}: must be a finite number") + return quantity + + +def _validated_hhmm(raw) -> str: + """Config cast: accept a HH:MM string, rejecting anything unparseable.""" + text = str(raw).strip() + _parse_hhmm(text) + return text + + +def _validated_market_calendar(raw) -> str: + """Config cast: accept an exchange name known to pandas_market_calendars.""" + name = str(raw).strip() + if not name: + raise ValueError("market_calendar must be a non-empty exchange name") + try: + import pandas_market_calendars as mcal + except ImportError as e: + raise ValueError( + "pandas_market_calendars is required to validate market_calendar; " + "install it in the plugin environment" + ) from e + try: + mcal.get_calendar(name) + except Exception as e: + raise ValueError( + f"market_calendar {name!r} is not accepted by " + f"pandas_market_calendars: {e}" + ) from e + return name + + +def _validated_market_timezone(raw) -> str: + """Config cast: accept an IANA timezone name.""" + name = str(raw).strip() + if not name: + raise ValueError("market_timezone must be a non-empty IANA timezone") + try: + ZoneInfo(name) + except Exception as e: + raise ValueError( + f"market_timezone {name!r} is not a valid IANA timezone: {e}" + ) from e + return name + + +VALIDATORS: list = [ + Validator( + "database", default=DEFAULT_DATABASE, + cast=lambda raw: str(raw).strip() or DEFAULT_DATABASE, + ), + Validator("portfolio", default="", cast=str), + Validator("categories", default="", cast=str), + Validator("write_during_closed_hours", default=True, cast=parse_bool), + Validator("mutual_fund_check_time", default="18:00", cast=_validated_hhmm), + Validator("market_calendar", default="NYSE", cast=_validated_market_calendar), + Validator( + "market_timezone", default="America/New_York", cast=_validated_market_timezone + ), +] + + def _normalize_quote_type(qt: Optional[str]) -> str: """Map yfinance fast_info.quote_type to our normalized asset_type tag.""" if not qt: @@ -182,10 +300,6 @@ class ResolvedConfig: market_timezone: str = "America/New_York" -# Default holdings -DEFAULT_PORTFOLIO = "AAPL:1|MSFT:1|GOOG:1" - - def parse_inline_portfolio(value: str) -> dict[str, list[Holding]]: """Parse the inline `portfolio=` trigger argument. @@ -200,16 +314,11 @@ def parse_inline_portfolio(value: str) -> dict[str, list[Holding]]: Raises ValueError on any malformed input. """ - if not value or not value.strip(): + entries = parse_delimited_list(value, sep="|") + if not entries: raise ValueError("portfolio argument is empty") result: dict[str, list[Holding]] = {} - for raw in value.split("|"): - raw = raw.strip() - if not raw: - raise ValueError( - f"invalid portfolio argument {value!r}: empty token " - f"(check for leading, trailing, or doubled '|')" - ) + for raw in entries: parts = raw.split(":") if len(parts) not in (2, 3): raise ValueError( @@ -218,12 +327,7 @@ def parse_inline_portfolio(value: str) -> dict[str, list[Holding]]: symbol = parts[0].strip().upper() if not symbol: raise ValueError(f"invalid holding spec {raw!r}: empty symbol") - try: - quantity = float(parts[1]) - except ValueError as e: - raise ValueError( - f"invalid quantity {parts[1]!r} in {raw!r}" - ) from e + quantity = _parse_quantity(parts[1], f"in {raw!r}") if len(parts) == 3 and parts[2].strip(): portfolio = parts[2].strip() else: @@ -242,23 +346,17 @@ def parse_inline_categories(value: str) -> dict[str, str]: Raises ValueError on any malformed input. """ - if not value or not value.strip(): + entries = parse_delimited_list(value, sep="|") + if not entries: raise ValueError("categories argument is empty") result: dict[str, str] = {} - for raw in value.split("|"): - raw = raw.strip() - if not raw: - raise ValueError( - f"invalid categories argument {value!r}: empty token " - f"(check for leading, trailing, or doubled '|')" - ) + for raw in entries: parts = raw.split(":") if len(parts) != 2: raise ValueError( f"invalid category spec {raw!r}: expected PORTFOLIO:CATEGORY" ) - portfolio_name = parts[0].strip() - category_name = parts[1].strip() + portfolio_name, category_name = parts[0].strip(), parts[1].strip() if not portfolio_name or not category_name: raise ValueError( f"invalid category spec {raw!r}: empty portfolio or category name" @@ -294,37 +392,10 @@ def _aggregate_duplicate_holdings( return result -def _validate_market_calendar(calendar_name: str) -> None: - """Fail fast if pandas_market_calendars does not know this calendar.""" - try: - import pandas_market_calendars as mcal - except ImportError as e: - raise ValueError( - "pandas_market_calendars is required to validate market_calendar; " - "install it in the plugin environment" - ) from e - - try: - mcal.get_calendar(calendar_name) - except Exception as e: - raise ValueError( - f"market_calendar {calendar_name!r} is not accepted by " - f"pandas_market_calendars: {e}" - ) from e - - -def load_toml_config(path: Path) -> tuple[dict, dict[str, list[Holding]]]: - """Load TOML config from `path`. - - Returns (top_level_data_dict, holdings_by_portfolio). The full raw - TOML top-level dict is returned so resolve_config can pick out - optional scalar keys (database, market_calendar, etc). - - Expected TOML shape: - database = "stocks" - write_during_closed_hours = true - mutual_fund_check_time = "18:00" +def parse_toml_holdings(holdings_section) -> dict[str, list[Holding]]: + """Parse the `[holdings.]` tables of the TOML config. + Expected shape: [holdings.401k] AAPL = 10 MSFT = 5 @@ -332,19 +403,12 @@ def load_toml_config(path: Path) -> tuple[dict, dict[str, list[Holding]]]: [holdings.brokerage] GOOG = 2.5 - Holdings may be empty (no [holdings.*] sections); the caller decides - how to handle that (e.g. fall back to a default portfolio). + Holdings may be empty (no [holdings.*] sections); the caller decides how to + handle that (e.g. fall back to a default portfolio). - Raises: - FileNotFoundError: path does not exist - tomllib.TOMLDecodeError: file is not valid TOML - ValueError: holdings is not a table, or a holding has an invalid quantity + Raises ValueError if holdings is not a table, or a holding has an invalid + quantity. """ - if not path.exists(): - raise FileNotFoundError(f"TOML config not found: {path}") - with open(path, "rb") as f: - data = tomllib.load(f) - holdings_section = data.get("holdings", {}) if not isinstance(holdings_section, dict): raise ValueError( f"[holdings] must be a table of portfolios; " @@ -365,13 +429,10 @@ def load_toml_config(path: Path) -> tuple[dict, dict[str, list[Holding]]]: f"ticker symbol, quote it in TOML, for example " f'"{dotted_hint}" = 1' ) - try: - qty = float(quantity) - except (TypeError, ValueError) as e: - raise ValueError( - f"invalid quantity {quantity!r} for symbol {symbol!r} " - f"in [holdings.{portfolio_name}]" - ) from e + qty = _parse_quantity( + quantity, + f"for symbol {symbol!r} in [holdings.{portfolio_name}]", + ) result.setdefault(portfolio_name, []).append( Holding( symbol=symbol.upper(), @@ -379,74 +440,81 @@ def load_toml_config(path: Path) -> tuple[dict, dict[str, list[Holding]]]: portfolio=portfolio_name, ) ) - return data, result + return result -def resolve_config_path(path: str, default_toml_path: Path) -> Path: - """Resolve TOML config path using the plugin-dir fallbacks used by plugins. +def resolve_toml_path(local, args: dict[str, str], task_id: str) -> Optional[Path]: + """Locate the TOML config file, or return None when there is none to load. - Absolute paths are used as-is. Relative paths are resolved from - INFLUXDB3_PLUGIN_DIR or PLUGIN_DIR when available, then VIRTUAL_ENV's - parent directory, and finally the supplied default TOML directory. + An explicit `config_path` must exist; the default file is loaded only when + present, so the plugin runs without any TOML config. """ - raw_path = Path(path) - if raw_path.is_absolute(): - return raw_path - - candidates: list[Path] = [] - if influxdb3_plugin_dir := os.environ.get("INFLUXDB3_PLUGIN_DIR"): - candidates.append(Path(influxdb3_plugin_dir)) - if plugin_dir := os.environ.get("PLUGIN_DIR"): - candidates.append(Path(plugin_dir)) - if virtual_env := os.environ.get("VIRTUAL_ENV"): - candidates.append(Path(virtual_env).parent) - candidates.append(default_toml_path.parent) - - for base in candidates: - candidate = base / raw_path - if candidate.exists(): - return candidate - return candidates[0] / raw_path - - -def resolve_config( - args: dict[str, str], default_toml_path: Path -) -> ResolvedConfig: - """Resolve final config from trigger args + TOML. + explicit = args.get("config_path") + try: + path = resolve_path(explicit or DEFAULT_CONFIG_FILE) + except ValueError as e: + if explicit: + raise ValueError(f"cannot resolve config_path {explicit!r}: {e}") from e + local.warn( + f"[{task_id}] stock_plugin: cannot resolve the plugin directory ({e}), " + f"so any {DEFAULT_CONFIG_FILE} is ignored; continuing with trigger arguments" + ) + return None + if path.exists(): + return path + if explicit: + raise ValueError(f"config_path was set but no TOML config found at {path}.") + return None - Precedence: - Holdings: inline `portfolio=` arg > TOML [holdings.*] > DEFAULT_PORTFOLIO - Database: `database=` arg > TOML `database` > default "stocks" - Config path: `config_path=` arg > default_toml_path. Relative paths - resolve from INFLUXDB3_PLUGIN_DIR, PLUGIN_DIR, VIRTUAL_ENV's parent, - or default_toml_path.parent. - Other TOML scalars (write_during_closed_hours, mutual_fund_check_time, - market_calendar, market_timezone) come only from TOML — there is no - inline-arg override for them. Defaults apply when the key is absent. +def resolve_config(local, args: dict[str, str], task_id: str) -> ResolvedConfig: + """Resolve final config from trigger args + TOML. - When no inline holdings and no TOML file are found, falls back to - DEFAULT_PORTFOLIO so the plugin runs without configuration. + Trigger arguments override TOML keys of the same name. Holdings come from the + inline `portfolio=` argument, else from the TOML `[holdings.*]` tables, else + from DEFAULT_PORTFOLIO, so the plugin runs without configuration; categories + follow the same order with `categories=` and `[portfolio_categories]`. - Raises ValueError if config_path is set but the file is missing, or if - any TOML scalar has an invalid value. + Raises ValueError if config_path is set but the file is missing, or if any + configuration value is invalid. """ - inline_portfolio = args.get("portfolio") - explicit_config_path = bool(args.get("config_path")) - config_path = resolve_config_path( - args.get("config_path") or str(default_toml_path), - default_toml_path, - ) + args = {key: value for key, value in (args or {}).items() if value not in (None, "")} toml_data: dict = {} - holdings: dict[str, list[Holding]] = {} - if inline_portfolio: - holdings = parse_inline_portfolio(inline_portfolio) - elif config_path.exists(): - toml_data, holdings = load_toml_config(config_path) - elif explicit_config_path: - raise ValueError("config_path was set but no TOML config found.") + toml_path = resolve_toml_path(local, args, task_id) + if toml_path: + loaded = load_plugin_config( + {"config_path": str(toml_path)}, + config_file_path_arg="config_path", + source="toml", + ) + toml_data = {key.lower(): value for key, value in loaded.as_dict().items()} + holdings_table = toml_data.get("holdings") + categories_table = toml_data.get("portfolio_categories") + settings = load_plugin_config( + { + **{ + key: value + for key, value in toml_data.items() + if key not in TOML_TABLE_KEYS + INLINE_ARG_KEYS + }, + **{ + key: value + for key, value in args.items() + if key not in TOML_TABLE_KEYS + }, + }, + validators=VALIDATORS, + config_file_path_arg="config_path", + source="args", + ) + config = {key.lower(): value for key, value in settings.as_dict().items()} + + if config["portfolio"]: + holdings = parse_inline_portfolio(config["portfolio"]) + else: + holdings = parse_toml_holdings({} if holdings_table is None else holdings_table) if not holdings: holdings = parse_inline_portfolio(DEFAULT_PORTFOLIO) @@ -457,66 +525,25 @@ def resolve_config( ) holdings = _aggregate_duplicate_holdings(holdings) - if args.get("database"): - database = args["database"] - elif toml_data.get("database"): - database = toml_data["database"] - else: - database = "stocks" - - write_closed = toml_data.get("write_during_closed_hours", True) - if not isinstance(write_closed, bool): - raise ValueError( - f"write_during_closed_hours must be true or false; got {write_closed!r}" - ) - - mf_check = toml_data.get("mutual_fund_check_time", "18:00") - if not isinstance(mf_check, str): - raise ValueError( - f"mutual_fund_check_time must be a HH:MM string; got {mf_check!r}" - ) - _parse_hhmm(mf_check) # validate format, raises ValueError on bad input - - market_calendar = toml_data.get("market_calendar", "NYSE") - if not isinstance(market_calendar, str) or not market_calendar.strip(): - raise ValueError( - f"market_calendar must be a non-empty exchange name string; got {market_calendar!r}" - ) - market_calendar = market_calendar.strip() - _validate_market_calendar(market_calendar) - - market_timezone = toml_data.get("market_timezone", "America/New_York") - if not isinstance(market_timezone, str) or not market_timezone.strip(): - raise ValueError( - f"market_timezone must be a non-empty IANA timezone string; got {market_timezone!r}" - ) - try: - ZoneInfo(market_timezone) - except Exception as e: - raise ValueError( - f"market_timezone {market_timezone!r} is not a valid IANA timezone: {e}" - ) from e - - # Resolve categories: trigger arg > TOML > {} - inline_categories = args.get("categories") - if inline_categories: - categories = parse_inline_categories(inline_categories) + if config["categories"]: + categories = parse_inline_categories(config["categories"]) else: - toml_cats = toml_data.get("portfolio_categories", {}) - if not isinstance(toml_cats, dict): + toml_categories = {} if categories_table is None else categories_table + if not isinstance(toml_categories, dict): raise ValueError( - f"[portfolio_categories] must be a TOML table; got {type(toml_cats).__name__}" + f"[portfolio_categories] must be a TOML table; " + f"got {type(toml_categories).__name__}" ) - categories = {str(k): str(v) for k, v in toml_cats.items()} + categories = {str(k): str(v) for k, v in toml_categories.items()} return ResolvedConfig( - database=database, + database=config["database"], holdings_by_portfolio=holdings, categories=categories, - write_during_closed_hours=write_closed, - mutual_fund_check_time=mf_check, - market_calendar=market_calendar, - market_timezone=market_timezone, + write_during_closed_hours=config["write_during_closed_hours"], + mutual_fund_check_time=config["mutual_fund_check_time"], + market_calendar=config["market_calendar"], + market_timezone=config["market_timezone"], ) @@ -627,15 +654,26 @@ def compute_category_totals( return rows +def _finite(value) -> Optional[float]: + """Coerce to float, treating None and non-finite values (NaN, inf) as missing.""" + if value is None: + return None + try: + number = float(value) + except (TypeError, ValueError): + return None + return number if math.isfinite(number) else None + + def fetch_quote(symbol: str) -> Quote: """Fetch current price + day OHL + previous close + currency for `symbol`. Uses yfinance.Ticker(symbol).fast_info — a single HTTP call that returns all the fields the plugin needs. Optional fields default to None when - yfinance returns None for them; the caller decides how to handle missing - fields when building line protocol. + yfinance returns None or a non-finite number for them; the caller decides + how to handle missing fields when building line protocol. - Raises ValueError if no last_price is returned. Raises whatever + Raises ValueError if no usable last_price is returned. Raises whatever yfinance raises on network/other failures (e.g. requests exceptions). yfinance is imported lazily so this module is importable / AST-parseable @@ -644,12 +682,15 @@ def fetch_quote(symbol: str) -> Quote: import yfinance as yf ticker = yf.Ticker(symbol) fi = ticker.fast_info - if fi.last_price is None: - raise ValueError(f"no last_price returned for {symbol}") + price = _finite(fi.last_price) + if price is None: + raise ValueError(f"no usable last_price returned for {symbol}") def _opt(attr: str) -> Optional[float]: - v = getattr(fi, attr, None) - return float(v) if v is not None else None + try: + return _finite(getattr(fi, attr, None)) + except (KeyError, AttributeError): + return None try: raw_currency = fi.currency @@ -663,7 +704,7 @@ def _opt(attr: str) -> Optional[float]: return Quote( symbol=symbol, - price=float(fi.last_price), + price=price, currency=str(raw_currency) if raw_currency else "USD", asset_type=_normalize_quote_type(raw_quote_type), previous_close=_opt("previous_close"), @@ -678,7 +719,6 @@ def _main( args: dict[str, str], fetcher, line_builder_cls, - plugin_dir: Path, now_ns: int, task_id: str, ) -> None: @@ -697,10 +737,9 @@ def _main( Per-symbol asset_type is auto-detected via yfinance.fast_info.quote_type on first fetch and cached (no TTL) so we don't re-query for type. """ - default_toml = plugin_dir / "stock_plugin.toml" try: - config = resolve_config(args, default_toml) - except (ValueError, OSError, tomllib.TOMLDecodeError) as e: + config = resolve_config(local, args, task_id) + except Exception as e: local.error(f"[{task_id}] stock_plugin: configuration error: {e}") return @@ -861,52 +900,57 @@ def _record_skip(holding: Holding, portfolio: str, reason: str) -> None: ) category_totals = compute_category_totals(totals, now_ns) - for r in successful: - lb = line_builder_cls("stock_holdings") - lb.tag("symbol", r.symbol) - lb.tag("portfolio", r.portfolio) - lb.tag("asset_type", r.asset_type) - if r.category: - lb.tag("category", r.category) - lb.float64_field("price", r.price) - lb.float64_field("quantity", r.quantity) - lb.float64_field("value", r.value) - lb.string_field("currency", r.currency) - if r.previous_close is not None: - lb.float64_field("previous_close", r.previous_close) - if r.day_open is not None: - lb.float64_field("day_open", r.day_open) - if r.day_high is not None: - lb.float64_field("day_high", r.day_high) - if r.day_low is not None: - lb.float64_field("day_low", r.day_low) - lb.time_ns(r.timestamp_ns) - local.write_to_db(config.database, lb) - - for t in totals: - lb = line_builder_cls("portfolio_totals") - lb.tag("portfolio", t.portfolio) - if t.category: - lb.tag("category", t.category) - lb.float64_field("value", t.value) - lb.int64_field("symbol_count", t.symbol_count) - lb.int64_field("missing_symbols", t.missing_symbols) - lb.int64_field("skipped_symbols", t.skipped_symbols) - lb.int64_field("carried_symbols", t.carried_symbols) - lb.time_ns(t.timestamp_ns) - local.write_to_db(config.database, lb) - - for c in category_totals: - lb = line_builder_cls("category_totals") - lb.tag("category", c.category) - lb.float64_field("value", c.value) - lb.int64_field("symbol_count", c.symbol_count) - lb.int64_field("portfolio_count", c.portfolio_count) - lb.int64_field("missing_symbols", c.missing_symbols) - lb.int64_field("skipped_symbols", c.skipped_symbols) - lb.int64_field("carried_symbols", c.carried_symbols) - lb.time_ns(c.timestamp_ns) - local.write_to_db(config.database, lb) + # Field types are inferred from the row values: prices are floats, counts ints. + # A None tag or field is omitted, as is an empty category. + lines: list = [] + try: + lines += [ + build_line( + line_builder_cls, + "stock_holdings", + tags={"symbol": r.symbol, "portfolio": r.portfolio, + "asset_type": r.asset_type, "category": r.category or None}, + fields={"price": r.price, "quantity": r.quantity, "value": r.value, + "currency": r.currency, "previous_close": r.previous_close, + "day_open": r.day_open, "day_high": r.day_high, + "day_low": r.day_low}, + time_ns=r.timestamp_ns, + ) + for r in successful + ] + lines += [ + build_line( + line_builder_cls, + "portfolio_totals", + tags={"portfolio": t.portfolio, "category": t.category or None}, + fields={"value": t.value, "symbol_count": t.symbol_count, + "missing_symbols": t.missing_symbols, + "skipped_symbols": t.skipped_symbols, + "carried_symbols": t.carried_symbols}, + time_ns=t.timestamp_ns, + ) + for t in totals + ] + lines += [ + build_line( + line_builder_cls, + "category_totals", + tags={"category": c.category}, + fields={"value": c.value, "symbol_count": c.symbol_count, + "portfolio_count": c.portfolio_count, + "missing_symbols": c.missing_symbols, + "skipped_symbols": c.skipped_symbols, + "carried_symbols": c.carried_symbols}, + time_ns=c.timestamp_ns, + ) + for c in category_totals + ] + write_data(local, lines, batch=True, retries=0, database=config.database) + except Exception as e: + local.error( + f"[{task_id}] stock_plugin: failed to write to {config.database}: {e}" + ) + return skipped_total = sum(skipped_by_portfolio.values()) parts = [f"fetched {len(successful)}/{total_symbols} symbols"] @@ -955,20 +999,14 @@ def process_scheduled_call(influxdb3_local, call_time, args): is unambiguous regardless of host timezone; the few-seconds offset from the scheduled tick boundary is negligible for a 15-minute polling cadence. - - The InfluxDB runtime executes plugins via exec(), so __file__ is not - defined. INFLUXDB3_PLUGIN_DIR is the env var the server itself uses - to locate the plugin directory. """ - plugin_dir = Path(os.environ.get("INFLUXDB3_PLUGIN_DIR", ".")) - now_ns = int(datetime.now(timezone.utc).timestamp() * 1_000_000_000) + now_ns = parse_timestamp_ns(datetime.now(timezone.utc), "datetime") task_id = uuid.uuid4().hex[:8] _main( local=influxdb3_local, args=args or {}, fetcher=fetch_quote, line_builder_cls=LineBuilder, # runtime-injected global - plugin_dir=plugin_dir, now_ns=now_ns, task_id=task_id, ) diff --git a/influxdata/stock_plugin/stock_plugin.toml.example b/influxdata/stock_plugin/stock_plugin.toml.example index 70c9aaa..aedd063 100644 --- a/influxdata/stock_plugin/stock_plugin.toml.example +++ b/influxdata/stock_plugin/stock_plugin.toml.example @@ -1,8 +1,10 @@ # Example stock_plugin configuration. # Copy to stock_plugin.toml in your InfluxDB plugin directory. +# The scalar keys below can also be passed as trigger arguments, which win over +# the values set here. The [holdings.*] and [portfolio_categories] tables have no +# trigger-argument spelling: pass portfolio=... and categories=... instead. # Target database for writes. Optional — defaults to "stocks". -# Overridden by --trigger-arguments database=... if set. database = "stocks" # Write rows even when US markets are closed? When false, stocks/ETFs diff --git a/influxdata/stock_plugin/test_stock_plugin.py b/influxdata/stock_plugin/test_stock_plugin.py new file mode 100644 index 0000000..f02e0cd --- /dev/null +++ b/influxdata/stock_plugin/test_stock_plugin.py @@ -0,0 +1,747 @@ +"""Unit and integration tests for the stock_plugin plugin.""" + +import json +import os +import sys +from collections import namedtuple +from datetime import datetime +from textwrap import dedent + +import pytest +from influxdata_plugin_utils import write as utils_write + +sys.path.insert(0, os.path.dirname(__file__)) +import stock_plugin as sp + +# Captured before the fixtures patch the module attribute. +REAL_IS_MARKET_OPEN = sp._is_market_open + + +def ns(iso: str) -> int: + return int(datetime.fromisoformat(iso).timestamp() * 1_000_000_000) + + +# 2023-11-15 is a Wednesday; the market timezone default is America/New_York. +BEFORE_NAV = ns("2023-11-15T15:00:00-05:00") +AFTER_NAV = ns("2023-11-15T19:00:00-05:00") +TODAY_LOCAL = "2023-11-15" + + +# --------------------------------------------------------------------------- +# Fakes +# --------------------------------------------------------------------------- + + +class FakeCache: + def __init__(self, initial=None): + self.store = dict(initial or {}) + + def get(self, key, default=None, use_global=None): + return self.store.get(key, default) + + def put(self, key, value, ttl=None, use_global=None): + self.store[key] = value + + +class FakeLineBuilder: + def __init__(self, measurement): + self.measurement = measurement + self.tags = {} + self.fields = {} + self.timestamp = None + + def tag(self, key, value): + self.tags[key] = value + return self + + def int64_field(self, key, value): + self.fields[key] = f"{value}i" + return self + + def uint64_field(self, key, value): + self.fields[key] = f"{value}u" + return self + + def float64_field(self, key, value): + self.fields[key] = repr(float(value)) + return self + + def bool_field(self, key, value): + self.fields[key] = "true" if value else "false" + return self + + def string_field(self, key, value): + self.fields[key] = f'"{value}"' + return self + + def time_ns(self, timestamp_ns): + self.timestamp = timestamp_ns + return self + + def build(self): + line = self.measurement + if self.tags: + line += "," + ",".join(f"{k}={v}" for k, v in self.tags.items()) + line += " " + ",".join(f"{k}={v}" for k, v in self.fields.items()) + return f"{line} {self.timestamp}" + + +Record = namedtuple("Record", ["measurement", "tags", "fields", "timestamp"]) + + +def _parse_field(raw): + if raw.startswith('"'): + return raw[1:-1] + if raw[-1] in ("i", "u"): + return int(raw[:-1]) + return float(raw) + + +def _parse_lp(line): + head, fields_str, timestamp = line.rsplit(" ", 2) + parts = head.split(",") + tags = dict(kv.split("=", 1) for kv in parts[1:]) + fields = { + key: _parse_field(value) + for key, value in (kv.split("=", 1) for kv in fields_str.split(",")) + } + return Record(parts[0], tags, fields, int(timestamp)) + + +class FakeLocal: + def __init__(self, cache=None, write_error=None): + self.cache = FakeCache(cache) + self.write_error = write_error + self.writes = [] # one (database, [Record]) per write call + self.infos = [] + self.warns = [] + self.errors = [] + + def write_to_db(self, database, batch): + if self.write_error is not None: + raise self.write_error + lines = [_parse_lp(lp) for lp in batch.build().split("\n")] + self.writes.append((database, lines)) + + def write(self, batch): + raise AssertionError("writes must target an explicit database") + + def info(self, message): + self.infos.append(message) + + def warn(self, message): + self.warns.append(message) + + def error(self, message): + self.errors.append(message) + + +@pytest.fixture(autouse=True) +def plugin_dir(tmp_path, monkeypatch): + """Point config resolution at an empty directory, never the real plugin dir.""" + monkeypatch.setenv("PLUGIN_DIR", str(tmp_path)) + monkeypatch.delenv("INFLUXDB3_PLUGIN_DIR", raising=False) + monkeypatch.setattr(sp, "LineBuilder", FakeLineBuilder, raising=False) + monkeypatch.setattr(utils_write.time, "sleep", lambda _: None) + return tmp_path + + +@pytest.fixture(autouse=True) +def market(monkeypatch): + """Market state, so gating tests do not depend on the real calendar or clock.""" + state = {"open": True} + monkeypatch.setattr(sp, "_is_market_open", lambda *_: state["open"]) + return state + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def write_toml(plugin_dir, body, name=sp.DEFAULT_CONFIG_FILE): + path = plugin_dir / name + path.write_text(dedent(body)) + return path + + +def quote_fetcher(prices=None, asset_types=None, failures=(), currency="USD"): + """A fetcher returning deterministic quotes and raising for `failures`.""" + + def fetch(symbol): + if symbol in failures: + raise RuntimeError(f"no data for {symbol}") + return sp.Quote( + symbol=symbol, + price=(prices or {}).get(symbol, 100.0), + currency=currency, + asset_type=(asset_types or {}).get(symbol, "equity"), + previous_close=99.0, + day_open=None, + day_high=101.0, + day_low=None, + ) + + return fetch + + +def run(args=None, cache=None, now_ns=AFTER_NAV, fetcher=None, local=None): + local = local or FakeLocal(cache) + sp._main( + local=local, + args=args or {}, + fetcher=fetcher or quote_fetcher(), + line_builder_cls=FakeLineBuilder, + now_ns=now_ns, + task_id="test", + ) + return local + + +def records(local, measurement=None): + rows = [row for _, lines in local.writes for row in lines] + if measurement: + rows = [row for row in rows if row.measurement == measurement] + return rows + + +def resolve_config(args=None, local=None): + return sp.resolve_config(local or FakeLocal(), args or {}, "test") + + +def holdings_of(config): + return { + portfolio: [(h.symbol, h.quantity) for h in holdings] + for portfolio, holdings in config.holdings_by_portfolio.items() + } + + +# --------------------------------------------------------------------------- +# M1 — plugin metadata +# --------------------------------------------------------------------------- + + +def test_docstring_metadata_covers_every_supported_argument(): + header = json.loads(sp.__doc__) + assert header["plugin_type"] == ["scheduled"] + names = [arg["name"] for arg in header["scheduled_args_config"]] + validated = {validator.names[0] for validator in sp.VALIDATORS} + # every validated key is documented, plus the TOML path itself + assert set(names) == validated | {"config_path"} + for entry in header["scheduled_args_config"]: + assert set(entry) == {"name", "example", "description", "required"} + + +# --------------------------------------------------------------------------- +# M2 — configuration resolution +# --------------------------------------------------------------------------- + + +def test_defaults_apply_without_any_configuration(): + config = resolve_config({}) + assert config.database == "stocks" + assert config.write_during_closed_hours is True + assert config.mutual_fund_check_time == "18:00" + assert config.market_calendar == "NYSE" + assert config.market_timezone == "America/New_York" + assert holdings_of(config) == {"main": [("AAPL", 1.0), ("MSFT", 1.0), ("GOOG", 1.0)]} + assert config.categories == {} + + +def test_toml_supplies_holdings_categories_and_scalars(plugin_dir): + write_toml( + plugin_dir, + """ + database = "portfolio" + write_during_closed_hours = false + mutual_fund_check_time = "20:30" + market_calendar = "LSE" + market_timezone = "Europe/London" + + [portfolio_categories] + "401k" = "Retirement" + + [holdings.401k] + AAPL = 10 + "VOD.L" = 2.5 + """, + ) + config = resolve_config({}) + assert config.database == "portfolio" + assert config.write_during_closed_hours is False + assert config.mutual_fund_check_time == "20:30" + assert config.market_calendar == "LSE" + assert config.market_timezone == "Europe/London" + assert holdings_of(config) == {"401k": [("AAPL", 10.0), ("VOD.L", 2.5)]} + assert config.categories == {"401k": "Retirement"} + + +def test_trigger_arguments_override_toml_keys(plugin_dir): + write_toml( + plugin_dir, + """ + database = "from_toml" + write_during_closed_hours = false + market_calendar = "LSE" + + [holdings.401k] + AAPL = 10 + """, + ) + config = resolve_config( + { + "database": "from_args", + "write_during_closed_hours": "true", + "market_calendar": "NYSE", + } + ) + assert config.database == "from_args" + assert config.write_during_closed_hours is True + assert config.market_calendar == "NYSE" + # holdings still come from the file + assert holdings_of(config) == {"401k": [("AAPL", 10.0)]} + + +def test_inline_holdings_replace_the_toml_tables_but_keep_toml_scalars(plugin_dir): + write_toml( + plugin_dir, + """ + database = "portfolio" + + [portfolio_categories] + "401k" = "Retirement" + + [holdings.brokerage] + GOOG = 5 + """, + ) + config = resolve_config({"portfolio": "AAPL:2:401k"}) + assert holdings_of(config) == {"401k": [("AAPL", 2.0)]} + assert config.database == "portfolio" + assert config.categories == {"401k": "Retirement"} + + +def test_explicit_config_path_must_exist(plugin_dir): + write_toml(plugin_dir, '[holdings.main]\nAAPL = 1\n', name="custom.toml") + assert holdings_of(resolve_config({"config_path": "custom.toml"})) == { + "main": [("AAPL", 1.0)] + } + with pytest.raises(ValueError, match="no TOML config found"): + resolve_config({"config_path": "absent.toml"}) + + +@pytest.mark.parametrize( + "args, fragment", + [ + ({"mutual_fund_check_time": "25:00"}, "out of range"), + ({"mutual_fund_check_time": "noon"}, "expected HH:MM"), + ({"write_during_closed_hours": "maybe"}, "Invalid boolean"), + ({"market_timezone": "Mars/Olympus"}, "not a valid IANA timezone"), + ({"market_calendar": "NOPE"}, "not accepted by pandas_market_calendars"), + ({"portfolio": "AAPL:1:_total"}, "'_total' is reserved"), + ], +) +def test_invalid_configuration_is_rejected(args, fragment): + with pytest.raises(ValueError, match=fragment): + resolve_config(args) + + +def test_duplicate_symbols_in_one_portfolio_are_aggregated(): + config = resolve_config({"portfolio": "AAPL:2:401k|MSFT:1:401k|AAPL:3:401k"}) + assert holdings_of(config) == {"401k": [("AAPL", 5.0), ("MSFT", 1.0)]} + + +# --------------------------------------------------------------------------- +# M3 — inline argument parsing +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "value, expected", + [ + ("AAPL:10", {"main": [("AAPL", 10.0)]}), + ("AAPL:10|MSFT:5", {"main": [("AAPL", 10.0), ("MSFT", 5.0)]}), + ("aapl:2.5:401k", {"401k": [("AAPL", 2.5)]}), + ("AAPL:1||MSFT:2", {"main": [("AAPL", 1.0), ("MSFT", 2.0)]}), + (" AAPL:1 | MSFT:2 ", {"main": [("AAPL", 1.0), ("MSFT", 2.0)]}), + ("AAPL:-1", {"main": [("AAPL", -1.0)]}), + ], +) +def test_parse_inline_portfolio(value, expected): + parsed = sp.parse_inline_portfolio(value) + assert {p: [(h.symbol, h.quantity) for h in hs] for p, hs in parsed.items()} == expected + + +@pytest.mark.parametrize( + "value, fragment", + [ + ("", "is empty"), + ("AAPL", "expected SYMBOL:QUANTITY"), + ("AAPL:1:401k:extra", "expected SYMBOL:QUANTITY"), + (":1", "empty symbol"), + ("AAPL:many", "invalid quantity"), + ("AAPL:inf", "must be a finite number"), + ("AAPL:nan", "must be a finite number"), + ("AAPL:1e400", "must be a finite number"), + ], +) +def test_parse_inline_portfolio_rejects_malformed_input(value, fragment): + with pytest.raises(ValueError, match=fragment): + sp.parse_inline_portfolio(value) + + +@pytest.mark.parametrize( + "value, expected", + [ + ("401k:Retirement", {"401k": "Retirement"}), + ("401k:Retirement|brokerage:Investment", + {"401k": "Retirement", "brokerage": "Investment"}), + ("401k:Retirement||brokerage:Investment", + {"401k": "Retirement", "brokerage": "Investment"}), + ], +) +def test_parse_inline_categories(value, expected): + assert sp.parse_inline_categories(value) == expected + + +@pytest.mark.parametrize( + "value, fragment", + [ + ("|", "is empty"), + ("401k", "expected PORTFOLIO:CATEGORY"), + ("401k:", "empty portfolio or category name"), + (":Retirement", "empty portfolio or category name"), + ("401k:Retirement:extra", "expected PORTFOLIO:CATEGORY"), + # the message quotes the user's own token, not the parsed halves + ("401k : Retirement : extra", r"'401k : Retirement : extra'"), + ], +) +def test_parse_inline_categories_rejects_malformed_input(value, fragment): + with pytest.raises(ValueError, match=fragment): + sp.parse_inline_categories(value) + + +# --------------------------------------------------------------------------- +# M4 — TOML holdings parsing +# --------------------------------------------------------------------------- + + +def test_unquoted_dotted_symbol_reports_how_to_fix_it(plugin_dir): + write_toml(plugin_dir, "[holdings.main]\nVOD.L = 10\n") + with pytest.raises(ValueError, match=r'"VOD.L" = 1'): + resolve_config({}) + + +@pytest.mark.parametrize( + "body, fragment", + [ + ('holdings = "AAPL"', "must be a table of portfolios"), + ("[holdings.main]\nAAPL = nan\n", "must be a finite number"), + ('[holdings.main]\nAAPL = "ten"\n', "invalid quantity"), + ], +) +def test_invalid_toml_holdings_are_rejected(plugin_dir, body, fragment): + write_toml(plugin_dir, body) + with pytest.raises(ValueError, match=fragment): + resolve_config({}) + + +# --------------------------------------------------------------------------- +# M5 — market and mutual-fund gating +# --------------------------------------------------------------------------- + +WARM_EQUITY = {"asset_type:AAPL": "equity", "last_price:AAPL": 90.0} +WARM_FUND = {"asset_type:VFIAX": "mutualfund", "last_price:VFIAX": 40.0} + + +def test_closed_market_skips_equities_when_closed_hour_writes_are_disabled(market): + market["open"] = False + local = run( + args={"portfolio": "AAPL:2", "write_during_closed_hours": "false"}, + cache=WARM_EQUITY, + ) + assert records(local, "stock_holdings") == [] + total = records(local, "portfolio_totals")[0] + # value carried forward from the cached last price + assert total.fields["value"] == 180.0 + assert total.fields["skipped_symbols"] == 1 + assert total.fields["carried_symbols"] == 1 + assert total.fields["missing_symbols"] == 0 + + +def test_closed_market_still_fetches_when_closed_hour_writes_are_enabled(market): + market["open"] = False + local = run(args={"portfolio": "AAPL:2"}, cache=WARM_EQUITY) + assert [r.tags["symbol"] for r in records(local, "stock_holdings")] == ["AAPL"] + assert records(local, "portfolio_totals")[0].fields["skipped_symbols"] == 0 + + +@pytest.mark.parametrize( + "cache, now_ns, expected_skip", + [ + ({**WARM_FUND, "last_mf_date:VFIAX": TODAY_LOCAL}, AFTER_NAV, "already-today"), + (WARM_FUND, BEFORE_NAV, "too-early"), + ], +) +def test_mutual_fund_is_fetched_once_a_day_after_the_check_time( + cache, now_ns, expected_skip +): + local = run(args={"portfolio": "VFIAX:3"}, cache=cache, now_ns=now_ns) + assert records(local, "stock_holdings") == [] + assert records(local, "portfolio_totals")[0].fields["carried_symbols"] == 1 + assert expected_skip in local.infos[-1] + + +def test_mutual_fund_is_fetched_once_the_check_time_has_passed(): + local = run( + args={"portfolio": "VFIAX:3"}, + cache={**WARM_FUND, "last_mf_date:VFIAX": "2023-11-14"}, + fetcher=quote_fetcher(asset_types={"VFIAX": "mutualfund"}), + ) + assert [r.tags["symbol"] for r in records(local, "stock_holdings")] == ["VFIAX"] + assert local.cache.get("last_mf_date:VFIAX") == TODAY_LOCAL + + +def test_a_symbol_that_would_be_skipped_is_fetched_while_its_price_is_uncached(market): + market["open"] = False + local = run( + args={"portfolio": "AAPL:2", "write_during_closed_hours": "false"}, + cache={"asset_type:AAPL": "equity"}, + ) + assert [r.tags["symbol"] for r in records(local, "stock_holdings")] == ["AAPL"] + assert "cold-cache bootstrap" in local.infos[-1] + + +def test_asset_type_and_last_price_are_cached_for_later_runs(): + local = run(args={"portfolio": "AAPL:2"}, fetcher=quote_fetcher(prices={"AAPL": 12.5})) + assert local.cache.get("asset_type:AAPL") == "equity" + assert local.cache.get("last_price:AAPL") == 12.5 + assert local.cache.get("last_mf_date:AAPL") is None + + +@pytest.mark.parametrize( + "moment, expected", + [ + ("2023-11-15T10:00:00-05:00", True), # Wednesday, mid-session + ("2023-11-15T20:00:00-05:00", False), # Wednesday, after the close + ("2023-11-11T10:00:00-05:00", False), # Saturday + ("2023-11-23T10:00:00-05:00", False), # Thanksgiving + ], +) +def test_real_nyse_calendar_decides_whether_the_session_is_open(moment, expected): + from zoneinfo import ZoneInfo + + now_utc = datetime.fromisoformat(moment).astimezone(ZoneInfo("UTC")) + tz = ZoneInfo("America/New_York") + assert REAL_IS_MARKET_OPEN(now_utc, "NYSE", tz) is expected + + +# --------------------------------------------------------------------------- +# M6 — totals and category roll-ups +# --------------------------------------------------------------------------- + + +def test_totals_roll_up_per_portfolio_then_into_a_grand_total(): + local = run( + args={ + "portfolio": "AAPL:2:401k|MSFT:1:401k|GOOG:1:brokerage", + "categories": "401k:Retirement", + }, + fetcher=quote_fetcher(prices={"AAPL": 10.0, "MSFT": 20.0, "GOOG": 30.0}), + ) + totals = {r.tags["portfolio"]: r for r in records(local, "portfolio_totals")} + assert totals["401k"].fields["value"] == 40.0 + assert totals["401k"].tags["category"] == "Retirement" + assert totals["brokerage"].fields["value"] == 30.0 + assert "category" not in totals["brokerage"].tags + assert totals["_total"].fields["value"] == 70.0 + assert totals["_total"].fields["symbol_count"] == 3 + assert "category" not in totals["_total"].tags + + +def test_category_totals_exclude_uncategorized_portfolios_and_the_grand_total(): + local = run( + args={ + "portfolio": "AAPL:1:401k|MSFT:1:ira|GOOG:1:brokerage", + "categories": "401k:Retirement|ira:Retirement", + }, + fetcher=quote_fetcher(prices={"AAPL": 10.0, "MSFT": 20.0, "GOOG": 30.0}), + ) + rows = records(local, "category_totals") + assert [r.tags["category"] for r in rows] == ["Retirement"] + assert rows[0].fields["value"] == 30.0 + assert rows[0].fields["portfolio_count"] == 2 + assert rows[0].fields["symbol_count"] == 2 + + +# --------------------------------------------------------------------------- +# M7 — line protocol output +# --------------------------------------------------------------------------- + + +def test_a_run_emits_one_batched_write_with_a_single_timestamp(): + local = run(args={"portfolio": "AAPL:1:401k", "categories": "401k:Retirement", + "database": "portfolio"}) + assert len(local.writes) == 1 + database, lines = local.writes[0] + assert database == "portfolio" + assert [r.measurement for r in lines] == [ + "stock_holdings", + "portfolio_totals", + "portfolio_totals", + "category_totals", + ] + assert {r.timestamp for r in lines} == {AFTER_NAV} + + +def test_holding_row_carries_the_quote_and_omits_unavailable_fields(): + local = run(args={"portfolio": "AAPL:2"}, fetcher=quote_fetcher(prices={"AAPL": 10.0})) + row = records(local, "stock_holdings")[0] + assert row.tags == {"symbol": "AAPL", "portfolio": "main", "asset_type": "equity"} + assert row.fields == { + "price": 10.0, + "quantity": 2.0, + "value": 20.0, + "currency": "USD", + "previous_close": 99.0, + "day_high": 101.0, + } + + +def test_counts_are_written_as_integer_fields(): + local = run(args={"portfolio": "AAPL:1"}) + total = records(local, "portfolio_totals")[0] + for name in ("symbol_count", "missing_symbols", "skipped_symbols", "carried_symbols"): + assert isinstance(total.fields[name], int) + assert isinstance(total.fields["value"], float) + + +# --------------------------------------------------------------------------- +# M8 — failure handling +# --------------------------------------------------------------------------- + + +def test_a_fetch_failure_is_reported_and_counted_as_missing(): + local = run( + args={"portfolio": "AAPL:1|BOOM:1"}, fetcher=quote_fetcher(failures={"BOOM"}) + ) + assert [r.tags["symbol"] for r in records(local, "stock_holdings")] == ["AAPL"] + total = records(local, "portfolio_totals")[0] + assert total.fields["missing_symbols"] == 1 + assert total.fields["carried_symbols"] == 0 + assert any("failed to fetch BOOM" in w for w in local.warns) + assert "Failed: BOOM" in local.infos[-1] + + +def test_calendar_failure_aborts_only_when_closed_hour_writes_are_disabled(monkeypatch): + def boom(*_): + raise RuntimeError("calendar unavailable") + + monkeypatch.setattr(sp, "_is_market_open", boom) + + aborted = run(args={"portfolio": "AAPL:1", "write_during_closed_hours": "false"}) + assert aborted.writes == [] + assert any("skipping run" in e for e in aborted.errors) + + continued = run(args={"portfolio": "AAPL:1"}) + assert len(continued.writes) == 1 + assert any("continuing because" in w for w in continued.warns) + + +def test_a_configuration_error_is_logged_and_nothing_is_written(): + local = run(args={"portfolio": "AAPL:inf"}) + assert local.writes == [] + assert len(local.errors) == 1 + assert "configuration error" in local.errors[0] + assert "must be a finite number" in local.errors[0] + + +def test_a_row_that_cannot_be_built_is_logged_instead_of_raising(): + # quantity and price are both finite, but their product overflows to inf + local = run( + args={"portfolio": "AAPL:1e200"}, fetcher=quote_fetcher(prices={"AAPL": 1e200}) + ) + assert local.writes == [] + assert any("failed to write" in e and "not finite" in e for e in local.errors) + + +def test_a_write_failure_is_logged_instead_of_raising(): + local = FakeLocal(write_error=RuntimeError("database gone")) + run(args={"portfolio": "AAPL:1"}, local=local) + assert local.writes == [] + assert any("failed to write" in e and "database gone" in e for e in local.errors) + + +def test_process_scheduled_call_uses_the_runtime_line_builder(monkeypatch): + captured = {} + + def fake_main(**kwargs): + captured.update(kwargs) + + monkeypatch.setattr(sp, "_main", fake_main) + sp.process_scheduled_call(FakeLocal(), "ignored call_time", {"portfolio": "AAPL:1"}) + + assert captured["line_builder_cls"] is FakeLineBuilder + assert captured["fetcher"] is sp.fetch_quote + assert captured["args"] == {"portfolio": "AAPL:1"} + assert captured["now_ns"] > 0 + assert len(captured["task_id"]) == 8 + + +# --------------------------------------------------------------------------- +# M9 — quote fetching +# --------------------------------------------------------------------------- + + +class FakeFastInfo: + """Stand-in for yfinance fast_info, which raises KeyError for keys it cannot fill.""" + + def __init__(self, values): + self._values = values + + def __getattr__(self, name): + if name not in self._values: + raise KeyError(name) + return self._values[name] + + +@pytest.fixture +def fake_yfinance(monkeypatch): + def install(values): + module = type(sys)("yfinance") + module.Ticker = lambda symbol: type( + "Ticker", (), {"fast_info": FakeFastInfo(values)} + )() + monkeypatch.setitem(sys.modules, "yfinance", module) + + return install + + +def test_a_quote_survives_optional_fields_the_api_cannot_fill(fake_yfinance): + fake_yfinance({"last_price": 10.0, "currency": "GBP", "quote_type": "ETF"}) + quote = sp.fetch_quote("VOD.L") + assert (quote.price, quote.currency, quote.asset_type) == (10.0, "GBP", "etf") + assert (quote.previous_close, quote.day_open, quote.day_high, quote.day_low) == ( + None, + None, + None, + None, + ) + + +@pytest.mark.parametrize("last_price", [None, float("nan"), float("inf")]) +def test_an_unusable_last_price_fails_the_fetch(fake_yfinance, last_price): + fake_yfinance({"last_price": last_price}) + with pytest.raises(ValueError, match="no usable last_price"): + sp.fetch_quote("AAPL") + + +def test_non_finite_optional_fields_are_dropped(fake_yfinance): + fake_yfinance( + {"last_price": 10.0, "previous_close": float("nan"), "day_high": 11.0} + ) + quote = sp.fetch_quote("AAPL") + assert quote.previous_close is None + assert quote.day_high == 11.0 + # no currency or quote_type in the payload + assert (quote.currency, quote.asset_type) == ("USD", "other") \ No newline at end of file diff --git a/influxdata/synthefy_forecasting/README.md b/influxdata/synthefy_forecasting/README.md index 5968e11..121070a 100644 --- a/influxdata/synthefy_forecasting/README.md +++ b/influxdata/synthefy_forecasting/README.md @@ -16,7 +16,7 @@ The Synthefy Forecasting Plugin integrates the Synthefy Forecasting API with Inf ## Configuration -Plugin parameters may be specified as key-value pairs in the `--trigger-arguments` flag (CLI) or in the `trigger_arguments` field (API) when creating a trigger, and/or in the JSON body of each HTTP request. Body values override trigger arguments. +Plugin parameters may be specified as key-value pairs in the `--trigger-arguments` flag (CLI) or in the `trigger_arguments` field (API) when creating a trigger, and/or in the JSON body of each HTTP request. Body values override trigger arguments. A `null` in the body counts as unset, so the trigger argument applies; send `{}` for `tags` or `[]` for `metadata_fields` to clear them. ### Plugin metadata @@ -38,11 +38,12 @@ If both are set, the header takes precedence. | `measurement` | string | required | Source measurement (table) containing historical data | | `field` | string | `"value"` | Field name to forecast | | `tags` | string \| dict | `""` | Tag filters. Trigger args: dot-separated string `key:val1@val2.key2:val3`. Request body: JSON object mapping tag name to a string or list of strings. See [Tag filter format](#tag-filter-format). | -| `time_range` | string | `"30d"` | Historical window. Format: ``. Units: `s`, `min`, `h`, `d`, `w`, `m`, `q`, `y` (`m`/`q`/`y` are approximate). | +| `time_range` | string | `"30d"` | Historical window. Format: ``. Units: `us`, `ms`, `s`, `min`, `h`, `d`, `w`, `m`, `q`, `y` (`m`/`q`/`y` are approximate). | | `forecast_horizon` | string | `"7d"` | Forecast duration. Format: `` (same units as `time_range`) or ` points`. | | `model` | string | `"sfm-tabular"` | Synthefy model identifier (e.g., `sfm-tabular`, `Migas-latest`) | | `output_measurement` | string | `"{measurement}_forecast"` | Destination measurement for forecast results | | `metadata_fields` | string \| list | `""` | Trigger args: space-separated list of field names (`"humidity pressure"`). Request body: JSON list of strings. Used as covariates. | +| `max_forecast_points`| integer | `10000` | Upper bound on the forecast points one request may produce. A time-based `forecast_horizon` is divided by the series' own step, so a dense series with a long horizon builds a very large payload. | | `database` | string | `""` | Optional override database for **writes only**. If unset, forecasts are written to the trigger's database. Reads always go to the trigger's database. | #### Tag filter format @@ -54,13 +55,14 @@ The plugin supports multi-value tag filters that are translated to `tag IN ('a', - `.` separates `key:value` pairs - `:` separates the tag name from its value(s) - `@` separates multiple values for the same tag -- Quote a value with `'...'` or `"..."` if it contains special characters such as `:`, `@`, `.` or `'` +- Quote a value with `'...'` or `"..."` if it contains `:`, `@` or `.`. A quote inside a value, as in `Bob's`, needs no escaping. An unclosed quote is rejected. Examples: ``` tags="room:Bedroom" tags="room:Bedroom@Kitchen.location:Hall" tags="room:'Some other room'@Bedroom.device:sensor1" +tags="owner:Bob's.room:Bedroom" ``` **Request body (JSON form)**: @@ -68,6 +70,8 @@ tags="room:'Some other room'@Bedroom.device:sensor1" { "tags": { "room": ["Bedroom", "Kitchen"], "location": "Hall" } } ``` +The body also accepts the string form above. Send `{}` to clear the tag filters configured on the trigger. + #### Forecast points and tags When a tag filter has a single value, that value is added as a tag on every forecast point. When it has multiple values (an `IN (...)` filter), no value is written for that tag — the response covers several tag values at once. @@ -76,9 +80,10 @@ When a tag filter has a single value, that value is added as a tag on every fore ### Dependencies -- Python 3.9 or higher +- Python 3.11 or higher - `pandas` — Data manipulation - `requests` — HTTP client for the Synthefy API +- `influxdata-plugin-utils` — Shared configuration, schema and write helpers ### Installation steps @@ -87,6 +92,7 @@ Using the InfluxDB 3 package manager: ```bash influxdb3 install package pandas influxdb3 install package requests +influxdb3 install package influxdata-plugin-utils ``` ### Prerequisites @@ -320,7 +326,7 @@ Check the [Synthefy documentation](https://docs.synthefy.com) for the most up-to - `process_request(influxdb3_local, query_parameters, request_headers, request_body, args=None)`: handles HTTP requests, merges trigger arguments with request-body overrides, validates input, calls Synthefy, and writes forecast points. - `build_history_query(measurement, field, metadata_fields, tag_filters, start_time)`: builds the parameterized SQL query for historical data. -- `dataframe_to_synthefy_request(df, field, forecast_horizon, metadata_fields, model, task_id)`: converts InfluxDB query rows into the Synthefy forecast request payload. +- `dataframe_to_synthefy_request(influxdb3_local, df, field, forecast_horizon, metadata_fields, model, max_forecast_points, task_id)`: converts InfluxDB query rows into the Synthefy forecast request payload and enforces the point limit. - `forecast_response_to_line_builders(influxdb3_local, forecast_response, output_measurement, tag_filters, model, field_name, task_id)`: converts Synthefy forecast results into InfluxDB line protocol builders. ## Troubleshooting @@ -345,7 +351,19 @@ Check the [Synthefy documentation](https://docs.synthefy.com) for the most up-to ### Invalid interval format -`Invalid interval format: ''. Expected ''.` — `time_range` and `forecast_horizon` must be `` (units: `s`, `min`, `h`, `d`, `w`, `m`, `q`, `y`) or, for `forecast_horizon`, ` points`. +`Invalid interval format: ''. Expected ''.` — `time_range` and `forecast_horizon` must be `` (units: `us`, `ms`, `s`, `min`, `h`, `d`, `w`, `m`, `q`, `y`) or, for `forecast_horizon`, ` points`. + +### Forecast horizon too large + +`forecast_horizon '' resolves to N points … above the max_forecast_points limit` — the horizon is divided by the interval between the last two historical points, so a dense series produces many points. Shorten `forecast_horizon`, use the ` points` form, or raise `max_forecast_points`. + +### Timestamps collapse onto each other + +`N history timestamps differ by less than a microsecond …` or `The series' step of … is finer than a microsecond …` — timestamps are sent to Synthefy with microsecond precision, so a series stepping in nanoseconds cannot be represented. Resample it to a coarser step. + +### History holds repeated timestamps + +`History holds N repeated timestamps, so the window covers more than one series …` — the query matched several tag series and their values are interleaved in one input sequence. Add a `tags` filter that selects a single series. ### Synthefy API errors @@ -361,7 +379,7 @@ If writes fail: - Ensure the database exists (the trigger database, or the override `database` if used) - Ensure the plugin has write permissions -- Check the `[task_id] Error writing forecasts attempt N/M: …` warnings in the InfluxDB logs +- Check the `[task_id] Failed to write forecasts after N attempts: …` error in the InfluxDB logs ### Query file limit exceeded (InfluxDB 3 Core) @@ -369,9 +387,10 @@ If you see "Query would scan X Parquet files, exceeding the file limit" errors, ## Limitations -- Currently supports a single time series per request (one `field` plus optional covariates). +- Currently supports a single time series per request (one `field` plus optional covariates). A request whose window matches several tag series is logged as a warning. - Forecast horizon calculation assumes regular time intervals. -- Tag values containing `:`, `@`, `.` or `'` are only fully supported via the JSON request body or quoted values in the trigger-arguments string form. +- Timestamps are exchanged with microsecond precision; a series whose step is finer is rejected. +- In the trigger-arguments string form, a tag value containing `:`, `@` or `.` must be quoted; the JSON request body needs no quoting. ## License diff --git a/influxdata/synthefy_forecasting/manifest.toml b/influxdata/synthefy_forecasting/manifest.toml index fbbe0a6..438a0e1 100644 --- a/influxdata/synthefy_forecasting/manifest.toml +++ b/influxdata/synthefy_forecasting/manifest.toml @@ -2,7 +2,7 @@ manifest_schema_version = "1.2" [plugin] name = "synthefy_forecasting" -version = "0.1.0" +version = "0.2.0" description = "Integrates Synthefy Forecasting API with InfluxDB 3 for on-demand time series forecasting via HTTP. Reads data from InfluxDB, generates forecasts with Synthefy models, and writes results back." triggers = ["process_request"] homepage = "https://www.influxdata.com/" @@ -16,4 +16,4 @@ exclude = [ [dependencies] database_version = ">=3.8.2" -python = ["pandas", "requests"] +python = ["pandas", "requests", "influxdata-plugin-utils>=0.3.0"] diff --git a/influxdata/synthefy_forecasting/requirements-dev.txt b/influxdata/synthefy_forecasting/requirements-dev.txt new file mode 100644 index 0000000..64e33b3 --- /dev/null +++ b/influxdata/synthefy_forecasting/requirements-dev.txt @@ -0,0 +1,4 @@ +pytest +pandas +requests +influxdata-plugin-utils>=0.3.0 diff --git a/influxdata/synthefy_forecasting/requirements.txt b/influxdata/synthefy_forecasting/requirements.txt index a94cf69..32e549d 100644 --- a/influxdata/synthefy_forecasting/requirements.txt +++ b/influxdata/synthefy_forecasting/requirements.txt @@ -1,2 +1,3 @@ requests -pandas \ No newline at end of file +pandas +influxdata-plugin-utils>=0.3.0 diff --git a/influxdata/synthefy_forecasting/synthefy_forecasting.py b/influxdata/synthefy_forecasting/synthefy_forecasting.py index baf5aaf..a5e69c4 100644 --- a/influxdata/synthefy_forecasting/synthefy_forecasting.py +++ b/influxdata/synthefy_forecasting/synthefy_forecasting.py @@ -23,13 +23,13 @@ { "name": "time_range", "example": "30d", - "description": "Historical data window. Format: '' where unit is one of s, min, h, d, w, m, q, y.", + "description": "Historical data window. Format: '' where unit is one of us, ms, s, min, h, d, w, m, q, y.", "required": false }, { "name": "forecast_horizon", "example": "7d", - "description": "Forecast duration. Format: '' (units: s, min, h, d, w, m, q, y) or ' points'.", + "description": "Forecast duration. Format: '' (units: us, ms, s, min, h, d, w, m, q, y) or ' points'.", "required": false }, { @@ -50,28 +50,111 @@ "description": "Space-separated list of metadata field names to use as covariates. In request body, may also be a JSON list of strings.", "required": false }, + { + "name": "max_forecast_points", + "example": "10000", + "description": "Maximum number of forecast points one request may produce (default: 10000). The horizon is converted to points using the series' own step, so a dense series with a long horizon would otherwise build a very large payload.", + "required": false + }, { "name": "database", "example": "mydb", "description": "Optional override database for writing forecasts. Reads always go to the trigger's database.", "required": false } + ], + "http_body_config": [ + { + "name": "measurement", + "example": "temperature", + "description": "InfluxDB measurement name to read from. Required unless set in the trigger arguments.", + "required": false + }, + { + "name": "field", + "example": "value", + "description": "Field name containing the time series values", + "required": false + }, + { + "name": "tags", + "example": "{'room': ['Bedroom', 'Kitchen'], 'location': 'Hall'}", + "description": "Tag filters as a JSON object mapping tag name to a value or list of values. The dot-separated string form of the trigger arguments is also accepted. Send {} to clear the filters configured on the trigger; null means 'not set', so the trigger argument applies.", + "required": false + }, + { + "name": "time_range", + "example": "30d", + "description": "Historical data window. Format: '' where unit is one of us, ms, s, min, h, d, w, m, q, y.", + "required": false + }, + { + "name": "forecast_horizon", + "example": "7d", + "description": "Forecast duration. Format: '' (units: us, ms, s, min, h, d, w, m, q, y) or ' points'.", + "required": false + }, + { + "name": "model", + "example": "sfm-tabular", + "description": "Synthefy model to use (e.g., 'sfm-tabular', 'Migas-latest'). See README for supported models.", + "required": false + }, + { + "name": "output_measurement", + "example": "temperature_forecast", + "description": "Output measurement name (default: '{measurement}_forecast')", + "required": false + }, + { + "name": "metadata_fields", + "example": "['humidity', 'pressure']", + "description": "JSON list of metadata field names to use as covariates. A space-separated string is also accepted. Send [] to clear the list configured on the trigger; null means 'not set', so the trigger argument applies.", + "required": false + }, + { + "name": "max_forecast_points", + "example": "10000", + "description": "Maximum number of forecast points one request may produce (default: 10000). Accepts a JSON number or a string.", + "required": false + }, + { + "name": "database", + "example": "mydb", + "description": "Optional override database for writing forecasts. Reads always go to the trigger's database.", + "required": false + } + ], + "http_headers_config": [ + { + "name": "X-Synthefy-Api-Key", + "example": "", + "description": "Synthefy API key. Required unless the SYNTHEFY_API_KEY environment variable is set on the InfluxDB process; the header wins when both are present.", + "required": false + } ] } """ import json +import math import os -import random import re -import time import uuid from datetime import datetime, timedelta, timezone from json import JSONDecodeError -from typing import Any, Iterable, Optional, Protocol +from typing import Any import pandas as pd import requests +from influxdata_plugin_utils.config import Validator, load_plugin_config +from influxdata_plugin_utils.introspection import get_field_names, get_tag_names +from influxdata_plugin_utils.parsing import ( + parse_delimited_list, + parse_int, + parse_timedelta, +) +from influxdata_plugin_utils.write import build_line, write_data # Note: LineBuilder is provided by the InfluxDB 3 plugin framework at runtime. @@ -79,35 +162,34 @@ API_KEY_HEADER = "X-Synthefy-Api-Key" API_KEY_ENV_VAR = "SYNTHEFY_API_KEY" +DEFAULT_MAX_FORECAST_POINTS = 10000 -class _LineBuilderInterface(Protocol): - def build(self) -> str: ... - - -class _BatchLines: - """ - Wraps multiple LineBuilder objects into a single object with a build() - method that returns a newline-separated string. Allows batched writes - through the write_sync / write_sync_to_db APIs. - """ +# Calendar units have no fixed length, so they are approximated in days. +CALENDAR_UNIT_DAYS = {"m": 30.42, "q": 91.25, "y": 365.0} - def __init__(self, line_builders: Iterable[_LineBuilderInterface]): - self._line_builders = list(line_builders) - self._built: Optional[str] = None +QUOTE_CHARS = ("'", '"') +# Separators after which a quoted tag value may start. +VALUE_START_CHARS = ":@" - def _coerce_builder(self, builder: _LineBuilderInterface) -> str: - build_fn = getattr(builder, "build", None) - if not callable(build_fn): - raise TypeError("line_builder is missing a callable build()") - return str(build_fn()) +# Synthefy accepts sub-second timestamps. +TIMESTAMP_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" - def build(self) -> str: - if self._built is None: - lines = [self._coerce_builder(b) for b in self._line_builders] - if not lines: - raise ValueError("batch_write received no lines to build") - self._built = "\n".join(lines) - return self._built +VALIDATORS: list = [ + Validator("measurement", default="", cast=str), + Validator("field", default="value", cast=str), + Validator("tags", default=None), + Validator("metadata_fields", default=None), + Validator("time_range", default="30d", cast=str), + Validator("forecast_horizon", default="7d", cast=str), + Validator("model", default="sfm-tabular", cast=str), + Validator("output_measurement", default="", cast=str), + Validator("database", default="", cast=str), + Validator( + "max_forecast_points", + default=DEFAULT_MAX_FORECAST_POINTS, + cast=lambda raw: parse_int(raw, minimum=1), + ), +] def quote_identifier(name: str) -> str: @@ -118,96 +200,95 @@ def escape_string_literal(value: str) -> str: return value.replace("'", "''") -def parse_time_interval(raw: str, task_id: str) -> timedelta: +def _load_config(args: dict | None, body: dict | None) -> dict: """ - Parse an interval string ('10min', '2d', '1y', ...) into a timedelta. + Merge trigger arguments with the request body and validate the result. - Supported units: s, min, h, d, w, m (≈30.42d), q (≈91.25d), y (365d). + Body values override trigger arguments. An explicit JSON null means "not set", + so the validator default applies. """ - unit_mapping = { - "s": "seconds", - "min": "minutes", - "h": "hours", - "d": "days", - "w": "weeks", - "m": "days", - "q": "days", - "y": "days", - } - day_conversions = { - "m": 30.42, - "q": 91.25, - "y": 365.0, + args = args or {} + body = body or {} + merged = { + **args, + **{key: value for key, value in body.items() if value is not None}, } + try: + settings = load_plugin_config(merged, validators=VALIDATORS, source="args") + except Exception as e: + raise Exception(f"Invalid configuration: {e}") from e + return {key.lower(): value for key, value in settings.as_dict().items()} + + +def parse_time_interval(raw: str, task_id: str) -> timedelta: + """ + Parse an interval string ('10min', '2d', '1y', ...) into a timedelta. + Supported units: us, ms, s, min, h, d, w, plus the approximate calendar units + m (30.42d), q (91.25d) and y (365d). + """ if not isinstance(raw, str): raise Exception( f"[{task_id}] Invalid interval type: expected string like '10min', got {type(raw).__name__}" ) - match = re.fullmatch(r"(\d+)([a-zA-Z]+)", raw.strip()) - if not match: - raise Exception( - f"[{task_id}] Invalid interval format: '{raw}'. Expected '', e.g. '10min', '2d'." - ) - - number_part, unit = match.groups() - magnitude = int(number_part) - unit = unit.lower() - if unit not in unit_mapping: - raise Exception(f"[{task_id}] Unsupported interval unit '{unit}' in '{raw}'.") - - if unit in day_conversions: - days_approx = int(magnitude * day_conversions[unit]) - if days_approx < 1: + match = re.fullmatch(r"\s*(\d+)\s*([a-zA-Z]+)\s*", raw) + if match and match.group(2).lower() in CALENDAR_UNIT_DAYS: + magnitude = int(match.group(1)) + unit = match.group(2).lower() + days = int(magnitude * CALENDAR_UNIT_DAYS[unit]) + if days < 1: raise Exception( f"[{task_id}] Computed days < 1 for {magnitude}{unit} in '{raw}'." ) - return timedelta(days=days_approx) - - if unit == "s": - return timedelta(seconds=magnitude) - if unit == "min": - return timedelta(minutes=magnitude) - if unit == "h": - return timedelta(hours=magnitude) - if unit == "d": - return timedelta(days=magnitude) - if unit == "w": - return timedelta(weeks=magnitude) - raise Exception(f"[{task_id}] Unsupported interval unit '{unit}' in '{raw}'.") - - -def get_tag_names(influxdb3_local, measurement: str, task_id: str) -> list[str]: - """Return tag column names for `measurement`, or an empty list if none/no schema.""" - query = """ - SELECT column_name - FROM information_schema.columns - WHERE table_name = $measurement - AND data_type = 'Dictionary(Int32, Utf8)' + return timedelta(days=days) + + try: + return parse_timedelta(raw) + except ValueError as e: + raise Exception( + f"[{task_id}] Invalid interval format: '{raw}' ({e}). " + f"Expected '', e.g. '10min', '2d', '1y'." + ) from e + + +def split_unquoted(text: str, separator: str) -> list[str]: """ - res = influxdb3_local.query(query, {"measurement": measurement}) - if not res: - influxdb3_local.info( - f"[{task_id}] No tags found for measurement '{measurement}'." - ) - return [] - return [row["column_name"] for row in res] + Split on `separator`, ignoring separators inside '...' or "..." quotes. + A quote is only special where a value may start: at the beginning of a part + or right after ':' or '@'. Elsewhere it is data, so "Bob's" stays intact. -def get_field_names(influxdb3_local, measurement: str, task_id: str) -> list[str]: - """Return non-tag, non-time field column names for `measurement`.""" - query = """ - SELECT column_name - FROM information_schema.columns - WHERE table_name = $measurement - AND data_type != 'Dictionary(Int32, Utf8)' - AND column_name != 'time' + Raises: + ValueError: if a quote is never closed. """ - res = influxdb3_local.query(query, {"measurement": measurement}) - if not res: - return [] - return [row["column_name"] for row in res] + parts: list[str] = [] + current: list[str] = [] + quote = "" + for char in text: + if quote: + current.append(char) + if char == quote: + quote = "" + elif char in QUOTE_CHARS and (not current or current[-1] in VALUE_START_CHARS): + quote = char + current.append(char) + elif char == separator: + parts.append("".join(current)) + current = [] + else: + current.append(char) + if quote: + raise ValueError(f"unterminated {quote} quote in '{text}'") + parts.append("".join(current)) + return parts + + +def strip_quotes(value: str) -> str: + """Remove one matching pair of surrounding single or double quotes.""" + if len(value) >= 2 and value[0] == value[-1] and value[0] in QUOTE_CHARS: + return value[1:-1] + return value def parse_tags_from_args( @@ -220,7 +301,9 @@ def parse_tags_from_args( - '.' separates pairs - ':' separates the tag key from its value(s) - '@' separates multiple values for one key - - quoted values ('...' or "...") are stripped of their quotes + - a value wrapped in '...' or "..." is stripped of its quotes, and any + separator inside the quotes is treated as part of the value + - a quote inside a value, as in "Bob's", needs no escaping """ if raw is None or raw == "": return {} @@ -230,21 +313,23 @@ def parse_tags_from_args( ) result: dict[str, list[str]] = {} - for pair in raw.split("."): + try: + pairs = split_unquoted(raw, ".") + except ValueError as e: + raise Exception( + f"[{task_id}] Invalid 'tags' string in trigger args: {e}." + ) from e + + for pair in pairs: if not pair: continue - parts = pair.split(":") + parts = split_unquoted(pair, ":") if len(parts) != 2: raise Exception( f"[{task_id}] Invalid tag-value pair: '{pair}' (must contain exactly one ':'; quote values containing ':')" ) tag_name, value_str = parts - values: list[str] = [] - for v in value_str.split("@"): - if len(v) >= 2 and v[0] == v[-1] and v[0] in ("'", '"'): - values.append(v[1:-1]) - else: - values.append(v) + values = [strip_quotes(value) for value in split_unquoted(value_str, "@")] if tag_name not in tag_names: influxdb3_local.warn( @@ -300,6 +385,23 @@ def parse_tags_from_body( return result +def parse_tags( + influxdb3_local, raw: Any, measurement: str, tag_names: list[str], task_id: str +) -> dict[str, list[str]]: + """Dispatch to the JSON-object form (request body) or the string form (trigger args).""" + if isinstance(raw, dict): + return parse_tags_from_body( + influxdb3_local, raw, measurement, tag_names, task_id + ) + if raw is None or isinstance(raw, str): + return parse_tags_from_args( + influxdb3_local, raw, measurement, tag_names, task_id + ) + raise Exception( + f"[{task_id}] Invalid 'tags' format: expected a string or JSON object, got {type(raw).__name__}." + ) + + def parse_metadata_fields( influxdb3_local, raw: Any, @@ -313,17 +415,13 @@ def parse_metadata_fields( """ if raw is None or raw == "": return [] - if isinstance(raw, str): - items = raw.split() - elif isinstance(raw, list): - items = [str(x) for x in raw] - else: + if not isinstance(raw, (str, list)): raise Exception( f"[{task_id}] Invalid 'metadata_fields' format: expected string or list, got {type(raw).__name__}." ) result: list[str] = [] - for item in items: + for item in parse_delimited_list(raw): if item not in field_names: influxdb3_local.warn( f"[{task_id}] Metadata field '{item}' does not exist in '{measurement}'; ignoring." @@ -351,7 +449,7 @@ def build_history_query( select_columns.append(quote_identifier(mf)) select_clause = ", ".join(select_columns) - start_iso = start_time.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + start_iso = start_time.astimezone(timezone.utc).strftime(TIMESTAMP_FORMAT) params: dict[str, Any] = {} where_parts = [f"time >= '{start_iso}'"] @@ -386,11 +484,13 @@ def build_history_query( def dataframe_to_synthefy_request( + influxdb3_local, df: pd.DataFrame, field: str, forecast_horizon: str, metadata_fields: list[str], model: str, + max_forecast_points: int, task_id: str, ) -> dict[str, Any]: """ @@ -405,9 +505,26 @@ def dataframe_to_synthefy_request( df["time"] = pd.to_datetime(df["time"]) df = df.sort_values("time").reset_index(drop=True) - history_timestamps = df["time"].dt.strftime("%Y-%m-%dT%H:%M:%SZ").tolist() + repeated_timestamps = int(df["time"].duplicated().sum()) + if repeated_timestamps: + influxdb3_local.warn( + f"[{task_id}] History holds {repeated_timestamps} repeated timestamps, so the " + f"window covers more than one series and their values are interleaved. " + f"Set 'tags' to select a single series." + ) + + history_timestamps = df["time"].dt.strftime(TIMESTAMP_FORMAT).tolist() history_values = [None if pd.isna(v) else v for v in df[field].tolist()] + collapsed = ( + len(history_timestamps) - len(set(history_timestamps)) - repeated_timestamps + ) + if collapsed > 0: + raise Exception( + f"[{task_id}] {collapsed} history timestamps differ by less than a microsecond " + f"and collapse onto each other. Resample the series to a coarser step." + ) + if len(df) >= 2: time_step = df["time"].iloc[-1] - df["time"].iloc[-2] if time_step <= timedelta(0): @@ -418,23 +535,34 @@ def dataframe_to_synthefy_request( fh = forecast_horizon.strip() if fh.endswith(" points"): try: - num_points = int(fh.replace(" points", "").strip()) - except ValueError: + num_points = parse_int(fh.removesuffix(" points"), minimum=1) + except ValueError as e: raise Exception( - f"[{task_id}] Invalid forecast_horizon: '{forecast_horizon}'." - ) - if num_points < 1: - raise Exception(f"[{task_id}] forecast_horizon must be >= 1 point.") + f"[{task_id}] Invalid forecast_horizon: '{forecast_horizon}' ({e})." + ) from e else: forecast_td = parse_time_interval(fh, task_id) num_points = max(1, int(forecast_td / time_step)) + if num_points > max_forecast_points: + raise Exception( + f"[{task_id}] forecast_horizon '{forecast_horizon}' resolves to {num_points} points " + f"at the series' step of {time_step}, above the max_forecast_points limit of " + f"{max_forecast_points}. Shorten the horizon or raise max_forecast_points." + ) + target_timestamps: list[str] = [] current_time = df["time"].iloc[-1] + time_step for _ in range(num_points): - target_timestamps.append(current_time.strftime("%Y-%m-%dT%H:%M:%SZ")) + target_timestamps.append(current_time.strftime(TIMESTAMP_FORMAT)) current_time += time_step + if len(set(target_timestamps)) != len(target_timestamps): + raise Exception( + f"[{task_id}] The series' step of {time_step} is finer than a microsecond, so " + f"forecast timestamps collapse onto each other. Resample to a coarser step." + ) + target_values = [None] * len(target_timestamps) forecast_sample = { @@ -502,6 +630,19 @@ def call_synthefy_api( raise +def _timestamp_ns(raw: Any) -> int: + """Convert a forecast timestamp to integer nanoseconds; naive values are UTC.""" + ts = pd.Timestamp(raw) + if ts.tz is None: + ts = ts.tz_localize("UTC") + return int(ts.value) + + +def _is_non_finite(value: Any) -> bool: + """True for NaN/inf floats, which would make InfluxDB reject the whole batch.""" + return isinstance(value, float) and not math.isfinite(value) + + def forecast_response_to_line_builders( influxdb3_local, forecast_response: dict[str, Any], @@ -527,7 +668,7 @@ def forecast_response_to_line_builders( forecast_row = forecasts[0] - forecast_payload: Optional[dict] = None + forecast_payload: dict | None = None for f in forecast_row: if isinstance(f, dict) and "timestamps" in f and "values" in f: forecast_payload = f @@ -543,90 +684,86 @@ def forecast_response_to_line_builders( quantiles = forecast_payload.get("quantiles") or {} output_field_name = field_name or forecast_payload.get("sample_id", "value") + static_tags = { + tag_key: tag_values[0] + for tag_key, tag_values in tag_filters.items() + if len(tag_values) == 1 + } + static_tags["model"] = model + builders: list[Any] = [] for i, (ts_str, value) in enumerate(zip(timestamps, values)): if value is None: continue + if _is_non_finite(value): + influxdb3_local.warn( + f"[{task_id}] Non-finite forecast value at '{ts_str}'; skipping point." + ) + continue try: - ts = pd.to_datetime(ts_str) - ts_ns = int(ts.timestamp() * 1e9) + ts_ns = _timestamp_ns(ts_str) except Exception: influxdb3_local.warn( f"[{task_id}] Could not parse timestamp '{ts_str}'; skipping point." ) continue - builder = LineBuilder(output_measurement) - builder.time_ns(ts_ns) - - for tag_key, tag_values in tag_filters.items(): - if len(tag_values) == 1: - builder.tag(tag_key, tag_values[0]) - builder.tag("model", model) - - _set_field(builder, output_field_name, value) + fields: dict[str, Any] = {output_field_name: value} for q_level, q_values in quantiles.items(): - if i < len(q_values) and q_values[i] is not None: - _set_field(builder, f"value_{q_level}", q_values[i]) - - builders.append(builder) + if i >= len(q_values): + continue + q_value = q_values[i] + if q_value is None or _is_non_finite(q_value): + continue + fields[f"value_{q_level}"] = q_value + + builders.append( + build_line( + LineBuilder, + output_measurement, + tags=static_tags, + fields=fields, + time_ns=ts_ns, + ) + ) return builders -def _set_field(builder: Any, name: str, value: Any) -> None: - if isinstance(value, bool): - builder.string_field(name, str(value)) - elif isinstance(value, int): - builder.int64_field(name, value) - elif isinstance(value, float): - builder.float64_field(name, value) - else: - builder.string_field(name, str(value)) - - def write_forecasts_to_influxdb( influxdb3_local, builders: list[Any], - database: Optional[str], + database: str | None, task_id: str, max_retries: int = 3, ) -> None: """ - Write forecast points using write_sync (or write_sync_to_db when `database` - is set), batched into a single line-protocol payload, with exponential backoff retries. + Write forecast points as a single batched, synchronous payload, retrying with + exponential backoff. Writes go to `database` when set, otherwise to the + trigger's own database. """ if not builders: influxdb3_local.warn(f"[{task_id}] No forecast points to write.") return + target = f"database {database}" if database else "trigger database" influxdb3_local.info( - f"[{task_id}] Writing {len(builders)} forecast points to " - f"{'database ' + database if database else 'trigger database'}." + f"[{task_id}] Writing {len(builders)} forecast points to {target}." ) - - batch = _BatchLines(builders) - for attempt in range(max_retries): - try: - if database: - influxdb3_local.write_sync_to_db(database, batch, no_sync=True) - else: - influxdb3_local.write_sync(batch, no_sync=True) - influxdb3_local.info( - f"[{task_id}] Wrote {len(builders)} forecast points (attempt {attempt + 1})." - ) - return - except Exception as e: - influxdb3_local.warn( - f"[{task_id}] Error writing forecasts attempt {attempt + 1}/{max_retries}: {e}" - ) - if attempt < max_retries - 1: - wait_time = (2**attempt) + random.random() - time.sleep(wait_time) - else: - influxdb3_local.error( - f"[{task_id}] Failed to write forecasts after {max_retries} attempts: {e}" - ) - raise + try: + write_data( + influxdb3_local, + builders, + batch=True, + retries=max_retries - 1, + no_sync=True, + database=database, + ) + except Exception as e: + influxdb3_local.error( + f"[{task_id}] Failed to write forecasts after {max_retries} attempts: {e}" + ) + raise + influxdb3_local.info(f"[{task_id}] Wrote {len(builders)} forecast points.") def _decode_request_body(request_body: Any, task_id: str) -> dict: @@ -635,18 +772,19 @@ def _decode_request_body(request_body: Any, task_id: str) -> dict: return {} if isinstance(request_body, dict): return request_body - if isinstance(request_body, bytes): - body_str = request_body.decode("utf-8") - elif isinstance(request_body, str): - body_str = request_body - else: + if not isinstance(request_body, (bytes, str)): raise Exception( f"[{task_id}] Unsupported request_body type: {type(request_body).__name__}" ) + body_str = ( + request_body.decode("utf-8") + if isinstance(request_body, bytes) + else request_body + ) return json.loads(body_str) -def _get_api_key(request_headers: Optional[dict]) -> Optional[str]: +def _get_api_key(request_headers: dict | None) -> str | None: """Return the API key from the request header or env var, or None.""" if request_headers: for key, value in request_headers.items(): @@ -660,7 +798,7 @@ def process_request( query_parameters: dict, request_headers: dict, request_body: Any, - args: Optional[dict] = None, + args: dict | None = None, ) -> dict: """ HTTP entry point. Reads historical data, calls Synthefy, writes the forecast back. @@ -696,24 +834,21 @@ def process_request( return {"message": "Missing API key"} try: - merged_args = {**args, **body_dict} + config = _load_config(args, body_dict) - measurement = merged_args.get("measurement") + measurement = config["measurement"] if not measurement: influxdb3_local.error(f"[{task_id}] 'measurement' argument is required.") return {"message": "'measurement' argument is required"} - field = merged_args.get("field", "value") - time_range_str = merged_args.get("time_range", "30d") - forecast_horizon_str = merged_args.get("forecast_horizon", "7d") - model = merged_args.get("model", "sfm-tabular") - output_measurement = ( - merged_args.get("output_measurement") or f"{measurement}_forecast" - ) - database = merged_args.get("database") or None + field = config["field"] + model = config["model"] + output_measurement = config["output_measurement"] or f"{measurement}_forecast" + database = config["database"] or None + max_forecast_points = config["max_forecast_points"] - field_names = get_field_names(influxdb3_local, measurement, task_id) - tag_names = get_tag_names(influxdb3_local, measurement, task_id) + field_names = get_field_names(influxdb3_local, measurement, use_cache=False) + tag_names = get_tag_names(influxdb3_local, measurement, use_cache=False) if not field_names and not tag_names: influxdb3_local.error( @@ -727,41 +862,14 @@ def process_request( ) return {"message": f"Field '{field}' does not exist in '{measurement}'"} - if "tags" in body_dict: - tag_filters = parse_tags_from_body( - influxdb3_local, - body_dict.get("tags"), - measurement, - tag_names, - task_id, - ) - else: - tag_filters = parse_tags_from_args( - influxdb3_local, - args.get("tags"), - measurement, - tag_names, - task_id, - ) - - if "metadata_fields" in body_dict: - metadata_fields = parse_metadata_fields( - influxdb3_local, - body_dict.get("metadata_fields"), - measurement, - field_names, - task_id, - ) - else: - metadata_fields = parse_metadata_fields( - influxdb3_local, - args.get("metadata_fields"), - measurement, - field_names, - task_id, - ) + tag_filters = parse_tags( + influxdb3_local, config["tags"], measurement, tag_names, task_id + ) + metadata_fields = parse_metadata_fields( + influxdb3_local, config["metadata_fields"], measurement, field_names, task_id + ) - time_range_td = parse_time_interval(time_range_str, task_id) + time_range_td = parse_time_interval(config["time_range"], task_id) start_time = datetime.now(timezone.utc) - time_range_td query, params = build_history_query( @@ -781,7 +889,14 @@ def process_request( return {"message": "No data found"} synthefy_request = dataframe_to_synthefy_request( - df, field, forecast_horizon_str, metadata_fields, model, task_id + influxdb3_local, + df, + field, + config["forecast_horizon"], + metadata_fields, + model, + max_forecast_points, + task_id, ) forecast_response = call_synthefy_api( influxdb3_local, synthefy_request, api_key, task_id diff --git a/influxdata/synthefy_forecasting/test_synthefy_forecasting.py b/influxdata/synthefy_forecasting/test_synthefy_forecasting.py new file mode 100644 index 0000000..79073cd --- /dev/null +++ b/influxdata/synthefy_forecasting/test_synthefy_forecasting.py @@ -0,0 +1,868 @@ +"""Unit and integration tests for the synthefy_forecasting plugin.""" + +import json +import os +import sys +from collections import namedtuple + +import pandas as pd +import pytest +from influxdata_plugin_utils import write as utils_write + +sys.path.insert(0, os.path.dirname(__file__)) +import synthefy_forecasting as sf + +TAG_TYPE = "Dictionary(Int32, Utf8)" + +COLUMNS = { + "time": "Timestamp(Nanosecond, None)", + "value": "Float64", + "humidity": "Float64", + "pressure": "Float64", + "room": TAG_TYPE, + "site": TAG_TYPE, +} + + +# --------------------------------------------------------------------------- +# Fakes +# --------------------------------------------------------------------------- + + +class FakeCache: + def __init__(self): + self.store = {} + + def get(self, key, default=None, use_global=None): + return self.store.get(key, default) + + def put(self, key, value, ttl=None, use_global=None): + self.store[key] = value + + def delete(self, key, use_global=None): + return self.store.pop(key, None) is not None + + +class FakeLineBuilder: + def __init__(self, measurement): + self.measurement = measurement + self.tags = [] + self.fields = {} + self.timestamp = None + + def tag(self, key, value): + self.tags.append((key, value)) + return self + + def int64_field(self, key, value): + self.fields[key] = f"{value}i" + return self + + def uint64_field(self, key, value): + self.fields[key] = f"{value}u" + return self + + def float64_field(self, key, value): + self.fields[key] = f"{int(value)}.0" if value % 1 == 0 else str(value) + return self + + def bool_field(self, key, value): + self.fields[key] = "true" if value else "false" + return self + + def string_field(self, key, value): + self.fields[key] = f'"{value}"' + return self + + def time_ns(self, timestamp_ns): + self.timestamp = timestamp_ns + return self + + def build(self): + line = self.measurement + if self.tags: + line += "," + ",".join(f"{k}={v}" for k, v in self.tags) + line += " " + ",".join(f"{k}={v}" for k, v in self.fields.items()) + if self.timestamp is not None: + line += f" {self.timestamp}" + return line + + +Record = namedtuple("Record", ["measurement", "tags", "fields", "timestamp"]) + + +def _parse_field(raw): + if raw.startswith('"'): + return raw[1:-1] + if raw in ("true", "false"): + return raw == "true" + if raw[-1] in ("i", "u"): + return int(raw[:-1]) + return float(raw) + + +def _parse_lp(line): + """Parse one line-protocol record (sufficient for this plugin's output).""" + head, fields_str, ts = line.rsplit(" ", 2) + parts = head.split(",") + tags = dict(kv.split("=", 1) for kv in parts[1:]) + fields = {k: _parse_field(v) for k, v in (kv.split("=", 1) for kv in fields_str.split(","))} + return Record(parts[0], tags, fields, int(ts)) + + +class FakeLocal: + def __init__(self, columns=None, rows=None, write_failures=0): + self.cache = FakeCache() + self.columns = COLUMNS if columns is None else columns + self.rows = [] if rows is None else rows + self.write_failures = write_failures + self.queries = [] + self.writes = [] # (db_name | None, Record) per emitted point + self.infos = [] + self.warns = [] + self.errors = [] + + def query(self, query, args=None): + self.queries.append((query, args)) + if "information_schema.columns" not in query: + return self.rows + rows = [{"column_name": n, "data_type": t} for n, t in self.columns.items()] + wanted = (args or {}).get("data_type") + return [r for r in rows if wanted is None or r["data_type"] == wanted] + + def _record_batch(self, db_name, batch): + # The plugin hands a BatchLines; the engine calls build(). Exercise that + # path, then expand back to one Record per line for assertions. + if self.write_failures: + self.write_failures -= 1 + raise RuntimeError("simulated write failure") + for lp in batch.build().split("\n"): + self.writes.append((db_name, _parse_lp(lp))) + + def info(self, *args): + self.infos.append(" ".join(str(a) for a in args)) + + def warn(self, *args): + self.warns.append(" ".join(str(a) for a in args)) + + def error(self, *args): + self.errors.append(" ".join(str(a) for a in args)) + + def write(self, *args): + raise AssertionError("buffered write must not be used") + + def write_sync(self, batch, no_sync=False): + self._record_batch(None, batch) + + def write_sync_to_db(self, db_name, batch, no_sync=False): + self._record_batch(db_name, batch) + + +class FakeResponse: + def __init__(self, payload): + self._payload = payload + + def raise_for_status(self): + return None + + def json(self): + return self._payload + + +class FakeRequests: + """Stand-in for the `requests` module inside the plugin.""" + + def __init__(self, payload=None, error=None): + self.payload = payload + self.error = error + self.calls = [] + + def post(self, url, json=None, headers=None, timeout=None): + self.calls.append({"url": url, "body": json, "headers": headers, "timeout": timeout}) + if self.error is not None: + raise self.error + return FakeResponse(self.payload) + + +@pytest.fixture(autouse=True) +def _plugin_env(monkeypatch): + monkeypatch.setattr(sf, "LineBuilder", FakeLineBuilder, raising=False) + monkeypatch.setattr(utils_write.time, "sleep", lambda _: None) + yield + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +START_NS = 1_700_000_000_000_000_000 +HOUR_NS = 3_600_000_000_000 + + +def history_rows(count=5, field="value", step_ns=HOUR_NS, extra=None): + rows = [] + for i in range(count): + row = {"time": START_NS + i * step_ns, field: float(i)} + row.update(extra or {}) + rows.append(row) + return rows + + +def forecast_response(timestamps, values, quantiles=None, sample_id="value"): + payload = {"sample_id": sample_id, "timestamps": timestamps, "values": values} + if quantiles is not None: + payload["quantiles"] = quantiles + return {"forecasts": [[payload]]} + + +def run(local, body=None, args=None, headers=None, requests_stub=None, monkeypatch=None): + if requests_stub is not None: + monkeypatch.setattr(sf, "requests", requests_stub) + return sf.process_request( + local, + {}, + {"X-Synthefy-Api-Key": "k"} if headers is None else headers, + json.dumps(body or {}), + args or {}, + ) + + +# --------------------------------------------------------------------------- +# M1 — plugin metadata +# --------------------------------------------------------------------------- + + +def test_docstring_header_is_valid_json_with_expected_args(): + header = json.loads(sf.__doc__) + assert header["plugin_type"] == ["http"] + names = [arg["name"] for arg in header["http_args_config"]] + assert set(names) == { + "measurement", "field", "tags", "time_range", "forecast_horizon", "model", + "output_measurement", "metadata_fields", "max_forecast_points", "database", + } + # every argument is also accepted in the request body, in the same order + assert [arg["name"] for arg in header["http_body_config"]] == names + assert [h["name"] for h in header["http_headers_config"]] == [sf.API_KEY_HEADER] + for section in ("http_args_config", "http_body_config", "http_headers_config"): + for entry in header[section]: + assert set(entry) == {"name", "example", "description", "required"} + + +# --------------------------------------------------------------------------- +# M2 — configuration +# --------------------------------------------------------------------------- + + +def test_config_defaults(): + cfg = sf._load_config({"measurement": "t"}, {}) + assert cfg["field"] == "value" + assert cfg["time_range"] == "30d" + assert cfg["forecast_horizon"] == "7d" + assert cfg["model"] == "sfm-tabular" + assert cfg["output_measurement"] == "" + assert cfg["database"] == "" + assert cfg["max_forecast_points"] == sf.DEFAULT_MAX_FORECAST_POINTS + + +def test_config_body_overrides_args_and_null_falls_back(): + cfg = sf._load_config({"measurement": "t", "model": "from-args"}, {"model": "from-body"}) + assert cfg["model"] == "from-body" + cfg = sf._load_config({"measurement": "t", "model": "from-args"}, {"model": None}) + assert cfg["model"] == "from-args" + + +def test_config_never_reads_a_toml_file(): + cfg = sf._load_config({"measurement": "t", "config_file_path": "/nonexistent.toml"}, {}) + assert "config_file_path" not in cfg + assert cfg["measurement"] == "t" + + +def test_config_leaves_dynaconf_tokens_literal(): + cfg = sf._load_config({"measurement": "@format {env[HOME]}"}, {}) + assert cfg["measurement"] == "@format {env[HOME]}" + + +@pytest.mark.parametrize( + "value, fragment", + [("0", "below minimum 1"), ("-5", "below minimum 1"), ("junk", "Invalid integer")], +) +def test_config_rejects_bad_max_forecast_points(value, fragment): + with pytest.raises(Exception) as excinfo: + sf._load_config({"measurement": "t", "max_forecast_points": value}, {}) + assert "Invalid configuration" in str(excinfo.value) + assert fragment in str(excinfo.value) + + +# --------------------------------------------------------------------------- +# M3 — interval parsing +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "raw, seconds", + [ + ("500us", 0.0005), + ("100ms", 0.1), + ("30s", 30), + ("10min", 600), + ("2h", 7200), + ("30d", 2_592_000), + ("1w", 604_800), + ("1m", 30 * 86_400), + ("2q", 182 * 86_400), + ("1y", 365 * 86_400), + ], +) +def test_parse_time_interval_units(raw, seconds): + assert sf.parse_time_interval(raw, "T").total_seconds() == pytest.approx(seconds) + + +@pytest.mark.parametrize( + "raw, fragment", + [ + ("5x", "Invalid interval format"), + ("abc", "Invalid interval format"), + ("", "Invalid interval format"), + ("0y", "Computed days < 1"), + (30, "Invalid interval type"), + ], +) +def test_parse_time_interval_rejections(raw, fragment): + with pytest.raises(Exception, match=fragment): + sf.parse_time_interval(raw, "T") + + +# --------------------------------------------------------------------------- +# M4 — tag filters +# --------------------------------------------------------------------------- + +TAG_NAMES = ["room", "site", "path"] + + +@pytest.mark.parametrize( + "raw, expected", + [ + ("", {}), + ("room:Bedroom", {"room": ["Bedroom"]}), + ("room:Bedroom@Kitchen.site:north", {"room": ["Bedroom", "Kitchen"], "site": ["north"]}), + ("room:'Some other room'@Bedroom", {"room": ["Some other room", "Bedroom"]}), + ("room:A.room:B", {"room": ["A", "B"]}), + # quoting protects every separator, as documented in the README + ("path:'a:b'", {"path": ["a:b"]}), + ("path:'a.b'", {"path": ["a.b"]}), + ("path:'a@b'", {"path": ["a@b"]}), + ('path:"a:b@c.d"', {"path": ["a:b@c.d"]}), + ("path:Bob's.room:A", {"path": ["Bob's"], "room": ["A"]}), + ('path:5".room:A', {"path": ['5"'], "room": ["A"]}), + ], +) +def test_parse_tags_from_args(raw, expected): + local = FakeLocal() + assert sf.parse_tags_from_args(local, raw, "m", TAG_NAMES, "T") == expected + + +def test_parse_tags_from_args_rejects_ambiguous_pair(): + with pytest.raises(Exception, match="Invalid tag-value pair"): + sf.parse_tags_from_args(FakeLocal(), "path:a:b:c", "m", TAG_NAMES, "T") + with pytest.raises(Exception, match="expected string"): + sf.parse_tags_from_args(FakeLocal(), ["room:A"], "m", TAG_NAMES, "T") + with pytest.raises(Exception, match="unterminated ' quote"): + sf.parse_tags_from_args(FakeLocal(), "room:'Living room", "m", TAG_NAMES, "T") + + +def test_parse_tags_from_args_warns_on_unknown_tag(): + local = FakeLocal() + assert sf.parse_tags_from_args(local, "nope:x.room:A", "m", TAG_NAMES, "T") == {"room": ["A"]} + assert any("Tag 'nope' does not exist" in w for w in local.warns) + + +@pytest.mark.parametrize( + "raw, expected", + [ + (None, {}), + ({}, {}), + ({"room": "Bedroom"}, {"room": ["Bedroom"]}), + ({"room": ["Bedroom", "Kitchen"]}, {"room": ["Bedroom", "Kitchen"]}), + ({"room": [1, 2]}, {"room": ["1", "2"]}), + ({"room": []}, {}), + ], +) +def test_parse_tags_from_body(raw, expected): + assert sf.parse_tags_from_body(FakeLocal(), raw, "m", TAG_NAMES, "T") == expected + + +@pytest.mark.parametrize( + "raw, fragment", + [(["Bedroom"], "expected JSON object"), ({"room": 5}, "expected string or list")], +) +def test_parse_tags_from_body_rejections(raw, fragment): + with pytest.raises(Exception, match=fragment): + sf.parse_tags_from_body(FakeLocal(), raw, "m", TAG_NAMES, "T") + + +def test_parse_tags_dispatches_on_the_value_type(): + local = FakeLocal() + assert sf.parse_tags(local, {"room": "A"}, "m", TAG_NAMES, "T") == {"room": ["A"]} + assert sf.parse_tags(local, "room:A", "m", TAG_NAMES, "T") == {"room": ["A"]} + assert sf.parse_tags(local, None, "m", TAG_NAMES, "T") == {} + with pytest.raises(Exception, match="expected a string or JSON object"): + sf.parse_tags(local, ["room:A"], "m", TAG_NAMES, "T") + + +# --------------------------------------------------------------------------- +# M5 — metadata fields +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "raw, expected", + [ + ("", []), + ("humidity pressure", ["humidity", "pressure"]), + (["humidity", "pressure"], ["humidity", "pressure"]), + ("humidity pressure", ["humidity", "pressure"]), + ], +) +def test_parse_metadata_fields(raw, expected): + names = ["humidity", "pressure"] + assert sf.parse_metadata_fields(FakeLocal(), raw, "m", names, "T") == expected + + +def test_parse_metadata_fields_drops_unknown_with_warning(): + local = FakeLocal() + result = sf.parse_metadata_fields(local, "humidity nope", "m", ["humidity"], "T") + assert result == ["humidity"] + assert any("Metadata field 'nope' does not exist" in w for w in local.warns) + + +def test_parse_metadata_fields_rejects_other_types(): + with pytest.raises(Exception, match="expected string or list"): + sf.parse_metadata_fields(FakeLocal(), 5, "m", [], "T") + + +# --------------------------------------------------------------------------- +# M6 — history query +# --------------------------------------------------------------------------- + + +def test_history_query_binds_tag_values_and_quotes_identifiers(): + start = pd.Timestamp("2026-01-01T00:00:00Z").to_pydatetime() + query, params = sf.build_history_query( + "temp", "va\"lue", ["humidity"], {"room": ["A"], "site": ["x", "y"]}, start + ) + assert '"va""lue"' in query and '"humidity"' in query + assert '"room" = $tag_val_0' in query + assert '"site" IN ($tag_val_1, $tag_val_2)' in query + assert params == {"tag_val_0": "A", "tag_val_1": "x", "tag_val_2": "y"} + assert "time >= '2026-01-01T00:00:00.000000Z'" in query + assert "ORDER BY time" in query + + +# --------------------------------------------------------------------------- +# M7 — Synthefy request payload +# --------------------------------------------------------------------------- + + +def test_request_payload_derives_step_and_targets(): + df = pd.DataFrame(history_rows(4)) + request = sf.dataframe_to_synthefy_request(FakeLocal(), df, "value", "3h", [], "sfm-tabular", 10_000, "T") + sample = request["samples"][0][0] + assert request["model"] == "sfm-tabular" + assert sample["forecast"] is True and sample["metadata"] is False + assert len(sample["history_timestamps"]) == 4 + assert sample["history_timestamps"][-1] == "2023-11-15T01:13:20.000000Z" + # the horizon continues the series at its own step, starting after the last point + assert sample["target_timestamps"] == [ + "2023-11-15T02:13:20.000000Z", + "2023-11-15T03:13:20.000000Z", + "2023-11-15T04:13:20.000000Z", + ] + assert sample["target_values"] == [None, None, None] + + +def test_request_payload_point_form_and_covariates(): + rows = history_rows(3, extra={"humidity": 1.0}) + request = sf.dataframe_to_synthefy_request( + FakeLocal(), pd.DataFrame(rows), "value", "2 points", ["humidity"], "m", 10_000, "T" + ) + samples = request["samples"][0] + assert len(samples[0]["target_timestamps"]) == 2 + assert len(samples) == 2 + assert samples[1]["sample_id"] == "humidity" + assert samples[1]["metadata"] is True and samples[1]["forecast"] is False + + +@pytest.mark.parametrize( + "horizon, cap, fragment", + [ + ("7d", 10_000, "above the max_forecast_points limit"), + ("50000 points", 10_000, "above the max_forecast_points limit"), + ("0 points", 10_000, "below minimum 1"), + ("many points", 10_000, "Invalid forecast_horizon"), + ("2 points points", 10_000, "Invalid forecast_horizon"), + ], +) +def test_request_payload_rejections(horizon, cap, fragment): + df = pd.DataFrame(history_rows(3, step_ns=1_000_000_000)) + with pytest.raises(Exception, match=fragment): + sf.dataframe_to_synthefy_request(FakeLocal(), df, "value", horizon, [], "m", cap, "T") + + +def test_request_payload_allows_a_raised_cap(): + df = pd.DataFrame(history_rows(3, step_ns=1_000_000_000)) + request = sf.dataframe_to_synthefy_request(FakeLocal(), df, "value", "1h", [], "m", 10_000, "T") + assert len(request["samples"][0][0]["target_timestamps"]) == 3600 + + +def test_request_payload_warns_when_the_window_holds_several_series(): + # two series without a tag filter: every timestamp appears twice + rows = history_rows(3) + history_rows(3, field="value") + local = FakeLocal() + sf.dataframe_to_synthefy_request(local, pd.DataFrame(rows), "value", "1 points", [], "m", 10, "T") + assert any("3 repeated timestamps" in w and "Set 'tags'" in w for w in local.warns) + + local = FakeLocal() + sf.dataframe_to_synthefy_request( + local, pd.DataFrame(history_rows(3)), "value", "1 points", [], "m", 10, "T" + ) + assert local.warns == [] + + +def test_request_payload_keeps_sub_second_steps(): + df = pd.DataFrame(history_rows(6, step_ns=100_000_000)) + sample = sf.dataframe_to_synthefy_request( + FakeLocal(), df, "value", "500ms", [], "m", 10_000, "T" + )["samples"][0][0] + assert sample["history_timestamps"][1].endswith(".100000Z") + assert len(set(sample["target_timestamps"])) == 5 + + +def test_request_payload_rejects_steps_finer_than_a_microsecond(): + df = pd.DataFrame(history_rows(4, step_ns=500)) + with pytest.raises(Exception, match="less than a microsecond"): + sf.dataframe_to_synthefy_request(FakeLocal(), df, "value", "2 points", [], "m", 10, "T") + + +# --------------------------------------------------------------------------- +# M8 — forecast response to line protocol +# --------------------------------------------------------------------------- + + +def test_response_writes_tags_quantiles_and_exact_nanoseconds(): + response = forecast_response( + ["2026-01-01T00:00:00.123456789Z", "2026-01-01T01:00:00Z"], + [1.5, 2.5], + quantiles={"0.1": [1.0, 2.0], "0.9": [2.0, 3.0]}, + ) + builders = sf.forecast_response_to_line_builders( + FakeLocal(), response, "temp_forecast", {"room": ["A"], "site": ["x", "y"]}, + "sfm-tabular", "temp", "T", + ) + first = _parse_lp(builders[0].build()) + assert first.measurement == "temp_forecast" + # a single-valued filter is written as a tag; a multi-valued one is not + assert first.tags == {"room": "A", "model": "sfm-tabular"} + assert first.fields == {"temp": 1.5, "value_0.1": 1.0, "value_0.9": 2.0} + assert first.timestamp == 1767225600123456789 + assert len(builders) == 2 + + +def test_response_treats_naive_timestamps_as_utc(): + aware = forecast_response(["2026-01-01T00:00:00Z"], [1.0]) + naive = forecast_response(["2026-01-01T00:00:00"], [1.0]) + build = lambda r: sf.forecast_response_to_line_builders( + FakeLocal(), r, "m", {}, "mdl", "v", "T" + )[0].timestamp + assert build(aware) == build(naive) + + +def test_response_skips_unusable_points_but_keeps_the_rest(): + local = FakeLocal() + response = forecast_response( + ["2026-01-01T00:00:00Z", "2026-01-01T01:00:00Z", "not-a-time", "2026-01-01T03:00:00Z"], + [1.0, None, 3.0, float("nan")], + ) + builders = sf.forecast_response_to_line_builders( + local, response, "m", {}, "mdl", "v", "T" + ) + assert len(builders) == 1 + assert any("Non-finite forecast value" in w for w in local.warns) + assert any("Could not parse timestamp" in w for w in local.warns) + + +def test_response_drops_non_finite_quantiles_only(): + response = forecast_response( + ["2026-01-01T00:00:00Z"], [1.0], quantiles={"0.1": [float("inf")], "0.9": [2.0]} + ) + builders = sf.forecast_response_to_line_builders( + FakeLocal(), response, "m", {}, "mdl", "v", "T" + ) + assert _parse_lp(builders[0].build()).fields == {"v": 1.0, "value_0.9": 2.0} + + +@pytest.mark.parametrize( + "response, fragment", + [ + ({}, "missing 'forecasts' field"), + ({"forecasts": []}, "No forecasts in response"), + ({"forecasts": [[{"nope": 1}]]}, "No forecast payload"), + ], +) +def test_response_rejections(response, fragment): + with pytest.raises(ValueError, match=fragment): + sf.forecast_response_to_line_builders( + FakeLocal(), response, "m", {}, "mdl", "v", "T" + ) + + +# --------------------------------------------------------------------------- +# M9 — writes +# --------------------------------------------------------------------------- + + +def _two_builders(): + return sf.forecast_response_to_line_builders( + FakeLocal(), + forecast_response(["2026-01-01T00:00:00Z", "2026-01-01T01:00:00Z"], [1.0, 2.0]), + "m", {}, "mdl", "v", "T", + ) + + +def test_write_batches_all_points_into_one_payload(): + local = FakeLocal() + calls = [] + original = local._record_batch + local._record_batch = lambda db, batch: (calls.append(db), original(db, batch)) + + sf.write_forecasts_to_influxdb(local, _two_builders(), None, "T") + assert calls == [None] # a single batched write, not one call per point + assert len(local.writes) == 2 + + +def test_write_routes_to_the_override_database(): + local = FakeLocal() + sf.write_forecasts_to_influxdb(local, _two_builders(), "other", "T") + assert [db for db, _ in local.writes] == ["other", "other"] + assert any("database other" in i for i in local.infos) + + +def test_write_retries_then_succeeds(): + local = FakeLocal(write_failures=2) + sf.write_forecasts_to_influxdb(local, _two_builders(), None, "T") + assert len(local.writes) == 2 + assert local.errors == [] + + +def test_write_reports_and_reraises_after_exhausting_retries(): + local = FakeLocal(write_failures=3) + with pytest.raises(RuntimeError): + sf.write_forecasts_to_influxdb(local, _two_builders(), None, "T") + assert any("Failed to write forecasts after 3 attempts" in e for e in local.errors) + + +def test_write_skips_an_empty_result(): + local = FakeLocal() + sf.write_forecasts_to_influxdb(local, [], None, "T") + assert local.writes == [] + assert any("No forecast points to write" in w for w in local.warns) + + +# --------------------------------------------------------------------------- +# M10 — request body decoding +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "body, expected", + [ + (None, {}), + ("", {}), + (b"", {}), + ({"a": 1}, {"a": 1}), + ('{"a": 1}', {"a": 1}), + (b'{"a": 1}', {"a": 1}), + ], +) +def test_decode_request_body(body, expected): + assert sf._decode_request_body(body, "T") == expected + + +def test_decode_request_body_rejects_unsupported_type(): + with pytest.raises(Exception, match="Unsupported request_body type"): + sf._decode_request_body(42, "T") + + +# --------------------------------------------------------------------------- +# M11 — process_request +# --------------------------------------------------------------------------- + + +def test_full_flow_reads_forecasts_and_writes(monkeypatch): + local = FakeLocal(rows=history_rows(5, extra={"humidity": 1.0})) + stub = FakeRequests( + forecast_response(["2026-01-01T00:00:00Z", "2026-01-01T01:00:00Z"], [10.0, 11.0]) + ) + result = run( + local, + body={ + "measurement": "sf_temp", + "tags": {"room": "Bedroom"}, + "metadata_fields": ["humidity"], + "forecast_horizon": "2 points", + }, + requests_stub=stub, + monkeypatch=monkeypatch, + ) + + assert result == { + "message": "Forecast generated and written to InfluxDB. 2 forecast points written." + } + call = stub.calls[0] + assert call["url"] == "https://forecast.synthefy.com/v2/forecast" + assert call["headers"]["X-API-Key"] == "k" + assert [s["sample_id"] for s in call["body"]["samples"][0]] == ["value", "humidity"] + + written = [record for _, record in local.writes] + assert [r.measurement for r in written] == ["sf_temp_forecast"] * 2 + assert written[0].tags == {"room": "Bedroom", "model": "sfm-tabular"} + assert [r.fields["value"] for r in written] == [10.0, 11.0] + assert local.errors == [] + + +def test_full_flow_honours_output_measurement_and_database(monkeypatch): + local = FakeLocal(rows=history_rows(3)) + stub = FakeRequests(forecast_response(["2026-01-01T00:00:00Z"], [10.0])) + run( + local, + args={"measurement": "sf_temp", "output_measurement": "my_fc", "database": "other"}, + body={"forecast_horizon": "1 points"}, + requests_stub=stub, + monkeypatch=monkeypatch, + ) + db, record = local.writes[0] + assert (db, record.measurement) == ("other", "my_fc") + + +def test_body_overrides_trigger_arguments(monkeypatch): + local = FakeLocal(rows=history_rows(3)) + stub = FakeRequests(forecast_response(["2026-01-01T00:00:00Z"], [10.0])) + run( + local, + args={"measurement": "sf_temp", "model": "from-args"}, + body={"model": "from-body", "forecast_horizon": "1 points"}, + requests_stub=stub, + monkeypatch=monkeypatch, + ) + assert stub.calls[0]["body"]["model"] == "from-body" + assert local.writes[0][1].tags["model"] == "from-body" + + +@pytest.mark.parametrize( + "body_extra, expected_room, expected_samples", + [ + ({}, "Bedroom", ["value", "humidity"]), + # a null means "not set", so the trigger argument still applies + ({"tags": None, "metadata_fields": None}, "Bedroom", ["value", "humidity"]), + # an empty value clears the trigger argument + ({"tags": {}, "metadata_fields": []}, None, ["value"]), + ({"tags": {"room": "Hall"}}, "Hall", ["value", "humidity"]), + # the body accepts the trigger-argument string form too + ({"tags": "room:Hall"}, "Hall", ["value", "humidity"]), + ], +) +def test_tags_and_covariates_merge_with_trigger_arguments( + body_extra, expected_room, expected_samples, monkeypatch +): + local = FakeLocal(rows=history_rows(3, extra={"humidity": 1.0})) + stub = FakeRequests(forecast_response(["2026-01-01T00:00:00Z"], [10.0])) + run( + local, + args={"measurement": "sf_temp", "tags": "room:Bedroom", "metadata_fields": "humidity"}, + body={"forecast_horizon": "1 points", **body_extra}, + requests_stub=stub, + monkeypatch=monkeypatch, + ) + assert local.writes[0][1].tags.get("room") == expected_room + assert [s["sample_id"] for s in stub.calls[0]["body"]["samples"][0]] == expected_samples + + +@pytest.mark.parametrize( + "body, message", + [ + ({}, "'measurement' argument is required"), + ({"measurement": "nope"}, "Measurement 'nope' not found"), + ({"measurement": "sf_temp", "field": "nope"}, + "Field 'nope' does not exist in 'sf_temp'"), + ], +) +def test_request_rejections_before_the_api_call(body, message, monkeypatch): + columns = {} if body.get("measurement") == "nope" else COLUMNS + local = FakeLocal(columns=columns, rows=history_rows(3)) + stub = FakeRequests(error=AssertionError("API must not be called")) + assert run(local, body=body, requests_stub=stub, monkeypatch=monkeypatch) == { + "message": message + } + assert stub.calls == [] + + +def test_missing_api_key_stops_before_touching_the_database(): + local = FakeLocal(rows=history_rows(3)) + result = sf.process_request(local, {}, {}, '{"measurement": "sf_temp"}', {}) + assert result == {"message": "Missing API key"} + assert local.queries == [] + + +def test_api_key_falls_back_to_the_environment(monkeypatch): + monkeypatch.setenv(sf.API_KEY_ENV_VAR, "env-key") + local = FakeLocal(rows=history_rows(3)) + stub = FakeRequests(forecast_response(["2026-01-01T00:00:00Z"], [10.0])) + run( + local, + body={"measurement": "sf_temp", "forecast_horizon": "1 points"}, + headers={}, + requests_stub=stub, + monkeypatch=monkeypatch, + ) + assert stub.calls[0]["headers"]["X-API-Key"] == "env-key" + + +def test_empty_history_returns_no_data_without_calling_the_api(monkeypatch): + local = FakeLocal(rows=[]) + stub = FakeRequests(error=AssertionError("API must not be called")) + result = run(local, body={"measurement": "sf_temp"}, requests_stub=stub, monkeypatch=monkeypatch) + assert result == {"message": "No data found"} + assert stub.calls == [] + + +def test_invalid_json_body_is_reported(): + local = FakeLocal() + result = sf.process_request(local, {}, {"X-Synthefy-Api-Key": "k"}, "{oops", {}) + assert result == {"message": "Invalid JSON in request body"} + assert any("Invalid JSON in request body" in e for e in local.errors) + + +def test_api_failure_is_logged_and_returned(monkeypatch): + local = FakeLocal(rows=history_rows(3)) + stub = FakeRequests(error=RuntimeError("503 Service Unavailable")) + result = run( + local, + body={"measurement": "sf_temp", "forecast_horizon": "1 points"}, + requests_stub=stub, + monkeypatch=monkeypatch, + ) + assert result == {"message": "Error: 503 Service Unavailable"} + assert any("Synthefy API call failed" in e for e in local.errors) + assert local.writes == [] + + +def test_configuration_error_is_returned_not_raised(monkeypatch): + local = FakeLocal(rows=history_rows(3)) + stub = FakeRequests(error=AssertionError("API must not be called")) + result = run( + local, + body={"measurement": "sf_temp", "max_forecast_points": "junk"}, + requests_stub=stub, + monkeypatch=monkeypatch, + ) + assert "Invalid configuration" in result["message"] + assert any("HTTP request forecast failed" in e for e in local.errors)