Multilog Analysis (Logx)

notebooks/Tutorial_[advanced].ipynb

@@ -157,6 +157,73 @@
     "fig.show()"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "run-collection-header",
+   "metadata": {},
+   "source": [
+    "#### Multi-Log Comparative Analysis (RunCollection)\n",
+    "\n",
+    "PERDA provides a high-level `RunCollection` utility to manage, filter, and compare multiple log files from a directory chronologically."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "run-collection-code-1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from perda.analyzer import RunCollection\n",
+    "from datetime import datetime\n",
+    "\n",
+    "# Index a directory recursively to scan logs in all subdirectories\n",
+    "col_recursive = RunCollection(\"csv_files\", recursive=True)\n",
+    "print(f\"Indexed {len(col_recursive)} runs recursively.\")\n",
+    "\n",
+    "# Alternatively, scan a mixed list of specific files and folders using from_paths\n",
+    "col = RunCollection([\n",
+    "    \"csv_files/LogBrowseTest/16thApr10-52-00.csv\",\n",
+    "    \"csv_files/LogBrowseTest\"\n",
+    "])\n",
+    "print(f\"Indexed {len(col)} runs via specific paths.\")\n",
+    "\n",
+    "# Filter logs chronologically by a date range\n",
+    "filtered_col = col.filter_by_date(start_date=datetime(2026, 4, 1), end_date=datetime(2026, 6, 1))\n",
+    "print(f\"Filtered to {len(filtered_col)} runs.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "run-collection-code-2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Find which logs in the collection hit a watchdog error condition (evaluates lazily to avoid memory leaks)\n",
+    "error_logs = col.check_any(\n",
+    "    \"pcm.watchdog.sensorTimeout\",\n",
+    "    lambda values: (values == 1.0).any()\n",
+    ")\n",
+    "print(f\"Runs with watchdog timeout error: {error_logs}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "run-collection-code-3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Generate statistical summaries of a variable across all runs\n",
+    "stats = col.compare_summary(\"pcm.wheelSpeeds.frontLeft\")\n",
+    "print(\"Stats Summary:\", stats)\n",
+    "\n",
+    "# Overlay plot of a variable across all runs, downsampled to 20,000 points per trace for fast rendering\n",
+    "fig = col.plot_comparison(\"pcm.wheelSpeeds.frontLeft\", max_points=20000)\n",
+    "fig.show()"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "9",

notebooks/Tutorial_[simple].ipynb

@@ -96,6 +96,29 @@
     "search_results = aly.search(\"vectornav\")"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Semantic Search\n",
+    "\n",
+    "PERDA supports hybrid semantic search. By passing `semantic=True`, the library uses a local bi-encoder model to understand the meaning of your query, finding relevant variables even if they don't share exact keywords (e.g. searching 'battery current limit' can match specific BMS stack parameters)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Run a semantic search for a query\n",
+    "semantic_results = aly.search(\"battery current limit\", semantic=True)\n",
+    "\n",
+    "# Print top 5 matches with descriptions\n",
+    "for r in semantic_results[:5]:\n",
+    "    print(f\"{r.score:.2f} | {r.cpp_name} - {r.descript}\")"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -167,6 +190,25 @@
     "aly.plot(variables)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Parameterized Downsampling\n",
+    "\n",
+    "To speed up rendering or save network bandwidth, you can optionally restrict the number of points plotted using the `max_points` parameter. This applies a memory-efficient zero-copy step downsampling."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Graph with a hard number limit of 10,000 points per trace\n",
+    "aly.plot(variables, max_points=10000)"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -307,4 +349,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 4
-}
+}

perda/analyzer/__init__.py

@@ -1,4 +1,7 @@
+# Submodule re-exports
 from .analyzer import *
 from .analyzer_factory import *
 from .concat_helpers import concat
+from .comparison import plot_comparison, compare_summary
 from .csv import *
+from .run_collection import RunCollection, RunMetadata

perda/analyzer/analyzer.py

@@ -101,6 +101,7 @@ def __init__(
             parsing_errors_limit=parsing_errors_limit,
             verbose=verbose,
         )
+        self.file_path = filepath
         if preprocessing:
             self.data = apply_preprocessing(self.data, preprocessing)
 
@@ -120,7 +121,9 @@ def __str__(self) -> str:
 
         return output
 
-    def search(self, query: str, top_n: int = 10) -> list[SearchResult]:
+    def search(
+        self, query: str, top_n: int = 10, semantic: bool = False
+    ) -> list[SearchResult]:
         """
         Natural language search for available variables in the parsed data.
 
@@ -132,6 +135,8 @@ def search(self, query: str, top_n: int = 10) -> list[SearchResult]:
             Free-text search query (e.g. "front wheel speed").
         top_n : int
             Maximum number of results to return and display (default 10).
+        semantic : bool
+            Whether to use semantic search embeddings (default False).
 
         Returns
         -------
@@ -143,10 +148,10 @@ def search(self, query: str, top_n: int = 10) -> list[SearchResult]:
         Examples
         --------
         >>> results = aly.search("front wheel speed")
-        >>> results = aly.search("front wheel speed", top_n=5)
+        >>> results = aly.search("front wheel speed", top_n=5, semantic=True)
         >>> names = [r.cpp_name for r in results]
         """
-        return search(self.data, query, top_n)
+        return search(self.data, query, top_n, semantic)
 
     def plot(
         self,
@@ -163,6 +168,7 @@ def plot(
         font_config: FontConfig = DEFAULT_FONT_CONFIG,
         layout_config: LayoutConfig = DEFAULT_LAYOUT_CONFIG,
         vline_config: VLineConfig = DEFAULT_VLINE_CONFIG,
+        max_points: int | None = None,
     ) -> go.Figure:
         """
         Display variables from the parsed data on an interactive Plotly plot.
@@ -192,6 +198,8 @@ def plot(
             Layout configuration for plot dimensions.
         vline_config : VLineConfig, optional
             Visual configuration for concat boundary lines.
+        max_points : int | None, optional
+            Maximum number of data points to plot per trace (enables zero-copy step downsampling). Default is None (full fidelity).
 
         Examples
         --------
@@ -237,6 +245,7 @@ def plot(
                 timestamp_unit=self.data.timestamp_unit,
                 vlines=vlines,
                 vline_config=vline_config,
+                max_points=max_points,
             )
         else:
             return plot_single_axis(
@@ -249,6 +258,7 @@ def plot(
                 timestamp_unit=self.data.timestamp_unit,
                 vlines=vlines,
                 vline_config=vline_config,
+                max_points=max_points,
             )
 
     def subplots(

perda/analyzer/comparison.py

@@ -0,0 +1,126 @@
+import os
+from typing import Any, Dict, List
+import numpy as np
+import plotly.graph_objects as go
+
+from ..plotting.plotting_constants import (
+    DEFAULT_FONT_CONFIG,
+    DEFAULT_LAYOUT_CONFIG,
+)
+from ..units import _to_seconds
+
+
+def plot_comparison(
+    analyzers: List[Any], cpp_name: str, max_points: int | None = None
+) -> go.Figure:
+    """Compare and overlay a variable from multiple runs on a single Plotly figure.
+
+    Aligns each run's timestamps to start at relative seconds (t = 0).
+
+    Parameters
+    ----------
+    analyzers : List[Analyzer]
+        List of Analyzer instances to compare.
+    cpp_name : str
+        C++ variable name to plot and overlay.
+    max_points : int | None, optional
+        Maximum number of data points to plot per run (enables zero-copy step downsampling). Default is None (full fidelity).
+
+    Returns
+    -------
+    go.Figure
+        Plotly Figure with overlaid traces from all matching runs.
+    """
+    fig = go.Figure()
+
+    for aly in analyzers:
+        if cpp_name not in aly.data:
+            continue
+        di = aly.data[cpp_name]
+        if len(di) == 0:
+            continue
+
+        # Shift timestamps relative to start time and convert to seconds
+        raw_start = aly.data.data_start_time
+        rel_timestamps_raw = di.timestamp_np - raw_start
+
+        # Downsample coordinates if data points exceed the limit
+        n_points = len(di)
+        if max_points is not None and n_points > max_points and max_points > 0:
+            step = (n_points + max_points - 1) // max_points
+            timestamps_s = _to_seconds(rel_timestamps_raw[::step], aly.data.timestamp_unit)
+            values_np = di.value_np[::step]
+        else:
+            timestamps_s = _to_seconds(rel_timestamps_raw, aly.data.timestamp_unit)
+            values_np = di.value_np
+
+        # Get file name for legend label
+        file_path = getattr(aly, "file_path", None) or getattr(aly, "filename", None) or "Run"
+        run_label = os.path.basename(str(file_path))
+
+        fig.add_trace(
+            go.Scattergl(
+                x=timestamps_s,
+                y=values_np,
+                mode="lines",
+                name=f"{run_label} ({di.label})",
+            )
+        )
+
+    fig.update_layout(
+        title=dict(
+            text=f"Multi-Log Comparison: {cpp_name}",
+            font=dict(size=DEFAULT_FONT_CONFIG.large)
+        ),
+        xaxis_title=dict(
+            text="Time (seconds from start)",
+            font=dict(size=DEFAULT_FONT_CONFIG.medium)
+        ),
+        yaxis_title=dict(
+            text=cpp_name,
+            font=dict(size=DEFAULT_FONT_CONFIG.medium)
+        ),
+        template="plotly_dark",
+        width=DEFAULT_LAYOUT_CONFIG.width,
+        height=DEFAULT_LAYOUT_CONFIG.height,
+        margin=DEFAULT_LAYOUT_CONFIG.margin,
+    )
+    return fig
+
+
+def compare_summary(analyzers: List[Any], cpp_name: str) -> Dict[str, Dict[str, float]]:
+    """Compare basic statistics of a variable across multiple logs.
+
+    Parameters
+    ----------
+    analyzers : List[Analyzer]
+        List of Analyzer instances to compare.
+    cpp_name : str
+        C++ variable name to summarize.
+
+    Returns
+    -------
+    Dict[str, Dict[str, float]]
+        A dictionary mapping run filenames to statistical descriptors:
+        {"run_filename.csv": {"min": x, "max": y, "mean": z, "std": w, "len": n}}
+    """
+    comparison = {}
+    for aly in analyzers:
+        if cpp_name not in aly.data:
+            continue
+        di = aly.data[cpp_name]
+        if len(di) == 0:
+            continue
+
+        vals = di.value_np
+        file_path = getattr(aly, "file_path", None) or getattr(aly, "filename", None) or "Run"
+        run_label = os.path.basename(str(file_path))
+
+        comparison[run_label] = {
+            "min": float(np.min(vals)),
+            "max": float(np.max(vals)),
+            "mean": float(np.mean(vals)),
+            "std": float(np.std(vals)),
+            "len": int(len(vals)),
+        }
+    return comparison