feat: Add capture buffer for unknown/error bus frames with web UI page

CHANGELOG.md

@@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 ### Added
+- Saving of unknown bus messages, added web UI to view them
 ### Changed
 - Note about CMD 0x05 observed with payload 0x00. Added to PROTOCOL.md and new sample trace.
 - Note about CMD 0x12 observed with payload 0x01 0x00. Added to PROTOCOL.md and new sample trace.

main/CMakeLists.txt

@@ -13,6 +13,7 @@ idf_component_register(
         "tcp_bridge.c"
         "message_decoder.c"
         "register_requester.c"
+        "unknown_buffer.c"
         "dns_server.c"
     INCLUDE_DIRS
          "."

main/config.h

@@ -19,7 +19,7 @@
 
 // HTTP Server
 #define HTTP_SERVER_PORT               80
-#define HTTP_MAX_URI_HANDLERS          14      // Number of endpoint handlers
+#define HTTP_MAX_URI_HANDLERS          20      // Number of endpoint handlers
 #define HTTP_RECV_TIMEOUT_SEC          10      // Timeout for receiving requests
 #define HTTP_SEND_TIMEOUT_SEC          10      // Timeout for sending responses
 #define HTTP_STACK_SIZE                8192    // Stack size for HTTP server task

main/main.c

@@ -19,6 +19,7 @@
 #include "tcp_bridge.h"
 #include "message_decoder.h"
 #include "register_requester.h"
+#include "unknown_buffer.h"
 
 // ==================== APPLICATION =====================
 // All configuration values are in config.h
@@ -53,7 +54,7 @@ static bool decode_wrapper(const uint8_t *data, int len)
     return decode_message(data, len, &s_decoder_context);
 }
 
-static void frame_error_wrapper(tcp_bridge_frame_error_t error)
+static void frame_error_wrapper(tcp_bridge_frame_error_t error, const uint8_t *data, int len)
 {
     if (xSemaphoreTake(s_pool_state_mutex, pdMS_TO_TICKS(MUTEX_TIMEOUT_MS)) != pdTRUE) {
         return;
@@ -65,6 +66,9 @@ static void frame_error_wrapper(tcp_bridge_frame_error_t error)
         case TCP_BRIDGE_FRAME_ERR_NO_END:      s_pool_state.errors_no_end++;      break;
     }
     xSemaphoreGive(s_pool_state_mutex);
+    if (data && len > 0) {
+        unknown_buffer_record(data, len, true);
+    }
 }
 
 // ======================================================
@@ -146,6 +150,7 @@ void app_main(void)
     s_decoder_context.state_mutex = s_pool_state_mutex;
 
     // Initialize hardware
+    unknown_buffer_init();
     bus_init();
     ESP_ERROR_CHECK(led_init());
     led_set_startup();

main/message_decoder.c

@@ -2,6 +2,7 @@
 #include "config.h"
 #include "mqtt_publish.h"
 #include "register_requester.h"
+#include "unknown_buffer.h"
 #include "esp_log.h"
 #include <stdlib.h>
 #include <string.h>
@@ -2883,6 +2884,7 @@ bool decode_message(const uint8_t *data, int len, message_decoder_context_t *ctx
             ctx->pool_state->messages_error_total++;
             xSemaphoreGive(ctx->state_mutex);
         }
+        unknown_buffer_record(data, len, true);
         return false;
     }
 
@@ -2950,16 +2952,23 @@ bool decode_message(const uint8_t *data, int len, message_decoder_context_t *ctx
 
     // Increment global and per-device counters. A frame with any validation
     // error counts as an error only, so decoded + unknown + errors = total.
+    bool record_frame = false;
+    bool record_is_error = false;
     if (ctx->state_mutex) {
         if (xSemaphoreTake(ctx->state_mutex, pdMS_TO_TICKS(MUTEX_TIMEOUT_MS)) == pdTRUE) {
             if (length_error || header_chk_error || data_chk_error) {
                 ctx->pool_state->messages_error_total++;
                 if (length_error)     ctx->pool_state->errors_length_mismatch++;
                 if (header_chk_error) ctx->pool_state->errors_header_checksum++;
                 if (data_chk_error)   ctx->pool_state->errors_data_checksum++;
+                record_frame = true;
+                record_is_error = true;
             } else {
                 if (decoded) ctx->pool_state->messages_decoded_total++;
-                else         ctx->pool_state->messages_unknown_total++;
+                else {
+                    ctx->pool_state->messages_unknown_total++;
+                    record_frame = true;
+                }
 
                 // Per-device counters (skip broadcast)
                 if (!(src_hi == 0xFF && src_lo == 0xFF)) {
@@ -2973,6 +2982,7 @@ bool decode_message(const uint8_t *data, int len, message_decoder_context_t *ctx
             xSemaphoreGive(ctx->state_mutex);
         }
     }
+    if (record_frame) unknown_buffer_record(data, len, record_is_error);
 
     return decoded;
 }

main/tcp_bridge.c

@@ -133,7 +133,7 @@ static bool extract_and_process_message(int client_sock)
             hex_str[hex_pos] = '\0';
             ESP_LOGW(TAG, "No start byte in buffer, discarding %d bytes: %s%s",
                      s_msg_buffer_len, hex_str, s_msg_buffer_len > 32 ? "..." : "");
-            if (s_config.on_frame_error) s_config.on_frame_error(TCP_BRIDGE_FRAME_ERR_NO_START);
+            if (s_config.on_frame_error) s_config.on_frame_error(TCP_BRIDGE_FRAME_ERR_NO_START, s_msg_buffer, s_msg_buffer_len);
         }
         s_msg_buffer_len = 0;
         return false;
@@ -161,7 +161,7 @@ static bool extract_and_process_message(int client_sock)
         hex_str[hex_pos] = '\0';
         ESP_LOGW(TAG, "Invalid control bytes: %02X %02X (expected 80 00), data: %s%s, discarding start byte",
                  s_msg_buffer[5], s_msg_buffer[6], hex_str, s_msg_buffer_len > 32 ? "..." : "");
-        if (s_config.on_frame_error) s_config.on_frame_error(TCP_BRIDGE_FRAME_ERR_BAD_CONTROL);
+        if (s_config.on_frame_error) s_config.on_frame_error(TCP_BRIDGE_FRAME_ERR_BAD_CONTROL, s_msg_buffer, s_msg_buffer_len);
         // Discard this start byte and look for next
         memmove(s_msg_buffer, &s_msg_buffer[1], s_msg_buffer_len - 1);
         s_msg_buffer_len--;
@@ -241,7 +241,7 @@ static bool extract_and_process_message(int client_sock)
         hex_str[hex_pos] = '\0';
         ESP_LOGW(TAG, "Buffer nearly full (%d bytes) without complete message, first 32 bytes: %s..., clearing",
                  s_msg_buffer_len, hex_str);
-        if (s_config.on_frame_error) s_config.on_frame_error(TCP_BRIDGE_FRAME_ERR_NO_END);
+        if (s_config.on_frame_error) s_config.on_frame_error(TCP_BRIDGE_FRAME_ERR_NO_END, s_msg_buffer, s_msg_buffer_len);
         s_msg_buffer_len = 0;
     }
 
@@ -347,7 +347,7 @@ static void tcp_bridge_task(void *pvParameters)
             } else {
                 ESP_LOGW(TAG, "Reassembly buffer overflow (%d + %d > %d), clearing",
                          s_msg_buffer_len, len, BUS_MESSAGE_MAX_SIZE);
-                if (s_config.on_frame_error) s_config.on_frame_error(TCP_BRIDGE_FRAME_ERR_NO_END);
+                if (s_config.on_frame_error) s_config.on_frame_error(TCP_BRIDGE_FRAME_ERR_NO_END, s_msg_buffer, s_msg_buffer_len);
                 s_msg_buffer_len = 0;
             }
 

main/tcp_bridge.h

@@ -31,8 +31,9 @@ typedef enum {
     TCP_BRIDGE_FRAME_ERR_NO_END,       // Buffer filled without a checksum+END match (too long / corrupt)
 } tcp_bridge_frame_error_t;
 
-// Called when the reassembly layer discards data due to a framing error
-typedef void (*tcp_bridge_frame_error_fn)(tcp_bridge_frame_error_t error);
+// Called when the reassembly layer discards data due to a framing error.
+// data/len are the raw bytes being discarded (may be NULL/0 for overflow cases).
+typedef void (*tcp_bridge_frame_error_fn)(tcp_bridge_frame_error_t error, const uint8_t *data, int len);
 
 /**
  * Configuration structure for TCP bridge

main/unknown_buffer.c

@@ -0,0 +1,109 @@
+#include "unknown_buffer.h"
+#include "freertos/FreeRTOS.h"
+#include "freertos/semphr.h"
+#include <string.h>
+#include <time.h>
+
+#define MUTEX_TIMEOUT_MS 100
+
+static unknown_entry_t s_entries[UNKNOWN_BUFFER_CAPACITY];
+static int              s_count = 0;
+static SemaphoreHandle_t s_mutex = NULL;
+
+void unknown_buffer_init(void)
+{
+    s_mutex = xSemaphoreCreateMutex();
+    s_count = 0;
+}
+
+// Adds `data` to the record of unknown data frames.
+//
+// If the data matches an existing record, the hit count and last seen time are updated.
+// Otherwise, a new record is created, evicting the least frequently used record if the
+// buffer is full.
+//
+// Note on matching: A match is defined as the same error status, the same length, and
+// the same first `UNKNOWN_BUFFER_MAX_RAW_BYTES` bytes. As we cap stored bytes to
+// `UNKNOWN_BUFFER_MAX_RAW_BYTES`, this means we can't distinguish between two different
+// frames that only differ in the bytes after `UNKNOWN_BUFFER_MAX_RAW_BYTES`. This is a
+// known limitation of this implementation.
+void unknown_buffer_record(const uint8_t *data, int len, bool is_error)
+{
+    if (!s_mutex || !data || len <= 0) return;
+
+    int store_len = (len < UNKNOWN_BUFFER_MAX_RAW_BYTES) ? len : UNKNOWN_BUFFER_MAX_RAW_BYTES;
+
+    time_t now = time(NULL);
+
+    if (xSemaphoreTake(s_mutex, pdMS_TO_TICKS(MUTEX_TIMEOUT_MS)) != pdTRUE) return;
+
+    for (int i = 0; i < s_count; i++) {
+        if (s_entries[i].is_error == is_error &&
+            s_entries[i].raw_len == (uint16_t)len &&
+            memcmp(s_entries[i].raw, data, (size_t)store_len) == 0) {
+            s_entries[i].hit_count++;
+            s_entries[i].last_seen = now;
+            xSemaphoreGive(s_mutex);
+            return;
+        }
+    }
+
+    int slot;
+    if (s_count < UNKNOWN_BUFFER_CAPACITY) {
+        slot = s_count++;
+    } else {
+        // LFU eviction: evict the entry with the fewest hits; break ties on oldest first_seen
+        slot = 0;
+        for (int i = 1; i < UNKNOWN_BUFFER_CAPACITY; i++) {
+            if (s_entries[i].hit_count < s_entries[slot].hit_count ||
+                (s_entries[i].hit_count == s_entries[slot].hit_count &&
+                 s_entries[i].first_seen < s_entries[slot].first_seen)) {
+                slot = i;
+            }
+        }
+    }
+
+    unknown_entry_t *e = &s_entries[slot];
+    e->raw_len   = (uint16_t)len;
+    memcpy(e->raw, data, (size_t)store_len);
+    e->hit_count  = 1;
+    e->first_seen = now;
+    e->last_seen  = now;
+    e->is_error   = is_error;
+
+    xSemaphoreGive(s_mutex);
+}
+
+void unknown_buffer_clear(void)
+{
+    if (!s_mutex) return;
+    if (xSemaphoreTake(s_mutex, pdMS_TO_TICKS(MUTEX_TIMEOUT_MS)) == pdTRUE) {
+        s_count = 0;
+        xSemaphoreGive(s_mutex);
+    }
+}
+
+locked_unknown_buffer_t unknown_buffer_lock_for_read(void)
+{
+    locked_unknown_buffer_t locked_buf = {
+        .entries = NULL,
+        .count = 0
+    };
+
+    if (!s_mutex) {
+        return locked_buf;
+    }
+
+    if (xSemaphoreTake(s_mutex, pdMS_TO_TICKS(MUTEX_TIMEOUT_MS)) != pdTRUE) {
+        return locked_buf;
+    }
+
+    locked_buf.entries = s_entries;
+    locked_buf.count = s_count;
+    return locked_buf;
+}
+
+void unknown_buffer_unlock_after_read(void)
+{
+    if (s_mutex) xSemaphoreGive(s_mutex);
+}

main/unknown_buffer.h

@@ -0,0 +1,38 @@
+#ifndef UNKNOWN_BUFFER_H
+#define UNKNOWN_BUFFER_H
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <time.h>
+#include "config.h"
+
+#define UNKNOWN_BUFFER_CAPACITY         100
+#define UNKNOWN_BUFFER_MAX_RAW_BYTES    64   // store at most the first 64 bytes of each frame
+
+typedef struct {
+    uint16_t raw_len;      // actual frame length (may exceed UNKNOWN_BUFFER_MAX_RAW_BYTES)
+    uint8_t  raw[UNKNOWN_BUFFER_MAX_RAW_BYTES];  // first bytes of the frame
+    uint32_t hit_count;
+    time_t   first_seen;   // UTC epoch seconds (0 if NTP not synced)
+    time_t   last_seen;    // UTC epoch seconds (0 if NTP not synced)
+    bool     is_error;     // true = framing/checksum error; false = no handler matched
+} unknown_entry_t;
+
+void unknown_buffer_init(void);
+
+// Record a frame. is_error=true for framing/checksum errors, false for unhandled frames.
+void unknown_buffer_record(const uint8_t *data, int len, bool is_error);
+
+void unknown_buffer_clear(void);
+
+typedef struct {
+    const unknown_entry_t *entries;
+    int                   count;
+} locked_unknown_buffer_t;
+
+// Acquires the buffer mutex and returns a read-only pointer to the entry array.
+// The caller MUST call unknown_buffer_unlock_after_read() when done.
+locked_unknown_buffer_t unknown_buffer_lock_for_read(void);
+void unknown_buffer_unlock_after_read(void);
+
+#endif // UNKNOWN_BUFFER_H