51 sql store

sagemic/helpers.py

@@ -2,14 +2,92 @@
 """
 import os
 import sys
+import sqlite3
+import platform
+from importlib.metadata import version
 import sounddevice as sd
 import yaml
 import numpy as np
 
 
+def insert_database(base_path, filepath, species, confidence, timestamp, coordinates, audio_device, sample_rate, bitrate):
+    """
+    Inserts detection entries into the database.
+    Called by sagemic_local.
+
+    Args:
+        base_path (str): path to file  storing directory
+        filepath (str): path to detection audio file
+        species (str): species detected
+        confidence (float): inference confidence
+        timestamp (str): timestamp of detection
+        coordinates (str): coordinates in config file
+        audio_device (str): name of microphone used
+        sample_rate (integer): user defined in config file
+        bitrate (real): audio bitrate calculated in main
+
+    """
+
+    try:
+        with open("/sys/firmware/devicetree/base/model", "r", encoding="utf-8") as f:
+            hardware = f.read().strip('\x00')
+    except FileNotFoundError:
+        hardware = "Unknown"
+
+    firmware = platform.release().split('+')[0]
+
+    model = f"birdnetlib_v{version('birdnetlib')}"
+
+    db_path = os.path.join(base_path, "detections.db")
+
+    with sqlite3.connect(db_path) as conn:
+        conn.execute(
+            "INSERT OR IGNORE INTO detections "
+            "(sent, filepath, species, confidence, timestamp, hardware, firmware, model, coordinates, audio_device, sample_rate, bitrate)"
+            "VALUES (0, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+            (filepath, species, confidence, timestamp, hardware, firmware, model, coordinates, audio_device, sample_rate, bitrate)
+        )
+
+
+def create_database(base_path):
+    """
+    Checks if database exists, if not then creates detections database.
+
+    Called by sagemic_local and sagemic_scansend
+
+    Args:
+        base_path (str): path to file storing directory
+    """
+    db_path = os.path.join(base_path, "detections.db")
+    with sqlite3.connect(db_path) as conn:
+        conn.execute('''
+            CREATE TABLE IF NOT EXISTS detections (
+                sent INTEGER DEFAULT 0,
+                filepath TEXT PRIMARY KEY,
+                species TEXT,
+                confidence REAL,
+                timestamp TEXT,
+                hardware TEXT,
+                firmware TEXT,
+                model TEXT,
+                coordinates TEXT,
+                audio_device TEXT,
+                sample_rate INTEGER,
+                bitrate INTEGER
+            )
+        ''')
+
+        conn.execute('''
+            CREATE INDEX IF NOT EXISTS idx_unsent_files
+            ON detections(sent)
+        ''')
+
+
 def check_path(date, base_path):
     """Create new folder for date to store detections.
 
+    Called by sagemic_local.py.
+
     Args:
         date (str): Current date in YYYY-MM-DD.
 
@@ -28,6 +106,8 @@ def check_path(date, base_path):
 def inmp441_check(device_id, samplerate):
     """ Checks if inmp441 is connected and listening
 
+    Called by get_device_id
+
     Args:
         device_id (int): id of inmp441 found from get_device_id()
         samplerate (int): samplerate of audio device from config

sagemic_local.py

@@ -21,7 +21,13 @@
 from birdnetlib import RecordingBuffer
 from birdnetlib.analyzer import Analyzer
 
-from sagemic.helpers import check_path, get_config, get_device_id
+from sagemic.helpers import (
+    insert_database,
+    create_database,
+    check_path,
+    get_config,
+    get_device_id
+)
 
 
 # callback defined here to use with config variables
@@ -60,6 +66,13 @@ def audio_callback(
     base_path = config["PATHS"]["BASE_PATH"]
     confidence_threshold = config["SETTINGS"]["CONFIDENCE_THRESHOLD"]
     sample_rate = config["SETTINGS"]["SAMPLERATE"]
+    dtype = config["SETTINGS"]["AUDIO_DTYPE"]
+
+    bitrate = sample_rate * int(dtype[3:])
+    audio_device = config["SETTINGS"]["AUDIO_DEVICE"]
+    latitude = config["SETTINGS"]["LATITUDE"]
+    longitude = config["SETTINGS"]["LONGITUDE"]
+    coordinates = f"{latitude}, {longitude}"
 
     timestamp = datetime.now(local_tz)
     date = timestamp.strftime('%Y-%m-%d')
@@ -98,6 +111,19 @@ def audio_callback(
                 os.rename(
                     temp_filename, final_filename
                 )  # to .wav for scansend when done
+
+                insert_database(
+                    base_path,
+                    final_filename,
+                    name,
+                    round(confidence, 2),
+                    date_time,
+                    coordinates,
+                    audio_device,
+                    sample_rate,
+                    bitrate
+                )
+
     else:
         print("No detections")
 
@@ -116,6 +142,9 @@ def main():
     args = parser.parse_args()
 
     config = get_config(args.config)
+    base_path = config["PATHS"]["BASE_PATH"]
+
+    create_database(base_path)
 
     # audio variables
     latitude = config["SETTINGS"]["LATITUDE"]

sagemic_scansend.py

@@ -1,3 +1,4 @@
+
 """
 Service to check new BirdNet Detection Files and send them via MQTTS.
 """
@@ -7,99 +8,10 @@
 import argparse
 import ssl
 import time  # for sending delays
-import bisect
+import sqlite3
 import paho.mqtt.client as mqtt
 
-from sagemic.helpers import get_config
-
-
-# function to write sent filepaths to log file
-def write_log_file(filepath, log_file):
-    """Logs a successfully sent file with its filepath.
-
-
-    Args:
-        filepath (str): Path to the .wav file written by sagemic_local.
-        - Also means the file has been sent by this script.
-
-        log_file (str): Path to the log file, defined in the config file.
-    """
-
-    with open(log_file, "a", encoding='utf-8') as f:  # append to bottom
-        f.write(f"{filepath}\n")
-
-
-def search_unsent(base_path, log_file):
-    """ Searches parent directory and date folders for unsent files.
-
-    Args:
-        base_path (str): Path to the base directory to audio files.
-        - Defined in config.
-
-        log_file (str): Path to the log file, defined in the config file.
-
-    Returns:
-        dict: Returns an dictionary of filepaths of unsent files.
-    """
-    sent_files = set()
-    files_to_send = []
-    start_folder = ""
-    start_file = ""
-
-    # Read the log file to get entire set and latest date read
-    if os.path.exists(log_file):
-        with open(log_file, "r", encoding="utf-8") as f:
-            lines = f.readlines()
-            if lines:
-                sent_files = set(line.strip() for line in lines)
-                last_line = lines[-1].strip()
-                start_folder = os.path.basename(os.path.dirname(last_line))
-                start_file = os.path.basename(last_line)
-
-    # filter out folders only in basepath
-    try:
-        folders = []
-        for f in os.listdir(base_path):
-            # check if item is a folder
-            # can be eliminated late for efficiency
-            folder_path = os.path.join(base_path, f)
-            if os.path.isdir(folder_path):
-                folders.append(f)
-        all_folders = sorted(folders)  # sorted for for loop later
-    except FileNotFoundError:
-        all_folders = []
-
-    # trim all_folders to start at start_folder
-    if not start_folder:
-        search_folders = all_folders
-    else:
-        # Using this instead of for loop and compare O(N) so its O(logN)
-        index = bisect.bisect_left(all_folders, start_folder)
-        search_folders = all_folders[index:]
-
-    # search in relevant folders
-    for folder in search_folders:
-        folder_path = os.path.join(base_path, folder)
-        wavs = []
-
-        for file in os.listdir(folder_path):
-            if file.endswith(".wav"):
-                wavs.append(file)
-        wav_files = sorted(wavs)
-
-        if folder == start_folder and start_file:
-            index = bisect.bisect_right(wav_files, start_file)
-            wav_files = wav_files[index:]
-
-        for file in wav_files:
-            filepath = os.path.join(folder_path, file)
-
-            if filepath not in sent_files:
-                files_to_send.append(filepath)
-
-    files_to_send.sort()  # so we send chronologically
-
-    return files_to_send
+from sagemic.helpers import get_config, create_database
 
 
 # script only runs every few minutes (loop)
@@ -119,9 +31,6 @@ def main():
 
     base_path = config["PATHS"]["BASE_PATH"]
 
-    # log file to track clips that have alr been sent (tracker)
-    log_file = os.path.join(base_path, "sent_clips.log")
-
     port = config["MQTT"]["PORT"]
     broker = config["MQTT"]["BROKER"]
     topic = config["MQTT"]["BASE_TOPIC"] + "/" + config["MQTT"]["DEVICE"]
@@ -130,15 +39,25 @@ def main():
     user = config["MQTT"]["USER"]
     password = config["MQTT"]["PASS"]
 
-    files_to_send = search_unsent(base_path, log_file)
+    db_path = os.path.join(base_path, "detections.db")
+
+    # get files to send
+    create_database(base_path)
+
+    with sqlite3.connect(db_path) as conn:
+        cursor = conn.cursor()
+        cursor.execute(
+            "SELECT filepath FROM detections WHERE sent = 0 ORDER BY filepath ASC"
+        )
+        files_to_send = [row[0] for row in cursor.fetchall()]
 
     if not files_to_send:
         print("No new clips we need to send")
         return
 
     # ensure only sending completed clip, (done in sagemic_local.py)
 
-    client = mqtt.Client(client_id=session_id)
+    client = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2, client_id=session_id)
 
     client.username_pw_set(user, password)
 
@@ -157,7 +76,9 @@ def main():
 
     # send new clips to broker over mqtt
     for filepath in files_to_send:
-        # add print statements here if needed later
+        if not os.path.exists(filepath):
+            print(f"Warning: {filepath} found in DB but missing on disk.")
+            continue
 
         # get .wav filename from filepath
         filename = os.path.basename(filepath)
@@ -175,7 +96,13 @@ def main():
                 )
                 result.wait_for_publish()
 
-                write_log_file(filepath, log_file)
+                with sqlite3.connect(db_path) as conn:
+                    conn.execute(
+                        "UPDATE detections SET sent = 1 WHERE filepath = ?",
+                        (filepath,)
+                    )
+
+                print(f"Successfully sent and logged: {filename}")
                 time.sleep(0.5)  # to prevent network flood
 
         except Exception as e:  # pylint: disable=broad-except