From 356386afae8f6659d684682388a39073067eeae2 Mon Sep 17 00:00:00 2001 From: Ellina Ho Date: Thu, 9 Jul 2026 15:32:03 -0700 Subject: [PATCH 1/5] store basic detection data in SQL, scansend search with SQL --- sagemic/helpers.py | 50 ++++++++++++++++++ sagemic_local.py | 12 +++-- sagemic_scansend.py | 120 +++++++++----------------------------------- 3 files changed, 82 insertions(+), 100 deletions(-) diff --git a/sagemic/helpers.py b/sagemic/helpers.py index 0f700d5..4f46af9 100644 --- a/sagemic/helpers.py +++ b/sagemic/helpers.py @@ -4,12 +4,60 @@ import sys import sounddevice as sd import yaml +import sqlite3 import numpy as np +def insert_database(base_path, filepath, species, confidence, timestamp): + """ + Inserts detection entries into the database. + Called by sagemic_local. + + Args: + base_path (str): path to file storing directory + filepath (str): path to detection audio file + species (str): species of detection + confidence (float): confidence of detection + timestamp (str): in YMD_HMS format + """ + db_path = os.path.join(base_path, "detections.db") + + with sqlite3.connect(db_path) as conn: + conn.execute( + "INSERT OR IGNORE INTO detections " + "(filepath, species, confidence, timestamp, sent) " + "VALUES (?, ?, ?, ?, 0)", + (filepath, species, round(confidence, 2), timestamp) + ) + + +def create_database(base_path): + """ + Checks if database exists, if not then creates detections database. + + Called by sagemic_local and sagemic_scansend + + Args: + base_path (str): path to file storing directory + """ + db_path = os.path.join(base_path, "detections.db") + with sqlite3.connect(db_path) as conn: + conn.execute(''' + CREATE TABLE IF NOT EXISTS detections ( + filepath TEXT PRIMARY KEY, + species TEXT, + confidence REAL, + time DATETIME, + sent INTEGER DEFAULT 0 + ) + ''') + + def check_path(date, base_path): """Create new folder for date to store detections. + Called by sagemic_local.py. + Args: date (str): Current date in YYYY-MM-DD. @@ -28,6 +76,8 @@ def check_path(date, base_path): def inmp441_check(device_id, samplerate): """ Checks if inmp441 is connected and listening + Called by get_device_id + Args: device_id (int): id of inmp441 found from get_device_id() samplerate (int): samplerate of audio device from config diff --git a/sagemic_local.py b/sagemic_local.py index 19f08eb..566905f 100644 --- a/sagemic_local.py +++ b/sagemic_local.py @@ -21,7 +21,7 @@ from birdnetlib import RecordingBuffer from birdnetlib.analyzer import Analyzer -from sagemic.helpers import check_path, get_config, get_device_id +from sagemic.helpers import insert_database, create_database, check_path, get_config, get_device_id # callback defined here to use with config variables @@ -62,7 +62,7 @@ def audio_callback( sample_rate = config["SETTINGS"]["SAMPLERATE"] timestamp = datetime.now(local_tz) - date = timestamp.strftime('%Y-%m-%d') + date = timestamp.strftime('%Y%m%d') path = check_path(date, base_path) # Flatten the data to a 1D array as expected by birdnetlib @@ -85,7 +85,7 @@ def audio_callback( confidence = detection["confidence"] # new for filenames w/ data + time - date_time = timestamp.strftime("%Y-%m-%d_%H-%M-%S") + date_time = timestamp.strftime("%Y%m%d_%H%M%S") print(f"** {name} Detected w/ (Confidence: {confidence:.2f})") @@ -98,6 +98,9 @@ def audio_callback( os.rename( temp_filename, final_filename ) # to .wav for scansend when done + + insert_database(base_path, final_filename, name, confidence, date_time) + else: print("No detections") @@ -116,6 +119,9 @@ def main(): args = parser.parse_args() config = get_config(args.config) + base_path = config["PATHS"]["BASE_PATH"] + + create_database(base_path) # audio variables latitude = config["SETTINGS"]["LATITUDE"] diff --git a/sagemic_scansend.py b/sagemic_scansend.py index 796e390..d454b7f 100644 --- a/sagemic_scansend.py +++ b/sagemic_scansend.py @@ -1,3 +1,4 @@ + """ Service to check new BirdNet Detection Files and send them via MQTTS. """ @@ -7,98 +8,10 @@ import argparse import ssl import time # for sending delays -import bisect +import sqlite3 import paho.mqtt.client as mqtt -from sagemic.helpers import get_config - - -# function to write sent filepaths to log file -def write_log_file(filepath, log_file): - """Logs a successfully sent file with its filepath. - - Args: - filepath (str): Path to the .wav file written by sagemic_local. - - Also means the file has been sent by this script. - - log_file (str): Path to the log file, defined in the config file. - """ - - with open(log_file, "a", encoding='utf-8') as f: # append to bottom - f.write(f"{filepath}\n") - - -def search_unsent(base_path, log_file): - """ Searches parent directory and date folders for unsent files. - - Args: - base_path (str): Path to the base directory to audio files. - - Defined in config. - - log_file (str): Path to the log file, defined in the config file. - - Returns: - dict: Returns an dictionary of filepaths of unsent files. - """ - sent_files = set() - files_to_send = [] - start_folder = "" - start_file = "" - - # Read the log file to get entire set and latest date read - if os.path.exists(log_file): - with open(log_file, "r", encoding="utf-8") as f: - lines = f.readlines() - if lines: - sent_files = set(line.strip() for line in lines) - last_line = lines[-1].strip() - start_folder = os.path.basename(os.path.dirname(last_line)) - start_file = os.path.basename(last_line) - - # filter out folders only in basepath - try: - folders = [] - for f in os.listdir(base_path): - # check if item is a folder - # can be eliminated late for efficiency - folder_path = os.path.join(base_path, f) - if os.path.isdir(folder_path): - folders.append(f) - all_folders = sorted(folders) # sorted for for loop later - except FileNotFoundError: - all_folders = [] - - # trim all_folders to start at start_folder - if not start_folder: - search_folders = all_folders - else: - # Using this instead of for loop and compare O(N) so its O(logN) - index = bisect.bisect_left(all_folders, start_folder) - search_folders = all_folders[index:] - - # search in relevant folders - for folder in search_folders: - folder_path = os.path.join(base_path, folder) - wavs = [] - - for file in os.listdir(folder_path): - if file.endswith(".wav"): - wavs.append(file) - wav_files = sorted(wavs) - - if folder == start_folder and start_file: - index = bisect.bisect_right(wav_files, start_file) - wav_files = wav_files[index:] - - for file in wav_files: - filepath = os.path.join(folder_path, file) - - if filepath not in sent_files: - files_to_send.append(filepath) - - files_to_send.sort() # so we send chronologically - - return files_to_send +from sagemic.helpers import get_config, create_database # script only runs every few minutes (loop) @@ -118,9 +31,6 @@ def main(): base_path = config["PATHS"]["BASE_PATH"] - # log file to track clips that have alr been sent (tracker) - log_file = os.path.join(base_path, "sent_clips.log") - port = config["MQTT"]["PORT"] broker = config["MQTT"]["BROKER"] topic = config["MQTT"]["BASE_TOPIC"] + "/" + config["MQTT"]["DEVICE"] @@ -129,7 +39,15 @@ def main(): user = config["MQTT"]["USER"] password = config["MQTT"]["PASS"] - files_to_send = search_unsent(base_path, log_file) + db_path = os.path.join(base_path, "detections.db") + + # get files to send + create_database(base_path) + + with sqlite3.connect(db_path) as conn: + cursor = conn.cursor() + cursor.execute("SELECT filepath FROM detections WHERE sent = 0 ORDER BY timestamp ASC") + files_to_send = [row[0] for row in cursor.fetchall()] if not files_to_send: print("No new clips we need to send") @@ -137,7 +55,7 @@ def main(): # ensure only sending completed clip, (done in sagemic_local.py) - client = mqtt.Client(client_id=session_id) + client = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2, client_id=session_id) client.username_pw_set(user, password) @@ -156,7 +74,9 @@ def main(): # send new clips to broker over mqtt for filepath in files_to_send: - # add print statements here if needed later + if not os.path.exists(filepath): + print(f"Warning: {filepath} found in DB but missing on disk.") + continue # get .wav filename from filepath filename = os.path.basename(filepath) @@ -174,7 +94,13 @@ def main(): ) result.wait_for_publish() - write_log_file(filepath, log_file) + with sqlite3.connect(db_path) as conn: + conn.execute( + "UPDATE detections SET sent = 1 WHERE filepath = ?", + (filepath,) + ) + + print(f"Successfully sent and logged: {filename}") time.sleep(0.5) # to prevent network flood except Exception as e: # pylint: disable=broad-except From 57e5d73c95e20d35621b2ce65bc4e8135a740cf0 Mon Sep 17 00:00:00 2001 From: Ellina Ho Date: Mon, 13 Jul 2026 11:30:36 -0700 Subject: [PATCH 2/5] Added device versions and bitrate samplerate metadata, removed redundant species/confidence for now --- sagemic/helpers.py | 37 ++++++++++++++++++++++++++----------- sagemic_local.py | 13 +++++++++++-- sagemic_scansend.py | 4 +++- 3 files changed, 40 insertions(+), 14 deletions(-) diff --git a/sagemic/helpers.py b/sagemic/helpers.py index 4f46af9..a0c3e5f 100644 --- a/sagemic/helpers.py +++ b/sagemic/helpers.py @@ -5,10 +5,12 @@ import sounddevice as sd import yaml import sqlite3 +import platform +from importlib.metadata import version import numpy as np -def insert_database(base_path, filepath, species, confidence, timestamp): +def insert_database(base_path, filepath, sample_rate, bitrate): """ Inserts detection entries into the database. Called by sagemic_local. @@ -16,18 +18,29 @@ def insert_database(base_path, filepath, species, confidence, timestamp): Args: base_path (str): path to file storing directory filepath (str): path to detection audio file - species (str): species of detection - confidence (float): confidence of detection - timestamp (str): in YMD_HMS format + sample_rate (integer): user defined in config file + bitrate (real): audio bitrate calculated in main + """ + + try: + with open("/sys/firmware/devicetree/base/model", "r") as f: + hardware = f.read().strip('\x00') + except FileNotFoundError: + hardware = "Unknown" + + firmware = platform.release().split('+')[0] + + model = f"birdnetlib_v{version('birdnetlib')}" + db_path = os.path.join(base_path, "detections.db") with sqlite3.connect(db_path) as conn: conn.execute( "INSERT OR IGNORE INTO detections " - "(filepath, species, confidence, timestamp, sent) " - "VALUES (?, ?, ?, ?, 0)", - (filepath, species, round(confidence, 2), timestamp) + "(sent, filepath, hardware, firmware, model, sample_rate, bitrate)" + "VALUES (0, ?, ?, ?, ?, ?, ?)", + (filepath, hardware, firmware, model, sample_rate, bitrate) ) @@ -44,11 +57,13 @@ def create_database(base_path): with sqlite3.connect(db_path) as conn: conn.execute(''' CREATE TABLE IF NOT EXISTS detections ( + sent INTEGER DEFAULT 0, filepath TEXT PRIMARY KEY, - species TEXT, - confidence REAL, - time DATETIME, - sent INTEGER DEFAULT 0 + hardware TEXT, + firmware TEXT, + model TEXT, + sample_rate INTEGER, + bitrate INTEGER ) ''') diff --git a/sagemic_local.py b/sagemic_local.py index 566905f..785ed80 100644 --- a/sagemic_local.py +++ b/sagemic_local.py @@ -21,7 +21,13 @@ from birdnetlib import RecordingBuffer from birdnetlib.analyzer import Analyzer -from sagemic.helpers import insert_database, create_database, check_path, get_config, get_device_id +from sagemic.helpers import ( + insert_database, + create_database, + check_path, + get_config, + get_device_id +) # callback defined here to use with config variables @@ -60,6 +66,9 @@ def audio_callback( base_path = config["PATHS"]["BASE_PATH"] confidence_threshold = config["SETTINGS"]["CONFIDENCE_THRESHOLD"] sample_rate = config["SETTINGS"]["SAMPLERATE"] + dtype = config["SETTINGS"]["AUDIO_DTYPE"] + + bitrate = sample_rate * int(dtype[3:]) timestamp = datetime.now(local_tz) date = timestamp.strftime('%Y%m%d') @@ -99,7 +108,7 @@ def audio_callback( temp_filename, final_filename ) # to .wav for scansend when done - insert_database(base_path, final_filename, name, confidence, date_time) + insert_database(base_path, final_filename, sample_rate, bitrate) else: print("No detections") diff --git a/sagemic_scansend.py b/sagemic_scansend.py index d454b7f..42308d3 100644 --- a/sagemic_scansend.py +++ b/sagemic_scansend.py @@ -46,7 +46,9 @@ def main(): with sqlite3.connect(db_path) as conn: cursor = conn.cursor() - cursor.execute("SELECT filepath FROM detections WHERE sent = 0 ORDER BY timestamp ASC") + cursor.execute( + "SELECT filepath FROM detections WHERE sent = 0 ORDER BY filepath ASC" + ) files_to_send = [row[0] for row in cursor.fetchall()] if not files_to_send: From 0d846d26f7c132fe58c6d1aaeff831caed4f6d45 Mon Sep 17 00:00:00 2001 From: Ellina Ho Date: Mon, 13 Jul 2026 15:26:56 -0700 Subject: [PATCH 3/5] Added indexing for faster search time --- sagemic/helpers.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sagemic/helpers.py b/sagemic/helpers.py index a0c3e5f..9475f4c 100644 --- a/sagemic/helpers.py +++ b/sagemic/helpers.py @@ -67,6 +67,11 @@ def create_database(base_path): ) ''') + conn.execute(''' + CREATE INDEX IF NOT EXISTS idx_unsent_files + ON detections(sent) + ''') + def check_path(date, base_path): """Create new folder for date to store detections. From a31786b8646566f9fa44e6449709f3003ecbf262 Mon Sep 17 00:00:00 2001 From: Ellina Ho Date: Tue, 14 Jul 2026 12:18:54 -0700 Subject: [PATCH 4/5] added species, confidence, time --- sagemic/helpers.py | 17 ++++++++++------- sagemic_local.py | 14 +++++++++++--- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/sagemic/helpers.py b/sagemic/helpers.py index 9475f4c..e4398f6 100644 --- a/sagemic/helpers.py +++ b/sagemic/helpers.py @@ -2,15 +2,15 @@ """ import os import sys -import sounddevice as sd -import yaml import sqlite3 import platform from importlib.metadata import version +import sounddevice as sd +import yaml import numpy as np -def insert_database(base_path, filepath, sample_rate, bitrate): +def insert_database(base_path, filepath, species, confidence, timestamp, sample_rate, bitrate): """ Inserts detection entries into the database. Called by sagemic_local. @@ -24,7 +24,7 @@ def insert_database(base_path, filepath, sample_rate, bitrate): """ try: - with open("/sys/firmware/devicetree/base/model", "r") as f: + with open("/sys/firmware/devicetree/base/model", "r", encoding="utf-8") as f: hardware = f.read().strip('\x00') except FileNotFoundError: hardware = "Unknown" @@ -38,9 +38,9 @@ def insert_database(base_path, filepath, sample_rate, bitrate): with sqlite3.connect(db_path) as conn: conn.execute( "INSERT OR IGNORE INTO detections " - "(sent, filepath, hardware, firmware, model, sample_rate, bitrate)" - "VALUES (0, ?, ?, ?, ?, ?, ?)", - (filepath, hardware, firmware, model, sample_rate, bitrate) + "(sent, filepath, species, confidence, timestamp, hardware, firmware, model, sample_rate, bitrate)" + "VALUES (0, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (filepath, species, confidence, timestamp, hardware, firmware, model, sample_rate, bitrate) ) @@ -59,6 +59,9 @@ def create_database(base_path): CREATE TABLE IF NOT EXISTS detections ( sent INTEGER DEFAULT 0, filepath TEXT PRIMARY KEY, + species TEXT, + confidence REAL, + timestamp TEXT, hardware TEXT, firmware TEXT, model TEXT, diff --git a/sagemic_local.py b/sagemic_local.py index 785ed80..b093abd 100644 --- a/sagemic_local.py +++ b/sagemic_local.py @@ -71,7 +71,7 @@ def audio_callback( bitrate = sample_rate * int(dtype[3:]) timestamp = datetime.now(local_tz) - date = timestamp.strftime('%Y%m%d') + date = timestamp.strftime('%Y-%m-%d') path = check_path(date, base_path) # Flatten the data to a 1D array as expected by birdnetlib @@ -94,7 +94,7 @@ def audio_callback( confidence = detection["confidence"] # new for filenames w/ data + time - date_time = timestamp.strftime("%Y%m%d_%H%M%S") + date_time = timestamp.strftime("%Y-%m-%d_%H-%M-%S") print(f"** {name} Detected w/ (Confidence: {confidence:.2f})") @@ -108,7 +108,15 @@ def audio_callback( temp_filename, final_filename ) # to .wav for scansend when done - insert_database(base_path, final_filename, sample_rate, bitrate) + insert_database( + base_path, + final_filename, + name, + round(confidence, 2), + date_time, + sample_rate, + bitrate + ) else: print("No detections") From 85eed398bbdaf1c4c390113709f9f4f4aec5825d Mon Sep 17 00:00:00 2001 From: Ellina Ho Date: Tue, 21 Jul 2026 15:57:25 -0700 Subject: [PATCH 5/5] Added mic name and coordinates --- sagemic/helpers.py | 15 +++++++++++---- sagemic_local.py | 6 ++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/sagemic/helpers.py b/sagemic/helpers.py index e4398f6..f6a99fa 100644 --- a/sagemic/helpers.py +++ b/sagemic/helpers.py @@ -10,7 +10,7 @@ import numpy as np -def insert_database(base_path, filepath, species, confidence, timestamp, sample_rate, bitrate): +def insert_database(base_path, filepath, species, confidence, timestamp, coordinates, audio_device, sample_rate, bitrate): """ Inserts detection entries into the database. Called by sagemic_local. @@ -18,6 +18,11 @@ def insert_database(base_path, filepath, species, confidence, timestamp, sample_ Args: base_path (str): path to file storing directory filepath (str): path to detection audio file + species (str): species detected + confidence (float): inference confidence + timestamp (str): timestamp of detection + coordinates (str): coordinates in config file + audio_device (str): name of microphone used sample_rate (integer): user defined in config file bitrate (real): audio bitrate calculated in main @@ -38,9 +43,9 @@ def insert_database(base_path, filepath, species, confidence, timestamp, sample_ with sqlite3.connect(db_path) as conn: conn.execute( "INSERT OR IGNORE INTO detections " - "(sent, filepath, species, confidence, timestamp, hardware, firmware, model, sample_rate, bitrate)" - "VALUES (0, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - (filepath, species, confidence, timestamp, hardware, firmware, model, sample_rate, bitrate) + "(sent, filepath, species, confidence, timestamp, hardware, firmware, model, coordinates, audio_device, sample_rate, bitrate)" + "VALUES (0, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (filepath, species, confidence, timestamp, hardware, firmware, model, coordinates, audio_device, sample_rate, bitrate) ) @@ -65,6 +70,8 @@ def create_database(base_path): hardware TEXT, firmware TEXT, model TEXT, + coordinates TEXT, + audio_device TEXT, sample_rate INTEGER, bitrate INTEGER ) diff --git a/sagemic_local.py b/sagemic_local.py index a8b0cdc..829b3f4 100644 --- a/sagemic_local.py +++ b/sagemic_local.py @@ -69,6 +69,10 @@ def audio_callback( dtype = config["SETTINGS"]["AUDIO_DTYPE"] bitrate = sample_rate * int(dtype[3:]) + audio_device = config["SETTINGS"]["AUDIO_DEVICE"] + latitude = config["SETTINGS"]["LATITUDE"] + longitude = config["SETTINGS"]["LONGITUDE"] + coordinates = f"{latitude}, {longitude}" timestamp = datetime.now(local_tz) date = timestamp.strftime('%Y-%m-%d') @@ -114,6 +118,8 @@ def audio_callback( name, round(confidence, 2), date_time, + coordinates, + audio_device, sample_rate, bitrate )