diff --git a/sagemic/helpers.py b/sagemic/helpers.py index 0f700d5..f6a99fa 100644 --- a/sagemic/helpers.py +++ b/sagemic/helpers.py @@ -2,14 +2,92 @@ """ import os import sys +import sqlite3 +import platform +from importlib.metadata import version import sounddevice as sd import yaml import numpy as np +def insert_database(base_path, filepath, species, confidence, timestamp, coordinates, audio_device, sample_rate, bitrate): + """ + Inserts detection entries into the database. + Called by sagemic_local. + + Args: + base_path (str): path to file storing directory + filepath (str): path to detection audio file + species (str): species detected + confidence (float): inference confidence + timestamp (str): timestamp of detection + coordinates (str): coordinates in config file + audio_device (str): name of microphone used + sample_rate (integer): user defined in config file + bitrate (real): audio bitrate calculated in main + + """ + + try: + with open("/sys/firmware/devicetree/base/model", "r", encoding="utf-8") as f: + hardware = f.read().strip('\x00') + except FileNotFoundError: + hardware = "Unknown" + + firmware = platform.release().split('+')[0] + + model = f"birdnetlib_v{version('birdnetlib')}" + + db_path = os.path.join(base_path, "detections.db") + + with sqlite3.connect(db_path) as conn: + conn.execute( + "INSERT OR IGNORE INTO detections " + "(sent, filepath, species, confidence, timestamp, hardware, firmware, model, coordinates, audio_device, sample_rate, bitrate)" + "VALUES (0, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (filepath, species, confidence, timestamp, hardware, firmware, model, coordinates, audio_device, sample_rate, bitrate) + ) + + +def create_database(base_path): + """ + Checks if database exists, if not then creates detections database. + + Called by sagemic_local and sagemic_scansend + + Args: + base_path (str): path to file storing directory + """ + db_path = os.path.join(base_path, "detections.db") + with sqlite3.connect(db_path) as conn: + conn.execute(''' + CREATE TABLE IF NOT EXISTS detections ( + sent INTEGER DEFAULT 0, + filepath TEXT PRIMARY KEY, + species TEXT, + confidence REAL, + timestamp TEXT, + hardware TEXT, + firmware TEXT, + model TEXT, + coordinates TEXT, + audio_device TEXT, + sample_rate INTEGER, + bitrate INTEGER + ) + ''') + + conn.execute(''' + CREATE INDEX IF NOT EXISTS idx_unsent_files + ON detections(sent) + ''') + + def check_path(date, base_path): """Create new folder for date to store detections. + Called by sagemic_local.py. + Args: date (str): Current date in YYYY-MM-DD. @@ -28,6 +106,8 @@ def check_path(date, base_path): def inmp441_check(device_id, samplerate): """ Checks if inmp441 is connected and listening + Called by get_device_id + Args: device_id (int): id of inmp441 found from get_device_id() samplerate (int): samplerate of audio device from config diff --git a/sagemic_local.py b/sagemic_local.py index 457a8e2..829b3f4 100644 --- a/sagemic_local.py +++ b/sagemic_local.py @@ -21,7 +21,13 @@ from birdnetlib import RecordingBuffer from birdnetlib.analyzer import Analyzer -from sagemic.helpers import check_path, get_config, get_device_id +from sagemic.helpers import ( + insert_database, + create_database, + check_path, + get_config, + get_device_id +) # callback defined here to use with config variables @@ -60,6 +66,13 @@ def audio_callback( base_path = config["PATHS"]["BASE_PATH"] confidence_threshold = config["SETTINGS"]["CONFIDENCE_THRESHOLD"] sample_rate = config["SETTINGS"]["SAMPLERATE"] + dtype = config["SETTINGS"]["AUDIO_DTYPE"] + + bitrate = sample_rate * int(dtype[3:]) + audio_device = config["SETTINGS"]["AUDIO_DEVICE"] + latitude = config["SETTINGS"]["LATITUDE"] + longitude = config["SETTINGS"]["LONGITUDE"] + coordinates = f"{latitude}, {longitude}" timestamp = datetime.now(local_tz) date = timestamp.strftime('%Y-%m-%d') @@ -98,6 +111,19 @@ def audio_callback( os.rename( temp_filename, final_filename ) # to .wav for scansend when done + + insert_database( + base_path, + final_filename, + name, + round(confidence, 2), + date_time, + coordinates, + audio_device, + sample_rate, + bitrate + ) + else: print("No detections") @@ -116,6 +142,9 @@ def main(): args = parser.parse_args() config = get_config(args.config) + base_path = config["PATHS"]["BASE_PATH"] + + create_database(base_path) # audio variables latitude = config["SETTINGS"]["LATITUDE"] diff --git a/sagemic_scansend.py b/sagemic_scansend.py index b5ebf65..42308d3 100644 --- a/sagemic_scansend.py +++ b/sagemic_scansend.py @@ -1,3 +1,4 @@ + """ Service to check new BirdNet Detection Files and send them via MQTTS. """ @@ -7,99 +8,10 @@ import argparse import ssl import time # for sending delays -import bisect +import sqlite3 import paho.mqtt.client as mqtt -from sagemic.helpers import get_config - - -# function to write sent filepaths to log file -def write_log_file(filepath, log_file): - """Logs a successfully sent file with its filepath. - - - Args: - filepath (str): Path to the .wav file written by sagemic_local. - - Also means the file has been sent by this script. - - log_file (str): Path to the log file, defined in the config file. - """ - - with open(log_file, "a", encoding='utf-8') as f: # append to bottom - f.write(f"{filepath}\n") - - -def search_unsent(base_path, log_file): - """ Searches parent directory and date folders for unsent files. - - Args: - base_path (str): Path to the base directory to audio files. - - Defined in config. - - log_file (str): Path to the log file, defined in the config file. - - Returns: - dict: Returns an dictionary of filepaths of unsent files. - """ - sent_files = set() - files_to_send = [] - start_folder = "" - start_file = "" - - # Read the log file to get entire set and latest date read - if os.path.exists(log_file): - with open(log_file, "r", encoding="utf-8") as f: - lines = f.readlines() - if lines: - sent_files = set(line.strip() for line in lines) - last_line = lines[-1].strip() - start_folder = os.path.basename(os.path.dirname(last_line)) - start_file = os.path.basename(last_line) - - # filter out folders only in basepath - try: - folders = [] - for f in os.listdir(base_path): - # check if item is a folder - # can be eliminated late for efficiency - folder_path = os.path.join(base_path, f) - if os.path.isdir(folder_path): - folders.append(f) - all_folders = sorted(folders) # sorted for for loop later - except FileNotFoundError: - all_folders = [] - - # trim all_folders to start at start_folder - if not start_folder: - search_folders = all_folders - else: - # Using this instead of for loop and compare O(N) so its O(logN) - index = bisect.bisect_left(all_folders, start_folder) - search_folders = all_folders[index:] - - # search in relevant folders - for folder in search_folders: - folder_path = os.path.join(base_path, folder) - wavs = [] - - for file in os.listdir(folder_path): - if file.endswith(".wav"): - wavs.append(file) - wav_files = sorted(wavs) - - if folder == start_folder and start_file: - index = bisect.bisect_right(wav_files, start_file) - wav_files = wav_files[index:] - - for file in wav_files: - filepath = os.path.join(folder_path, file) - - if filepath not in sent_files: - files_to_send.append(filepath) - - files_to_send.sort() # so we send chronologically - - return files_to_send +from sagemic.helpers import get_config, create_database # script only runs every few minutes (loop) @@ -119,9 +31,6 @@ def main(): base_path = config["PATHS"]["BASE_PATH"] - # log file to track clips that have alr been sent (tracker) - log_file = os.path.join(base_path, "sent_clips.log") - port = config["MQTT"]["PORT"] broker = config["MQTT"]["BROKER"] topic = config["MQTT"]["BASE_TOPIC"] + "/" + config["MQTT"]["DEVICE"] @@ -130,7 +39,17 @@ def main(): user = config["MQTT"]["USER"] password = config["MQTT"]["PASS"] - files_to_send = search_unsent(base_path, log_file) + db_path = os.path.join(base_path, "detections.db") + + # get files to send + create_database(base_path) + + with sqlite3.connect(db_path) as conn: + cursor = conn.cursor() + cursor.execute( + "SELECT filepath FROM detections WHERE sent = 0 ORDER BY filepath ASC" + ) + files_to_send = [row[0] for row in cursor.fetchall()] if not files_to_send: print("No new clips we need to send") @@ -138,7 +57,7 @@ def main(): # ensure only sending completed clip, (done in sagemic_local.py) - client = mqtt.Client(client_id=session_id) + client = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2, client_id=session_id) client.username_pw_set(user, password) @@ -157,7 +76,9 @@ def main(): # send new clips to broker over mqtt for filepath in files_to_send: - # add print statements here if needed later + if not os.path.exists(filepath): + print(f"Warning: {filepath} found in DB but missing on disk.") + continue # get .wav filename from filepath filename = os.path.basename(filepath) @@ -175,7 +96,13 @@ def main(): ) result.wait_for_publish() - write_log_file(filepath, log_file) + with sqlite3.connect(db_path) as conn: + conn.execute( + "UPDATE detections SET sent = 1 WHERE filepath = ?", + (filepath,) + ) + + print(f"Successfully sent and logged: {filename}") time.sleep(0.5) # to prevent network flood except Exception as e: # pylint: disable=broad-except