From 95fcda499d4aaeb8a96082c64a7976667dcadea8 Mon Sep 17 00:00:00 2001 From: David Date: Thu, 4 Jun 2026 19:37:22 -0600 Subject: [PATCH 01/21] feat: add ulog2ros2bag script Does basic conversion from ulog to ros2 bag --- pyproject.toml | 1 + pyulog/ulog2ros2bag.py | 192 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 193 insertions(+) create mode 100644 pyulog/ulog2ros2bag.py diff --git a/pyproject.toml b/pyproject.toml index 69dfa89..84961de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ ulog_params = "pyulog.params:main" ulog2csv = "pyulog.ulog2csv:main" ulog2kml = "pyulog.ulog2kml:main" ulog2rosbag = "pyulog.ulog2rosbag:main" +ulog2ros2bag = "pyulog.ulog2ros2bag:main" ulog_migratedb = "pyulog.migrate_db:main" [project.urls] diff --git a/pyulog/ulog2ros2bag.py b/pyulog/ulog2ros2bag.py new file mode 100644 index 0000000..39b9174 --- /dev/null +++ b/pyulog/ulog2ros2bag.py @@ -0,0 +1,192 @@ +#! /usr/bin/env python + +""" +Convert a ULog file into a ROS2 bag. Inspired by ulog2rosbag.py. +""" + +from collections import defaultdict +import argparse +import re +from rclpy.serialization import serialize_message # pylint: disable=import-error +import rosbag2_py # pylint: disable=import-error +from px4_msgs import msg as px4_msgs # pylint: disable=import-error +import numpy as np + +# TODO: temporary for typing +# from .core import ULog +from pyulog import ULog + +EXCEPTIONS = { + "estimator_aid_src_baro_hgt": "EstimatorAidSource1d", + "estimator_aid_src_ev_pos": "EstimatorAidSource2d", + "estimator_aid_src_fake_hgt": "EstimatorAidSource1d", + "estimator_aid_src_fake_pos": "EstimatorAidSource2d", + "estimator_aid_src_gnss_hgt": "EstimatorAidSource1d", + "estimator_aid_src_gnss_pos": "EstimatorAidSource2d", + "estimator_aid_src_gnss_vel": "EstimatorAidSource3d", + "estimator_aid_src_gravity": "EstimatorAidSource3d", + "estimator_aid_src_mag": "EstimatorAidSource3d", + "estimator_aid_src_rng_hgt": "EstimatorAidSource1d", + "estimator_attitude": "VehicleAttitude", + "estimator_baro_bias": "EstimatorBias", + "estimator_global_position": "VehicleGlobalPosition", + "estimator_gnss_hgt_bias": "EstimatorBias", + "estimator_innovation_test_ratios": "EstimatorInnovations", + "estimator_innovation_variances": "EstimatorInnovations", + "estimator_local_position": "VehicleLocalPosition", + "estimator_odometry": "VehicleOdometry", + "px4io_status": "Px4ioStatus", + "vehicle_gps_position": "SensorGps", + "vehicle_visual_odometry": "VehicleOdometry", +} + +# pylint: disable=too-many-locals, invalid-name + + +def main(): + """Command line interface""" + + parser = argparse.ArgumentParser(description="Convert ULog to rosbag") + parser.add_argument("filename", metavar="file.ulg", help="ULog input file") + parser.add_argument("bag", metavar="rosbag_file", help="rosbag output file") + + parser.add_argument( + "-m", + "--messages", + dest="messages", + help=( + "Only consider given messages. Must be a comma-separated list of" + " names, like 'sensor_combined,vehicle_gps_position'" + ), + ) + + parser.add_argument( + "-i", + "--ignore", + dest="ignore", + action="store_true", + help="Ignore string parsing exceptions", + default=False, + ) + + args = parser.parse_args() + + convert_ulog2rosbag(args.filename, args.bag, args.messages, args.ignore) + + +# https://stackoverflow.com/questions/19053707/converting-snake-case-to-lower-camel-case-lowercamelcase +def to_camel_case(snake_str): + """Convert snake case string to camel case""" + components = snake_str.split("_") + return "".join(x.title() for x in components) + + +def convert_ulog2rosbag( + ulog_file_name: str, rosbag_name: str, messages: str, disable_str_exceptions=False +): + """ + Coverts and ULog file to a CSV file. + + :param ulog_file_name: The ULog filename to open and read + :param rosbag_name: The rosbag filename to open and write + :param messages: A list of message names + + :return: No + """ + + msg_filter = messages.split(",") if messages else None + + ulog = ULog(ulog_file_name, msg_filter, disable_str_exceptions) + + multiids: dict[str, set[int]] = defaultdict(set) + for topic in ulog.data_list: + multiids[topic.name].add(topic.multi_id) + + # ROS2 boilerplate + writer = rosbag2_py.SequentialWriter() + + storage_options = rosbag2_py.StorageOptions(uri=rosbag_name, storage_id="sqlite3") + converter_options = rosbag2_py.ConverterOptions( + input_serialization_format="cdr", output_serialization_format="cdr" + ) + writer.open(storage_options, converter_options) + + topic_count, message_count = 0, 0 + for ulg_topic in ulog.data_list: + topic_message_count = len(ulg_topic.data["timestamp"]) + # Determine ROS2 topic name + if multiids[ulg_topic.name] == {0}: + ros2_topic = "/px4/{}".format(ulg_topic.name) + else: + ros2_topic = "/px4/{}_{}".format(ulg_topic.name, ulg_topic.multi_id) + + # Determine ROS2 message type (px4_msgs) + direct_name = to_camel_case(ulg_topic.name) + if hasattr(px4_msgs, direct_name): + MsgType = getattr(px4_msgs, direct_name) + elif ulg_topic.name in EXCEPTIONS: + # Exception + MsgType = getattr(px4_msgs, EXCEPTIONS[ulg_topic.name]) + else: + print( + f"Message type '{to_camel_case(ulg_topic.name)}' for {ulg_topic.name} not found in px4_msgs, skipping." + ) + continue + + # Check if it is a composed message type + if any( + re.compile(r"(.*?)\.(.*?)").match(field.field_name) + for field in ulg_topic.field_data + ): + # TODO: add support for composed message types + print(f"Message type for {ulg_topic.name} is composed, skipping.") + continue + + # Register topic in rosbag + topic_info = rosbag2_py.TopicMetadata( + name=ros2_topic, + type=f"px4_msgs/msg/{MsgType.__name__}", + serialization_format="cdr", + ) + writer.create_topic(topic_info) + + # Write each message + for i in range(len(ulg_topic.data["timestamp"])): + msg = MsgType() + for field in ulg_topic.field_data: + array_condition = re.compile(r"(.*?)\[(.*?)\]") + array_match = array_condition.match(field.field_name) + value = ulg_topic.data[field.field_name][i] + if array_match: + field_name, array_index = array_match.groups() + array_index = int(array_index) + if value.dtype == np.int8: + value = bool(value) + getattr(msg, field_name)[array_index] = value + else: + try: + if value.dtype == np.int8: + value = bool(value) + else: + value = value.item() + setattr(msg, field.field_name, value) + except Exception as e: + print( + f"Exception when setting field '{field.field_name}' from topic '{ulg_topic.name}' with type: {type(value)}" + ) + raise e + + ts = ulg_topic.data["timestamp"][i] * 1000 # us -> ns + writer.write( + ros2_topic, + serialize_message(msg), + int(ts), + ) + topic_count += 1 + message_count += topic_message_count + + writer.close() + print(f"Wrote rosbag '{rosbag_name}' with {topic_count} topics and {message_count} messages.") + +if __name__ == "__main__": + main() From 75a597fac9468566c5b4e0b330a91274c0cb6697 Mon Sep 17 00:00:00 2001 From: David Date: Thu, 4 Jun 2026 20:34:34 -0600 Subject: [PATCH 02/21] feat: add support for Ubuntu Noble to ulog2ros2bag --- pyulog/ulog2ros2bag.py | 65 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 9 deletions(-) diff --git a/pyulog/ulog2ros2bag.py b/pyulog/ulog2ros2bag.py index 39b9174..c297689 100644 --- a/pyulog/ulog2ros2bag.py +++ b/pyulog/ulog2ros2bag.py @@ -8,6 +8,7 @@ import argparse import re from rclpy.serialization import serialize_message # pylint: disable=import-error +from importlib.metadata import version import rosbag2_py # pylint: disable=import-error from px4_msgs import msg as px4_msgs # pylint: disable=import-error import numpy as np @@ -71,7 +72,7 @@ def main(): args = parser.parse_args() - convert_ulog2rosbag(args.filename, args.bag, args.messages, args.ignore) + convert_ulog2ros2bag(args.filename, args.bag, args.messages, args.ignore) # https://stackoverflow.com/questions/19053707/converting-snake-case-to-lower-camel-case-lowercamelcase @@ -81,7 +82,7 @@ def to_camel_case(snake_str): return "".join(x.title() for x in components) -def convert_ulog2rosbag( +def convert_ulog2ros2bag( ulog_file_name: str, rosbag_name: str, messages: str, disable_str_exceptions=False ): """ @@ -111,6 +112,29 @@ def convert_ulog2rosbag( ) writer.open(storage_options, converter_options) + # Support rosbag2_py topic sequence number in ROS2 versions greater than Humble + if rosbag_write_uses_seqnum(): + rosbag_write = lambda topic, msg, timestamp: writer.write( + topic, msg, timestamp, 0 + ) + else: + rosbag_write = lambda topic, msg, timestamp: writer.write(topic, msg, timestamp) + + # Support rosbag2_py TopicMetadata ID in ROS2 versions greater than Humble + if rosbag_topicmetadata_uses_id(): + topic_metadata = lambda name, type: rosbag2_py.TopicMetadata( + 0, + name=name, + type=type, + serialization_format="cdr", + ) + else: + topic_metadata = lambda name, type: rosbag2_py.TopicMetadata( + name=name, + type=type, + serialization_format="cdr", + ) + topic_count, message_count = 0, 0 for ulg_topic in ulog.data_list: topic_message_count = len(ulg_topic.data["timestamp"]) @@ -143,12 +167,9 @@ def convert_ulog2rosbag( continue # Register topic in rosbag - topic_info = rosbag2_py.TopicMetadata( - name=ros2_topic, - type=f"px4_msgs/msg/{MsgType.__name__}", - serialization_format="cdr", + writer.create_topic( + topic_metadata(ros2_topic, f"px4_msgs/msg/{MsgType.__name__}") ) - writer.create_topic(topic_info) # Write each message for i in range(len(ulg_topic.data["timestamp"])): @@ -177,7 +198,7 @@ def convert_ulog2rosbag( raise e ts = ulg_topic.data["timestamp"][i] * 1000 # us -> ns - writer.write( + rosbag_write( ros2_topic, serialize_message(msg), int(ts), @@ -186,7 +207,33 @@ def convert_ulog2rosbag( message_count += topic_message_count writer.close() - print(f"Wrote rosbag '{rosbag_name}' with {topic_count} topics and {message_count} messages.") + print( + f"Wrote rosbag '{rosbag_name}' with {topic_count} topics and {message_count} messages." + ) + + +def rosbag_topicmetadata_uses_id(): + """ + rosbag2_py PR #1538 adds a parameter to TopicMetadata.__init__() for a topic ID, + breaking the interface. This attempts to add compability with both versions. + """ + try: + return version("rosbag2_py") >= "0.25.0" + except: + return False # Default: <= Humble + + +def rosbag_write_uses_seqnum(): + """ + rosbag2_py commit 62e5f77 adds a parameter to SequentialWriter.write() for a + topic sequence number, breaking the interface. This attempts to add compatibility + with both versions. + """ + try: + return version("rosbag2_py") >= "0.32.0" + except: + return False # Default: <= Humble + if __name__ == "__main__": main() From c34840bd49d626d7d255b36ed9c3b1758c1cd18c Mon Sep 17 00:00:00 2001 From: David Date: Fri, 5 Jun 2026 16:14:02 -0500 Subject: [PATCH 03/21] feat: dynamically determine some message types --- pyulog/ulog2ros2bag.py | 59 ++++++++++++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/pyulog/ulog2ros2bag.py b/pyulog/ulog2ros2bag.py index c297689..fd96ad4 100644 --- a/pyulog/ulog2ros2bag.py +++ b/pyulog/ulog2ros2bag.py @@ -18,22 +18,10 @@ from pyulog import ULog EXCEPTIONS = { - "estimator_aid_src_baro_hgt": "EstimatorAidSource1d", - "estimator_aid_src_ev_pos": "EstimatorAidSource2d", - "estimator_aid_src_fake_hgt": "EstimatorAidSource1d", - "estimator_aid_src_fake_pos": "EstimatorAidSource2d", - "estimator_aid_src_gnss_hgt": "EstimatorAidSource1d", - "estimator_aid_src_gnss_pos": "EstimatorAidSource2d", - "estimator_aid_src_gnss_vel": "EstimatorAidSource3d", - "estimator_aid_src_gravity": "EstimatorAidSource3d", - "estimator_aid_src_mag": "EstimatorAidSource3d", - "estimator_aid_src_rng_hgt": "EstimatorAidSource1d", "estimator_attitude": "VehicleAttitude", "estimator_baro_bias": "EstimatorBias", "estimator_global_position": "VehicleGlobalPosition", "estimator_gnss_hgt_bias": "EstimatorBias", - "estimator_innovation_test_ratios": "EstimatorInnovations", - "estimator_innovation_variances": "EstimatorInnovations", "estimator_local_position": "VehicleLocalPosition", "estimator_odometry": "VehicleOdometry", "px4io_status": "Px4ioStatus", @@ -124,7 +112,7 @@ def convert_ulog2ros2bag( if rosbag_topicmetadata_uses_id(): topic_metadata = lambda name, type: rosbag2_py.TopicMetadata( 0, - name=name, + name=name, # type: ignore type=type, serialization_format="cdr", ) @@ -145,12 +133,9 @@ def convert_ulog2ros2bag( ros2_topic = "/px4/{}_{}".format(ulg_topic.name, ulg_topic.multi_id) # Determine ROS2 message type (px4_msgs) - direct_name = to_camel_case(ulg_topic.name) - if hasattr(px4_msgs, direct_name): - MsgType = getattr(px4_msgs, direct_name) - elif ulg_topic.name in EXCEPTIONS: - # Exception - MsgType = getattr(px4_msgs, EXCEPTIONS[ulg_topic.name]) + msg_type_name = calc_msgtype(ulg_topic.name) + if msg_type_name is not None: + MsgType = getattr(px4_msgs, msg_type_name) else: print( f"Message type '{to_camel_case(ulg_topic.name)}' for {ulg_topic.name} not found in px4_msgs, skipping." @@ -235,5 +220,41 @@ def rosbag_write_uses_seqnum(): return False # Default: <= Humble +def calc_msgtype(topic_name: str) -> str | None: + """Calculate message type from topic name, if possible""" + direct_name = to_camel_case(topic_name) + global EXCEPTIONS + + if hasattr(px4_msgs, direct_name): + return direct_name + elif topic_name.startswith("estimator_aid_src_"): + if any( + topic_name.endswith(suffix) + for suffix in ["hgt", "airspeed", "slideslip", "yaw"] + ): + return "EstimatorAidSource1d" + elif any( + topic_name.endswith(suffix) + for suffix in [ + "pos", + "aux_global_position", + "aux_vel", + "optical_flow", + "drag", + ] + ): + return "EstimatorAidSource2d" + elif any(topic_name.endswith(suffix) for suffix in ["vel", "gravity", "mag"]): + return "EstimatorAidSource3d" + else: + return None + elif topic_name.startswith("estimator_innovation"): + return "EstimatorInnovations" + elif topic_name in EXCEPTIONS: + return EXCEPTIONS[topic_name] + else: + return None + + if __name__ == "__main__": main() From 17bd316bfa044096756e992931b157d0da29793e Mon Sep 17 00:00:00 2001 From: David Date: Fri, 5 Jun 2026 16:46:41 -0500 Subject: [PATCH 04/21] feat: check px4_msgs compatibiltiy to avoid crashing --- pyulog/ulog2ros2bag.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pyulog/ulog2ros2bag.py b/pyulog/ulog2ros2bag.py index fd96ad4..18a0099 100644 --- a/pyulog/ulog2ros2bag.py +++ b/pyulog/ulog2ros2bag.py @@ -151,6 +151,18 @@ def convert_ulog2ros2bag( print(f"Message type for {ulg_topic.name} is composed, skipping.") continue + # Verify message type + ros2_fields = MsgType.get_fields_and_field_types().keys() + good = True + for data in ulg_topic.field_data: + px4_field = data.field_name + if px4_field not in ros2_fields: + good = False + if not good: + print(f"Message type '{MsgType.__name__}' in px4_msgs does not match version found in ulog, skipping. Please check your version of px4_msgs.") + print(f"PX4 firmware version from ulog: {ulog.get_version_info_str()}") + continue + # Register topic in rosbag writer.create_topic( topic_metadata(ros2_topic, f"px4_msgs/msg/{MsgType.__name__}") From 1a727816134b1689ffd402a46cd4aa711eec61f2 Mon Sep 17 00:00:00 2001 From: David Date: Sun, 7 Jun 2026 13:48:27 -0500 Subject: [PATCH 05/21] feat: add verbose flag --- pyulog/ulog2ros2bag.py | 65 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 57 insertions(+), 8 deletions(-) diff --git a/pyulog/ulog2ros2bag.py b/pyulog/ulog2ros2bag.py index 18a0099..f70d201 100644 --- a/pyulog/ulog2ros2bag.py +++ b/pyulog/ulog2ros2bag.py @@ -7,8 +7,9 @@ from collections import defaultdict import argparse import re -from rclpy.serialization import serialize_message # pylint: disable=import-error +from os import environ from importlib.metadata import version +from rclpy.serialization import serialize_message # pylint: disable=import-error import rosbag2_py # pylint: disable=import-error from px4_msgs import msg as px4_msgs # pylint: disable=import-error import numpy as np @@ -58,9 +59,20 @@ def main(): default=False, ) + parser.add_argument( + "-v", + "--verbose", + dest="verbose", + action="store_true", + help="Print extra debugging information", + default=False, + ) + args = parser.parse_args() - convert_ulog2ros2bag(args.filename, args.bag, args.messages, args.ignore) + convert_ulog2ros2bag( + args.filename, args.bag, args.messages, args.ignore, args.verbose + ) # https://stackoverflow.com/questions/19053707/converting-snake-case-to-lower-camel-case-lowercamelcase @@ -71,7 +83,11 @@ def to_camel_case(snake_str): def convert_ulog2ros2bag( - ulog_file_name: str, rosbag_name: str, messages: str, disable_str_exceptions=False + ulog_file_name: str, + rosbag_name: str, + messages: str, + disable_str_exceptions=False, + verbose=False, ): """ Coverts and ULog file to a CSV file. @@ -123,32 +139,60 @@ def convert_ulog2ros2bag( serialization_format="cdr", ) + if verbose: + print(f"D: ROS2 Distro: {environ.get('ROS_DISTRO') or 'not detected'}") + px4_msgs_share_dir = None + try: + from ament_index_python.packages import ( + get_package_share_directory, + ) # pylint: disable=import-error + + px4_msgs_share_dir = get_package_share_directory("px4_msgs") + except: + pass + print(f"D: ROS2 px4_msgs: {px4_msgs_share_dir or 'not found'}") + print(f"D: PX4 firmware version from ulog: {ulog.get_version_info_str()}") + print("") + topic_count, message_count = 0, 0 for ulg_topic in ulog.data_list: topic_message_count = len(ulg_topic.data["timestamp"]) + # Determine ROS2 topic name if multiids[ulg_topic.name] == {0}: ros2_topic = "/px4/{}".format(ulg_topic.name) else: ros2_topic = "/px4/{}_{}".format(ulg_topic.name, ulg_topic.multi_id) + if verbose: + if multiids[ulg_topic.name] == {0}: + ulg_topic_name = ulg_topic.name + else: + ulg_topic_name = f"{ulg_topic.name} ({ulg_topic.multi_id})" + print( + f"D: Processing ulog topic {ulg_topic_name} with {topic_message_count} messages and ROS2 topic {ros2_topic}" + ) + # Determine ROS2 message type (px4_msgs) msg_type_name = calc_msgtype(ulg_topic.name) if msg_type_name is not None: MsgType = getattr(px4_msgs, msg_type_name) else: print( - f"Message type '{to_camel_case(ulg_topic.name)}' for {ulg_topic.name} not found in px4_msgs, skipping." + f"W: Message type '{to_camel_case(ulg_topic.name)}' for {ulg_topic.name} not found in px4_msgs, skipping." ) continue + if verbose: + print(f"D: Found ROS2 message type px4_msgs/msg/{MsgType.__name__}") + # Check if it is a composed message type if any( re.compile(r"(.*?)\.(.*?)").match(field.field_name) for field in ulg_topic.field_data ): # TODO: add support for composed message types - print(f"Message type for {ulg_topic.name} is composed, skipping.") + print(f"W: Message type for {ulg_topic.name} is composed, skipping.") continue # Verify message type @@ -157,10 +201,15 @@ def convert_ulog2ros2bag( for data in ulg_topic.field_data: px4_field = data.field_name if px4_field not in ros2_fields: + if verbose: + print( + f"D: field {px4_field} of {ulg_topic.name} not found in {MsgType.__name__}" + ) good = False if not good: - print(f"Message type '{MsgType.__name__}' in px4_msgs does not match version found in ulog, skipping. Please check your version of px4_msgs.") - print(f"PX4 firmware version from ulog: {ulog.get_version_info_str()}") + print( + f"W: Message type px4_msgs/msg/{MsgType.__name__} does not match topic '{ulg_topic.name}' in ulog, skipping. Please check your version of px4_msgs." + ) continue # Register topic in rosbag @@ -205,7 +254,7 @@ def convert_ulog2ros2bag( writer.close() print( - f"Wrote rosbag '{rosbag_name}' with {topic_count} topics and {message_count} messages." + f"\nWrote rosbag '{rosbag_name}' with {topic_count} topics and {message_count} messages." ) From f3a315d2cadfd6f138b3741dced52671a2a228db Mon Sep 17 00:00:00 2001 From: David Date: Sun, 7 Jun 2026 16:49:06 -0500 Subject: [PATCH 06/21] fix: correctly handle array fields during message type checking --- pyulog/ulog2ros2bag.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/pyulog/ulog2ros2bag.py b/pyulog/ulog2ros2bag.py index f70d201..78cfba8 100644 --- a/pyulog/ulog2ros2bag.py +++ b/pyulog/ulog2ros2bag.py @@ -140,7 +140,7 @@ def convert_ulog2ros2bag( ) if verbose: - print(f"D: ROS2 Distro: {environ.get('ROS_DISTRO') or 'not detected'}") + print(f"I: ROS2 Distro: {environ.get('ROS_DISTRO') or 'not detected'}") px4_msgs_share_dir = None try: from ament_index_python.packages import ( @@ -150,8 +150,8 @@ def convert_ulog2ros2bag( px4_msgs_share_dir = get_package_share_directory("px4_msgs") except: pass - print(f"D: ROS2 px4_msgs: {px4_msgs_share_dir or 'not found'}") - print(f"D: PX4 firmware version from ulog: {ulog.get_version_info_str()}") + print(f"I: ROS2 px4_msgs: {px4_msgs_share_dir or 'not found'}") + print(f"I: PX4 firmware version from ulog: {ulog.get_version_info_str()}") print("") topic_count, message_count = 0, 0 @@ -197,19 +197,23 @@ def convert_ulog2ros2bag( # Verify message type ros2_fields = MsgType.get_fields_and_field_types().keys() - good = True - for data in ulg_topic.field_data: - px4_field = data.field_name - if px4_field not in ros2_fields: - if verbose: - print( - f"D: field {px4_field} of {ulg_topic.name} not found in {MsgType.__name__}" - ) - good = False - if not good: + px4_fields = [ + re.sub(r"\[\d+\]$", "", data.field_name) for data in ulg_topic.field_data + ] # strip array indices + px4_fields = list(set(px4_fields)) # remove duplicates + if not all([px4_field in ros2_fields for px4_field in px4_fields]): print( f"W: Message type px4_msgs/msg/{MsgType.__name__} does not match topic '{ulg_topic.name}' in ulog, skipping. Please check your version of px4_msgs." ) + if verbose: + for px4_field in [ + px4_field + for px4_field in px4_fields + if px4_field not in ros2_fields + ]: + print( + f"D: field {px4_field} of {ulg_topic.name} not found in {MsgType.__name__}" + ) continue # Register topic in rosbag @@ -221,7 +225,7 @@ def convert_ulog2ros2bag( for i in range(len(ulg_topic.data["timestamp"])): msg = MsgType() for field in ulg_topic.field_data: - array_condition = re.compile(r"(.*?)\[(.*?)\]") + array_condition = re.compile(r"(.*?)\[\d+\]") array_match = array_condition.match(field.field_name) value = ulg_topic.data[field.field_name][i] if array_match: From 6a4b3323137c1aa1a7ce9c78891fcf07d4ff1cc4 Mon Sep 17 00:00:00 2001 From: David Date: Sun, 7 Jun 2026 17:43:45 -0500 Subject: [PATCH 07/21] fix: correctly handle grouping of array fields --- pyulog/ulog2ros2bag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyulog/ulog2ros2bag.py b/pyulog/ulog2ros2bag.py index 78cfba8..822b497 100644 --- a/pyulog/ulog2ros2bag.py +++ b/pyulog/ulog2ros2bag.py @@ -225,7 +225,7 @@ def convert_ulog2ros2bag( for i in range(len(ulg_topic.data["timestamp"])): msg = MsgType() for field in ulg_topic.field_data: - array_condition = re.compile(r"(.*?)\[\d+\]") + array_condition = re.compile(r"(.*?)\[(\d+)\]") array_match = array_condition.match(field.field_name) value = ulg_topic.data[field.field_name][i] if array_match: From 96657ca8fb6fe9a21f577021ece629363f47f4da Mon Sep 17 00:00:00 2001 From: David Date: Sun, 7 Jun 2026 21:10:04 -0500 Subject: [PATCH 08/21] refactor: determine message type using regex dict --- pyulog/ulog2ros2bag.py | 95 ++++++++++++++++++++---------------------- 1 file changed, 45 insertions(+), 50 deletions(-) diff --git a/pyulog/ulog2ros2bag.py b/pyulog/ulog2ros2bag.py index 822b497..8498701 100644 --- a/pyulog/ulog2ros2bag.py +++ b/pyulog/ulog2ros2bag.py @@ -18,18 +18,6 @@ # from .core import ULog from pyulog import ULog -EXCEPTIONS = { - "estimator_attitude": "VehicleAttitude", - "estimator_baro_bias": "EstimatorBias", - "estimator_global_position": "VehicleGlobalPosition", - "estimator_gnss_hgt_bias": "EstimatorBias", - "estimator_local_position": "VehicleLocalPosition", - "estimator_odometry": "VehicleOdometry", - "px4io_status": "Px4ioStatus", - "vehicle_gps_position": "SensorGps", - "vehicle_visual_odometry": "VehicleOdometry", -} - # pylint: disable=too-many-locals, invalid-name @@ -151,7 +139,9 @@ def convert_ulog2ros2bag( except: pass print(f"I: ROS2 px4_msgs: {px4_msgs_share_dir or 'not found'}") - print(f"I: PX4 firmware version from ulog: {ulog.get_version_info_str()}") + print( + f"I: PX4 firmware version from ulog: {ulog.get_version_info_str() or 'not found'}" + ) print("") topic_count, message_count = 0, 0 @@ -160,26 +150,25 @@ def convert_ulog2ros2bag( # Determine ROS2 topic name if multiids[ulg_topic.name] == {0}: - ros2_topic = "/px4/{}".format(ulg_topic.name) + ros2_topic = f"/px4/{ulg_topic.name}" else: - ros2_topic = "/px4/{}_{}".format(ulg_topic.name, ulg_topic.multi_id) + ros2_topic = f"/px4/{ulg_topic.name}_{ulg_topic.multi_id}" if verbose: - if multiids[ulg_topic.name] == {0}: - ulg_topic_name = ulg_topic.name - else: - ulg_topic_name = f"{ulg_topic.name} ({ulg_topic.multi_id})" + ulg_topic_name = ulg_topic.name + if multiids[ulg_topic.name] != {0}: + ulg_topic_name += f" ({ulg_topic.multi_id})" print( f"D: Processing ulog topic {ulg_topic_name} with {topic_message_count} messages and ROS2 topic {ros2_topic}" ) # Determine ROS2 message type (px4_msgs) msg_type_name = calc_msgtype(ulg_topic.name) - if msg_type_name is not None: + if msg_type_name is not None and hasattr(px4_msgs, msg_type_name): MsgType = getattr(px4_msgs, msg_type_name) else: print( - f"W: Message type '{to_camel_case(ulg_topic.name)}' for {ulg_topic.name} not found in px4_msgs, skipping." + f"W: Message type '{msg_type_name or to_camel_case(ulg_topic.name)}' for {ulg_topic.name} not found in px4_msgs, skipping." ) continue @@ -287,38 +276,44 @@ def rosbag_write_uses_seqnum(): def calc_msgtype(topic_name: str) -> str | None: """Calculate message type from topic name, if possible""" - direct_name = to_camel_case(topic_name) - global EXCEPTIONS + REGEX_EXCEPTIONS = { + r"^actuator_controls_status_\d$": "ActuatorControlsStatus", + r"^actuator_outputs\w*": "ActuatorOutputs", + r"^config_overrides\w*": "ConfigOverrides", + r"^estimator_aid_src_((gnss_yaw)|(ev_yaw)|(fake_hgt)|(airspeed)|(sideslip)|(\w+_hgt))$": "EstimatorAidSource1d", + r"^estimator_aid_src_((drag)|(aux_vel)|(optical_flow)|(\w+_pos)|(aux_global_position))$": "EstimatorAidSource2d", + r"^estimator_aid_src_((ev_vel)|(gnss_vel)|(gravity)|(mag))$": "EstimatorAidSource3d", + r"^estimator_ev_pos_bias": "EstimatorBias3d", + r"^estimator_((baro)|(gnss_hgt))_bias": "EstimatorBias", + r"^estimator_innovation\w*": "EstimatorInnovations", + r"^px4io_status$": "Px4ioStatus", + r"^vehicle_gps_position$": "SensorGps", + r"^sensors_status_\w+": "SensorsStatus", + r"^vehicle_angular_velocity\w*": "VehicleAngularVelocity", + r"^\w+?attitude(_groundtruth)?$": "VehicleAttitude", + r"^\w+?attitude_setpoint$": "VehicleAttitudeSetpoint", + r"^\w+?_command\w*": "VehicleCommand", + r"^\w+?_control_\w+": "VehicleControlMode", + r"^((aux_)|(estimator_)|(vehicle_)|(external_ins_))global_position(_groundtruth)?$": "VehicleGlobalPosition", + r"^\w+?_local_position(_groundtruth)?$": "VehicleLocalPosition", + r"^\w+?_odometry$": "VehicleOdometry", + r"^((estimator)|(vehicle))_optical_flow_vel$": "VehicleOpticalFlowVel", + r"^vehicle_thrust_setpoint\w*": "VehicleThrustSetpoint", + r"^vehicle_torque_setpoint\w*": "VehicleTorqueSetpoint", + r"^(estimator_)?wind$": "Wind", + } + direct_name = to_camel_case(topic_name) if hasattr(px4_msgs, direct_name): return direct_name - elif topic_name.startswith("estimator_aid_src_"): - if any( - topic_name.endswith(suffix) - for suffix in ["hgt", "airspeed", "slideslip", "yaw"] - ): - return "EstimatorAidSource1d" - elif any( - topic_name.endswith(suffix) - for suffix in [ - "pos", - "aux_global_position", - "aux_vel", - "optical_flow", - "drag", - ] - ): - return "EstimatorAidSource2d" - elif any(topic_name.endswith(suffix) for suffix in ["vel", "gravity", "mag"]): - return "EstimatorAidSource3d" - else: - return None - elif topic_name.startswith("estimator_innovation"): - return "EstimatorInnovations" - elif topic_name in EXCEPTIONS: - return EXCEPTIONS[topic_name] - else: - return None + + result = None + for pattern, type_name in REGEX_EXCEPTIONS.items(): + if re.match(pattern, topic_name): + result = type_name + break + + return result if __name__ == "__main__": From d69d11270b5f84eea2507f7f2baa88f13767f622 Mon Sep 17 00:00:00 2001 From: David Date: Sun, 7 Jun 2026 21:24:48 -0500 Subject: [PATCH 09/21] feat: make output dest argument optional --- pyulog/ulog2ros2bag.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pyulog/ulog2ros2bag.py b/pyulog/ulog2ros2bag.py index 8498701..d1d2bba 100644 --- a/pyulog/ulog2ros2bag.py +++ b/pyulog/ulog2ros2bag.py @@ -8,6 +8,7 @@ import argparse import re from os import environ +from pathlib import Path from importlib.metadata import version from rclpy.serialization import serialize_message # pylint: disable=import-error import rosbag2_py # pylint: disable=import-error @@ -26,7 +27,9 @@ def main(): parser = argparse.ArgumentParser(description="Convert ULog to rosbag") parser.add_argument("filename", metavar="file.ulg", help="ULog input file") - parser.add_argument("bag", metavar="rosbag_file", help="rosbag output file") + parser.add_argument( + "-o", "--output", dest="bag", help="rosbag output folder", default=None, + ) parser.add_argument( "-m", @@ -72,7 +75,7 @@ def to_camel_case(snake_str): def convert_ulog2ros2bag( ulog_file_name: str, - rosbag_name: str, + rosbag_name: str | None, messages: str, disable_str_exceptions=False, verbose=False, @@ -98,6 +101,11 @@ def convert_ulog2ros2bag( # ROS2 boilerplate writer = rosbag2_py.SequentialWriter() + # Default rosbag name from ulg file + rosbag_name = rosbag_name or "rosbag_" + re.sub( + r"\.ulg$", "", Path(ulog_file_name).name + ) + storage_options = rosbag2_py.StorageOptions(uri=rosbag_name, storage_id="sqlite3") converter_options = rosbag2_py.ConverterOptions( input_serialization_format="cdr", output_serialization_format="cdr" From 8add357e2418ce05e59fa81fe43ab8f060d7172f Mon Sep 17 00:00:00 2001 From: David Date: Sun, 7 Jun 2026 21:26:57 -0500 Subject: [PATCH 10/21] docs: add ulog2ros2bag to README --- README.md | 27 +++++++++++++++++++++++++++ pyulog/ulog2ros2bag.py | 7 ++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0571289..260f29d 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@ The provided [command line scripts](#scripts) are: - `ulog_params`: extract parameters from an ULog file. - `ulog2csv`: convert ULog to CSV files. - `ulog2kml`: convert ULog to KML files. +- `ulog2rosbag`: convert ULog to rosbag files. +- `ulog2ros2bag`: convert ULog to ROS2 bag files. ## Installation @@ -233,6 +235,31 @@ optional arguments: separated list of names, like 'sensor_combined,vehicle_gps_position' ``` + +### Convert ULog to ROS2 bag files (ulog2ros2bag) + +> **Note** You need a ROS2 environment with the corresponding version of `px4_msgs` built and sourced. + +Usage: +``` +usage: ulog2ros2bag.py [-h] [-o BAG] [-m MESSAGES] [-i] [-v] file.ulg + +Convert ULog to rosbag + +positional arguments: + file.ulg ULog input file + +options: + -h, --help show this help message and exit + -o BAG, --output BAG rosbag output folder + -m MESSAGES, --messages MESSAGES + Only consider given messages. Must be a comma- + separated list of names, like + 'sensor_combined,vehicle_gps_position' + -i, --ignore Ignore string parsing exceptions + -v, --verbose Print extra debugging information +``` + ### Migrate/setup the database for use with the DatabaseULog class (ulog_migratedb) > **Warning** This command must be run whenever the schema changes, otherwise DatabaseULog won't function. diff --git a/pyulog/ulog2ros2bag.py b/pyulog/ulog2ros2bag.py index d1d2bba..7e2ab63 100644 --- a/pyulog/ulog2ros2bag.py +++ b/pyulog/ulog2ros2bag.py @@ -27,8 +27,13 @@ def main(): parser = argparse.ArgumentParser(description="Convert ULog to rosbag") parser.add_argument("filename", metavar="file.ulg", help="ULog input file") + parser.add_argument( - "-o", "--output", dest="bag", help="rosbag output folder", default=None, + "-o", + "--output", + dest="bag", + help="rosbag output folder", + default=None, ) parser.add_argument( From c1cb0dd442fd15d51c1c5ff1169fd2671449535e Mon Sep 17 00:00:00 2001 From: David Date: Sun, 7 Jun 2026 22:11:30 -0500 Subject: [PATCH 11/21] feat: move ros2 imports for QoL --- pyulog/ulog2ros2bag.py | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/pyulog/ulog2ros2bag.py b/pyulog/ulog2ros2bag.py index 7e2ab63..ca9a018 100644 --- a/pyulog/ulog2ros2bag.py +++ b/pyulog/ulog2ros2bag.py @@ -10,9 +10,6 @@ from os import environ from pathlib import Path from importlib.metadata import version -from rclpy.serialization import serialize_message # pylint: disable=import-error -import rosbag2_py # pylint: disable=import-error -from px4_msgs import msg as px4_msgs # pylint: disable=import-error import numpy as np # TODO: temporary for typing @@ -95,6 +92,28 @@ def convert_ulog2ros2bag( :return: No """ + # Wait until now to import ROS2 packages so they don't block the help message. + try: + from rclpy.serialization import ( + serialize_message, + ) # pylint: disable=import-error + import rosbag2_py # pylint: disable=import-error + except ImportError as e: + print( + f"Error: Could not import ROS2 packages. Make sure you have sourced your ROS2 installation." + ) + print("Actual error:", e) + return + + try: + from px4_msgs import msg as px4_msgs # pylint: disable=import-error + except ImportError as e: + print( + f"Error: Could not import px4_msgs. Make sure you have built and sourced the correct version of px4_msgs." + ) + print("Actual error:", e) + return + msg_filter = messages.split(",") if messages else None ulog = ULog(ulog_file_name, msg_filter, disable_str_exceptions) @@ -289,6 +308,8 @@ def rosbag_write_uses_seqnum(): def calc_msgtype(topic_name: str) -> str | None: """Calculate message type from topic name, if possible""" + from px4_msgs import msg as px4_msgs # pylint: disable=import-error + REGEX_EXCEPTIONS = { r"^actuator_controls_status_\d$": "ActuatorControlsStatus", r"^actuator_outputs\w*": "ActuatorOutputs", From 1705b317cacab45f8406e2c50bb780c33b1ca904 Mon Sep 17 00:00:00 2001 From: David Date: Sun, 7 Jun 2026 22:31:14 -0500 Subject: [PATCH 12/21] fix: correctly import ULog --- pyulog/ulog2ros2bag.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyulog/ulog2ros2bag.py b/pyulog/ulog2ros2bag.py index ca9a018..194e40b 100644 --- a/pyulog/ulog2ros2bag.py +++ b/pyulog/ulog2ros2bag.py @@ -12,9 +12,7 @@ from importlib.metadata import version import numpy as np -# TODO: temporary for typing -# from .core import ULog -from pyulog import ULog +from .core import ULog # pylint: disable=too-many-locals, invalid-name From 1c5b207e4d04a7383dd7d735ad1e42f57f4105c8 Mon Sep 17 00:00:00 2001 From: David Date: Sun, 7 Jun 2026 23:00:06 -0500 Subject: [PATCH 13/21] refactor: no re.compile() --- pyulog/ulog2ros2bag.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pyulog/ulog2ros2bag.py b/pyulog/ulog2ros2bag.py index 194e40b..6d4a7a7 100644 --- a/pyulog/ulog2ros2bag.py +++ b/pyulog/ulog2ros2bag.py @@ -207,7 +207,7 @@ def convert_ulog2ros2bag( # Check if it is a composed message type if any( - re.compile(r"(.*?)\.(.*?)").match(field.field_name) + re.match(r"(.*?)\.(.*?)", field.field_name) for field in ulg_topic.field_data ): # TODO: add support for composed message types @@ -244,8 +244,7 @@ def convert_ulog2ros2bag( for i in range(len(ulg_topic.data["timestamp"])): msg = MsgType() for field in ulg_topic.field_data: - array_condition = re.compile(r"(.*?)\[(\d+)\]") - array_match = array_condition.match(field.field_name) + array_match = re.match(r"(.*?)\[(\d+)\]", field.field_name) value = ulg_topic.data[field.field_name][i] if array_match: field_name, array_index = array_match.groups() From d5d1e728d6ac9c7d57323ae105ee2137392312ad Mon Sep 17 00:00:00 2001 From: David Date: Sun, 7 Jun 2026 23:15:56 -0500 Subject: [PATCH 14/21] style: minor cleanup --- pyulog/ulog2ros2bag.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/pyulog/ulog2ros2bag.py b/pyulog/ulog2ros2bag.py index 6d4a7a7..36abf5a 100644 --- a/pyulog/ulog2ros2bag.py +++ b/pyulog/ulog2ros2bag.py @@ -98,7 +98,8 @@ def convert_ulog2ros2bag( import rosbag2_py # pylint: disable=import-error except ImportError as e: print( - f"Error: Could not import ROS2 packages. Make sure you have sourced your ROS2 installation." + "Error: Could not import ROS2 packages. Make sure you have sourced" + " your ROS2 installation." ) print("Actual error:", e) return @@ -107,7 +108,8 @@ def convert_ulog2ros2bag( from px4_msgs import msg as px4_msgs # pylint: disable=import-error except ImportError as e: print( - f"Error: Could not import px4_msgs. Make sure you have built and sourced the correct version of px4_msgs." + "Error: Could not import px4_msgs. Make sure you have built and sourced" + " the correct version of px4_msgs." ) print("Actual error:", e) return @@ -189,7 +191,8 @@ def convert_ulog2ros2bag( if multiids[ulg_topic.name] != {0}: ulg_topic_name += f" ({ulg_topic.multi_id})" print( - f"D: Processing ulog topic {ulg_topic_name} with {topic_message_count} messages and ROS2 topic {ros2_topic}" + f"D: Processing ulog topic {ulg_topic_name} with {topic_message_count}" + f" messages and ROS2 topic {ros2_topic}" ) # Determine ROS2 message type (px4_msgs) @@ -198,7 +201,8 @@ def convert_ulog2ros2bag( MsgType = getattr(px4_msgs, msg_type_name) else: print( - f"W: Message type '{msg_type_name or to_camel_case(ulg_topic.name)}' for {ulg_topic.name} not found in px4_msgs, skipping." + f"W: Message type '{msg_type_name or to_camel_case(ulg_topic.name)}'" + f" for {ulg_topic.name} not found in px4_msgs, skipping." ) continue @@ -222,7 +226,8 @@ def convert_ulog2ros2bag( px4_fields = list(set(px4_fields)) # remove duplicates if not all([px4_field in ros2_fields for px4_field in px4_fields]): print( - f"W: Message type px4_msgs/msg/{MsgType.__name__} does not match topic '{ulg_topic.name}' in ulog, skipping. Please check your version of px4_msgs." + f"W: Message type px4_msgs/msg/{MsgType.__name__} does not match topic" + f" '{ulg_topic.name}' in ulog, skipping. Please check your version of px4_msgs." ) if verbose: for px4_field in [ @@ -261,7 +266,8 @@ def convert_ulog2ros2bag( setattr(msg, field.field_name, value) except Exception as e: print( - f"Exception when setting field '{field.field_name}' from topic '{ulg_topic.name}' with type: {type(value)}" + f"Exception when setting field '{field.field_name}' from" + f" topic '{ulg_topic.name}' with type: {type(value)}" ) raise e @@ -285,10 +291,7 @@ def rosbag_topicmetadata_uses_id(): rosbag2_py PR #1538 adds a parameter to TopicMetadata.__init__() for a topic ID, breaking the interface. This attempts to add compability with both versions. """ - try: - return version("rosbag2_py") >= "0.25.0" - except: - return False # Default: <= Humble + return version("rosbag2_py") >= "0.25.0" def rosbag_write_uses_seqnum(): @@ -297,10 +300,7 @@ def rosbag_write_uses_seqnum(): topic sequence number, breaking the interface. This attempts to add compatibility with both versions. """ - try: - return version("rosbag2_py") >= "0.32.0" - except: - return False # Default: <= Humble + return version("rosbag2_py") >= "0.32.0" def calc_msgtype(topic_name: str) -> str | None: From 7db58fb43547533137c17ec638908cd4f9c67823 Mon Sep 17 00:00:00 2001 From: David Date: Tue, 9 Jun 2026 00:18:16 -0500 Subject: [PATCH 15/21] refactor: move ros2 imports back to the top --- pyulog/ulog2ros2bag.py | 53 +++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/pyulog/ulog2ros2bag.py b/pyulog/ulog2ros2bag.py index 36abf5a..7f50ba6 100644 --- a/pyulog/ulog2ros2bag.py +++ b/pyulog/ulog2ros2bag.py @@ -12,6 +12,29 @@ from importlib.metadata import version import numpy as np +# Do not exit on failing to import ROS2 packages so they don't block the help message. +ros2_available = True +try: + from rclpy.serialization import serialize_message # pylint: disable=import-error + import rosbag2_py # pylint: disable=import-error +except ImportError as e: + print( + "Error: Could not import ROS2 packages. Make sure you have sourced" + " your ROS2 installation." + ) + print("Actual error:", e) + ros2_available = False + +try: + from px4_msgs import msg as px4_msgs # pylint: disable=import-error +except ImportError as e: + print( + "Error: Could not import px4_msgs. Make sure you have built and sourced" + " the correct version of px4_msgs." + ) + print("Actual error:", e) + ros2_available = False + from .core import ULog # pylint: disable=too-many-locals, invalid-name @@ -61,6 +84,9 @@ def main(): args = parser.parse_args() + if not ros2_available: + return # Error messages printed in 'except ImportError' blocks above + convert_ulog2ros2bag( args.filename, args.bag, args.messages, args.ignore, args.verbose ) @@ -90,30 +116,6 @@ def convert_ulog2ros2bag( :return: No """ - # Wait until now to import ROS2 packages so they don't block the help message. - try: - from rclpy.serialization import ( - serialize_message, - ) # pylint: disable=import-error - import rosbag2_py # pylint: disable=import-error - except ImportError as e: - print( - "Error: Could not import ROS2 packages. Make sure you have sourced" - " your ROS2 installation." - ) - print("Actual error:", e) - return - - try: - from px4_msgs import msg as px4_msgs # pylint: disable=import-error - except ImportError as e: - print( - "Error: Could not import px4_msgs. Make sure you have built and sourced" - " the correct version of px4_msgs." - ) - print("Actual error:", e) - return - msg_filter = messages.split(",") if messages else None ulog = ULog(ulog_file_name, msg_filter, disable_str_exceptions) @@ -305,7 +307,6 @@ def rosbag_write_uses_seqnum(): def calc_msgtype(topic_name: str) -> str | None: """Calculate message type from topic name, if possible""" - from px4_msgs import msg as px4_msgs # pylint: disable=import-error REGEX_EXCEPTIONS = { r"^actuator_controls_status_\d$": "ActuatorControlsStatus", @@ -335,7 +336,7 @@ def calc_msgtype(topic_name: str) -> str | None: } direct_name = to_camel_case(topic_name) - if hasattr(px4_msgs, direct_name): + if hasattr(px4_msgs, direct_name): # type: ignore return direct_name result = None From 8649d2a8b36d0814308a59df208388003d4babc6 Mon Sep 17 00:00:00 2001 From: David Date: Fri, 12 Jun 2026 13:01:35 -0500 Subject: [PATCH 16/21] fix: int8 not always bool --- pyulog/ulog2ros2bag.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/pyulog/ulog2ros2bag.py b/pyulog/ulog2ros2bag.py index 7f50ba6..f0ff129 100644 --- a/pyulog/ulog2ros2bag.py +++ b/pyulog/ulog2ros2bag.py @@ -50,7 +50,7 @@ def main(): "-o", "--output", dest="bag", - help="rosbag output folder", + help="rosbag output destination", default=None, ) @@ -256,15 +256,11 @@ def convert_ulog2ros2bag( if array_match: field_name, array_index = array_match.groups() array_index = int(array_index) - if value.dtype == np.int8: - value = bool(value) + value = ros2ify_value(field_name, value, MsgType) getattr(msg, field_name)[array_index] = value else: try: - if value.dtype == np.int8: - value = bool(value) - else: - value = value.item() + value = ros2ify_value(field.field_name, value, MsgType) setattr(msg, field.field_name, value) except Exception as e: print( @@ -348,5 +344,19 @@ def calc_msgtype(topic_name: str) -> str | None: return result +def ros2ify_value(field_name: str, value, MsgType: object): + value_type: np.signedinteger | np.unsignedinteger | np.floating = value.dtype + + # int8 could either be int8_t, bool, or char + # check the destination type + ros2_type: str = MsgType.get_fields_and_field_types()[field_name] # type: ignore + + # startswith() accounts for array types + if value_type == np.int8 and ros2_type.startswith("bool"): + return bool(value) + else: + return value.item() + + if __name__ == "__main__": main() From 64d161058b71de5ae007413cce0bbdaa689f41a2 Mon Sep 17 00:00:00 2001 From: David Date: Fri, 12 Jun 2026 13:18:12 -0500 Subject: [PATCH 17/21] feat!: remove ros1 bag converter --- README.ko-KR.md | 13 +++--- README.md | 23 ----------- pyproject.toml | 1 - pyulog/ulog2ros2bag.py | 5 ++- pyulog/ulog2rosbag.py | 94 ------------------------------------------ 5 files changed, 11 insertions(+), 125 deletions(-) delete mode 100644 pyulog/ulog2rosbag.py diff --git a/README.ko-KR.md b/README.ko-KR.md index 9ec7eab..6f91b69 100644 --- a/README.ko-KR.md +++ b/README.ko-KR.md @@ -9,6 +9,7 @@ - `ulog_params`: ULog 파일에 저장된 파라미터들을 추출합니다. - `ulog2csv`: ULog 파일을 CSV 파일로 변환합니다. - `ulog2kml`: ULog 파일을 KML 파일로 변환합니다. +- `ulog2ros2bag`: ULog 파일을 ros2bag 파일로 변환. ## 설치 @@ -212,26 +213,28 @@ optional arguments: Camera trigger topic name (e.g. camera_capture) ``` -### ULog 파일을 rosbag 파일로 변환 (ulog2rosbag) +### ULog 파일을 ros2bag 파일로 변환 (ulog2ros2bag) -> **Note** `px4_msgs`가 설치된 ROS 환경이 필요합니다. +> **Note** `px4_msgs`가 설치된 ROS2 환경이 필요합니다. 사용: ``` -usage: ulog2rosbag [-h] [-m MESSAGES] file.ulg result.bag +usage: ulog2ros2bag.py [-h] [-o BAG] [-m MESSAGES] [-i] [-v] file.ulg Convert ULog to rosbag positional arguments: file.ulg ULog input file - result.ulg rosbag output file -optional arguments: +options: -h, --help show this help message and exit + -o BAG, --output BAG rosbag output folder -m MESSAGES, --messages MESSAGES Only consider given messages. Must be a comma- separated list of names, like 'sensor_combined,vehicle_gps_position' + -i, --ignore Ignore string parsing exceptions + -v, --verbose Print extra debugging information ``` ### Migrate/setup the database for use with the DatabaseULog class (ulog_migratedb) diff --git a/README.md b/README.md index 260f29d..631c525 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,6 @@ The provided [command line scripts](#scripts) are: - `ulog_params`: extract parameters from an ULog file. - `ulog2csv`: convert ULog to CSV files. - `ulog2kml`: convert ULog to KML files. -- `ulog2rosbag`: convert ULog to rosbag files. - `ulog2ros2bag`: convert ULog to ROS2 bag files. @@ -214,28 +213,6 @@ optional arguments: Camera trigger topic name (e.g. camera_capture) ``` -### Convert ULog to rosbag files (ulog2rosbag) - -> **Note** You need a ROS environment with `px4_msgs` built and sourced. - -Usage: -``` -usage: ulog2rosbag [-h] [-m MESSAGES] file.ulg result.bag - -Convert ULog to rosbag - -positional arguments: - file.ulg ULog input file - result.ulg rosbag output file - -optional arguments: - -h, --help show this help message and exit - -m MESSAGES, --messages MESSAGES - Only consider given messages. Must be a comma- - separated list of names, like - 'sensor_combined,vehicle_gps_position' -``` - ### Convert ULog to ROS2 bag files (ulog2ros2bag) > **Note** You need a ROS2 environment with the corresponding version of `px4_msgs` built and sourced. diff --git a/pyproject.toml b/pyproject.toml index 84961de..a265dcb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,6 @@ ulog_messages = "pyulog.messages:main" ulog_params = "pyulog.params:main" ulog2csv = "pyulog.ulog2csv:main" ulog2kml = "pyulog.ulog2kml:main" -ulog2rosbag = "pyulog.ulog2rosbag:main" ulog2ros2bag = "pyulog.ulog2ros2bag:main" ulog_migratedb = "pyulog.migrate_db:main" diff --git a/pyulog/ulog2ros2bag.py b/pyulog/ulog2ros2bag.py index f0ff129..a5163ad 100644 --- a/pyulog/ulog2ros2bag.py +++ b/pyulog/ulog2ros2bag.py @@ -1,7 +1,7 @@ #! /usr/bin/env python """ -Convert a ULog file into a ROS2 bag. Inspired by ulog2rosbag.py. +Convert a ULog file into a ROS2 bag. """ from collections import defaultdict @@ -35,7 +35,8 @@ print("Actual error:", e) ros2_available = False -from .core import ULog +#from .core import ULog +from pyulog import ULog # pylint: disable=too-many-locals, invalid-name diff --git a/pyulog/ulog2rosbag.py b/pyulog/ulog2rosbag.py deleted file mode 100644 index 03a3ee5..0000000 --- a/pyulog/ulog2rosbag.py +++ /dev/null @@ -1,94 +0,0 @@ -#! /usr/bin/env python - -""" -Convert a ULog file into rosbag file(s) -""" - -from collections import defaultdict -import argparse -import re -import rospy # pylint: disable=import-error -import rosbag # pylint: disable=import-error -from px4_msgs import msg as px4_msgs # pylint: disable=import-error - -from .core import ULog - -#pylint: disable=too-many-locals, invalid-name - -def main(): - """Command line interface""" - - parser = argparse.ArgumentParser(description='Convert ULog to rosbag') - parser.add_argument('filename', metavar='file.ulg', help='ULog input file') - parser.add_argument('bag', metavar='file.bag', help='rosbag output file') - - parser.add_argument( - '-m', '--messages', dest='messages', - help=("Only consider given messages. Must be a comma-separated list of" - " names, like 'sensor_combined,vehicle_gps_position'")) - - parser.add_argument('-i', '--ignore', dest='ignore', action='store_true', - help='Ignore string parsing exceptions', default=False) - - args = parser.parse_args() - - convert_ulog2rosbag(args.filename, args.bag, args.messages, args.ignore) - -# https://stackoverflow.com/questions/19053707/converting-snake-case-to-lower-camel-case-lowercamelcase -def to_camel_case(snake_str): - """ Convert snake case string to camel case """ - components = snake_str.split("_") - return ''.join(x.title() for x in components) - -def convert_ulog2rosbag(ulog_file_name, rosbag_file_name, messages, disable_str_exceptions=False): - """ - Coverts and ULog file to a CSV file. - - :param ulog_file_name: The ULog filename to open and read - :param rosbag_file_name: The rosbag filename to open and write - :param messages: A list of message names - - :return: No - """ - - array_pattern = re.compile(r"(.*?)\[(.*?)\]") - msg_filter = messages.split(',') if messages else None - - ulog = ULog(ulog_file_name, msg_filter, disable_str_exceptions) - data = ulog.data_list - - multiids = defaultdict(set) - for d in data: - multiids[d.name].add(d.multi_id) - - with rosbag.Bag(rosbag_file_name, 'w') as bag: - items = [] - for d in data: - if multiids[d.name] == {0}: - topic = "/px4/{}".format(d.name) - else: - topic = "/px4/{}_{}".format(d.name, d.multi_id) - msg_type = getattr(px4_msgs, to_camel_case(d.name)) - - for i in range(len(d.data['timestamp'])): - msg = msg_type() - for f in d.field_data: - result = array_pattern.match(f.field_name) - value = d.data[f.field_name][i] - if result: - field, array_index = result.groups() - array_index = int(array_index) - if isinstance(getattr(msg, field), bytes): - attr = bytearray(getattr(msg, field)) - attr[array_index] = value - setattr(msg, field, bytes(attr)) - else: - getattr(msg, field)[array_index] = value - else: - setattr(msg, f.field_name, value) - ts = rospy.Time(nsecs=d.data['timestamp'][i]*1000) - items.append((topic, msg, ts)) - items.sort(key=lambda x: x[2]) - for topic, msg, ts in items: - bag.write(topic, msg, ts) - From 08268c92273bbf784cb37c4baebf33711ed7a729 Mon Sep 17 00:00:00 2001 From: David Date: Fri, 12 Jun 2026 13:34:53 -0500 Subject: [PATCH 18/21] refactor: make pylint happier --- pyulog/ulog2ros2bag.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/pyulog/ulog2ros2bag.py b/pyulog/ulog2ros2bag.py index a5163ad..92be35f 100644 --- a/pyulog/ulog2ros2bag.py +++ b/pyulog/ulog2ros2bag.py @@ -12,6 +12,9 @@ from importlib.metadata import version import numpy as np +from .core import ULog +# from pyulog import ULog + # Do not exit on failing to import ROS2 packages so they don't block the help message. ros2_available = True try: @@ -35,9 +38,6 @@ print("Actual error:", e) ros2_available = False -#from .core import ULog -from pyulog import ULog - # pylint: disable=too-many-locals, invalid-name @@ -168,10 +168,10 @@ def convert_ulog2ros2bag( try: from ament_index_python.packages import ( get_package_share_directory, - ) # pylint: disable=import-error + ) # pylint: disable=import-error, import-outside-toplevel px4_msgs_share_dir = get_package_share_directory("px4_msgs") - except: + except: # pylint: disable=bare-except pass print(f"I: ROS2 px4_msgs: {px4_msgs_share_dir or 'not found'}") print( @@ -309,8 +309,8 @@ def calc_msgtype(topic_name: str) -> str | None: r"^actuator_controls_status_\d$": "ActuatorControlsStatus", r"^actuator_outputs\w*": "ActuatorOutputs", r"^config_overrides\w*": "ConfigOverrides", - r"^estimator_aid_src_((gnss_yaw)|(ev_yaw)|(fake_hgt)|(airspeed)|(sideslip)|(\w+_hgt))$": "EstimatorAidSource1d", - r"^estimator_aid_src_((drag)|(aux_vel)|(optical_flow)|(\w+_pos)|(aux_global_position))$": "EstimatorAidSource2d", + r"^estimator_aid_src_((gnss_yaw)|(ev_yaw)|(fake_hgt)|(airspeed)|(sideslip)|(\w+_hgt))$": "EstimatorAidSource1d", # pylint: disable=line-too-long + r"^estimator_aid_src_((drag)|(aux_vel)|(optical_flow)|(\w+_pos)|(aux_global_position))$": "EstimatorAidSource2d", # pylint: disable=line-too-long r"^estimator_aid_src_((ev_vel)|(gnss_vel)|(gravity)|(mag))$": "EstimatorAidSource3d", r"^estimator_ev_pos_bias": "EstimatorBias3d", r"^estimator_((baro)|(gnss_hgt))_bias": "EstimatorBias", @@ -323,7 +323,7 @@ def calc_msgtype(topic_name: str) -> str | None: r"^\w+?attitude_setpoint$": "VehicleAttitudeSetpoint", r"^\w+?_command\w*": "VehicleCommand", r"^\w+?_control_\w+": "VehicleControlMode", - r"^((aux_)|(estimator_)|(vehicle_)|(external_ins_))global_position(_groundtruth)?$": "VehicleGlobalPosition", + r"^((aux_)|(estimator_)|(vehicle_)|(external_ins_))global_position(_groundtruth)?$": "VehicleGlobalPosition", # pylint: disable=line-too-long r"^\w+?_local_position(_groundtruth)?$": "VehicleLocalPosition", r"^\w+?_odometry$": "VehicleOdometry", r"^((estimator)|(vehicle))_optical_flow_vel$": "VehicleOpticalFlowVel", @@ -346,6 +346,7 @@ def calc_msgtype(topic_name: str) -> str | None: def ros2ify_value(field_name: str, value, MsgType: object): + """Translate a pyulog numpy value to ros2 plain python value""" value_type: np.signedinteger | np.unsignedinteger | np.floating = value.dtype # int8 could either be int8_t, bool, or char @@ -355,8 +356,8 @@ def ros2ify_value(field_name: str, value, MsgType: object): # startswith() accounts for array types if value_type == np.int8 and ros2_type.startswith("bool"): return bool(value) - else: - return value.item() + + return value.item() if __name__ == "__main__": From f5783ade4fd430711fcdadaf403ca9e3f19de206 Mon Sep 17 00:00:00 2001 From: David Date: Fri, 12 Jun 2026 14:01:44 -0500 Subject: [PATCH 19/21] fix: resolve all pylint warnings except lambda and branches --- pyulog/ulog2ros2bag.py | 60 +++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 27 deletions(-) diff --git a/pyulog/ulog2ros2bag.py b/pyulog/ulog2ros2bag.py index 92be35f..179b8a9 100644 --- a/pyulog/ulog2ros2bag.py +++ b/pyulog/ulog2ros2bag.py @@ -13,7 +13,6 @@ import numpy as np from .core import ULog -# from pyulog import ULog # Do not exit on failing to import ROS2 packages so they don't block the help message. ros2_available = True @@ -166,9 +165,9 @@ def convert_ulog2ros2bag( print(f"I: ROS2 Distro: {environ.get('ROS_DISTRO') or 'not detected'}") px4_msgs_share_dir = None try: - from ament_index_python.packages import ( - get_package_share_directory, - ) # pylint: disable=import-error, import-outside-toplevel + # fmt: off + from ament_index_python.packages import get_package_share_directory # pylint: disable=import-error, import-outside-toplevel + # fmt: on px4_msgs_share_dir = get_package_share_directory("px4_msgs") except: # pylint: disable=bare-except @@ -200,15 +199,15 @@ def convert_ulog2ros2bag( # Determine ROS2 message type (px4_msgs) msg_type_name = calc_msgtype(ulg_topic.name) - if msg_type_name is not None and hasattr(px4_msgs, msg_type_name): - MsgType = getattr(px4_msgs, msg_type_name) - else: + if msg_type_name is None or not hasattr(px4_msgs, msg_type_name): print( f"W: Message type '{msg_type_name or to_camel_case(ulg_topic.name)}'" f" for {ulg_topic.name} not found in px4_msgs, skipping." ) continue + MsgType = getattr(px4_msgs, msg_type_name) + if verbose: print(f"D: Found ROS2 message type px4_msgs/msg/{MsgType.__name__}") @@ -227,7 +226,7 @@ def convert_ulog2ros2bag( re.sub(r"\[\d+\]$", "", data.field_name) for data in ulg_topic.field_data ] # strip array indices px4_fields = list(set(px4_fields)) # remove duplicates - if not all([px4_field in ros2_fields for px4_field in px4_fields]): + if not all(px4_field in ros2_fields for px4_field in px4_fields): print( f"W: Message type px4_msgs/msg/{MsgType.__name__} does not match topic" f" '{ulg_topic.name}' in ulog, skipping. Please check your version of px4_msgs." @@ -250,25 +249,7 @@ def convert_ulog2ros2bag( # Write each message for i in range(len(ulg_topic.data["timestamp"])): - msg = MsgType() - for field in ulg_topic.field_data: - array_match = re.match(r"(.*?)\[(\d+)\]", field.field_name) - value = ulg_topic.data[field.field_name][i] - if array_match: - field_name, array_index = array_match.groups() - array_index = int(array_index) - value = ros2ify_value(field_name, value, MsgType) - getattr(msg, field_name)[array_index] = value - else: - try: - value = ros2ify_value(field.field_name, value, MsgType) - setattr(msg, field.field_name, value) - except Exception as e: - print( - f"Exception when setting field '{field.field_name}' from" - f" topic '{ulg_topic.name}' with type: {type(value)}" - ) - raise e + msg = ros2_msg_from_ulg_topic(ulg_topic, i, MsgType) ts = ulg_topic.data["timestamp"][i] * 1000 # us -> ns rosbag_write( @@ -360,5 +341,30 @@ def ros2ify_value(field_name: str, value, MsgType: object): return value.item() +def ros2_msg_from_ulg_topic(ulg_topic: ULog.Data, i: int, MsgType): + """Create a ROS2 message object from a ulg topic message""" + msg = MsgType() + for field in ulg_topic.field_data: + array_match = re.match(r"(.*?)\[(\d+)\]", field.field_name) + value = ulg_topic.data[field.field_name][i] + if array_match: + field_name, array_index = array_match.groups() + array_index = int(array_index) + value = ros2ify_value(field_name, value, MsgType) + getattr(msg, field_name)[array_index] = value + else: + try: + value = ros2ify_value(field.field_name, value, MsgType) + setattr(msg, field.field_name, value) + except Exception as e: + print( + f"Exception when setting field '{field.field_name}' from" + f" topic '{ulg_topic.name}' with type: {type(value)}" + ) + raise e + + return msg + + if __name__ == "__main__": main() From 62adfc3e62d8861d1639362f2c9c30da621658aa Mon Sep 17 00:00:00 2001 From: David Date: Mon, 15 Jun 2026 10:52:32 -0500 Subject: [PATCH 20/21] fix: resolve last pylint warnings --- pyulog/ulog2ros2bag.py | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/pyulog/ulog2ros2bag.py b/pyulog/ulog2ros2bag.py index 179b8a9..883134d 100644 --- a/pyulog/ulog2ros2bag.py +++ b/pyulog/ulog2ros2bag.py @@ -14,6 +14,8 @@ from .core import ULog +# pylint: disable=too-many-locals, invalid-name, too-many-branches + # Do not exit on failing to import ROS2 packages so they don't block the help message. ros2_available = True try: @@ -37,8 +39,6 @@ print("Actual error:", e) ros2_available = False -# pylint: disable=too-many-locals, invalid-name - def main(): """Command line interface""" @@ -140,26 +140,28 @@ def convert_ulog2ros2bag( # Support rosbag2_py topic sequence number in ROS2 versions greater than Humble if rosbag_write_uses_seqnum(): - rosbag_write = lambda topic, msg, timestamp: writer.write( - topic, msg, timestamp, 0 - ) + def rosbag_write(topic, msg, timestamp): + writer.write(topic, msg, timestamp, 0) else: - rosbag_write = lambda topic, msg, timestamp: writer.write(topic, msg, timestamp) + def rosbag_write(topic, msg, timestamp): + writer.write(topic, msg, timestamp) # Support rosbag2_py TopicMetadata ID in ROS2 versions greater than Humble if rosbag_topicmetadata_uses_id(): - topic_metadata = lambda name, type: rosbag2_py.TopicMetadata( - 0, - name=name, # type: ignore - type=type, - serialization_format="cdr", - ) + def topic_metadata(name, topic_type): + return rosbag2_py.TopicMetadata( + 0, + name=name, # type: ignore + type=topic_type, + serialization_format="cdr", + ) else: - topic_metadata = lambda name, type: rosbag2_py.TopicMetadata( - name=name, - type=type, - serialization_format="cdr", - ) + def topic_metadata(name, topic_type): + return rosbag2_py.TopicMetadata( + name=name, # type: ignore + type=topic_type, + serialization_format="cdr", + ) if verbose: print(f"I: ROS2 Distro: {environ.get('ROS_DISTRO') or 'not detected'}") From 87945e330987d8321a283e44dbb31849e1bc4365 Mon Sep 17 00:00:00 2001 From: David Date: Mon, 15 Jun 2026 11:54:22 -0500 Subject: [PATCH 21/21] feat: add support for mcap --- pyulog/ulog2ros2bag.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/pyulog/ulog2ros2bag.py b/pyulog/ulog2ros2bag.py index 883134d..08f3183 100644 --- a/pyulog/ulog2ros2bag.py +++ b/pyulog/ulog2ros2bag.py @@ -54,6 +54,14 @@ def main(): default=None, ) + parser.add_argument( + "--mcap", + dest="mcap", + action="store_true", + help="Use mcap storage in rosbag instead of sqlite3", + default=False, + ) + parser.add_argument( "-m", "--messages", @@ -88,7 +96,7 @@ def main(): return # Error messages printed in 'except ImportError' blocks above convert_ulog2ros2bag( - args.filename, args.bag, args.messages, args.ignore, args.verbose + args.filename, args.bag, args.mcap, args.messages, args.ignore, args.verbose ) @@ -102,6 +110,7 @@ def to_camel_case(snake_str): def convert_ulog2ros2bag( ulog_file_name: str, rosbag_name: str | None, + mcap: bool, messages: str, disable_str_exceptions=False, verbose=False, @@ -132,7 +141,9 @@ def convert_ulog2ros2bag( r"\.ulg$", "", Path(ulog_file_name).name ) - storage_options = rosbag2_py.StorageOptions(uri=rosbag_name, storage_id="sqlite3") + storage_options = rosbag2_py.StorageOptions( + uri=rosbag_name, storage_id="mcap" if mcap else "sqlite3" + ) converter_options = rosbag2_py.ConverterOptions( input_serialization_format="cdr", output_serialization_format="cdr" ) @@ -140,14 +151,18 @@ def convert_ulog2ros2bag( # Support rosbag2_py topic sequence number in ROS2 versions greater than Humble if rosbag_write_uses_seqnum(): + def rosbag_write(topic, msg, timestamp): writer.write(topic, msg, timestamp, 0) + else: + def rosbag_write(topic, msg, timestamp): writer.write(topic, msg, timestamp) # Support rosbag2_py TopicMetadata ID in ROS2 versions greater than Humble if rosbag_topicmetadata_uses_id(): + def topic_metadata(name, topic_type): return rosbag2_py.TopicMetadata( 0, @@ -155,7 +170,9 @@ def topic_metadata(name, topic_type): type=topic_type, serialization_format="cdr", ) + else: + def topic_metadata(name, topic_type): return rosbag2_py.TopicMetadata( name=name, # type: ignore