From 95d942a113741c55ffe84228b829674088be0f7a Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Wed, 18 Mar 2026 17:44:20 -0400 Subject: [PATCH 01/64] Move static methods to new Convert class to avoid circular imports and make them more widely available and testable. --- examples/example_cdi_access.py | 3 +- openlcb/convert.py | 166 +++++++++++++++++++++++++++++++++ openlcb/memoryservice.py | 157 +++---------------------------- tests/test_convert.py | 104 +++++++++++++++++++++ tests/test_memoryservice.py | 97 +------------------ 5 files changed, 285 insertions(+), 242 deletions(-) create mode 100644 openlcb/convert.py create mode 100644 tests/test_convert.py diff --git a/examples/example_cdi_access.py b/examples/example_cdi_access.py index d87130de..a931f0ae 100644 --- a/examples/example_cdi_access.py +++ b/examples/example_cdi_access.py @@ -21,6 +21,7 @@ from examples_settings import Settings # do 1st to fix path if no pip install from openlcb import precise_sleep +from openlcb.convert import Convert from openlcb.xmldataprocessor import attrs_to_dict from openlcb.tcplink.tcpsocket import TcpSocket settings = Settings() @@ -137,7 +138,7 @@ def memoryReadSuccess(memo): resultingCDI += memo.data logger.debug( f"[{memo.address}] successful read" - f" {MemoryService.arrayToString(memo.data, len(memo.data))}" + f" {Convert.arrayToString(memo.data, len(memo.data))}" "; next = address + 64") # update the address memo.address = memo.address+64 diff --git a/openlcb/convert.py b/openlcb/convert.py new file mode 100644 index 00000000..9f205eb6 --- /dev/null +++ b/openlcb/convert.py @@ -0,0 +1,166 @@ +''' +based on part of MemoryService.swift + +Created by Bob Jacobsen on 6/1/22. + +These parts moved to a separate class so callers of static methods don't +depend on MemoryService(DatagramService). + +''' + +from logging import getLogger +from typing import ( + List, # in case list doesn't support `[` in this Python version + Union, # in case `|` doesn't support 'type' in this Python version +) + +logger = getLogger(__name__) + + +class Convert: + + @staticmethod + def spaceDecode(space): + """Convert from a space number to either + False and command byte or True and standard memory space + + Args: + space (int): Encoded memory space identifier, where values: + - 0xFF to 0xFD are special spaces, and only the least significant + 2 bits are relevant. + - 0x00 to 0xFC represent standard memory spaces directly. + + Returns: + tuple(bool, byte): (False, 1-3 for in command byte) : + spaces 0xFF - 0xFD + or (True, space number) : spaces 0 - 0xFC + (NOTE: type of space may affect type of output) + """ + # TODO: Maybe check type of space & raise TypeError if not + # something valid, whether byte, int, or what is ok [add + # more _description_ to space in docstring]. + if space >= 0xFD: + return (False, space & 0x03) + return (True, space) + + @staticmethod + def arrayToInt(data: Union[bytes, bytearray, List[int]]) -> int: + """Convert an array in MSB-first order to an integer + + Args: + data (Union[bytes,bytearray,list[int]]): MSB-first order + encoded 32-bit int + + Returns: + int: The converted data as a number. + """ + result = 0 + for index in range(0, len(data)): + result = result << 8 + result = result | data[index] + return result + + @staticmethod + def arrayToUInt64(data): + """Parse a MSB-first order 64-bit integer + (Python auto-sizes int, so this is same as arrayToInt). + """ + return Convert.arrayToInt(data) + + @staticmethod + def arrayToString(data, length): + """Decode utf-8 bytes to string + up to the 1st zero byte or given length, + whichever is fewer characters. + + Args: + data (Union[bytearray, bytes]): A string encoded as bytes. + length (int): The used length the data. + + Returns: + str: Data decoded as text. + """ + if not isinstance(data, bytearray): + raise TypeError("Expected bytearray (formerly list[int]), got {}" + .format(type(data).__name__)) + zeroIndex = len(data) + try: + temp = data.index(0) + zeroIndex = temp + except KeyboardInterrupt: + raise + except: + pass + + byteCount = min(zeroIndex, length) + + if byteCount == 0: + return "" + + result = data[:byteCount].decode('utf-8') + return result + + @staticmethod + def intToArray(value, length): + """Convert an integer into an array of given length + + Args: + value (int): any value + length (int): Byte count (1, 2, 4, or 8). + + Returns: + bytearray: The value encoded in big-endian format. + """ + if value >= (1 << (length * 8)): # TODO: ? also exclude value < 0 ? + raise ValueError("Value {} cannot fit in {} bytes." + .format(value, length)) + if length == 1: + return bytearray([ + (value & 0xff) + ]) + if length == 2: + return bytearray([ + ((value >> 8) & 0xff), (value & 0xff) + ]) + if length == 4: + return bytearray([ + ((value >> 24) & 0xff), ((value >> 16) & 0xff), + ((value >> 8) & 0xff), (value & 0xff) + ]) + if length == 8: + return bytearray([ + ((value >> 56) & 0xff), ((value >> 48) & 0xff), + ((value >> 40) & 0xff), ((value >> 32) & 0xff), + ((value >> 24) & 0xff), ((value >> 16) & 0xff), + ((value >> 8) & 0xff), (value & 0xff) + ]) + logger.error("integer length {} is not implemented.".format(length)) + return bytearray() + + @staticmethod + def uInt64ToArray(value, length): + '''Convert a 64-bit integer into an array of given length + (Python auto-sizes int, so this is same as intToArray) + ''' + return Convert.intToArray(value, length) + + @staticmethod + def stringToArray(value, length): + '''Converts a string to an array of given length + padding with 0 bytes as needed + ''' + strToUInt8 = value.encode('utf-8') + byteCount = min(length, len(strToUInt8)) + # convert to bytearray since bytes is immutable: + contentPart = bytearray(strToUInt8[:byteCount]) + if len(contentPart) >= length: + if len(contentPart) > length: + logger.warning( + "MemoryService stringToArray: len(value)=={}" + " exceeds length {}".format(len(value), length)) + # TODO: Truncate (or is any length ok for the caller)? + return contentPart + # list[int] is compatible bytearray extend but not `+` so cast + # to bytearray after getting list[int] of remaining length: + padding = bytearray([0] * (length-len(contentPart))) + return contentPart + padding diff --git a/openlcb/memoryservice.py b/openlcb/memoryservice.py index f683d188..c61e51ba 100644 --- a/openlcb/memoryservice.py +++ b/openlcb/memoryservice.py @@ -35,6 +35,7 @@ DatagramWriteMemo, DatagramService, ) +from openlcb.convert import Convert logger = getLogger(__name__) @@ -45,15 +46,24 @@ class MemorySpace(Enum): uses this to track what data type and format is to be assumed in a received Message. It is assumed to have the same space as the request (MemoryReadMemo). + - A datagram's `space` attribute's type should be `int` not + MemorySpace, because CDI specifies variables' space arbitrarily. Attributes: Uninitialized: No data (memory read request response) is expected. CDI: The data expected from the memory read is CDI XML. FDI: The data expected from the memory read is FDI XML. + All: All memory of the device, where all is defined by its designer + (See OpenLCB Memory Configuration Standard 4.2). + Configuration: A writeable basic configuration space, with + the structure of the 32-bit space defined by the designer + (See OpenLCB Memory Configuration Standard 4.2). """ Uninitialized = -1 CDI = 0xFF # decodes to 0x03 FDI = 0xFA + All = 0xFE + Configuration = 0xFD class MemoryReadMemo: @@ -146,29 +156,6 @@ def __init__(self, service: DatagramService): self.datagramReceivedListener ) - def spaceDecode(self, space): - """Convert from a space number to either - False and command byte or True and standard memory space - - Args: - space (int): Encoded memory space identifier, where values: - - 0xFF to 0xFD are special spaces, and only the least significant - 2 bits are relevant. - - 0x00 to 0xFC represent standard memory spaces directly. - - Returns: - tuple(bool, byte): (False, 1-3 for in command byte) : - spaces 0xFF - 0xFD - or (True, space number) : spaces 0 - 0xFC - (NOTE: type of space may affect type of output) - """ - # TODO: Maybe check type of space & raise TypeError if not - # something valid, whether byte, int, or what is ok [add - # more _description_ to space in docstring]. - if space >= 0xFD: - return (False, space & 0x03) - return (True, space) - def requestMemoryRead(self, memo): # type: (MemoryReadMemo) -> None '''Request a read operation start. @@ -196,7 +183,7 @@ def requestMemoryReadNext(self, memo): """ byte6 = False flag = 0 - (byte6, flag) = self.spaceDecode(memo.space) + (byte6, flag) = Convert.spaceDecode(memo.space) spaceFlag = 0x40 if byte6 else (flag | 0x40) addr2 = ((memo.address >> 24) & 0xFF) addr3 = ((memo.address >> 16) & 0xFF) @@ -325,7 +312,7 @@ def requestMemoryWrite(self, memo: MemoryWriteMemo): # create & send a write datagram byte6 = False flag = 0 - (byte6, flag) = self.spaceDecode(memo.space) + (byte6, flag) = Convert.spaceDecode(memo.space) spaceFlag = 0x00 if byte6 else (flag | 0x00) addr2 = ((memo.address >> 24) & 0xFF) addr3 = ((memo.address >> 16) & 0xFF) @@ -372,123 +359,3 @@ def requestSpaceLength(self, space, nodeID, callback): ]) ) self.service.sendDatagram(dgReqMemo) - - def arrayToInt(self, data: Union[bytes, bytearray, List[int]]) -> int: - """Convert an array in MSB-first order to an integer - - Args: - data (Union[bytes,bytearray,list[int]]): MSB-first order - encoded 32-bit int - - Returns: - int: The converted data as a number. - """ - result = 0 - for index in range(0, len(data)): - result = result << 8 - result = result | data[index] - return result - - def arrayToUInt64(self, data): - """Parse a MSB-first order 64-bit integer - (Python auto-sizes int, so this is same as arrayToInt). - """ - return self.arrayToInt(data) - - @staticmethod - def arrayToString(data, length): - """Decode utf-8 bytes to string - up to the 1st zero byte or given length, - whichever is fewer characters. - - Args: - data (Union[bytearray, bytes]): A string encoded as bytes. - length (int): The used length the data. - - Returns: - str: Data decoded as text. - """ - if not isinstance(data, bytearray): - raise TypeError("Expected bytearray (formerly list[int]), got {}" - .format(type(data).__name__)) - zeroIndex = len(data) - try: - temp = data.index(0) - zeroIndex = temp - except KeyboardInterrupt: - raise - except: - pass - - byteCount = min(zeroIndex, length) - - if byteCount == 0: - return "" - - result = data[:byteCount].decode('utf-8') - return result - - @staticmethod - def intToArray(value, length): - """Convert an integer into an array of given length - - Args: - value (int): any value - length (int): Byte count (1, 2, 4, or 8). - - Returns: - bytearray: The value encoded in big-endian format. - """ - if value >= (1 << (length * 8)): # TODO: ? also exclude value < 0 ? - raise ValueError("Value {} cannot fit in {} bytes." - .format(value, length)) - if length == 1: - return bytearray([ - (value & 0xff) - ]) - if length == 2: - return bytearray([ - ((value >> 8) & 0xff), (value & 0xff) - ]) - if length == 4: - return bytearray([ - ((value >> 24) & 0xff), ((value >> 16) & 0xff), - ((value >> 8) & 0xff), (value & 0xff) - ]) - if length == 8: - return bytearray([ - ((value >> 56) & 0xff), ((value >> 48) & 0xff), - ((value >> 40) & 0xff), ((value >> 32) & 0xff), - ((value >> 24) & 0xff), ((value >> 16) & 0xff), - ((value >> 8) & 0xff), (value & 0xff) - ]) - logger.error("integer length {} is not implemented.".format(length)) - return bytearray() - - @staticmethod - def uInt64ToArray(value, length): - '''Convert a 64-bit integer into an array of given length - (Python auto-sizes int, so this is same as intToArray) - ''' - return MemoryService.intToArray(value, length) - - @staticmethod - def stringToArray(value, length): - '''Converts a string to an array of given length - padding with 0 bytes as needed - ''' - strToUInt8 = value.encode('utf-8') - byteCount = min(length, len(strToUInt8)) - # convert to bytearray since bytes is immutable: - contentPart = bytearray(strToUInt8[:byteCount]) - if len(contentPart) >= length: - if len(contentPart) > length: - logger.warning( - "MemoryService stringToArray: len(value)=={}" - " exceeds length {}".format(len(value), length)) - # TODO: Truncate (or is any length ok for the caller)? - return contentPart - # list[int] is compatible bytearray extend but not `+` so cast - # to bytearray after getting list[int] of remaining length: - padding = bytearray([0] * (length-len(contentPart))) - return contentPart + padding diff --git a/tests/test_convert.py b/tests/test_convert.py new file mode 100644 index 00000000..e89b019d --- /dev/null +++ b/tests/test_convert.py @@ -0,0 +1,104 @@ + +import struct +import unittest + +from openlcb.convert import Convert + + +class TestConvertClass(unittest.TestCase): + + def testReturnCyrillicStrings(self): + # See also testReturnCyrillicStrings in test_snip + # If you have characters specific to UTF-8 (either in code or comment) + # add the following as the 1st or 2nd line of the py file: + # -*- coding: utf-8 -*- + data = bytearray([0xd0, 0x94, 0xd0, 0xbc, 0xd0, 0xb8, 0xd1, 0x82, 0xd1, 0x80, 0xd0, 0xb8, 0xd0, 0xb9]) # Cyrillic spelling of the name Dmitry (7 characters becomes 14 bytes) # noqa: E501 + self.assertEqual(Convert.arrayToString(data, len(data)), "Дмитрий") # Cyrillic spelling of the name Dmitry. This string should appear as 7 Cyrillic characters like Cyrillic-demo-Dmitry.png in doc (14 bytes in a hex editor), otherwise your editor does not support utf-8 and editing this file with it could break it. # noqa:E501 + # TODO: Russian version is Дми́трий according to . See Cyrillic-demo-Dmitry-Russian.png in doc. # noqa:E501 + + def testArrayToString(self): + sut = Convert.arrayToString(bytearray([0x41, 0x42, 0x43, 0x44]), 4) # noqa:E501 + self.assertEqual(sut, "ABCD") + + sut = Convert.arrayToString(bytearray([0x41, 0x42, 0, 0x44]), 4) + self.assertEqual(sut, "AB") + + sut = Convert.arrayToString(bytearray([0x41, 0x42, 0x43, 0x44]), 2) # noqa:E501 + self.assertEqual(sut, "AB") + + sut = Convert.arrayToString(bytearray([0x41, 0x42, 0x43, 0]), 4) + self.assertEqual(sut, "ABC") + + sut = Convert.arrayToString(bytearray([0x41, 0x42, 0x31, 0x32]), 8) # noqa:E501 + self.assertEqual(sut, "AB12") + + def testStringToArray(self): + aut = Convert.stringToArray("ABCD", 4) + self.assertEqual(aut, bytearray([0x41, 0x42, 0x43, 0x44])) + + aut = Convert.stringToArray("ABCD", 2) + self.assertEqual(aut, bytearray([0x41, 0x42])) + + aut = Convert.stringToArray("ABCD", 6) + self.assertEqual(aut, bytearray([0x41, 0x42, 0x43, 0x44, 0x00, 0x00])) + + def testIntToArray(self): + test_metas = [ + { + 'value': 65536, # not a short (1 over max) + 'length': 8, + # good_bytes: b'\x00\x00\x00\x00\x00\x01\x00\x00' + }, + { + 'value': 65536, + 'length': 4, + # good_bytes: b'\x00\x01\x00\x00', + }, + { + 'value': 281470681743360, # 65535 << 32 + 'length': 8, + # 'good_bytes': b'\x00\x00\xff\xff\x00\x00\x00\x00', + } + ] + for test_meta in test_metas: + value = test_meta['value'] + length = test_meta['length'] + good_bytes = struct.pack(">{}s".format(length), + value.to_bytes(length, 'big')) + self.assertEqual(Convert.intToArray(value, length), + good_bytes) + + def testIntToArrayFail(self): + test_metas = [ + { + 'value': 65536, # not a short (1 over max) + 'length': 2, + # good_bytes: b'\x00\x00\x00\x00\x00\x01\x00\x00' + }, + { + 'value': 281470681743360, # 65535 << 32 + 'length': 4, + # 'good_bytes': b'\x00\x00\xff\xff\x00\x00\x00\x00', + } + ] + for test_meta in test_metas: + value = test_meta['value'] + length = test_meta['length'] + with self.assertRaises(ValueError): + Convert.intToArray(value, length) + + def testSpaceDecode(self): + byte6 = False + space = 0x00 + + (byte6, space) = Convert.spaceDecode(0xF8) + self.assertEqual(space, 0xF8) + self.assertTrue(byte6) + + (byte6, space) = Convert.spaceDecode(0xFF) + self.assertEqual(space, 0x03) + self.assertFalse(byte6) + + (byte6, space) = Convert.spaceDecode(0xFD) + self.assertEqual(space, 0x01) + self.assertFalse(byte6) diff --git a/tests/test_memoryservice.py b/tests/test_memoryservice.py index 07052143..8033d20a 100644 --- a/tests/test_memoryservice.py +++ b/tests/test_memoryservice.py @@ -6,6 +6,7 @@ from logging import getLogger +from openlcb.convert import Convert from openlcb.physicallayer import PhysicalLayer if __name__ == "__main__": logger = getLogger(__file__) @@ -80,15 +81,6 @@ def setUp(self): ) self.mService = MemoryService(self.dService) - def testReturnCyrillicStrings(self): - # See also testReturnCyrillicStrings in test_snip - # If you have characters specific to UTF-8 (either in code or comment) - # add the following as the 1st or 2nd line of the py file: - # -*- coding: utf-8 -*- - data = bytearray([0xd0, 0x94, 0xd0, 0xbc, 0xd0, 0xb8, 0xd1, 0x82, 0xd1, 0x80, 0xd0, 0xb8, 0xd0, 0xb9]) # Cyrillic spelling of the name Dmitry (7 characters becomes 14 bytes) # noqa: E501 - self.assertEqual(self.mService.arrayToString(data, len(data)), "Дмитрий") # Cyrillic spelling of the name Dmitry. This string should appear as 7 Cyrillic characters like Cyrillic-demo-Dmitry.png in doc (14 bytes in a hex editor), otherwise your editor does not support utf-8 and editing this file with it could break it. # noqa:E501 - # TODO: Russian version is Дми́трий according to . See Cyrillic-demo-Dmitry-Russian.png in doc. # noqa:E501 - def testSingleRead(self): memMemo = MemoryReadMemo(NodeID(123), 64, 0xFD, 0, self.callbackR, self.callbackR) @@ -191,93 +183,6 @@ def testMultipleRead(self): self.assertEqual(len(LinkMockLayer.sentMessages), 5) # read reply datagram reply sent and next datagram sent # noqa: E501 self.assertEqual(len(self.returnedMemoryReadMemo), 2) # memory read returned # noqa: E501 - def testArrayToString(self): - sut = MemoryService.arrayToString(bytearray([0x41, 0x42, 0x43, 0x44]), 4) # noqa:E501 - self.assertEqual(sut, "ABCD") - - sut = MemoryService.arrayToString(bytearray([0x41, 0x42, 0, 0x44]), 4) - self.assertEqual(sut, "AB") - - sut = MemoryService.arrayToString(bytearray([0x41, 0x42, 0x43, 0x44]), 2) # noqa:E501 - self.assertEqual(sut, "AB") - - sut = MemoryService.arrayToString(bytearray([0x41, 0x42, 0x43, 0]), 4) - self.assertEqual(sut, "ABC") - - sut = MemoryService.arrayToString(bytearray([0x41, 0x42, 0x31, 0x32]), 8) # noqa:E501 - self.assertEqual(sut, "AB12") - - def testStringToArray(self): - aut = MemoryService.stringToArray("ABCD", 4) - self.assertEqual(aut, bytearray([0x41, 0x42, 0x43, 0x44])) - - aut = MemoryService.stringToArray("ABCD", 2) - self.assertEqual(aut, bytearray([0x41, 0x42])) - - aut = MemoryService.stringToArray("ABCD", 6) - self.assertEqual(aut, bytearray([0x41, 0x42, 0x43, 0x44, 0x00, 0x00])) - - def testIntToArray(self): - test_metas = [ - { - 'value': 65536, # not a short (1 over max) - 'length': 8, - # good_bytes: b'\x00\x00\x00\x00\x00\x01\x00\x00' - }, - { - 'value': 65536, - 'length': 4, - # good_bytes: b'\x00\x01\x00\x00', - }, - { - 'value': 281470681743360, # 65535 << 32 - 'length': 8, - # 'good_bytes': b'\x00\x00\xff\xff\x00\x00\x00\x00', - } - ] - for test_meta in test_metas: - value = test_meta['value'] - length = test_meta['length'] - good_bytes = struct.pack(">{}s".format(length), - value.to_bytes(length, 'big')) - self.assertEqual(MemoryService.intToArray(value, length), - good_bytes) - - def testIntToArrayFail(self): - test_metas = [ - { - 'value': 65536, # not a short (1 over max) - 'length': 2, - # good_bytes: b'\x00\x00\x00\x00\x00\x01\x00\x00' - }, - { - 'value': 281470681743360, # 65535 << 32 - 'length': 4, - # 'good_bytes': b'\x00\x00\xff\xff\x00\x00\x00\x00', - } - ] - for test_meta in test_metas: - value = test_meta['value'] - length = test_meta['length'] - with self.assertRaises(ValueError): - MemoryService.intToArray(value, length) - - def testSpaceDecode(self): - byte6 = False - space = 0x00 - - (byte6, space) = self.mService.spaceDecode(0xF8) - self.assertEqual(space, 0xF8) - self.assertTrue(byte6) - - (byte6, space) = self.mService.spaceDecode(0xFF) - self.assertEqual(space, 0x03) - self.assertFalse(byte6) - - (byte6, space) = self.mService.spaceDecode(0xFD) - self.assertEqual(space, 0x01) - self.assertFalse(byte6) - if __name__ == '__main__': unittest.main() From 10fe25a6e6274eee85aa2e8bdc0a335cfc640004 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Wed, 18 Mar 2026 18:29:24 -0400 Subject: [PATCH 02/64] Handle CDIVar size as max for "string" and add tests for edge cases. --- openlcb/cdivar.py | 32 +++++-- python-openlcb.code-workspace | 1 + tests/test_cdivar.py | 13 +-- tests/test_cdivar_edge_cases.py | 151 ++++++++++++++++++++++++++++++++ 4 files changed, 186 insertions(+), 11 deletions(-) create mode 100644 tests/test_cdivar_edge_cases.py diff --git a/openlcb/cdivar.py b/openlcb/cdivar.py index 80259050..9aea7ef0 100644 --- a/openlcb/cdivar.py +++ b/openlcb/cdivar.py @@ -1,15 +1,17 @@ import struct -from openlcb import emit_cast +from logging import getLogger from typing import List, Type, Union +from openlcb import emit_cast from openlcb.eventid import EventID from openlcb.openlcbaction import OpenLCBAction +logger = getLogger(__name__) NUM_TYPES = {'int': int, 'float': float} # type: dict[str, Type] -# Assumes "IEEE" in LCC CDI Standard means IEEE 754-2008: +# Assumes "IEEE" in OpenLCB CDI Standard means IEEE 754-2008: FLOAT_MAXIMUMS = {16: 65504.0, 32: 3.40e38, 64: 1.80e308} # type: dict[int, float] # noqa: E501 CLASSNAME_TYPES = {'int': int, 'float': float, 'string': str, 'blob': bytearray, 'eventid': EventID, @@ -45,7 +47,7 @@ class CDIVar: (for className == "float"). signed (bool): Whether the value is signed (False unless min is negative). Defaults to True. - See LCC "Configuration Description Information" Standard. + See OpenLCB "Configuration Description Information" Standard. _data (bytes): The value read from the device or ready to write. Only None if not read yet, otherwise length must be .size. @@ -139,8 +141,13 @@ def stringToData(self, value: str) -> bytes: return value.encode("utf-8") def setString(self, value: str): - self.data = self.stringToData(value) - self.size = len(self.data) + # self.data = self.stringToData(value) + # self.size = len(self.data) + # assert self.className == "string" + encoded = value.encode("utf-8") + assert self.size is not None + assert len(encoded) + 1 <= self.size # size is max *only* if "string" + self.data = encoded + b"\x00" # null-terminated for OpenLCB network def dataToInt(self, data) -> Union[int, None]: assert self.className == "int" @@ -173,4 +180,17 @@ def dataToString(self, data) -> Union[str, None]: return data.decode("utf-8") def getString(self) -> Union[str, None]: - return self.dataToString(self.data) + # return self.dataToString(self.data) + if self.data is None or len(self.data) == 0: + return None + # Return content up to (but not including) first null + null_pos = self.data.find(b"\x00") + if null_pos == -1: + logger.error(f"No null terminator in {repr(self.data)}") + content = self.data + else: + content = self.data[:null_pos] + # try: + return content.decode("utf-8") + # except UnicodeDecodeError: + # return None # or raise diff --git a/python-openlcb.code-workspace b/python-openlcb.code-workspace index 160c1587..a7c562d2 100644 --- a/python-openlcb.code-workspace +++ b/python-openlcb.code-workspace @@ -75,6 +75,7 @@ "offvalue", "onvalue", "openlcb", + "openlcbaction", "openlcbnetwork", "padx", "pady", diff --git a/tests/test_cdivar.py b/tests/test_cdivar.py index 22af1852..51682dc6 100644 --- a/tests/test_cdivar.py +++ b/tests/test_cdivar.py @@ -18,12 +18,15 @@ def test_initialization_valid(self): self.assertEqual(cdivar_float.max, 100.0) self.assertEqual(cdivar_float.size, 4) + maxSize = 100 cdivar_string = CDIVar(className='string', - _default=bytearray(b'Hello')) + _default=bytearray(b'Hello'), + _size=maxSize) self.assertEqual(cdivar_string.className, 'string') self.assertEqual(cdivar_string.default, bytearray(b'Hello')) assert cdivar_string.default is not None - self.assertEqual(cdivar_string.size, len(cdivar_string.default)) + # self.assertEqual(cdivar_string.size, len(cdivar_string.default)) + self.assertEqual(cdivar_string.size, maxSize) def test_initialization_invalid_classname(self): with self.assertRaises(AssertionError): @@ -60,7 +63,7 @@ def test_set_get_float(self): self.assertAlmostEqual(got, 3.14, places=6) def test_set_get_string(self): - cdivar_string = CDIVar(className='string') + cdivar_string = CDIVar(className='string', _size=100) cdivar_string.setString("Hello") self.assertEqual(cdivar_string.getString(), "Hello") @@ -75,8 +78,8 @@ def test_invalid_set_float(self): cdivar_float.setFloat("not a float") # type:ignore (assertRaises) def test_invalid_set_string(self): - cdivar_string = CDIVar(className='string') - with self.assertRaises(AssertionError): + cdivar_string = CDIVar(className='string', _size=100) + with self.assertRaises(AttributeError): # number has no attribute 'encode' cdivar_string.setString(12345) # type:ignore (assertRaises) diff --git a/tests/test_cdivar_edge_cases.py b/tests/test_cdivar_edge_cases.py new file mode 100644 index 00000000..7820c43b --- /dev/null +++ b/tests/test_cdivar_edge_cases.py @@ -0,0 +1,151 @@ +from typing import List, Tuple +import unittest + +from openlcb.cdivar import CDIVar + + +class TestCDIVarNumericConversions(unittest.TestCase): + + def assertBytesEqual(self, expected_hex: List[int], actual: bytes, + msg: str = ""): + expected = bytes(expected_hex) + self.assertEqual( + expected, + actual, + (f"{msg}\n Expected: {expected.hex(' ').upper()}" + f"\n Got: {actual.hex(' ').upper()}") + ) + + # ------------------------------------------------------------------------- + # Basic signed int conversions — edge cases (4 bytes) + # ------------------------------------------------------------------------- + def test_setInt_getInt_4byte_edge_cases(self): + cases: List[Tuple[int, List[int]]] = [ + (-1, [0xFF, 0xFF, 0xFF, 0xFF]), + (-2147483648, [0x80, 0x00, 0x00, 0x00]), # INT32_MIN + (2147483647, [0x7F, 0xFF, 0xFF, 0xFF]), + (0, [0x00, 0x00, 0x00, 0x00]), + (300, [0x00, 0x00, 0x01, 0x2C]), + (0x12345678, [0x12, 0x34, 0x56, 0x78]), + ] + + for value, expected_bytes in cases: + with self.subTest(f"int {value} → bytes"): + var = CDIVar("int", _size=4, _min=-1) # signed + var.setInt(value) + assert var.data is not None + self.assertBytesEqual(expected_bytes, var.data) + + restored = var.getInt() + self.assertEqual(value, restored) + + # ------------------------------------------------------------------------- + # Smaller sizes — sign extension behavior + # ------------------------------------------------------------------------- + def test_small_int_sizes_sign_extension(self): + cases = [ + # value, size, signed, bytes, expected getInt + (-100, 2, True, [0xFF, 0x9C], -100), + (0xABCD, 2, False, [0xAB, 0xCD], 0xABCD), + (-128, 4, True, [0xFF, 0xFF, 0xFF, 0x80], -128), + (0x5A, 1, False, [0x5A], 0x5A), + ] + + for val, size, signed, exp_bytes, exp_restored in cases: + with self.subTest(f"{val} @ {size} bytes signed={signed}"): + var = CDIVar("int", _size=size, _min=-1 if signed else 0) + var.setInt(val) + assert var.data is not None + self.assertBytesEqual(exp_bytes, var.data) + + restored = var.getInt() + self.assertEqual(exp_restored, restored) + + # ------------------------------------------------------------------------- + # Strict IEEE 754 binary16 (half-precision) bit-exact tests + # ------------------------------------------------------------------------- + def test_float16_strict_bit_exact(self): + cases = [ # noqa: E501 + # value expected [high, low] description + (0.0, [0x00, 0x00], "+0.0"), + (5.9604644775390625e-8, [0x00, 0x01], "smallest positive subnormal"), # noqa: E501 + (-5.9604644775390625e-8, [0x80, 0x01], "smallest negative subnormal"), # noqa: E501 + (6.103515625e-5, [0x04, 0x00], "smallest positive normal"), # noqa: E501 + (-6.103515625e-5, [0x84, 0x00], "smallest negative normal"), # noqa: E501 + (1.0, [0x3C, 0x00], "1.0 exact"), + (-1.0, [0xBC, 0x00], "-1.0"), + (0.5, [0x38, 0x00], "0.5"), + (-0.5, [0xB8, 0x00], "-0.5"), + (65504.0, [0x7B, 0xFF], "max finite"), + (-65504.0, [0xFB, 0xFF], "max negative finite"), # noqa: E501 + (float("inf"), [0x7C, 0x00], "+Inf"), + (float("-inf"), [0xFC, 0x00], "-Inf"), + # (float("nan"), [0x7E, 0x00], "canonical quiet NaN"), # noqa: E501 + # (65536.0, [0x7C, 0x00], "overflow → +Inf"), # noqa: E501 + # (1.00048828125, [0x3C, 0x01], "ties-to-even example"), # noqa: E501 + (float("nan"), [0x7E, 0x00], "canonical quiet NaN"), # noqa: E501 + # 65536.0 removed — Python struct raises OverflowError (expected) + # (1.00048828125, [0x3C, 0x00], "ties-to-even rounds to even (down in this case)"), # noqa: E501 + # ^ becomes 1.0 due to float16 precision, so commented + (1.0009765625, [0x3C, 0x01], "1 + 2⁻¹⁰ = exact in float16"), # noqa: E501 + # (1.00048828125 + 1e-12, [0x3C, 0x01], "slightly above midpoint → rounds up"), # noqa: E501 + # ^ AssertionError: 1.000488281251 != 1.0009765625 : Round-trip mismatch: 1.000488281251 → 1.0009765625 # noqa: E501 + # due to float16 precision + ] + + for val, expected, message in cases: + with self.subTest(f"float16 {val}"): + var = CDIVar("float", _size=2) + var.setFloat(val) + assert var.data is not None + self.assertBytesEqual(expected, var.data, + f"setFloat 16 ({val}) {message} failed") # noqa: E501 + + # round-trip check + restored = var.getFloat() + assert restored is not None + if val != val: # NaN + self.assertTrue(restored != restored) + elif abs(val) == float("inf"): + self.assertTrue( + abs(restored) == float("inf") and (restored > 0) == (val > 0), # noqa: E501 + f"setFloat 16 {message} failed" + ) + else: + # For representable values → should be bit-exact round-trip + self.assertEqual( + val, restored, + f"Round-trip mismatch: {val} → {restored}" + ) + + # ------------------------------------------------------------------------- + # Basic null-terminated string behavior (modified methods) + # ------------------------------------------------------------------------- + def test_string_null_terminated(self): + cases = [ + ("hello", b"hello\x00"), + ("", b"\x00"), + ("café π", "café π".encode("utf-8") + b"\x00"), + ] + + for s, expected_bytes in cases: + with self.subTest(f"setString({s!r})"): + var = CDIVar("string", _size=100) + var.setString(s) + self.assertEqual(expected_bytes, var.data) + + restored = var.getString() + self.assertEqual(s, restored) + + # Extra data after null is ignored + var = CDIVar("string") + var.data = b"test\x00junk" + self.assertEqual("test", var.getString()) + + # No null → whole content + var.data = b"no-null-here" + self.assertEqual("no-null-here", var.getString()) + + +if __name__ == "__main__": + unittest.main(verbosity=2) \ No newline at end of file From 9b29e37e1bbe36a0355bf02f54989aaf5c2e3e14 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Wed, 22 Apr 2026 15:46:09 -0400 Subject: [PATCH 03/64] Fix: Handle "." at end of nodeid range prefix. --- openlcb/conventions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openlcb/conventions.py b/openlcb/conventions.py index bd2af5ce..842cf97f 100644 --- a/openlcb/conventions.py +++ b/openlcb/conventions.py @@ -248,6 +248,7 @@ def generate_node_id_str(id_range_prefix: str, increment: bool = False) -> str: lastParts = [f"{p:02X}" for p in generate_last_three_octets(increment=increment)] # noqa: E501 assert len(lastParts) == 3 + id_range_prefix = id_range_prefix.rstrip(".") prefixParts = id_range_prefix.split(".") if len(prefixParts) < 3: raise ValueError( From adccc26dcce026f2e46cd1702ce35069874d09ee Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Wed, 22 Apr 2026 15:47:17 -0400 Subject: [PATCH 04/64] Fix: Handle memo using new OO-defined onStatusMemo (formerly _onElement). Add memoryService property so client code can register handlers. --- examples/tkexamples/cdiform.py | 5 +++-- openlcb/openlcbnetwork.py | 8 ++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/examples/tkexamples/cdiform.py b/examples/tkexamples/cdiform.py index b6702b50..ce9ba7f4 100644 --- a/examples/tkexamples/cdiform.py +++ b/examples/tkexamples/cdiform.py @@ -226,8 +226,9 @@ def onStartDownload(self): def onStatusMemo(self, cm: CDIMemo) -> bool: """Handler for incoming CDI tag - Use this for callback in downloadCDI, which sets parser - (_dataProcessor)'s _onElement. + Use this for callback in downloadCDI + (onStatusMemo replaces _dataProcessor's _onElement + formerly set by downloadCDI). Args: cm (CDIMemo): Document parsing state info diff --git a/openlcb/openlcbnetwork.py b/openlcb/openlcbnetwork.py index 28bd45a2..c1374379 100644 --- a/openlcb/openlcbnetwork.py +++ b/openlcb/openlcbnetwork.py @@ -93,6 +93,10 @@ def __init__(self, localNodeID: Union[str, bytearray, int, NodeID]): self._memoryService = MemoryService(self._datagramService) self._dataProcessor: XMLDataProcessor = None + @property + def memoryService(self): + return self._memoryService + def setConnectHandler(self, handler: Callable[[CDIMemo], None]): """Deprecated in favor of a Message handler, Since it is the socket loop's responsibility to call @@ -292,8 +296,8 @@ def _listen(self): cm = CDIMemo() cm.error = formatted_ex(ex) cm.done = True # stop progress in gui/other main thread - if self._dataProcessor._onElement: - self._dataProcessor._onElement(cm) + if self._dataProcessor.onStatusMemo: + self._dataProcessor.onStatusMemo(cm) raise # re-raise since incomplete (prevent done OK state) finally: self.physicalLayer.physicalLayerDown() # Link_Layer_Down, setState From 472549776758f7d1706758eac291cb3c35fa5246 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Wed, 22 Apr 2026 16:20:30 -0400 Subject: [PATCH 05/64] Enforce onStartDownload via OO. --- examples/examples_gui.py | 1 - examples/tkexamples/cdiform.py | 4 ++-- openlcb/openlcbnetwork.py | 2 +- openlcb/xmldataprocessor.py | 9 +++++++++ 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/examples/examples_gui.py b/examples/examples_gui.py index fab718ec..a38338e3 100644 --- a/examples/examples_gui.py +++ b/examples/examples_gui.py @@ -632,7 +632,6 @@ def downloadCDI(self, farNodeID: str): self.setStatus("Downloading CDI...") assert self.cdi_form is not None assert self.network is not None - self.cdi_form.onStartDownload() try: self.network.download(farNodeID, MemorySpace.CDI, self.cdi_form) diff --git a/examples/tkexamples/cdiform.py b/examples/tkexamples/cdiform.py index ce9ba7f4..09fce4f3 100644 --- a/examples/tkexamples/cdiform.py +++ b/examples/tkexamples/cdiform.py @@ -221,8 +221,8 @@ def getStatus(self): def onStartDownload(self): """Initialize variables used by element handler(s).""" - self.onStart() - self._resetTree() + XMLDataProcessor.onStartDownload(self) + # TODO: clear tree? def onStatusMemo(self, cm: CDIMemo) -> bool: """Handler for incoming CDI tag diff --git a/openlcb/openlcbnetwork.py b/openlcb/openlcbnetwork.py index c1374379..9a362cc3 100644 --- a/openlcb/openlcbnetwork.py +++ b/openlcb/openlcbnetwork.py @@ -156,6 +156,7 @@ def _startMemoryRead(self, farNodeID: Union[NodeID, int, str, bytearray]): """ # read 64 bytes from the CDI space starting at address zero assert isinstance(self._dataProcessor.space, MemorySpace) + self._dataProcessor.onStartDownload() memMemo = MemoryReadMemo(NodeID(farNodeID), 64, self._dataProcessor.space.value, 0, # incremented on _memoryReadSuccess @@ -193,7 +194,6 @@ def download(self, farNodeID: str, space: MemorySpace, assert isinstance(space, MemorySpace) self._dataProcessor = dataProcessor self._dataProcessor._space = space - self._dataProcessor._stringTerminated = False self._startMemoryRead(farNodeID) # ^ Following this, _memoryReadSuccess callback will diff --git a/openlcb/xmldataprocessor.py b/openlcb/xmldataprocessor.py index 539308a0..02acfc0f 100644 --- a/openlcb/xmldataprocessor.py +++ b/openlcb/xmldataprocessor.py @@ -181,6 +181,15 @@ def onPopScope(self, cm: CDIMemo) -> bool: """ return False + def onStartDownload(self): + """Initialize variables used by element handler(s). + If subclass is a GUI, reimplement this to reset GUI, + but also call onStart or super().onStartDownload(). + """ + self._stringTerminated = False + self._resetTree() + self.onStart() + def onStart(self): # self._cdi_offset = 0 # Instead see memo.address (which is # incremented on _memoryReadSuccess or custom memory read From ae1d1cfdb06a16e91831066db92d63a894d1edd5 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Wed, 22 Apr 2026 18:35:05 -0400 Subject: [PATCH 06/64] Add high-level (size-aware if set manually) DataProcessor progress. Add a type hint. Reduce logging (use info level for memory read). --- openlcb/cdimemo.py | 4 ++++ openlcb/dataprocessor.py | 5 ++++- openlcb/memoryservice.py | 3 ++- openlcb/xmldataprocessor.py | 30 +++++++++++++++++++++++++++++- python-openlcb.code-workspace | 2 ++ 5 files changed, 41 insertions(+), 3 deletions(-) diff --git a/openlcb/cdimemo.py b/openlcb/cdimemo.py index 78ee35af..3cc8b15d 100644 --- a/openlcb/cdimemo.py +++ b/openlcb/cdimemo.py @@ -62,6 +62,10 @@ def __init__(self, tag: Union[str, None] = None, self.address = None # type: int|None self.cdivar = None # type: CDIVar|None self.children = [] # type: List[CDIMemo] + # Set by DataProcessor such as XMLDataProcessor: + self.progress_ratio = None # type: float|None + self.progress_count = None # type: int|None + self.expected_size = None # type: int|None def getTag(self): if self.element is None: diff --git a/openlcb/dataprocessor.py b/openlcb/dataprocessor.py index 75a8f962..d9fab76a 100644 --- a/openlcb/dataprocessor.py +++ b/openlcb/dataprocessor.py @@ -12,4 +12,7 @@ class DataProcessor: Superclass for data listeners. """ def __init__(self): - pass + # Members used to construct space memo such as CDIMemo: + self.progress_ratio = None # type: float|None + self.progress_count = None # type: int|None + self.expected_size = None # type: int|None diff --git a/openlcb/memoryservice.py b/openlcb/memoryservice.py index c61e51ba..f4f70246 100644 --- a/openlcb/memoryservice.py +++ b/openlcb/memoryservice.py @@ -328,7 +328,8 @@ def requestMemoryWrite(self, memo: MemoryWriteMemo): dgWriteMemo = DatagramWriteMemo(memo.nodeID, data) self.service.sendDatagram(dgWriteMemo) - def requestSpaceLength(self, space, nodeID, callback): + def requestSpaceLength(self, space: int, nodeID: NodeID, + callback: Callable[[int], None]): '''Request the length of a specific memory space from a remote node. Args: diff --git a/openlcb/xmldataprocessor.py b/openlcb/xmldataprocessor.py index 02acfc0f..860b6165 100644 --- a/openlcb/xmldataprocessor.py +++ b/openlcb/xmldataprocessor.py @@ -199,6 +199,7 @@ def onStart(self): "A previous downloadCDI operation is in progress" " or failed (Set _data to None first if failed)") self._data = bytearray() + self.progress_count = 0 def onStop(self): self._format = DataFormat.EOF # no data expected @@ -213,11 +214,22 @@ def _fireStatus(self, status, if callback is None: callback = self.onStatusMemo if callback: - print("OpenLCBNetwork callback_msg({})".format(repr(status))) + logger.info("OpenLCBNetwork callback_msg({})".format(repr(status))) callback(CDIMemo(status=status)) else: logger.warning("No callback, but set status: {}".format(status)) + def _fireStatusMemo(self, statusMemo, + callback: Union[Callable[[CDIMemo], bool], None] = None): # noqa: E501 + """Fire status handlers with the given status.""" + if callback is None: + callback = self.onStatusMemo + if callback: + logger.info(f"OpenLCBNetwork callback_msg({statusMemo})") + callback(statusMemo) + else: + logger.warning(f"No callback, but set status: {statusMemo}") + def _feedNext(self, memo: MemoryReadMemo): """Handle partial CDI XML (any packet except last) The last packet is not yet reached, so don't parse (but @@ -229,9 +241,14 @@ def _feedNext(self, memo: MemoryReadMemo): """ assert self._data is not None self._data += memo.data + self.progress_count = len(self._data) partial_str = memo.data.decode("utf-8") if self._realtime: self._parser.feed(partial_str) # may call startElement/endElement + cm = CDIMemo() + cm.progress_count = self.progress_count + cm.expected_size = self.expected_size + self.onStatusMemo(cm) def _feedLast(self, memo: MemoryReadMemo): """Handle end of CDI XML (last packet) @@ -255,6 +272,13 @@ def _feedLast(self, memo: MemoryReadMemo): if null_i > -1: terminate_i = min(null_i, terminate_i) partial_str = memo.data[:terminate_i].decode("utf-8") + assert self.progress_count is not None + self.progress_count += terminate_i + cm = CDIMemo() + cm.done = True # 'done' and not 'error' means got all + cm.progress_count = self.progress_count + cm.expected_size = self.expected_size + self.onStatusMemo(cm) else: # *not* realtime (but got to end, so parse all at once) cdiString = "" @@ -263,6 +287,9 @@ def _feedLast(self, memo: MemoryReadMemo): if null_i > -1: terminate_i = min(null_i, terminate_i) cdiString = self._data[:terminate_i].decode("utf-8") + assert self.progress_count is not None + self.progress_count += terminate_i + # print (cdiString) # self.parse(cdiString) # no such method # self._parser.parse(cdiString) # urllib.error.URLError @@ -273,6 +300,7 @@ def _feedLast(self, memo: MemoryReadMemo): # self._fireStatus("Done loading CDI.") cm = CDIMemo() cm.done = True # 'done' and not 'error' means got all + cm.progress_count = self.progress_count self.onStatusMemo(cm) if self._realtime: self._parser.feed(partial_str) # may call startElement/endElement diff --git a/python-openlcb.code-workspace b/python-openlcb.code-workspace index a7c562d2..bf30c553 100644 --- a/python-openlcb.code-workspace +++ b/python-openlcb.code-workspace @@ -41,6 +41,7 @@ "cdiform", "cdimemo", "cdivar", + "cdivars", "columnspan", "controlframe", "datagram", @@ -108,6 +109,7 @@ "usbmodem", "WASI", "winnative", + "wraplength", "xmldataprocessor", "xscrollcommand", "zeroconf" From 7192c264e753fbbe2d542f01bc80de510d2c2af5 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Wed, 22 Apr 2026 19:55:07 -0400 Subject: [PATCH 07/64] Fix: Provide size to CDIVar constructor. Add CDIVar serialization and name (Set externally optionally). --- openlcb/cdimemo.py | 34 +++++++++++++++++++++++++++------- openlcb/cdivar.py | 28 ++++++++++++++++++++++++++-- 2 files changed, 53 insertions(+), 9 deletions(-) diff --git a/openlcb/cdimemo.py b/openlcb/cdimemo.py index 3cc8b15d..2d4a75ef 100644 --- a/openlcb/cdimemo.py +++ b/openlcb/cdimemo.py @@ -156,17 +156,37 @@ def toCDIVar(self): """Create a CDIVar from descriptors (child elements of self). See LCC "Configuration Description Information" Standard. """ - result = CDIVar(self.tag) + # result = CDIVar(self.tag) assert (self.tag is not None) and (self.tag.strip()) - result.className = self.tag.lower() + className = self.tag.lower() + result_floatFormat = None if self.element: - result.floatFormat = self.element.attrib.get('floatFormat') + result_floatFormat = self.element.attrib.get('floatFormat') this_t = NUM_TYPES.get(self.tag) if self.tag else None + result_min = None + result_max = None + result_default = None + result_size = self.getSize() if this_t is not None: - result.min = self.getChildContentN("min", result.className) - result.max = self.getChildContentN("max", result.className) - result.default = self.getChildContentN("default", result.className) - result.size = self.getSize() + result_min = self.getChildContentN("min", className) + result_max = self.getChildContentN("max", className) + default_n = self.getChildContentN("default", className) + if default_n is not None: + default_var = CDIVar(className, _size=result_size) + if isinstance(default_n, int): + assert self.tag == "int" + default_var.setInt(default_n) + else: + assert self.tag == "float" + default_var.setFloat(default_n) + assert default_var.data is not None + result_default = bytearray(default_var.data) + # Size must be gotten ahead of time since CDIVar constructor + # enforces size: + result = CDIVar(self.tag, _min=result_min, _max=result_max, + _size=result_size, _default=result_default) + result.floatFormat = result_floatFormat + result.name = self.getChildContent("name") if result.className == "int": if result.min is None: diff --git a/openlcb/cdivar.py b/openlcb/cdivar.py index 9aea7ef0..0474ed34 100644 --- a/openlcb/cdivar.py +++ b/openlcb/cdivar.py @@ -1,4 +1,6 @@ +import base64 +from collections import OrderedDict import struct from logging import getLogger @@ -61,6 +63,7 @@ def __init__(self, className, _min=None, _max=None, assert className, f"Expected {CLASSNAME_TYPES.keys()} got {className}" assert className in CLASSNAME_TYPES, \ f"Expected {list(CLASSNAME_TYPES.keys())} got {className}" + self.name = None # type: str|None self.className = className # type: str self.data = None # type: bytes|None self.min = _min # type: int|float|None @@ -85,9 +88,11 @@ def standardSizes(self) -> Union[List[int], None]: def assertNumberFormat(self, assertWhat=""): if self.className == "int": - assert self.size in (1, 2, 4, 8) + assert self.size in (1, 2, 4, 8), \ + f"Expected size (1, 2, 4, 8) for int, got {self.size}" elif self.className == "float": - assert self.size in (2, 4, 8) + assert self.size in (2, 4, 8), \ + f"Expected size (2, 4, 8) for float, got {self.size}" else: if not assertWhat: assertWhat = f"Expected float/int size {STANDARD_SIZES}" @@ -124,6 +129,25 @@ def intToData(self, value: int) -> bytes: assert isinstance(value, int) return struct.pack(self.packFormat(), value) + def getSerializable(self): + """Get a value in the corresponding Python type""" + if self.className == "int": + return self.getInt() + elif self.className == "float": + return self.getFloat() + elif self.className == "string": + return self.getString() + assert self.className in ("blob", "eventid", "action") + return base64.b64encode(self.data) + + def getDict(self, add_name=True): + result = OrderedDict() + if add_name and self.name: + result['name'] = self.name + result['className'] = self.className + result['value'] = self.getSerializable() + return result + def setInt(self, value: int): self.data = self.intToData(value) From 921f518523925ab66607cb752189f6abf4ea8d64 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Wed, 22 Apr 2026 20:06:12 -0400 Subject: [PATCH 08/64] Save the memo tree in XMLDataProcessor (formerly had to be collected by callbacks if necessary). --- openlcb/xmldataprocessor.py | 44 +++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/openlcb/xmldataprocessor.py b/openlcb/xmldataprocessor.py index 860b6165..6df7dcd8 100644 --- a/openlcb/xmldataprocessor.py +++ b/openlcb/xmldataprocessor.py @@ -105,6 +105,8 @@ class XMLDataProcessor(xml.sax.handler.ContentHandler, DataProcessor): def __init__(self, linkLayer: CanLink, space: MemorySpace): self.canLink: CanLink = linkLayer caches_dir = SysDirs.Cache + self._root_memos = None # type: list[CDIMemo]|None + self._root_memo = None # type: CDIMemo|None self._space: Union[MemorySpace, None] = None self._openEl: Union[ET.Element, None] = None self._top_tag = "cdi" # cdi or fdi (detected in startElement) @@ -136,6 +138,42 @@ def __init__(self, linkLayer: CanLink, space: MemorySpace): # endregion ContentHandler self.acdi = False + def getRootMemo(self): + """Get the root memo object if any. + This should only be called after the entire file is parsed such + as when cm.done is True in onStatusMemo(cm) callback. Set + callback manually if necessary and if using realtime parsing + (_feed) mode. + """ + if not self._root_memos: + return None + if len(self._root_memos) > 1: + summaries = [] + cdi_roots = [] + tag = None + for memo in self._root_memos: + tag = memo.getTag() + if tag is not None: + tag = tag.lower() + summaries.append(memo.getTag()) + if tag in ("cdi", "fdi"): + cdi_roots.append(memo) + if len(cdi_roots) == 1: + return cdi_roots[0] + if tag not in ("cdi", "fdi"): + logger.warning( + f"Got more than one XML root: {summaries};" + " expected cdi/fdi") + else: + logger.warning(f"Got more than one XML root: {summaries}") + return self._root_memos[-1] + tag = self._root_memos[0].getTag() + if tag is not None: + tag = tag.lower() + if tag not in ("cdi", "fdi"): + logger.warning(f"Only XML root is {repr(tag)} not cdi/fdi") + return self._root_memos[0] + def setSpace(self, space: MemorySpace): self._space = space self._format = format_of_space(space) @@ -200,6 +238,8 @@ def onStart(self): " or failed (Set _data to None first if failed)") self._data = bytearray() self.progress_count = 0 + self._root_memos = [] # list of roots + self._root_memo = None def onStop(self): self._format = DataFormat.EOF # no data expected @@ -379,6 +419,10 @@ def startElement(self, name: str, cm.address = self._tmp_address # May be None if after /segment self.onPushScope(cm) + if len(self._tag_stack) < 1: + self._root_memos.append(cm) + if cm.tag == "cdi": + self._root_memo = cm # self._callback_msg( # "loaded: {}{}".format(tab, ET.tostring(el, encoding="unicode"))) From db12c3dee52948cacdc1a434fd146c3aa9e6de7f Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Thu, 23 Apr 2026 11:01:19 -0400 Subject: [PATCH 09/64] Add load feature (Load cached CDI XML file). Make caching configurable. Add fromNumber. Change cache_cdi_path to classmethod. Add fromNumber to MemorySpace enum. Add force_end option in _memoryReadSuccess (reserved for future use). --- openlcb/dataprocessor.py | 4 ++ openlcb/memoryservice.py | 11 +++++ openlcb/openlcbnetwork.py | 5 +- openlcb/xmldataprocessor.py | 96 ++++++++++++++++++++++++++++++++++--- 4 files changed, 107 insertions(+), 9 deletions(-) diff --git a/openlcb/dataprocessor.py b/openlcb/dataprocessor.py index d9fab76a..d21e99a9 100644 --- a/openlcb/dataprocessor.py +++ b/openlcb/dataprocessor.py @@ -10,8 +10,12 @@ class DataFormat(Enum): class DataProcessor: """Collect & process consecutive data from each incoming MemoryReadMemo. Superclass for data listeners. + + Attributes: + enable_cache (bool): Defaults to False (May differ in subclass). """ def __init__(self): + self.enable_cache = False # type: bool # Members used to construct space memo such as CDIMemo: self.progress_ratio = None # type: float|None self.progress_count = None # type: int|None diff --git a/openlcb/memoryservice.py b/openlcb/memoryservice.py index f4f70246..698bf8df 100644 --- a/openlcb/memoryservice.py +++ b/openlcb/memoryservice.py @@ -65,6 +65,17 @@ class MemorySpace(Enum): All = 0xFE Configuration = 0xFD + @classmethod + def fromNumber(cls, num: int): + """Return the MemorySpace member with the given numeric value, + or None if no match is found. + """ + assert isinstance(num, int) + for member in cls: + if member.value == num: + return member + return None + class MemoryReadMemo: """Memo carries request and reply. diff --git a/openlcb/openlcbnetwork.py b/openlcb/openlcbnetwork.py index 9a362cc3..fd70f729 100644 --- a/openlcb/openlcbnetwork.py +++ b/openlcb/openlcbnetwork.py @@ -363,7 +363,7 @@ def _fireStatus(self, status, else: logger.warning("No callback, but set status: {}".format(status)) - def _memoryReadSuccess(self, memo: MemoryReadMemo): + def _memoryReadSuccess(self, memo: MemoryReadMemo, force_end=False): """Handle a successful read Invoked when the memory read successfully returns, this queues a new read until the entire CDI has been @@ -373,7 +373,8 @@ def _memoryReadSuccess(self, memo: MemoryReadMemo): memo (MemoryReadMemo): Successful MemoryReadMemo """ # print("successful memory read: {}".format(memo.data)) - if len(memo.data) == 64 and 0 not in memo.data: # *not* last chunk + if (not force_end) and (len(memo.data) == 64 and 0 not in memo.data): + # *not* last chunk self._dataProcessor._stringTerminated = False if self._dataProcessor.format != DataFormat.EOF: # save content diff --git a/openlcb/xmldataprocessor.py b/openlcb/xmldataprocessor.py index 6df7dcd8..b1c304b3 100644 --- a/openlcb/xmldataprocessor.py +++ b/openlcb/xmldataprocessor.py @@ -59,13 +59,15 @@ def attrs_to_ordered(attrs: xml.sax.xmlreader.AttributesImpl): return od -def format_of_space(space): +def format_of_space(space, unknown_raises=True): assert isinstance(space, MemorySpace) if space == MemorySpace.CDI: return DataFormat.XML elif space == MemorySpace.FDI: return DataFormat.XML - raise NotImplementedError(emit_cast(space)) + if unknown_raises: + raise NotImplementedError(emit_cast(space)) + return None class XMLDataProcessor(xml.sax.handler.ContentHandler, DataProcessor): @@ -101,16 +103,19 @@ class XMLDataProcessor(xml.sax.handler.ContentHandler, DataProcessor): of start tags). """ XML_TOP_TAGS = ("cdi", "fdi") + DEFAULT_CACHES_DIR = SysDirs.Cache + DEFAULT_CACHE_DIR = os.path.join(DEFAULT_CACHES_DIR, "python-openlcb") def __init__(self, linkLayer: CanLink, space: MemorySpace): self.canLink: CanLink = linkLayer - caches_dir = SysDirs.Cache + # caches_dir = SysDirs.Cache self._root_memos = None # type: list[CDIMemo]|None self._root_memo = None # type: CDIMemo|None self._space: Union[MemorySpace, None] = None self._openEl: Union[ET.Element, None] = None self._top_tag = "cdi" # cdi or fdi (detected in startElement) - self._myCacheDir = os.path.join(caches_dir, "python-openlcb") + # self._myCacheDir = os.path.join(caches_dir, "python-openlcb") + self._myCacheDir = XMLDataProcessor.DEFAULT_CACHE_DIR self._tmp_space = None # type: int|None self._tmp_address = None # type: int|None assert isinstance(space, MemorySpace) @@ -120,6 +125,7 @@ def __init__(self, linkLayer: CanLink, space: MemorySpace): # prepare these for _callback_msg. xml.sax.ContentHandler.__init__(self) DataProcessor.__init__(self) + self.enable_cache = True self._stringTerminated = None # type: Union[bool, None] # ^ None means no read is occurring. if self._format != DataFormat.XML: @@ -197,6 +203,7 @@ def onStatusMemo(self, cm: CDIMemo) -> bool: Returns: bool: True if handled. """ + logger.warning("Default onStatusMemo ran.") return False def onPushScope(self, cm: CDIMemo) -> bool: @@ -290,13 +297,82 @@ def _feedNext(self, memo: MemoryReadMemo): cm.expected_size = self.expected_size self.onStatusMemo(cm) - def _feedLast(self, memo: MemoryReadMemo): + def load(self, node_id: NodeID, path, space: Union[MemorySpace, int], + memo: Union[MemoryReadMemo, None] = None, + format: Union[DataFormat, None] = None): + """Load instead of downloading.""" + assert not self._data + self.onStartDownload() + assert isinstance(space, (MemorySpace, int)) + if isinstance(space, int): + try_space = MemorySpace.fromNumber(space) + if try_space is not None: + space = try_space + if isinstance(space, MemorySpace): + self.setSpace(space) + else: + if format is None: + raise ValueError(f"Using device-specific space: {space}" + " but format not specified") + else: + assert isinstance(format, DataFormat) + self._format = format + self._space = space # int in device-specific case + logger.warning(f"Using device-specific space: {space}") + data = None + with open(path, "rb") as stream: + data = stream.read() # type:ignore + if self._format is DataFormat.XML: + if memo is not None: + assert isinstance(memo, MemoryReadMemo) + else: + def memoryReadSuccess(memo: MemoryReadMemo): + # See further down + print("Fallback memoryReadSuccess ran.") + pass + + def memoryReadFail(memo: MemoryReadMemo): + raise RuntimeError( + "Offline parse failure (should never happen)") + + assert data is not None + # Based on _startMemoryRead in OpenLCBNetwork: + memo = MemoryReadMemo(node_id, len(data), + self.getSpaceValue(), 0, + memoryReadFail, memoryReadSuccess) + + assert data is not None + memo.data = data # type: ignore + self._data = bytearray() # Since _feedLast adds memo.data to it + memo.size = len(data) + # based on "else" (done) case in _memoryReadSuccess + # in OpenLCBNetwork: + self._stringTerminated = True + self._feedLast(memo, enable_cache=False) + self.onStop() # sets self._format to DataFormat.EOF + else: + logger.warning(f"Custom DataFormat {self._format}" + f" (space={space}): not parsed automatically.") + + def getSpaceValue(self): + # type: () -> int|None + if self._space is None: + return None + if isinstance(self._space, MemorySpace): + return self._space.value + assert isinstance(self._space, int) + return self._space + + def _feedLast(self, memo: MemoryReadMemo, enable_cache=None): """Handle end of CDI XML (last packet) End of data, so parse (or feed if self._realtime) Args: memo (MemoryReadMemo): successful read memo containing data. + enable_cache (bool): Defaults to self.enable_cache. """ + if enable_cache is None: + enable_cache = self.enable_cache partial_str = memo.data.decode("utf-8") # save content assert self._data is not None @@ -353,8 +429,14 @@ def _feedLast(self, memo: MemoryReadMemo): print('Saved {}'.format(repr(path))) self._data = None # Ensure isn't reused for more than one doc - def cache_cdi_path(self, item_id: Union[NodeID, str]): - cdi_cache_dir = os.path.join(self._myCacheDir, "cdi") + def cache_cdi_path_scoped(self, item_id: Union[NodeID, str]): + type(self).cache_cdi_path(item_id, my_cache_dir=self._myCacheDir) + + @classmethod + def cache_cdi_path(cls, item_id: Union[NodeID, str], my_cache_dir=None): + if my_cache_dir is None: + my_cache_dir = cls.DEFAULT_CACHE_DIR + cdi_cache_dir = os.path.join(my_cache_dir, "cdi") if not os.path.isdir(cdi_cache_dir): os.makedirs(cdi_cache_dir) # TODO: add hardware name and firmware version and from SNIP to From 03ebb935e703389dbf23e14f1d0def9dcc3167e2 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Thu, 23 Apr 2026 19:01:42 -0400 Subject: [PATCH 10/64] Fix: Set size 8 for eventid. Enforce valid 'action' sizes. Add expandedTree, extractCDIVarMemos and related features. Add setData and enforce valid size(s). --- openlcb/__init__.py | 16 ++++ openlcb/cdimemo.py | 8 +- openlcb/cdivar.py | 44 ++++++++++- openlcb/xmldataprocessor.py | 150 +++++++++++++++++++++++++++++++++--- 4 files changed, 204 insertions(+), 14 deletions(-) diff --git a/openlcb/__init__.py b/openlcb/__init__.py index 0eebe1bb..788de048 100644 --- a/openlcb/__init__.py +++ b/openlcb/__init__.py @@ -131,3 +131,19 @@ def from_hex_bytes(b: bytearray, start: int, stop: int, def from_all_hex_bytes(b: bytearray) -> bytearray: return from_hex_bytes(b, 0, len(b)) + + +def hr_repr(value, always_quote: bool = False) -> str: + """Represent value with double quotes + (Human-readable repr). + """ + repr_value = repr(value) + if repr_value.startswith("'") and repr_value.endswith("'"): + return '"' + repr_value[1:-1].replace('"', '\\"') + '"' + elif always_quote: + return '"' + repr_value.replace('"', '\\"') + '"' + return repr(value) + + +def d_quote(value) -> str: + return hr_repr(value, always_quote=True) diff --git a/openlcb/cdimemo.py b/openlcb/cdimemo.py index 2d4a75ef..81124008 100644 --- a/openlcb/cdimemo.py +++ b/openlcb/cdimemo.py @@ -1,4 +1,5 @@ from collections import OrderedDict +import copy import json import math import xml.etree.ElementTree @@ -102,7 +103,10 @@ def getChildContent(self, tag) -> Union[str, None]: def copy(self): cm = CDIMemo() for k, v in self.__dict__.items(): - setattr(cm, k, v) + if isinstance(v, (list, dict, OrderedDict)): + setattr(cm, k, copy.deepcopy(v)) + else: + setattr(cm, k, v) return cm def getBranch(self, default=None) -> Union[str, None]: @@ -218,6 +222,8 @@ def toCDIVar(self): return result def getSize(self): + if self.tag == "eventid": + return 8 if self.element is None: return None size = self.element.attrib.get('size') diff --git a/openlcb/cdivar.py b/openlcb/cdivar.py index 0474ed34..2c667b03 100644 --- a/openlcb/cdivar.py +++ b/openlcb/cdivar.py @@ -1,10 +1,11 @@ import base64 from collections import OrderedDict +import copy import struct from logging import getLogger -from typing import List, Type, Union +from typing import Any, List, Type, Union from openlcb import emit_cast from openlcb.eventid import EventID @@ -18,6 +19,8 @@ CLASSNAME_TYPES = {'int': int, 'float': float, 'string': str, 'blob': bytearray, 'eventid': EventID, 'action': OpenLCBAction} +SIZED_CONSTRUCTION_TYPES = copy.deepcopy(CLASSNAME_TYPES) +SIZED_CONSTRUCTION_TYPES['eventid'] = bytearray SUBTYPE_FORMATS = { 'int8': "b", 'uint8': "B", 'int16': ">h", 'uint16': ">H", @@ -30,6 +33,8 @@ STANDARD_SIZES = { 'int': (1, 2, 4, 8), 'float': (2, 4, 8), + 'eventid': (8,), + 'action': (1, 2, 4, 8), } @@ -53,6 +58,8 @@ class CDIVar: _data (bytes): The value read from the device or ready to write. Only None if not read yet, otherwise length must be .size. + element (xml.etree.Element): An associated element in an XML + tree. """ TYPED_KEYS = ['min', 'max', 'default'] @@ -73,12 +80,47 @@ def __init__(self, className, _min=None, _max=None, self.max = _max # type: int|float|None self.default = _default # type: bytearray|None self.size = _size # type: int|None + self.branch_size = None # type: int|None # size including children if self.size is None: if self.default is not None: self.size = len(self.default) if self.className in ("int", "float"): self.assertNumberFormat() + elif self.className == "eventid": + if (_size is not None) and (_size != 8): + logger.error( + f'Specified eventid size="{_size}" but 8 is required.') + self.size = 8 + sizes = STANDARD_SIZES.get(self.className) + if sizes is not None: + assert self.size in sizes, \ + (f"Expected size in {sizes}" + f" for {self.className} but got {self.size}") self.floatFormat = None # type: str|None + self.address = None # type: int|None + self.element = None # type: Any|None + + def setData(self, data: Union[bytes, bytearray]): + assert isinstance(data, (bytes, bytearray)) + if isinstance(data, bytes): + data = bytearray(data) + if self.className == "eventid": + assert len(data) == 8 + elif self.className == "blob": + # FIXME: enforce blob + pass + elif self.className == "string": + assert self.size + assert len(data) <= self.size + elif self.className in SIZED_CONSTRUCTION_TYPES: + assert self.size + assert len(data) == self.size + else: + raise NotImplementedError(f"Type {self.className} not implemented") + self.data = data + + def getData(self): + return self.data def isNumber(self): return self.className in ("int", "float") diff --git a/openlcb/xmldataprocessor.py b/openlcb/xmldataprocessor.py index b1c304b3..3054efc5 100644 --- a/openlcb/xmldataprocessor.py +++ b/openlcb/xmldataprocessor.py @@ -6,10 +6,10 @@ from logging import getLogger from typing import Callable, List, Union -# from xml.sax.xmlreader import AttributesImpl # for type hints, for autocomplete only in this case +# from xml.sax.xmlreader import AttributesImpl # for type hints, for autocomplete only in this case # noqa:E501 import xml.sax.xmlreader # for type hints, for autocomplete only in this case -from openlcb import emit_cast +from openlcb import d_quote, emit_cast from openlcb.canbus.canlink import CanLink from openlcb.cdimemo import CDIMemo from openlcb.dataprocessor import DataFormat, DataProcessor @@ -23,6 +23,10 @@ MemorySpace, ) # from openlcb.remotenodeprocessor import RemoteNodeProcessor +from openlcb.cdivar import ( + CDIVar, + CLASSNAME_TYPES, +) if __name__ == "__main__": @@ -109,6 +113,7 @@ class XMLDataProcessor(xml.sax.handler.ContentHandler, DataProcessor): def __init__(self, linkLayer: CanLink, space: MemorySpace): self.canLink: CanLink = linkLayer # caches_dir = SysDirs.Cache + self.expanded_root = None # type: xml.etree.ElementTree|None self._root_memos = None # type: list[CDIMemo]|None self._root_memo = None # type: CDIMemo|None self._space: Union[MemorySpace, None] = None @@ -317,7 +322,7 @@ def load(self, node_id: NodeID, path, space: Union[MemorySpace, int], else: assert isinstance(format, DataFormat) self._format = format - self._space = space # int in device-specific case + self._space = space # type:ignore # int if device-specific logger.warning(f"Using device-specific space: {space}") data = None with open(path, "rb") as stream: @@ -421,7 +426,7 @@ def _feedLast(self, memo: MemoryReadMemo, enable_cache=None): if self._realtime: self._parser.feed(partial_str) # may call startElement/endElement # memo = MemoryReadMemo(memo) - path = self.cache_cdi_path(memo.nodeID) + path = self.cacheFilePath(memo.nodeID) with open(path, 'w') as stream: if cdiString is None: cdiString = self._data.rstrip(b'\0').decode("utf-8") @@ -429,21 +434,35 @@ def _feedLast(self, memo: MemoryReadMemo, enable_cache=None): print('Saved {}'.format(repr(path))) self._data = None # Ensure isn't reused for more than one doc - def cache_cdi_path_scoped(self, item_id: Union[NodeID, str]): - type(self).cache_cdi_path(item_id, my_cache_dir=self._myCacheDir) + def cacheFilePathCustom(self, item_id: Union[NodeID, str], **kwargs): + if 'my_cache_dir' not in kwargs: + kwargs['my_cache_dir'] = self._myCacheDir + type(self).cacheFilePath(item_id, **kwargs) @classmethod - def cache_cdi_path(cls, item_id: Union[NodeID, str], my_cache_dir=None): + def cacheFilePath(cls, item_id: Union[NodeID, str], my_cache_dir=None, + subFolder="cdi", name=None, ext=".xml"): if my_cache_dir is None: my_cache_dir = cls.DEFAULT_CACHE_DIR - cdi_cache_dir = os.path.join(my_cache_dir, "cdi") + if subFolder: + cdi_cache_dir = os.path.join(my_cache_dir, subFolder) + else: + cdi_cache_dir = my_cache_dir if not os.path.isdir(cdi_cache_dir): os.makedirs(cdi_cache_dir) # TODO: add hardware name and firmware version and from SNIP to # name file to avoid cache file from a different # device/version. - item_id = str(item_id) # Convert NodeID or other - clean_name = clean_file_name(item_id.replace(":", ".")) + if not name: + item_id = str(item_id) # Convert NodeID or other + clean_name = clean_file_name(item_id.replace(":", ".")) + clean_name += ext + else: + clean_name = clean_file_name(name) + if clean_name != name: + logger.warning( + "[cacheFilePath]" + f" changed name {repr(name)} to {repr(clean_name)}") # ^ replace ":" to avoid converting that one to default "_" # ^ will raise error if path instead of name path = os.path.join(cdi_cache_dir, clean_name) @@ -451,7 +470,7 @@ def cache_cdi_path(cls, item_id: Union[NodeID, str], my_cache_dir=None): # just to be safe, even though clean_file_name # should prevent. If this occurs, fix clean_file_name. raise ValueError("Cannot specify absolute path.") - return path + ".xml" + return path def startElement(self, name: str, attrs: xml.sax.xmlreader.AttributesImpl): @@ -471,7 +490,7 @@ def startElement(self, name: str, if offset is not None: parts.append(f"offset={offset}") logger.debug(*parts) - if attrs is not None and attrs : + if (attrs is not None) and attrs.getNames(): logger.debug(tab, " Attributes: ", attrs.getNames()) # el = ET.Element(name, attrs) @@ -502,6 +521,7 @@ def startElement(self, name: str, self.onPushScope(cm) if len(self._tag_stack) < 1: + assert self._root_memos is not None, "onStart must run first" self._root_memos.append(cm) if cm.tag == "cdi": self._root_memo = cm @@ -623,3 +643,109 @@ def characters(self, content: str): raise TypeError( "Expected str, got {}".format(type(content).__name__)) self._chunks.append(content) + + def expandedTree(self) -> ET.Element: + """Build an expanded XML tree with replication and addresses. + + Starting from the root CDIMemo (via :meth:`getRootMemo`), this + method creates a new ElementTree. Replication is expanded, + addresses are calculated per the OpenLCB CDI standard, and + ``address`` attributes are added where required. + + The ``replication`` attribute is removed from all copied group + elements in the new tree. The original tree is left unchanged. + + Returns: + ET.Element: Root of the new expanded tree. + """ + root_memo = self.getRootMemo() + assert root_memo is not None and root_memo.element is not None + + new_root = ET.Element(root_memo.element.tag) + new_root.attrib.update(root_memo.element.attrib) + + self.address = 0 + + self._expanded_tree_recursive(root_memo, new_root) + return new_root + + def _expanded_tree_recursive( + self, memo: CDIMemo, new_parent: ET.Element + ) -> None: + """Recursive helper for :meth:`expandedTree`. + + Copies the element, handles replication, sets addresses, and + recurses into children. Removes ``replication`` attribute from + copied group elements. + """ + assert memo.element is not None + tag = memo.getTag() or memo.element.tag + tag_lower = tag.lower() + + replication_str = memo.element.attrib.get("replication") + count = int(replication_str) if replication_str is not None else 1 + + if tag_lower == "group": + origin = memo.element.attrib.get("origin") + self.address = int(origin) if origin is not None else 0 + + for idx in range(count): + elem_copy = ET.Element(tag) + elem_copy.attrib.update(memo.element.attrib) + + # Remove replication from the expanded copy + if "replication" in elem_copy.attrib: + del elem_copy.attrib["replication"] + + if tag_lower == "group" or memo.tag in CLASSNAME_TYPES: + elem_copy.set("address", str(self.address)) + + if replication_str is not None and tag_lower == "group": + elem_copy.set("replication_index", str(idx)) + + new_parent.append(elem_copy) + + # Determine size for address advancement + if tag_lower == "eventid": + size = 8 + elif "size" in memo.element.attrib: + size = int(memo.element.attrib["size"]) + else: + size = 0 + + # Recurse into children (replication handled at this level) + for child_memo in memo.children: + self._expanded_tree_recursive(child_memo, elem_copy) + + # Advance address for leaf variables + if tag_lower in CLASSNAME_TYPES or tag_lower == "eventid": + self.address += size + + def extractCDIVarMemos(self) -> List[CDIMemo]: + """Build a flat list of CDIMemo objects for all variables. + + Uses :meth:`expandedTree` internally so replication is fully + expanded and stores it as ``self.expanded_root``. + + Returns: + List of CDIMemo objects (with elements from the expanded + tree). + """ + if not hasattr(self, "etree") or self.etree is None: + logger.error("processor has no etree") + return [] + + self.expanded_root = self.expandedTree() + + cdivar_memos: List[CDIMemo] = [] + + def traverse(element: ET.Element) -> None: + tag_lower = element.tag.lower() + if tag_lower in CLASSNAME_TYPES: + memo = CDIMemo(tag=element.tag, element=element) + cdivar_memos.append(memo) + for child in element: + traverse(child) + + traverse(self.expanded_root) + return cdivar_memos From 1682c765c40774fbdae343ac371b13cebd6e2dbe Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Fri, 24 Apr 2026 20:04:05 -0400 Subject: [PATCH 11/64] Make a DataProcessorMemo superclass for parsing messages that are not XML nodes. --- openlcb/dataprocessormemo.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 openlcb/dataprocessormemo.py diff --git a/openlcb/dataprocessormemo.py b/openlcb/dataprocessormemo.py new file mode 100644 index 00000000..32b28695 --- /dev/null +++ b/openlcb/dataprocessormemo.py @@ -0,0 +1,32 @@ +from typing import Union + +from openlcb.message import Message + + +class DataProcessorMemo: + """Store parsing state info. + This superclass can be used for progress notification. + + Attributes: + done (bool): If True, download such as downloadCDI is finished. + Though document itself may be incomplete if 'error' is also + set, stop tracking status of download regardless. + end (bool): False to start a deeper scope, or True for end tag, + which exits current scope (last created Treeview branch in + this case, or top if getBranch() would be None). + error (str): Message of failure (requires 'done' if stopped). + message (Message): Associated network/internal message. + name (str): Name (determined by `name` child element content). + status (str): Status message. + """ + def __init__(self, status: Union[str, None] = None): + self.done = False # type: bool + self.end = False # type: bool + self.error = None # type: str|None + self.message: Union[Message, None] = None # type: Message|None + self.status = status # type: str|None + # region set by DataProcessor such as XMLDataProcessor + self.progress_ratio = None # type: float|None + self.progress_count = None # type: int|None + self.expected_size = None # type: int|None + # end region set by DataProcessor such as XMLDataProcessor From f54ee8c7260869549b013923825956738decad6f Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Fri, 24 Apr 2026 20:04:34 -0400 Subject: [PATCH 12/64] Improve emit_cast. --- openlcb/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openlcb/__init__.py b/openlcb/__init__.py index 788de048..309c515f 100644 --- a/openlcb/__init__.py +++ b/openlcb/__init__.py @@ -36,6 +36,8 @@ def only_hex_pairs(value: str) -> Union[re.Match[bytes], re.Match[str], None]: def emit_cast(value) -> str: """Get type and value, such as for debug output.""" + if value is None: + return "None" repr_str = repr(value) if isinstance(value, Enum): repr_str = "{}".format(value.value) From 3ac3776516277f5872fa5c65ac7ea7579b943d66 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Fri, 24 Apr 2026 20:32:20 -0400 Subject: [PATCH 13/64] Fix replication recursion. Fix: Only set CDIMemo address during replication. --- examples/example_cdi_access.py | 6 +- openlcb/cdimemo.py | 101 ++++++++++--- openlcb/cdivar.py | 1 + openlcb/openlcbnetwork.py | 36 +++-- openlcb/xmldataprocessor.py | 249 ++++++++++++++++++++++----------- 5 files changed, 279 insertions(+), 114 deletions(-) diff --git a/examples/example_cdi_access.py b/examples/example_cdi_access.py index a931f0ae..4bbf5f27 100644 --- a/examples/example_cdi_access.py +++ b/examples/example_cdi_access.py @@ -190,10 +190,8 @@ class MyHandler(xml.sax.handler.ContentHandler): _chunks (list[str]): Collects chunks of data. This is implementation-specific, and not required if streaming (parser.feed). - _tmp_address (int|None): Where we are in the memory space (starting - at origin, and calculated using offset and/or size of start - tags). - _tmp_space (int|None): What space we are currently on. + _tmp_address (int|None): For sanity check, not actual address. + See expandedTree docstring. """ def __init__(self): diff --git a/openlcb/cdimemo.py b/openlcb/cdimemo.py index 81124008..9803d1f1 100644 --- a/openlcb/cdimemo.py +++ b/openlcb/cdimemo.py @@ -5,10 +5,14 @@ import xml.etree.ElementTree # import xml.etree.ElementTree as ET -from typing import List, Optional, Union +from typing import Dict, List, Optional, Union +from logging import getLogger -from openlcb.cdivar import FLOAT_MAXIMUMS, NUM_TYPES, CDIVar +from openlcb.cdivar import CLASSNAME_TYPES, FLOAT_MAXIMUMS, NUM_TYPES, CDIVar from openlcb.message import Message +from openlcb.dataprocessormemo import DataProcessorMemo + +logger = getLogger(__name__) def element_ordered(el: xml.etree.ElementTree.Element): @@ -18,7 +22,7 @@ def element_ordered(el: xml.etree.ElementTree.Element): return od -class CDIMemo: +class CDIMemo(DataProcessorMemo): """Store parsing state info as a tree (This is a tree node) Attributes: @@ -41,32 +45,34 @@ class CDIMemo: error (str): Message of failure (requires 'done' if stopped). iid (str): Treeview branch id (no parent when top of Treeview) name (str): Name (determined by `name` child element content). + space (int|None): The memory space address (May be one in + MemorySpace values, or not if vendor-specific such as + defined in CDI etc. See expandedTree in XMLDataProcessor). stray (bool): The end tag is misplaced (doesn't match a start tag) due to bad xml or incorrect parsing. + tail (str|None): Content following the end tag (not used in + OpenLCB CDI/FDI standards). """ def __init__(self, tag: Union[str, None] = None, element: Union[xml.etree.ElementTree.Element, None] = None, status: Union[str, None] = None, - parent: Optional['CDIMemo'] = None): + parent: Optional['CDIMemo'] = None, + document: Optional['XMLDocumentProcessor'] = None): + DataProcessorMemo.__init__(self) self.tag = tag # type: str|None # self.name = None # type: str|None self.element = element # type: xml.etree.ElementTree.Element|None - self.status = status # type: str|None - self.error = None # type: str|None - self.done = False # type: bool - self.end = False # type: bool self.parent = parent # type: CDIMemo|None self.stray = False # type: bool self.content = None # type: str|None - self.message: Union[Message, None] = None # type: Message|None + self.tail = None # type: str|None + # TODO: Set tail (unused in OpenLCB CDI/FDI standards, but allowed in XML) self.iid = None # type: str|None self.address = None # type: int|None + self.space = None # type: int|None self.cdivar = None # type: CDIVar|None self.children = [] # type: List[CDIMemo] - # Set by DataProcessor such as XMLDataProcessor: - self.progress_ratio = None # type: float|None - self.progress_count = None # type: int|None - self.expected_size = None # type: int|None + self.document: Union[Optional['XMLDocumentProcessor'], None] = document def getTag(self): if self.element is None: @@ -101,12 +107,30 @@ def getChildContent(self, tag) -> Union[str, None]: return None def copy(self): + return self.__copy__() + + def __copy__(self): cm = CDIMemo() for k, v in self.__dict__.items(): - if isinstance(v, (list, dict, OrderedDict)): - setattr(cm, k, copy.deepcopy(v)) - else: - setattr(cm, k, v) + setattr(cm, k, v) + return cm + + def __deepcopy__(self, memo: dict): + """Allow deepcopy on this class. + Place id of new object in memo dict + (prevents infinite recursion). + See . + """ + cm = type(self)() + memo[id(self)] = cm + for k, v in self.__dict__.items(): + if k == 'parent': + # prevent invalid container + continue + if k == "document": + # prevent un-pickle-able object (& invalid container) + continue + setattr(cm, k, copy.deepcopy(v, memo)) return cm def getBranch(self, default=None) -> Union[str, None]: @@ -147,6 +171,8 @@ def to_dict(cm): # continue if k == 'parent': continue + if k == 'document': + continue if isinstance(v, xml.etree.ElementTree.Element): d[k] = element_ordered(v) continue @@ -157,8 +183,12 @@ def __str__(self): return json.dumps(CDIMemo.to_dict(self), default=CDIMemo.to_dict) def toCDIVar(self): + # type: () -> CDIVar """Create a CDIVar from descriptors (child elements of self). See LCC "Configuration Description Information" Standard. + + NOTE: The `address` is only correct if this CDIMemo has been + replicated (such as in expandedTree or self.expanded_root). """ # result = CDIVar(self.tag) assert (self.tag is not None) and (self.tag.strip()) @@ -189,8 +219,12 @@ def toCDIVar(self): # enforces size: result = CDIVar(self.tag, _min=result_min, _max=result_max, _size=result_size, _default=result_default) + result.address = self.address # only set in expandedTree() + result.space = self.space result.floatFormat = result_floatFormat result.name = self.getChildContent("name") + if not result.name and (self.tag in CLASSNAME_TYPES): + raise NotImplementedError(f"Can't get name for {self}") if result.className == "int": if result.min is None: @@ -230,3 +264,36 @@ def getSize(self): if size is None: return None return int(size) + + def addChildren(self) -> None: + """Recursively build the full CDIMemo tree from self.element. + + Populates ``self.children`` with proper CDIMemo instances + (one per direct child element). Each child memo also gets + its own children built recursively. + + Preserves original ``.content`` from the parsed tree. + """ + if self.element is None: + self.children = [] + return + + self.children = [] + if self.element.text: + self.content = self.element.text + elif self.element.tag.lower() in ("name", "description"): + logger.warning( + f"{self.element.tag} has no content.") + + if self.element.tail: + self.tail = self.element.tail + + for child_elem in list(self.element): # list() to avoid modification issues + child_memo = CDIMemo( + tag=child_elem.tag, + element=child_elem, + parent=self, + document=self.document + ) + child_memo.addChildren() # recursive + self.children.append(child_memo) diff --git a/openlcb/cdivar.py b/openlcb/cdivar.py index 2c667b03..3a51db90 100644 --- a/openlcb/cdivar.py +++ b/openlcb/cdivar.py @@ -99,6 +99,7 @@ def __init__(self, className, _min=None, _max=None, self.floatFormat = None # type: str|None self.address = None # type: int|None self.element = None # type: Any|None + self.space = None # type: int|None def setData(self, data: Union[bytes, bytearray]): assert isinstance(data, (bytes, bytearray)) diff --git a/openlcb/openlcbnetwork.py b/openlcb/openlcbnetwork.py index fd70f729..33f28067 100644 --- a/openlcb/openlcbnetwork.py +++ b/openlcb/openlcbnetwork.py @@ -28,6 +28,7 @@ from openlcb.cdimemo import CDIMemo from openlcb.datagramservice import DatagramReadMemo, DatagramService from openlcb.dataprocessor import DataFormat +from openlcb.dataprocessormemo import DataProcessorMemo from openlcb.memoryservice import MemoryReadMemo, MemoryService, MemorySpace from openlcb.message import Message from openlcb.xmldataprocessor import XMLDataProcessor @@ -54,7 +55,7 @@ class OpenLCBNetwork: is a MemorySpace) """ def __init__(self, localNodeID: Union[str, bytearray, int, NodeID]): - self._onConnect: Union[Callable[[CDIMemo], None], None] = None + self._onConnect: Union[Callable[[DataProcessorMemo], None], None] = None self._port: PortInterface = None self.physicalLayer: CanPhysicalLayerGridConnect = None self.canLink: CanLink = None @@ -289,23 +290,28 @@ def _listen(self): # manually. # - Usually "socket connection broken" due to no more # bytes to read, but ok if "\0" terminator was reached. - if ((self._dataProcessor._data is not None) - and (not self._dataProcessor._stringTerminated)): - # This boolean is managed by the memoryReadSuccess - # callback. - cm = CDIMemo() - cm.error = formatted_ex(ex) - cm.done = True # stop progress in gui/other main thread - if self._dataProcessor.onStatusMemo: - self._dataProcessor.onStatusMemo(cm) - raise # re-raise since incomplete (prevent done OK state) + if self._dataProcessor is not None: + if ((self._dataProcessor._data is not None) + and (not self._dataProcessor._stringTerminated)): + # This boolean is managed by the memoryReadSuccess + # callback. + cm = DataProcessorMemo() + cm.error = formatted_ex(ex) + cm.done = True # stop progress in gui/other main thread + if self._dataProcessor.onStatusMemo: + self._dataProcessor.onStatusMemo(cm) + raise # re-raise since incomplete (prevent done OK state) + else: + logger.warning( + "Listen loop ended, but _dataProcessor not set" + " (DataProcessorMemo will not be used to notify caller).") finally: self.physicalLayer.physicalLayerDown() # Link_Layer_Down, setState self._listenThread: Union[threading.Thread, None] = None # If we got here, the RuntimeError was ok since the # null terminator '\0' was reached (otherwise re-raise occurs above) - cm = CDIMemo() + cm = DataProcessorMemo() cm.error = ("Listen loop stopped (caught_ex={})." .format(formatted_ex(caught_ex))) cm.done = True @@ -336,7 +342,7 @@ def _handleMessage(self, message: Message): logger.debug("[_handleMessage] message.mti={}".format(message.mti)) if message.mti == MTI.Link_Layer_Down: if self._onConnect: - cm = CDIMemo() + cm = DataProcessorMemo() cm.done = True cm.error = "Disconnected" cm.message = message @@ -345,7 +351,7 @@ def _handleMessage(self, message: Message): return True elif message.mti == MTI.Link_Layer_Up: if self._onConnect: - cm = CDIMemo() + cm = DataProcessorMemo() cm.done = True # 'done' without error indicates connected. cm.message = message self._onConnect(cm) @@ -406,7 +412,7 @@ def _memoryReadFail(self, memo: MemoryReadMemo): if len(self._dataProcessor._tag_stack): cm = self._dataProcessor._tag_stack[-1] else: - cm = CDIMemo() + cm = DataProcessorMemo() cm.error = error cm.done = True # stop progress in gui/other main thread self._dataProcessor._onElement(cm) diff --git a/openlcb/xmldataprocessor.py b/openlcb/xmldataprocessor.py index 3054efc5..f8bf8a81 100644 --- a/openlcb/xmldataprocessor.py +++ b/openlcb/xmldataprocessor.py @@ -1,17 +1,18 @@ from collections import OrderedDict +import copy import os import xml.sax # noqa: E402 import xml.sax.handler import xml.etree.ElementTree as ET from logging import getLogger -from typing import Callable, List, Union +from typing import Callable, List, Tuple, Union # from xml.sax.xmlreader import AttributesImpl # for type hints, for autocomplete only in this case # noqa:E501 import xml.sax.xmlreader # for type hints, for autocomplete only in this case from openlcb import d_quote, emit_cast from openlcb.canbus.canlink import CanLink -from openlcb.cdimemo import CDIMemo +from openlcb.cdimemo import CDIMemo, DataProcessorMemo from openlcb.dataprocessor import DataFormat, DataProcessor from openlcb.nodeid import NodeID from openlcb.platformextras import ( @@ -101,10 +102,9 @@ class XMLDataProcessor(xml.sax.handler.ContentHandler, DataProcessor): _space (int): Space containing the CDI itself (not data described by CDI). _tmp_space (int|None): What space we are currently on - (of data described by Element(s), not of XML data itself) - _tmp_address (int|None): Where we are in the memory space - (starting at origin, and calculated using offset and/or size - of start tags). + (of data described by Element(s), not of XML data itself). + _tmp_address (int|None): For sanity check, not actual address + (no replication)! See expandedTree docstring. """ XML_TOP_TAGS = ("cdi", "fdi") DEFAULT_CACHES_DIR = SysDirs.Cache @@ -113,7 +113,8 @@ class XMLDataProcessor(xml.sax.handler.ContentHandler, DataProcessor): def __init__(self, linkLayer: CanLink, space: MemorySpace): self.canLink: CanLink = linkLayer # caches_dir = SysDirs.Cache - self.expanded_root = None # type: xml.etree.ElementTree|None + self.expanded_root = None # type: ET.Element|None + self.expanded_root_memo = None # type: CDIMemo|None self._root_memos = None # type: list[CDIMemo]|None self._root_memo = None # type: CDIMemo|None self._space: Union[MemorySpace, None] = None @@ -202,7 +203,7 @@ def space(self) -> MemorySpace: assert isinstance(self._space, MemorySpace) return self._space - def onStatusMemo(self, cm: CDIMemo) -> bool: + def onStatusMemo(self, cm: DataProcessorMemo) -> bool: """Handle memo with status that doesn't affect tag stack/scope. (Implement in subclass) Returns: @@ -297,7 +298,7 @@ def _feedNext(self, memo: MemoryReadMemo): partial_str = memo.data.decode("utf-8") if self._realtime: self._parser.feed(partial_str) # may call startElement/endElement - cm = CDIMemo() + cm = DataProcessorMemo() cm.progress_count = self.progress_count cm.expected_size = self.expected_size self.onStatusMemo(cm) @@ -395,7 +396,7 @@ def _feedLast(self, memo: MemoryReadMemo, enable_cache=None): partial_str = memo.data[:terminate_i].decode("utf-8") assert self.progress_count is not None self.progress_count += terminate_i - cm = CDIMemo() + cm = DataProcessorMemo() cm.done = True # 'done' and not 'error' means got all cm.progress_count = self.progress_count cm.expected_size = self.expected_size @@ -419,7 +420,7 @@ def _feedLast(self, memo: MemoryReadMemo, enable_cache=None): # ?xml version="1.0" encoding="utf-8"?> xml.sax.parseString(cdiString, self) # self._fireStatus("Done loading CDI.") - cm = CDIMemo() + cm = DataProcessorMemo() cm.done = True # 'done' and not 'error' means got all cm.progress_count = self.progress_count self.onStatusMemo(cm) @@ -501,7 +502,7 @@ def startElement(self, name: str, parent_cm = None if self._tag_stack: parent_cm = self._tag_stack[-1] - cm = CDIMemo(tag=name, element=el, parent=parent_cm) + cm = CDIMemo(tag=name, element=el, parent=parent_cm, document=self) if name == "segment": self._tmp_space = attrib.get('space') self._tmp_address = int(attrib.get('origin', 0)) @@ -517,7 +518,7 @@ def startElement(self, name: str, raise AttributeError( f"Node specifies {name} offset before segment origin") self._tmp_address += offset - cm.address = self._tmp_address # May be None if after /segment + # NOTE: ^ Sanity check only! For real address see expandedTree. self.onPushScope(cm) if len(self._tag_stack) < 1: @@ -583,7 +584,7 @@ def endElement(self, name: str): top_cm.tag = name cm = top_cm else: - cm = CDIMemo(tag=name) + cm = CDIMemo(tag=name, document=self) cm.stray = True cm.end = True @@ -644,7 +645,7 @@ def characters(self, content: str): "Expected str, got {}".format(type(content).__name__)) self._chunks.append(content) - def expandedTree(self) -> ET.Element: + def expandedTree(self) -> Tuple[CDIMemo, ET.Element]: """Build an expanded XML tree with replication and addresses. Starting from the root CDIMemo (via :meth:`getRootMemo`), this @@ -661,91 +662,183 @@ def expandedTree(self) -> ET.Element: root_memo = self.getRootMemo() assert root_memo is not None and root_memo.element is not None - new_root = ET.Element(root_memo.element.tag) + new_root = ET.Element("cdi") # always new: children added from memos new_root.attrib.update(root_memo.element.attrib) - self.address = 0 - - self._expanded_tree_recursive(root_memo, new_root) - return new_root + new_root_memo = copy.deepcopy(root_memo) # deepcopy to edit children! + new_root_memo.document = self + size = self._expanded_tree_recursive(new_root_memo, new_root, address=0) + if size < 1: + logger.warning(f"No space used by CDI after replication (size={size})") + return new_root_memo, new_root def _expanded_tree_recursive( - self, memo: CDIMemo, new_parent: ET.Element - ) -> None: + self, parent: CDIMemo, parent_el: ET.Element, + allow_non_standard=False, + address: int = 0, + space: Union[int, None] = None, + ) -> int: """Recursive helper for :meth:`expandedTree`. Copies the element, handles replication, sets addresses, and recurses into children. Removes ``replication`` attribute from copied group elements. """ - assert memo.element is not None - tag = memo.getTag() or memo.element.tag - tag_lower = tag.lower() - - replication_str = memo.element.attrib.get("replication") - count = int(replication_str) if replication_str is not None else 1 - - if tag_lower == "group": - origin = memo.element.attrib.get("origin") - self.address = int(origin) if origin is not None else 0 - - for idx in range(count): - elem_copy = ET.Element(tag) - elem_copy.attrib.update(memo.element.attrib) - - # Remove replication from the expanded copy - if "replication" in elem_copy.attrib: - del elem_copy.attrib["replication"] - - if tag_lower == "group" or memo.tag in CLASSNAME_TYPES: - elem_copy.set("address", str(self.address)) - - if replication_str is not None and tag_lower == "group": - elem_copy.set("replication_index", str(idx)) - - new_parent.append(elem_copy) - - # Determine size for address advancement - if tag_lower == "eventid": - size = 8 - elif "size" in memo.element.attrib: - size = int(memo.element.attrib["size"]) - else: - size = 0 - - # Recurse into children (replication handled at this level) - for child_memo in memo.children: - self._expanded_tree_recursive(child_memo, elem_copy) - - # Advance address for leaf variables - if tag_lower in CLASSNAME_TYPES or tag_lower == "eventid": - self.address += size - - def extractCDIVarMemos(self) -> List[CDIMemo]: + assert address is not None + assert parent.element is not None + parent_tag = parent.getTag() or parent.element.tag + parent_tag_lower = parent_tag.lower() + if parent_el.text: + parent.content = parent_el.text + elif parent.content: # new_root + parent_el.text = parent.content + elif parent_tag_lower in ("name", "description"): + logger.warning( + f"expanded {parent_tag_lower} has no content.") + if parent_el.tail: + parent.tail = parent_el.tail + elif parent.tail: # # new_root + parent_el.tail = parent.tail + + # Recurse into children (replication handled at this level) + new_children = [] + # new_child_elements = [] + for child_memo in parent.children: + replication_str = parent.element.attrib.get("replication") + count = int(replication_str) if replication_str is not None else 1 + child_tag = child_memo.getTag() + assert child_tag + c_tag_lower = child_tag.lower() + child_el = child_memo.element + assert child_el is not None + if c_tag_lower == "segment": + space_str = child_el.attrib.get("space") + assert space_str, "expected space in segment" + space = int(space_str) + address = 0 # as per standard, 1st is at 0 else use "group" + if c_tag_lower == "group": + origin = child_el.attrib.get("origin") + address = int(origin) if (origin is not None) else 0 + for idx in range(count): + # if count > 1: + copy_child_el = ET.Element(child_el.tag) + copy_child_el.attrib.update(child_el.attrib) + copy_child_el.text = child_el.text + copy_child_el.tail = child_el.tail + copy_child_memo = copy.deepcopy(child_memo) + copy_child_memo.parent = parent + copy_child_memo.document = self + copy_child_memo.element = copy_child_el + # else: + # copy_child_el = child_el + # copy_child_memo = child_memo + # NOTE: ^ Why commented: We don't want to modify + # self.etree children (if we modify expandedTree + # result such as self.expanded_root)! + # - Don't even chance it by keeping the memo + # (otherwise child_memo.element would be from tree). + # - Also, we always add child to parent_el below. + + new_children.append(copy_child_memo) + # for child_el in new_parent: + # copy_parent_el.append(child_el) + + # Remove replication from the expanded copy + if "replication" in copy_child_el.attrib: + del copy_child_el.attrib["replication"] + + if c_tag_lower == "group" or c_tag_lower in CLASSNAME_TYPES: + copy_child_el.set("address", str(address)) + copy_child_el.set("space", str(space)) + if c_tag_lower in CLASSNAME_TYPES: + copy_child_memo.address = address + copy_child_memo.space = space + + if replication_str is not None: + if c_tag_lower != "group": + el_error = \ + f"unexpected replication for {c_tag_lower} tag" + if allow_non_standard: + logger.warning(el_error) + else: + raise SyntaxError(el_error) + copy_child_el.set('replication_index', str(idx)) + # ^ optimized attrib[key] = value + + parent_el.append(copy_child_el) + # parent: Use new_children below (can't change while iterating) + + # Determine size for address advancement + if c_tag_lower == "eventid": + size = 8 + elif "size" in copy_child_el.attrib: + size = int(copy_child_el.attrib["size"]) + else: + size = 0 + + # Advance address *before* leaf variables + if size: + if c_tag_lower in CLASSNAME_TYPES: + assert size, f"expected size for {c_tag_lower}" + address += size + else: + el_error = ( + f"size is not expected for {c_tag_lower}" + f" size={size}") + if allow_non_standard: + logger.warning(el_error) + address += size + else: + assert not size, el_error + + address = self._expanded_tree_recursive( + copy_child_memo, copy_child_el, address=address, + space=space) + if c_tag_lower == "segment": + space = None # undefined after section + + parent.children = new_children # Same references if no replication + return address + + def extractCDIVarMemos(self, expanded_root=None, root_memo=None) -> List[CDIMemo]: # noqa: E501 + # type: (ET.Element|None, CDIMemo|None) -> List[CDIMemo] """Build a flat list of CDIMemo objects for all variables. - Uses :meth:`expandedTree` internally so replication is fully - expanded and stores it as ``self.expanded_root``. - - Returns: - List of CDIMemo objects (with elements from the expanded - tree). + Uses the expanded tree (replication expanded, replication + attribute removed, addresses set). Returns original-style + memos (with .content) but with .element pointing into the + expanded tree so that modifications (setData etc.) affect + the saved XML. """ + # TODO: Implement ACDI vars if present (See OpenLCB + # "Configuration Description Information" Standard) if not hasattr(self, "etree") or self.etree is None: logger.error("processor has no etree") return [] + if expanded_root is not None: + if root_memo is not None: # reserved + assert isinstance(root_memo, CDIMemo) + assert isinstance(expanded_root, ET.Element) + root_memo = root_memo # reserved + root_el = expanded_root + else: + root_memo, root_el = self.expandedTree() + self.expanded_root = root_el + self.expanded_root_memo = root_memo - self.expanded_root = self.expandedTree() + assert isinstance(self.expanded_root_memo, CDIMemo) cdivar_memos: List[CDIMemo] = [] - def traverse(element: ET.Element) -> None: - tag_lower = element.tag.lower() + def traverse(memo: CDIMemo) -> None: + tag = memo.getTag() + tag_lower = tag.lower() if tag else "" if tag_lower in CLASSNAME_TYPES: - memo = CDIMemo(tag=element.tag, element=element) + # Use the existing expanded memo (has correct .content) cdivar_memos.append(memo) - for child in element: + for child in memo.children: traverse(child) - traverse(self.expanded_root) + traverse(self.expanded_root_memo) + assert root_el is self.expanded_root # concurrent modification check return cdivar_memos From a7d0cd664ec827a7b339b3084f42623fb95d66f1 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Mon, 27 Apr 2026 14:13:45 -0400 Subject: [PATCH 14/64] Fix: Correctly separate tail of XML--prevents formatting from modifying values (text content). --- openlcb/xmldataprocessor.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/openlcb/xmldataprocessor.py b/openlcb/xmldataprocessor.py index f8bf8a81..bc003a62 100644 --- a/openlcb/xmldataprocessor.py +++ b/openlcb/xmldataprocessor.py @@ -88,6 +88,12 @@ class XMLDataProcessor(xml.sax.handler.ContentHandler, DataProcessor): _openEl (SubElement): Tracks currently-open tag (no `` yet) during parsing, or if no tags are open then equals etree. + _ended_memo (CDIMemo|None): The memo most recently popped, + where "tail" (text after end tag) should be set during + "characters" when a new tag hasn't been started yet. + TODO: Put child element's tail in parent (Not part of + Standard as of 2026-05, but technically possible, such + as "World" in `Hello
World`). _tag_stack (list[SubElement]): Tracks scope during parse since self.etree doesn't have awareness of whether end tag is finished (and therefore doesn't know which element is the @@ -121,6 +127,7 @@ def __init__(self, linkLayer: CanLink, space: MemorySpace): self._openEl: Union[ET.Element, None] = None self._top_tag = "cdi" # cdi or fdi (detected in startElement) # self._myCacheDir = os.path.join(caches_dir, "python-openlcb") + self._ended_memo = None # type: CDIMemo|None self._myCacheDir = XMLDataProcessor.DEFAULT_CACHE_DIR self._tmp_space = None # type: int|None self._tmp_address = None # type: int|None @@ -482,6 +489,15 @@ def startElement(self, name: str, self._top_tag = name.lower() elif name.lower() == "acdi": self.acdi = True + tail = self._flushCharBuffer() if self._chunks else None + if tail is not None: + if self._ended_memo is not None: + self._ended_memo.tail = tail + else: + logger.warning( + f"Stray characters before {repr(name)}: {repr(tail)}") + if self._ended_memo is not None: + self._ended_memo = None attrib = attrs_to_dict(attrs) origin = attrib.get('origin') offset = attrib.get('offset') @@ -618,6 +634,7 @@ def endElement(self, name: str): cm.parent.children.append(cm) _ = self.checkDone(cm) cm.content = self._flushCharBuffer() + self._ended_memo = cm self.onPopScope(cm) def _flushCharBuffer(self): @@ -697,7 +714,7 @@ def _expanded_tree_recursive( f"expanded {parent_tag_lower} has no content.") if parent_el.tail: parent.tail = parent_el.tail - elif parent.tail: # # new_root + elif parent.tail: # new_root parent_el.tail = parent.tail # Recurse into children (replication handled at this level) @@ -724,7 +741,11 @@ def _expanded_tree_recursive( copy_child_el = ET.Element(child_el.tag) copy_child_el.attrib.update(child_el.attrib) copy_child_el.text = child_el.text + if child_el.text is not None: + copy_child_el.text = child_el.text.strip() copy_child_el.tail = child_el.tail + if child_el.tail is not None: + copy_child_el.tail = child_el.tail.strip() copy_child_memo = copy.deepcopy(child_memo) copy_child_memo.parent = parent copy_child_memo.document = self From d705a20e91aae34865915cd6371903acf8ffc7d1 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Mon, 27 Apr 2026 16:11:30 -0400 Subject: [PATCH 15/64] Fix: Correctly handle origin and offset. Fix: Correctly preserve XML formatting. --- openlcb/xmldataprocessor.py | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/openlcb/xmldataprocessor.py b/openlcb/xmldataprocessor.py index bc003a62..82a3cb89 100644 --- a/openlcb/xmldataprocessor.py +++ b/openlcb/xmldataprocessor.py @@ -489,13 +489,21 @@ def startElement(self, name: str, self._top_tag = name.lower() elif name.lower() == "acdi": self.acdi = True - tail = self._flushCharBuffer() if self._chunks else None - if tail is not None: + content = self._flushCharBuffer() if self._chunks else None + if content is not None: if self._ended_memo is not None: - self._ended_memo.tail = tail + self._ended_memo.tail = content else: - logger.warning( - f"Stray characters before {repr(name)}: {repr(tail)}") + if self._tag_stack: + # Text in parent before this + # (typically "\n", possibly indentation). + if self._tag_stack[-1].content is None: + self._tag_stack[-1].content = content + else: + self._tag_stack[-1].content += content + else: + logger.warning( + f"Stray characters before {repr(name)}: {repr(content)}") if self._ended_memo is not None: self._ended_memo = None attrib = attrs_to_dict(attrs) @@ -721,7 +729,7 @@ def _expanded_tree_recursive( new_children = [] # new_child_elements = [] for child_memo in parent.children: - replication_str = parent.element.attrib.get("replication") + replication_str = parent.element.attrib.get('replication') count = int(replication_str) if replication_str is not None else 1 child_tag = child_memo.getTag() assert child_tag @@ -729,13 +737,15 @@ def _expanded_tree_recursive( child_el = child_memo.element assert child_el is not None if c_tag_lower == "segment": - space_str = child_el.attrib.get("space") + space_str = child_el.attrib.get('space') assert space_str, "expected space in segment" space = int(space_str) - address = 0 # as per standard, 1st is at 0 else use "group" - if c_tag_lower == "group": - origin = child_el.attrib.get("origin") + origin = child_el.attrib.get('origin') address = int(origin) if (origin is not None) else 0 + if c_tag_lower == "group": + offset = child_el.attrib.get('offset') + if offset: + address += int(offset) for idx in range(count): # if count > 1: copy_child_el = ET.Element(child_el.tag) @@ -769,8 +779,8 @@ def _expanded_tree_recursive( del copy_child_el.attrib["replication"] if c_tag_lower == "group" or c_tag_lower in CLASSNAME_TYPES: - copy_child_el.set("address", str(address)) - copy_child_el.set("space", str(space)) + copy_child_el.set('address', str(address)) + copy_child_el.set('space', str(space)) if c_tag_lower in CLASSNAME_TYPES: copy_child_memo.address = address copy_child_memo.space = space From de593f951daab0daf8d5f2c2bcf2f5bd7f0aa8af Mon Sep 17 00:00:00 2001 From: Bob Jacobsen Date: Thu, 30 Apr 2026 10:30:40 -0400 Subject: [PATCH 16/64] only act on datagram replies matched to our requests --- openlcb/datagramservice.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/openlcb/datagramservice.py b/openlcb/datagramservice.py index dd70f4f1..8b0ae749 100644 --- a/openlcb/datagramservice.py +++ b/openlcb/datagramservice.py @@ -246,6 +246,9 @@ def handleDatagramReceivedOK(self, message: Message): # match to the memo and remove from queue memo = self.matchToWriteMemo(message) # type: DatagramWriteMemo|None + # check for whether a match was found, indicating this was for us + if memo is None : return + # check of tracking logic if self.currentOutstandingMemo != memo: logger.error( From 45783bd732a27e45a11e146b0a8fddec9e81945d Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Thu, 30 Apr 2026 15:41:10 -0400 Subject: [PATCH 17/64] Improve hr_repr. Separate cacheFileName code. --- openlcb/__init__.py | 10 ++++++---- openlcb/xmldataprocessor.py | 18 ++++++++++++------ 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/openlcb/__init__.py b/openlcb/__init__.py index 309c515f..325ed0fa 100644 --- a/openlcb/__init__.py +++ b/openlcb/__init__.py @@ -139,10 +139,12 @@ def hr_repr(value, always_quote: bool = False) -> str: """Represent value with double quotes (Human-readable repr). """ - repr_value = repr(value) - if repr_value.startswith("'") and repr_value.endswith("'"): - return '"' + repr_value[1:-1].replace('"', '\\"') + '"' - elif always_quote: + # repr_value = repr(value) + # repr_value = repr_value.replace("\\\\", "\\") + # if repr_value.startswith("'") and repr_value.endswith("'"): + # return '"' + repr_value[1:-1].replace('"', '\\"') + '"' + repr_value = str(value) + if always_quote or isinstance(value, str): return '"' + repr_value.replace('"', '\\"') + '"' return repr(value) diff --git a/openlcb/xmldataprocessor.py b/openlcb/xmldataprocessor.py index 82a3cb89..d08ed62e 100644 --- a/openlcb/xmldataprocessor.py +++ b/openlcb/xmldataprocessor.py @@ -447,13 +447,21 @@ def cacheFilePathCustom(self, item_id: Union[NodeID, str], **kwargs): kwargs['my_cache_dir'] = self._myCacheDir type(self).cacheFilePath(item_id, **kwargs) + @classmethod + def cacheFileName(cls, item_id: Union[NodeID, str], ext=".cdi.xml"): + item_id = str(item_id) # Convert NodeID or other + clean_name = clean_file_name(item_id.replace(":", ".")) + clean_name += ext + return clean_name + @classmethod def cacheFilePath(cls, item_id: Union[NodeID, str], my_cache_dir=None, - subFolder="cdi", name=None, ext=".xml"): + subfolder: Union[str, None] = "cdi", name=None, + ext=".cdi.xml"): if my_cache_dir is None: my_cache_dir = cls.DEFAULT_CACHE_DIR - if subFolder: - cdi_cache_dir = os.path.join(my_cache_dir, subFolder) + if subfolder: + cdi_cache_dir = os.path.join(my_cache_dir, subfolder) else: cdi_cache_dir = my_cache_dir if not os.path.isdir(cdi_cache_dir): @@ -462,9 +470,7 @@ def cacheFilePath(cls, item_id: Union[NodeID, str], my_cache_dir=None, # name file to avoid cache file from a different # device/version. if not name: - item_id = str(item_id) # Convert NodeID or other - clean_name = clean_file_name(item_id.replace(":", ".")) - clean_name += ext + clean_name = cls.cacheFileName(item_id, ext=ext) else: clean_name = clean_file_name(name) if clean_name != name: From 34a17cc1c07b752cef62283c9edcbac1ae5e6c68 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Thu, 30 Apr 2026 16:47:25 -0400 Subject: [PATCH 18/64] Make default extension specific to DataProcessor subclass. --- openlcb/dataprocessor.py | 2 ++ openlcb/tcplink/mdnsconventions.py | 3 ++- openlcb/xmldataprocessor.py | 9 +++++++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/openlcb/dataprocessor.py b/openlcb/dataprocessor.py index d21e99a9..660e3501 100644 --- a/openlcb/dataprocessor.py +++ b/openlcb/dataprocessor.py @@ -14,6 +14,8 @@ class DataProcessor: Attributes: enable_cache (bool): Defaults to False (May differ in subclass). """ + DEFAULT_EXT = ".bin" # override in subclass + def __init__(self): self.enable_cache = False # type: bool # Members used to construct space memo such as CDIMemo: diff --git a/openlcb/tcplink/mdnsconventions.py b/openlcb/tcplink/mdnsconventions.py index 6cf4380d..ff70cafb 100644 --- a/openlcb/tcplink/mdnsconventions.py +++ b/openlcb/tcplink/mdnsconventions.py @@ -1,4 +1,5 @@ from logging import getLogger +from typing import Union from openlcb import only_hex_pairs from openlcb.conventions import hex_to_dotted_lcc_id @@ -7,7 +8,7 @@ logger = getLogger(__name__) -def id_from_tcp_service_name(service_name): +def id_from_tcp_service_name(service_name) -> Union[str, None]: """Scrape an MDNS TCP service name, assuming it uses conventions (`"{org}_{model}_{id}._openlcb-can.{protocol}.{tld}".format(...)` where: diff --git a/openlcb/xmldataprocessor.py b/openlcb/xmldataprocessor.py index d08ed62e..bb0be8df 100644 --- a/openlcb/xmldataprocessor.py +++ b/openlcb/xmldataprocessor.py @@ -113,6 +113,7 @@ class XMLDataProcessor(xml.sax.handler.ContentHandler, DataProcessor): (no replication)! See expandedTree docstring. """ XML_TOP_TAGS = ("cdi", "fdi") + DEFAULT_EXT = ".cdi.xml" # override in subclass DEFAULT_CACHES_DIR = SysDirs.Cache DEFAULT_CACHE_DIR = os.path.join(DEFAULT_CACHES_DIR, "python-openlcb") @@ -448,7 +449,9 @@ def cacheFilePathCustom(self, item_id: Union[NodeID, str], **kwargs): type(self).cacheFilePath(item_id, **kwargs) @classmethod - def cacheFileName(cls, item_id: Union[NodeID, str], ext=".cdi.xml"): + def cacheFileName(cls, item_id: Union[NodeID, str], ext=None): + if ext is None: + ext = cls.DEFAULT_EXT item_id = str(item_id) # Convert NodeID or other clean_name = clean_file_name(item_id.replace(":", ".")) clean_name += ext @@ -457,7 +460,9 @@ def cacheFileName(cls, item_id: Union[NodeID, str], ext=".cdi.xml"): @classmethod def cacheFilePath(cls, item_id: Union[NodeID, str], my_cache_dir=None, subfolder: Union[str, None] = "cdi", name=None, - ext=".cdi.xml"): + ext=None): + if ext is None: + ext = cls.DEFAULT_EXT if my_cache_dir is None: my_cache_dir = cls.DEFAULT_CACHE_DIR if subfolder: From 2feca63f9d38954523fe6724e675436437294e3e Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Fri, 1 May 2026 11:17:07 -0400 Subject: [PATCH 19/64] Make missing name issue more clear. --- openlcb/platformextras.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openlcb/platformextras.py b/openlcb/platformextras.py index 8cb6b613..142e881c 100644 --- a/openlcb/platformextras.py +++ b/openlcb/platformextras.py @@ -53,6 +53,7 @@ def clean_file_name_char(c: str, placeholder: Union[str, None] = None) -> str: def clean_file_name(name: str, placeholder: Union[str, None] = None) -> str: + assert name is not None assert isinstance(name, str) if (os.path.sep in name) or ("/" in name): # or "/" since Python uses that even on Windows From 7c9a1911edc275727ba765dd80e402d2b76c9386 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Fri, 1 May 2026 13:06:46 -0400 Subject: [PATCH 20/64] Add import used for type hint. --- openlcb/memoryservice.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openlcb/memoryservice.py b/openlcb/memoryservice.py index 698bf8df..bc1fbef7 100644 --- a/openlcb/memoryservice.py +++ b/openlcb/memoryservice.py @@ -36,6 +36,7 @@ DatagramService, ) from openlcb.convert import Convert +from openlcb.nodeid import NodeID logger = getLogger(__name__) From e55fa784d1fd39a87448e6f6d66839aaafffeed5 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Fri, 1 May 2026 13:54:48 -0400 Subject: [PATCH 21/64] Check if rejectedReply should be discarded as well (follow-up to similar okReply commit for issue #88). --- openlcb/datagramservice.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/openlcb/datagramservice.py b/openlcb/datagramservice.py index 8b0ae749..743cc21a 100644 --- a/openlcb/datagramservice.py +++ b/openlcb/datagramservice.py @@ -248,7 +248,7 @@ def handleDatagramReceivedOK(self, message: Message): # check for whether a match was found, indicating this was for us if memo is None : return - + # check of tracking logic if self.currentOutstandingMemo != memo: logger.error( @@ -267,6 +267,10 @@ def handleDatagramRejected(self, message: Message): # match to the memo and remove from queue memo = self.matchToWriteMemo(message) + # check for whether a match was found, indicating this was for us + if memo is None: + return + # check of tracking logic if self.currentOutstandingMemo != memo: logger.error( From 87ffcd2276efc3d9f4a6e22fe6c78892f2df756c Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Fri, 1 May 2026 13:59:54 -0400 Subject: [PATCH 22/64] Implement stream modes for Memory Configuration. --- openlcb/convert.py | 10 +++++++--- openlcb/memoryservice.py | 40 +++++++++++++++++++++++++++++----------- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/openlcb/convert.py b/openlcb/convert.py index 9f205eb6..6ab21e1a 100644 --- a/openlcb/convert.py +++ b/openlcb/convert.py @@ -31,9 +31,13 @@ def spaceDecode(space): - 0x00 to 0xFC represent standard memory spaces directly. Returns: - tuple(bool, byte): (False, 1-3 for in command byte) : - spaces 0xFF - 0xFD - or (True, space number) : spaces 0 - 0xFC + tuple(bool, byte): (is custom space, command | space) + - (False, 1-3 for in command byte) : + spaces 0xFF - 0xFD (Except bits beyond 0x00000011 + differ for each datagram type. See 4.2 Address + Space Selection in OpenLCB Memory Configuration + Standard) + - or (True, space number) : spaces 0 - 0xFC (NOTE: type of space may affect type of output) """ # TODO: Maybe check type of space & raise TypeError if not diff --git a/openlcb/memoryservice.py b/openlcb/memoryservice.py index bc1fbef7..840533a2 100644 --- a/openlcb/memoryservice.py +++ b/openlcb/memoryservice.py @@ -25,7 +25,8 @@ from logging import getLogger from typing import ( Callable, - List, # in case list doesn't support `[` in this Python version + List, + Optional, # in case list doesn't support `[` in this Python version Union, # in case `|` doesn't support 'type' in this Python version ) @@ -168,8 +169,8 @@ def __init__(self, service: DatagramService): self.datagramReceivedListener ) - def requestMemoryRead(self, memo): - # type: (MemoryReadMemo) -> None + def requestMemoryRead(self, memo, stream: Bool = False): + # type: (MemoryReadMemo, Optional[bool]) -> None '''Request a read operation start. - If okReply in the memo is triggered, it will be followed by a @@ -180,23 +181,31 @@ def requestMemoryRead(self, memo): Args: memo (MemoryReadMemo): Request to enqueue. ''' + assert isinstance(stream, bool) # preserve the request self.readMemos.append(memo) if len(self.readMemos) == 1: - self.requestMemoryReadNext(memo) + self.requestMemoryReadNext(memo, stream=stream) - def requestMemoryReadNext(self, memo): - # type: (MemoryReadMemo) -> None + def requestMemoryReadNext(self, memo, stream: bool = False): + # type: (MemoryReadMemo, Optional[bool]) -> None """send the read request Args: memo (MemoryReadMemo): Request to send. """ - byte6 = False + assert isinstance(stream, bool) + byte6 = False # if custom space is defined in byte 6 flag = 0 (byte6, flag) = Convert.spaceDecode(memo.space) - spaceFlag = 0x40 if byte6 else (flag | 0x40) + if stream: + # Encoding: 0x60=custom, 0x61=0xFD, 0x62=0xFE, 0x63=0xFF + spaceFlag = 0x60 if byte6 else (flag | 0x60) + else: + # Encoding: 0x40=custom, 0x41=0xFD, 0x42=0xFE, 0x43=0xFF + spaceFlag = 0x40 if byte6 else (flag | 0x40) # | 0b11111100 + # ^ In else case, flag is 1-3, so re-add 0xFC (0b11111100) addr2 = ((memo.address >> 24) & 0xFF) addr3 = ((memo.address >> 16) & 0xFF) addr4 = ((memo.address >> 8) & 0xFF) @@ -206,6 +215,7 @@ def requestMemoryReadNext(self, memo): addr2, addr3, addr4, addr5]) # NOTE: list[int] is ok for bytearray extend (`+` requires cast) if byte6: + assert memo.space <= 0xFF, f"Space {memo.space} out of byte range" data.extend([(memo.space & 0xFF)]) data.extend([memo.size]) logger.debug( @@ -313,19 +323,26 @@ def datagramReceivedListener(self, dmemo: DatagramReadMemo) -> bool: return True - def requestMemoryWrite(self, memo: MemoryWriteMemo): + def requestMemoryWrite(self, memo: MemoryWriteMemo, stream: bool = False): + # type: (MemoryWriteMemo, Optional[bool]) -> None """Request memory write. Args: memo (MemoryWriteMemo): information to send """ + assert isinstance(stream, bool) # preserve the request self.writeMemos.append(memo) # create & send a write datagram - byte6 = False + byte6 = False # if custom space is defined in byte 6 flag = 0 (byte6, flag) = Convert.spaceDecode(memo.space) - spaceFlag = 0x00 if byte6 else (flag | 0x00) + if stream: + # Encoding: 0x20=custom, 0x21=0xFD, 0x22=0xFE, 0x23=0xFF + spaceFlag = 0x20 if byte6 else (flag | 0x20) + else: + # Encoding: 0x00=custom, 0x01=0xFD, 0x02=0xFE, 0x03=0xFF + spaceFlag = 0x00 if byte6 else (flag | 0x00) addr2 = ((memo.address >> 24) & 0xFF) addr3 = ((memo.address >> 16) & 0xFF) addr4 = ((memo.address >> 8) & 0xFF) @@ -335,6 +352,7 @@ def requestMemoryWrite(self, memo: MemoryWriteMemo): addr2, addr3, addr4, addr5 ]) if byte6: + assert memo.space <= 0xFF, f"Space {memo.space} out of byte range" data.extend([(memo.space & 0xFF)]) data.extend(memo.data) dgWriteMemo = DatagramWriteMemo(memo.nodeID, data) From 177237039e7eda7fa5027900cf089dd6105380a7 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Fri, 1 May 2026 16:17:12 -0400 Subject: [PATCH 23/64] Rename spaceDecode to serializeSpace for clarity, and add opposite method: deserializeMC2ndByte. Decode errorCode and collect error string (if present) from Datagram in rejectedReply. Add getBeforeNull for deserializing strings. --- openlcb/convert.py | 35 ++++++++-- openlcb/memoryservice.py | 140 +++++++++++++++++++++++++++++++++++---- tests/test_convert.py | 8 +-- 3 files changed, 159 insertions(+), 24 deletions(-) diff --git a/openlcb/convert.py b/openlcb/convert.py index 6ab21e1a..58754777 100644 --- a/openlcb/convert.py +++ b/openlcb/convert.py @@ -20,19 +20,30 @@ class Convert: @staticmethod - def spaceDecode(space): + def deserializeMC2ndByte(datagramByte1): + """Decode byte[1] (2nd) of Memory Configuration Datagram""" + has_byte6 = False + if datagramByte1 & 0x03 == 0: + has_byte6 = True + return has_byte6, datagramByte1 & 0xFC + # ^ 0xFC = 11111100 + + # formerly spaceDecode, but it serializes a space for datagram byte2 + @staticmethod + def serializeSpace(space): """Convert from a space number to either - False and command byte or True and standard memory space + False and control number or True and standard memory space + for use in a Datagram. Args: - space (int): Encoded memory space identifier, where values: + space (int): Sequential memory space identifier, where values: - 0xFF to 0xFD are special spaces, and only the least significant - 2 bits are relevant. + 2 bits will be used in a datagram. - 0x00 to 0xFC represent standard memory spaces directly. Returns: - tuple(bool, byte): (is custom space, command | space) - - (False, 1-3 for in command byte) : + tuple(bool, byte): (is custom space, control | space) + - (False, control number 1 to 3 inclusive) : spaces 0xFF - 0xFD (Except bits beyond 0x00000011 differ for each datagram type. See 4.2 Address Space Selection in OpenLCB Memory Configuration @@ -168,3 +179,15 @@ def stringToArray(value, length): # to bytearray after getting list[int] of remaining length: padding = bytearray([0] * (length-len(contentPart))) return contentPart + padding + + @staticmethod + def getBeforeNull(data: Union[bytes, bytearray], start): + null_idx = -1 + for i in range(start, len(data)): + assert isinstance(data[i], int) + if data[i] == 0: + null_idx = i + break + if null_idx > -1: + return data[start:null_idx] + return data[start:] diff --git a/openlcb/memoryservice.py b/openlcb/memoryservice.py index 840533a2..2b982245 100644 --- a/openlcb/memoryservice.py +++ b/openlcb/memoryservice.py @@ -30,6 +30,9 @@ Union, # in case `|` doesn't support 'type' in this Python version ) +from openlcb import ( + emit_cast, +) from openlcb.datagramservice import ( # DatagramReadMemo, DatagramReadMemo, @@ -42,6 +45,26 @@ logger = getLogger(__name__) +MODE_BYTES = { + 'Read_Command': {0x40, 0x41, 0x42, 0x43}, + 'Read_Reply': {0x50, 0x51, 0x52, 0x53}, + 'Read_Stream_Command': {0x60, 0x61, 0x62, 0x63}, + 'Read_Stream_Reply': {0x70, 0x71, 0x72, 0x73}, + 'Write_Command': {0x00, 0x01, 0x02, 0x03}, + 'Write_Reply': {0x10, 0x11, 0x12, 0x13}, + 'Write_Under_Mask_Command': {0x08, 0x09, 0x0A, 0x0B}, + 'Write_Stream_Command': {0x20, 0x21, 0x22, 0x23}, + 'Write_Stream_Reply': {0x30, 0x31, 0x32, 0x33}, +} + +MODE_ERROR_BYTES = { + 'Read_Reply': {0x58, 0x59, 0x5A, 0x5B}, + 'Read_Stream_Reply': {0x78, 0x79, 0x7A, 0x7B}, + 'Write_Reply': {0x18, 0x19, 0x1A, 0x1B}, + 'Write_Stream_Reply': {0x38, 0x39, 0x3A, 0x3B}, +} + + class MemorySpace(Enum): """The memory space to read. In practice, XMLDataProcessor (or a non-XML parser if necessary) @@ -105,8 +128,12 @@ class MemoryReadMemo: Attributes: data(bytearray): The data that was read. """ - def __init__(self, nodeID, size, space, address, rejectedReply, dataReply): + def __init__(self, nodeID: NodeID, size: int, space: int, address: int, + rejectedReply: Callable[['MemoryReadMemo'], None], + dataReply: Callable[['MemoryReadMemo'], None]): # For args see class docstring. + self.error = None # type: str|None + self.errorCode = None # type: int|None self.nodeID = nodeID self.size = size self.space = space @@ -116,6 +143,7 @@ def __init__(self, nodeID, size, space, address, rejectedReply, dataReply): # for convenience, data can be added or updated after creation of the # memo self.data = bytearray() + assertMemoOK(self) class MemoryWriteMemo: @@ -138,9 +166,13 @@ class MemoryWriteMemo: memory address. """ - def __init__(self, nodeID, okReply, rejectedReply, size, space, address, - data): + def __init__(self, nodeID: NodeID, + okReply: Callable[['MemoryWriteMemo'], None], + rejectedReply: Callable[['MemoryWriteMemo'], None], + size: int, space: int, address: int, data: bytearray): # For args see class docstring. + self.error = None # type: str|None + self.errorCode = None # type: int|None self.nodeID = nodeID self.okReply = okReply self.rejectedReply = rejectedReply @@ -148,6 +180,84 @@ def __init__(self, nodeID, okReply, rejectedReply, size, space, address, self.space = space self.address = address self.data = data + assertMemoOK(self) + + +def assertMemoOK(memo: Union[MemoryReadMemo, MemoryWriteMemo]): + assert isinstance(memo.space, int), \ + f"Expected int or MemorySpace.value, got space={emit_cast(memo.space)}" + assert isinstance(memo.size, int), \ + f"Expected int, got size={emit_cast(memo.size)}" + assert memo.size <= 64, \ + f"Expected <= 64, got size={memo.size}" + assert isinstance(memo.address, int), \ + f"Expected int, got address={emit_cast(memo.address)}" + assert isinstance(memo.data, Union[bytes, bytearray]), \ + f"Expected bytearray, got data={emit_cast(memo.data)}" + + +def parseReplyDatagram(memo: Union[MemoryReadMemo, MemoryWriteMemo], + dmemo: Union[DatagramReadMemo, DatagramWriteMemo]): + """Parse dmemo and set errorCode and/or error attributes of memo""" + if not dmemo.data or dmemo.data[0] != 0x20: + logger.warning( + "Datagram type is not memory configuration (0x20)" + f" it is {hex(dmemo.data[0])}") + return + if len(dmemo.data) < 2: + logger.warning( + "Datagram is truncated to 1 byte:" + f" it is {hex(dmemo.data[0])}") + return + (hasByte6, _) = Convert.deserializeMC2ndByte(dmemo.data[1]) + offset = 6 + error = None + if hasByte6: + offset = 7 + memo.error = None + memo.errorCode = None + if (dmemo.data[1] & 0x08 == 0): + # ok reply + return + else: + pass + # 0x08 (0b00001000) is error bit + # mode = None + # for k, values in MODE_ERROR_BYTES.items(): + # if dmemo.data[1] in values: + # mode = k + # break + # if mode is not None: + # error = f"No {mode} error code." + # else: + # error = f"No error code for unknown mode {hex(dmemo.data[1])}." + code_idx = offset + + if len(dmemo.data) < code_idx + 2: + memo.error = error + if len(dmemo.data) == code_idx + 1: + memo.error = ( + f"malformed error code {hex(dmemo.data[code_idx])}" + " (expected 2 bytes)") + memo.errorCode = dmemo.data[code_idx] + return + error = None + # Decode big-endian number: + memo.errorCode = dmemo.data[code_idx] << 8 + dmemo.data[code_idx+1] + message_idx = code_idx + 2 + if len(dmemo.data) > message_idx: + error_bytes = Convert.getBeforeNull(dmemo.data, message_idx) + error = error_bytes.decode("utf-8") + if len(error) == 1: + error += f" ({hex(dmemo.data[message_idx])})" + elif len(error) == 0 and (len(dmemo.data) - message_idx > 0): + error += f" ({list(dmemo.data[message_idx:])})" + else: + error = f"(2nd byte = {hex(dmemo.data[1])})" + error += f" (hasByte6={hasByte6})" + if hasByte6: + error += f" (space={hex(dmemo.data[6])})" + memo.error = error class MemoryService: @@ -196,15 +306,15 @@ def requestMemoryReadNext(self, memo, stream: bool = False): memo (MemoryReadMemo): Request to send. """ assert isinstance(stream, bool) - byte6 = False # if custom space is defined in byte 6 + hasByte6 = False # if custom space is defined in byte 6 flag = 0 - (byte6, flag) = Convert.spaceDecode(memo.space) + (hasByte6, flag) = Convert.serializeSpace(memo.space) if stream: # Encoding: 0x60=custom, 0x61=0xFD, 0x62=0xFE, 0x63=0xFF - spaceFlag = 0x60 if byte6 else (flag | 0x60) + spaceFlag = 0x60 if hasByte6 else (flag | 0x60) else: # Encoding: 0x40=custom, 0x41=0xFD, 0x42=0xFE, 0x43=0xFF - spaceFlag = 0x40 if byte6 else (flag | 0x40) # | 0b11111100 + spaceFlag = 0x40 if hasByte6 else (flag | 0x40) # | 0b11111100 # ^ In else case, flag is 1-3, so re-add 0xFC (0b11111100) addr2 = ((memo.address >> 24) & 0xFF) addr3 = ((memo.address >> 16) & 0xFF) @@ -214,7 +324,7 @@ def requestMemoryReadNext(self, memo, stream: bool = False): DatagramService.ProtocolID.MemoryOperation.value, spaceFlag, addr2, addr3, addr4, addr5]) # NOTE: list[int] is ok for bytearray extend (`+` requires cast) - if byte6: + if hasByte6: assert memo.space <= 0xFF, f"Space {memo.space} out of byte range" data.extend([(memo.space & 0xFF)]) data.extend([memo.size]) @@ -272,6 +382,7 @@ def datagramReceivedListener(self, dmemo: DatagramReadMemo) -> bool: if len(self.readMemos) > 0: self.requestMemoryReadNext(self.readMemos[0]) + parseReplyDatagram(tMemoryMemo, dmemo) # fill data for call-back to requestor if len(dmemo.data) > offset: tMemoryMemo.data = dmemo.data[offset:] @@ -295,6 +406,7 @@ def datagramReceivedListener(self, dmemo: DatagramReadMemo) -> bool: if self.writeMemos[index].nodeID == dmemo.srcID: writeMemo = self.writeMemos[index] # type: MemoryWriteMemo del self.writeMemos[index] + parseReplyDatagram(writeMemo, dmemo) if dmemo.data[1] & 0x08 == 0 : writeMemo.okReply(writeMemo) else: @@ -334,15 +446,15 @@ def requestMemoryWrite(self, memo: MemoryWriteMemo, stream: bool = False): # preserve the request self.writeMemos.append(memo) # create & send a write datagram - byte6 = False # if custom space is defined in byte 6 + hasByte6 = False # if custom space is defined in byte 6 flag = 0 - (byte6, flag) = Convert.spaceDecode(memo.space) + (hasByte6, flag) = Convert.serializeSpace(memo.space) if stream: # Encoding: 0x20=custom, 0x21=0xFD, 0x22=0xFE, 0x23=0xFF - spaceFlag = 0x20 if byte6 else (flag | 0x20) + spaceFlag = 0x20 if hasByte6 else (flag | 0x20) else: # Encoding: 0x00=custom, 0x01=0xFD, 0x02=0xFE, 0x03=0xFF - spaceFlag = 0x00 if byte6 else (flag | 0x00) + spaceFlag = 0x00 if hasByte6 else (flag | 0x00) addr2 = ((memo.address >> 24) & 0xFF) addr3 = ((memo.address >> 16) & 0xFF) addr4 = ((memo.address >> 8) & 0xFF) @@ -351,7 +463,7 @@ def requestMemoryWrite(self, memo: MemoryWriteMemo, stream: bool = False): DatagramService.ProtocolID.MemoryOperation.value, spaceFlag, addr2, addr3, addr4, addr5 ]) - if byte6: + if hasByte6: assert memo.space <= 0xFF, f"Space {memo.space} out of byte range" data.extend([(memo.space & 0xFF)]) data.extend(memo.data) @@ -365,7 +477,7 @@ def requestSpaceLength(self, space: int, nodeID: NodeID, Args: space (int): Encoded memory space identifier. This can be a value within a specific range, as defined in the - `spaceDecode` method. + `serializeSpace` method. nodeID (NodeID): ID of remote node from which the memory space length is requested. callback (Callable): Callback function that will receive the diff --git a/tests/test_convert.py b/tests/test_convert.py index e89b019d..4271edd5 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -87,18 +87,18 @@ def testIntToArrayFail(self): with self.assertRaises(ValueError): Convert.intToArray(value, length) - def testSpaceDecode(self): + def testSerializeSpace(self): byte6 = False space = 0x00 - (byte6, space) = Convert.spaceDecode(0xF8) + (byte6, space) = Convert.serializeSpace(0xF8) self.assertEqual(space, 0xF8) self.assertTrue(byte6) - (byte6, space) = Convert.spaceDecode(0xFF) + (byte6, space) = Convert.serializeSpace(0xFF) self.assertEqual(space, 0x03) self.assertFalse(byte6) - (byte6, space) = Convert.spaceDecode(0xFD) + (byte6, space) = Convert.serializeSpace(0xFD) self.assertEqual(space, 0x01) self.assertFalse(byte6) From b5a54dc710e17ae8abe8532501bf983eacc9f867 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Fri, 1 May 2026 16:43:10 -0400 Subject: [PATCH 24/64] Allow length request reply to have arbitrary size. --- openlcb/memoryservice.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/openlcb/memoryservice.py b/openlcb/memoryservice.py index 2b982245..98f9a15a 100644 --- a/openlcb/memoryservice.py +++ b/openlcb/memoryservice.py @@ -188,10 +188,12 @@ def assertMemoOK(memo: Union[MemoryReadMemo, MemoryWriteMemo]): f"Expected int or MemorySpace.value, got space={emit_cast(memo.space)}" assert isinstance(memo.size, int), \ f"Expected int, got size={emit_cast(memo.size)}" - assert memo.size <= 64, \ - f"Expected <= 64, got size={memo.size}" + # TODO: > 64 is only ok for a length request (?) + # assert memo.size <= 64, \ + # f"Expected <= 64, got size={memo.size}" assert isinstance(memo.address, int), \ f"Expected int, got address={emit_cast(memo.address)}" + assert len(memo.data) <= 64 assert isinstance(memo.data, Union[bytes, bytearray]), \ f"Expected bytearray, got data={emit_cast(memo.data)}" From 0074507377ec8344969008378539912d8a127bba Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Fri, 1 May 2026 17:12:25 -0400 Subject: [PATCH 25/64] Fix error code deserialization (order of operation issue; result now matches JMRI). --- openlcb/memoryservice.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openlcb/memoryservice.py b/openlcb/memoryservice.py index 98f9a15a..f6c2cd93 100644 --- a/openlcb/memoryservice.py +++ b/openlcb/memoryservice.py @@ -245,7 +245,7 @@ def parseReplyDatagram(memo: Union[MemoryReadMemo, MemoryWriteMemo], return error = None # Decode big-endian number: - memo.errorCode = dmemo.data[code_idx] << 8 + dmemo.data[code_idx+1] + memo.errorCode = (dmemo.data[code_idx] << 8) + dmemo.data[code_idx+1] message_idx = code_idx + 2 if len(dmemo.data) > message_idx: error_bytes = Convert.getBeforeNull(dmemo.data, message_idx) From 1f9afcd4d12e1da3255e46468e64368bd673b43a Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Fri, 1 May 2026 17:12:32 -0400 Subject: [PATCH 26/64] Add debug output for unmatched datagrams. --- openlcb/datagramservice.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/openlcb/datagramservice.py b/openlcb/datagramservice.py index 743cc21a..ff412f4b 100644 --- a/openlcb/datagramservice.py +++ b/openlcb/datagramservice.py @@ -247,7 +247,11 @@ def handleDatagramReceivedOK(self, message: Message): memo = self.matchToWriteMemo(message) # type: DatagramWriteMemo|None # check for whether a match was found, indicating this was for us - if memo is None : return + if memo is None: + logger.debug( + f"Unrelated OK reply discarded: from" + f" {message.source} to {message.destination}") + return # check of tracking logic if self.currentOutstandingMemo != memo: @@ -269,6 +273,9 @@ def handleDatagramRejected(self, message: Message): # check for whether a match was found, indicating this was for us if memo is None: + logger.debug( + f"Unrelated Rejected reply discarded: from" + f" {message.source} to {message.destination}") return # check of tracking logic From c4f69bd812f5477e0fc58a6a09f65b2871d7dce5 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Tue, 5 May 2026 15:43:46 -0400 Subject: [PATCH 27/64] Track whether CDI was loaded from cache (file). Add assert to clarify _space value instead of causing unclear downstream error. --- openlcb/dataprocessor.py | 8 ++++++++ openlcb/xmldataprocessor.py | 6 +++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/openlcb/dataprocessor.py b/openlcb/dataprocessor.py index 660e3501..f3f71090 100644 --- a/openlcb/dataprocessor.py +++ b/openlcb/dataprocessor.py @@ -17,8 +17,16 @@ class DataProcessor: DEFAULT_EXT = ".bin" # override in subclass def __init__(self): + self._is_from_cache = False # type: bool + self._path = None # type: str|None self.enable_cache = False # type: bool # Members used to construct space memo such as CDIMemo: self.progress_ratio = None # type: float|None self.progress_count = None # type: int|None self.expected_size = None # type: int|None + + def getPath(self): + return self._path + + def isFromCache(self): + return self._is_from_cache diff --git a/openlcb/xmldataprocessor.py b/openlcb/xmldataprocessor.py index bb0be8df..ed34fa4f 100644 --- a/openlcb/xmldataprocessor.py +++ b/openlcb/xmldataprocessor.py @@ -316,6 +316,7 @@ def load(self, node_id: NodeID, path, space: Union[MemorySpace, int], format: Union[DataFormat, None] = None): """Load instead of downloading.""" assert not self._data + self._is_from_cache = True self.onStartDownload() assert isinstance(space, (MemorySpace, int)) if isinstance(space, int): @@ -336,6 +337,7 @@ def load(self, node_id: NodeID, path, space: Union[MemorySpace, int], data = None with open(path, "rb") as stream: data = stream.read() # type:ignore + self._path = path if self._format is DataFormat.XML: if memo is not None: assert isinstance(memo, MemoryReadMemo) @@ -351,8 +353,10 @@ def memoryReadFail(memo: MemoryReadMemo): assert data is not None # Based on _startMemoryRead in OpenLCBNetwork: + _space = self.getSpaceValue() # self._space is set above + assert _space is not None memo = MemoryReadMemo(node_id, len(data), - self.getSpaceValue(), 0, + _space, 0, memoryReadFail, memoryReadSuccess) assert data is not None From c2df1850bd669348ac965d9a75454fc9b22f962d Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Tue, 5 May 2026 17:48:58 -0400 Subject: [PATCH 28/64] Fix: Respect cache setting (Only save CDI file in that case). --- openlcb/xmldataprocessor.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/openlcb/xmldataprocessor.py b/openlcb/xmldataprocessor.py index ed34fa4f..ec9a4e30 100644 --- a/openlcb/xmldataprocessor.py +++ b/openlcb/xmldataprocessor.py @@ -439,12 +439,13 @@ def _feedLast(self, memo: MemoryReadMemo, enable_cache=None): if self._realtime: self._parser.feed(partial_str) # may call startElement/endElement # memo = MemoryReadMemo(memo) - path = self.cacheFilePath(memo.nodeID) - with open(path, 'w') as stream: - if cdiString is None: - cdiString = self._data.rstrip(b'\0').decode("utf-8") - stream.write(cdiString) - print('Saved {}'.format(repr(path))) + if enable_cache: + path = self.cacheFilePath(memo.nodeID) + with open(path, 'w') as stream: + if cdiString is None: + cdiString = self._data.rstrip(b'\0').decode("utf-8") + stream.write(cdiString) + print('Saved {}'.format(repr(path))) self._data = None # Ensure isn't reused for more than one doc def cacheFilePathCustom(self, item_id: Union[NodeID, str], **kwargs): From 8b687db6e295f5a94ae86d5af1188071b6c8cac3 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Mon, 11 May 2026 16:18:20 -0400 Subject: [PATCH 29/64] Fix non-subscripable type (Change to PEP8 commented type hint). --- openlcb/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openlcb/__init__.py b/openlcb/__init__.py index 325ed0fa..ca124dce 100644 --- a/openlcb/__init__.py +++ b/openlcb/__init__.py @@ -23,11 +23,14 @@ ORD_z = 0x7A -def only_hex_pairs(value: str) -> Union[re.Match[bytes], re.Match[str], None]: +def only_hex_pairs(value: str): + # type: (str) -> Union[re.Match[bytes], re.Match[str], None] """Check if string contains only machine-readable hex pairs. See openlcb.conventions submodule for LCC ID dot notation functions (less restrictive). """ + # ^ PEP8 (instead of Python) type hint is used to avoid + # "TypeError: 'type' object is not subscriptable" if isinstance(value, (bytearray, bytes)): return hex_pairs_brc.fullmatch(value) assert isinstance(value, str) From bcef09b40c09ef940663e1d76e774918dd54d75c Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Mon, 11 May 2026 16:19:03 -0400 Subject: [PATCH 30/64] Add clear option such as for reconnect if client code doesn't want to retain the remote node list. --- openlcb/nodestore.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openlcb/nodestore.py b/openlcb/nodestore.py index cb423ba1..498e8b50 100644 --- a/openlcb/nodestore.py +++ b/openlcb/nodestore.py @@ -38,6 +38,10 @@ def store(self, node: Node) : self.nodes.sort(key=lambda x: x.snip.userProvidedNodeName, reverse=True) + def clear(self): + self.byIdMap.clear() + self.nodes.clear() + def isPresent(self, nodeID: NodeID) -> bool: return self.byIdMap.get(nodeID) is not None From 1d6b3ff75d8f204a6c479f067989de4d2d1b7fb9 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Tue, 19 May 2026 14:05:48 -0400 Subject: [PATCH 31/64] (NOOP) Fix some type hints and a docstring. --- openlcb/__init__.py | 5 +++++ openlcb/memoryservice.py | 2 +- openlcb/xmldataprocessor.py | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/openlcb/__init__.py b/openlcb/__init__.py index ca124dce..dc8ae025 100644 --- a/openlcb/__init__.py +++ b/openlcb/__init__.py @@ -140,6 +140,7 @@ def from_all_hex_bytes(b: bytearray) -> bytearray: def hr_repr(value, always_quote: bool = False) -> str: """Represent value with double quotes + if str, otherwise as an unquoted string. (Human-readable repr). """ # repr_value = repr(value) @@ -153,4 +154,8 @@ def hr_repr(value, always_quote: bool = False) -> str: def d_quote(value) -> str: + """Represent any type of value in double-quotes + (with any already present escaped) such as for emitting XML + attribute debug messages or any other technical/literary use. + """ return hr_repr(value, always_quote=True) diff --git a/openlcb/memoryservice.py b/openlcb/memoryservice.py index f6c2cd93..a65ad148 100644 --- a/openlcb/memoryservice.py +++ b/openlcb/memoryservice.py @@ -281,7 +281,7 @@ def __init__(self, service: DatagramService): self.datagramReceivedListener ) - def requestMemoryRead(self, memo, stream: Bool = False): + def requestMemoryRead(self, memo, stream: bool = False): # type: (MemoryReadMemo, Optional[bool]) -> None '''Request a read operation start. diff --git a/openlcb/xmldataprocessor.py b/openlcb/xmldataprocessor.py index ec9a4e30..8c0ed7ad 100644 --- a/openlcb/xmldataprocessor.py +++ b/openlcb/xmldataprocessor.py @@ -158,7 +158,7 @@ def __init__(self, linkLayer: CanLink, space: MemorySpace): # endregion ContentHandler self.acdi = False - def getRootMemo(self): + def getRootMemo(self) -> Union[CDIMemo, None]: """Get the root memo object if any. This should only be called after the entire file is parsed such as when cm.done is True in onStatusMemo(cm) callback. Set From d8b06807ba0676099b3c23e20c2ed2a67a34074f Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Tue, 19 May 2026 18:14:15 -0400 Subject: [PATCH 32/64] Begin implementing local node memory (FIXME: responding to memory configuration protocol from another node is not implemented). --- examples/example_node_implementation.py | 6 +- .../example_node_memory_implementation.py | 248 ++++++++++++++++++ openlcb/__init__.py | 14 + openlcb/cdimemo.py | 6 +- openlcb/cdivar.py | 3 +- openlcb/localnode.py | 227 ++++++++++++++++ openlcb/memoryservice.py | 2 +- openlcb/xmldataprocessor.py | 43 ++- 8 files changed, 536 insertions(+), 13 deletions(-) create mode 100644 examples/example_node_memory_implementation.py create mode 100644 openlcb/localnode.py diff --git a/examples/example_node_implementation.py b/examples/example_node_implementation.py index 768f09f7..b9e9416c 100644 --- a/examples/example_node_implementation.py +++ b/examples/example_node_implementation.py @@ -79,7 +79,11 @@ def printMessage(message): print("RM: {} from {}".format(message, message.source)) -canLink = CanLink(physicalLayer, NodeID(settings['localNodeID'])) +localNodeID = NodeID(settings['localNodeID']) +print() +print(f"[example_node_memory_implementation] localNodeID: {localNodeID}") + +canLink = CanLink(physicalLayer, localNodeID) canLink.registerMessageReceivedListener(printMessage) datagramService = DatagramService(canLink) diff --git a/examples/example_node_memory_implementation.py b/examples/example_node_memory_implementation.py new file mode 100644 index 00000000..d5011c11 --- /dev/null +++ b/examples/example_node_memory_implementation.py @@ -0,0 +1,248 @@ +''' +Demo of creating a virtual node to represent the application +(other local nodes are possible, but at least one is necessary +for the application to announce itself and provide SNIP info), +in this case with memory to allow another node to change settings +(could also be used to for a second virtual node such as to +represent/emulate a non-LCC train, but a separate +virtual node from the Configuration Tool is recommended in +that case). + +based on example_node_implementation from python-openlcb examples. + +Usage: +python3 example_node_memory_implementation.py [host|host:port] + +Options: +host|host:port (optional) Set the address (or using a colon, + the address and port). Defaults to a hard-coded test + address and port. +''' +import os +import socket +import struct + +# region same code as other examples +from examples_settings import Settings +from openlcb.localnode import LocalNode # do 1st to fix path if no pip install +settings = Settings() + +if __name__ == "__main__": + settings.load_cli_args(docstring=__doc__) +# endregion same code as other examples + +from openlcb import emit_cast, get_config_dir, precise_sleep # noqa: E402 +from openlcb.tcplink.tcpsocket import TcpSocket # noqa: E402 + +from openlcb.canbus.canphysicallayergridconnect import ( # noqa: E402 + CanPhysicalLayerGridConnect, +) +from openlcb.canbus.canlink import CanLink # noqa: E402 +from openlcb.nodeid import NodeID # noqa: E402 +from openlcb.datagramservice import DatagramService # noqa: E402 +from openlcb.memoryservice import MemoryService # noqa: E402 +from openlcb.message import Message # noqa: E402 +from openlcb.mti import MTI # noqa: E402 + +from openlcb.localnodeprocessor import LocalNodeProcessor # noqa: E402 +from openlcb.pip import PIP # noqa: E402 +from openlcb.snip import SNIP # noqa: E402 +from openlcb.node import Node # noqa: E402 + +# specify connection information +# region moved to settings +# host = "192.168.16.212" +# port = 12021 +# localNodeID = "05.01.01.01.03.01" +# farNodeID = "09.00.99.03.00.35" +# endregion moved to settings + +sock = TcpSocket() +# s.settimeout(30) +try: + sock.connect(settings['host'], settings['port']) +except socket.gaierror: + print("Failure accessing {}:{}" + .format(settings.get('host'), settings.get('port'))) + raise + +print("RR, SR are raw socket interface receive and send;" + " RL, SL are link interface; RM, SM are message interface") + + +# def sendToSocket(frame: CanFrame): +# string = frame.encodeAsString() +# print(" SR: {}".format(string.strip())) +# sock.sendString(string) +# physicalLayer.onFrameSent(frame) + + +def printFrame(frame): + print(" RL: {}".format(frame)) + + +physicalLayer = CanPhysicalLayerGridConnect() +physicalLayer.registerFrameReceivedListener(printFrame) + + +def printMessage(message): + print("RM: {} from {}".format(message, message.source)) + + +localNodeID = NodeID(settings['localNodeID']) +print() +print(f"[example_node_memory_implementation] localNodeID: {localNodeID}") +canLink = CanLink(physicalLayer, localNodeID) +canLink.registerMessageReceivedListener(printMessage) + +datagramService = DatagramService(canLink) +canLink.registerMessageReceivedListener(datagramService.process) + +spaces = { # big endian (most significant byte sent first) as per openlcb + # 0: bytearray([ + # 0x01, 0x00, # 0x1000 = 4096 (unsigned int 16) + # ]) + 0: bytearray(struct.pack(">H", 12021)), +} +# bytearray allows in-place append (from pack bytes does not) +# H: short (capitalized means unsigned) +# >: big endian (required for openlcb) +# e: float16 (IEEE 754 binary16, 2-bytes) +# For other symbols see Python documentation or SUBTYPE_FORMATS in cdivar.py. + +spaces[0] += struct.pack(">e", 0.5) # save at address 3 (size 2) +# NOTE: 0.5 can be stored precisely, as b'\x008' +# but not all numbers can be represented by IEEE float. +# For example, 2.4 is stored as b'\xcd@' which is ~2.400390625 + +# Additional pack examples: +neg2_float_ba = bytearray(b'\xc0\x00') +neg2_float_b = struct.pack(">e", -2) +assert bytes(neg2_float_ba) == bytes(neg2_float_b), \ + f"expected b'\xc0\x00', b'\xc0\x00', got {neg2_float_ba}, {neg2_float_b}" + +cdi = """ + + + python-openlcb example authors + example_node_memory_implementation + 1.0 + 1.0 + + + + + Port + Network port of remote hub (2-byte unsigned short) + 12021 + + + Timeout + Network timeout (2-byte binary16 value). + 0.5 + + + +""" # noqa: E501 + + +def handleDatagram(memo): + """create a call-back to print datagram contents when received + + Args: + memo (DatagramReadMemo): The datagram received + + Returns: + bool: Always False (True would mean we sent a reply to the datagram, + but let the MemoryService do that). + """ + print(f"Datagram receive call back: {emit_cast(memo)}") + return False + + +datagramService.registerDatagramReceivedListener(handleDatagram) + +memoryService = MemoryService(datagramService) + + +# callbacks to get results of memory read + +def memoryReadSuccess(memo): + print("successful memory read: {}".format(memo.data)) + + +def memoryReadFail(memo): + print("memory read failed: {}".format(memo.data)) + + +# create a node and connect it update +# This is a very minimal node, which just takes part in the low-level common +# protocols +localNode = LocalNode( + NodeID(settings['localNodeID']), + SNIP("python-openlcb example authors", + "example_node_memory_implementation", + "1.0", "1.0", "Custom Name Here", "Custom Description Here"), + set([ + PIP.SIMPLE_NODE_IDENTIFICATION_PROTOCOL, + PIP.DATAGRAM_PROTOCOL, + PIP.CONFIGURATION_DESCRIPTION_INFORMATION, + PIP.ADCDI_PROTOCOL, + PIP.MEMORY_CONFIGURATION_PROTOCOL, + ]), + canLink +) +my_conf_dir = os.path.join(get_config_dir("python-openlcb")) +backup_name = "example_node_memory_implementation.cdi.xml" +backup_path = os.path.join(my_conf_dir, backup_name) + +localNode.loadCDIString(cdi, backup_path) + +# localNodeProcessor = LocalNodeProcessor(canLink, localNode) +# canLink.registerMessageReceivedListener(localNodeProcessor.process) +localNodeProcessor = localNode.localNodeProcessor + + +def displayOtherNodeIds(message) : + """Listener to identify connected nodes + + Args: + message (Message): A response from the network + """ + print("[displayOtherNodeIds] type(message): {}" + "".format(type(message).__name__)) + if message.mti == MTI.Verified_NodeID : + print("Detected farNodeID is {}".format(message.source)) + + +canLink.registerMessageReceivedListener(displayOtherNodeIds) + + +####################### + +# have the socket layer report up to bring the link layer up and get an alias + +print(" SL : link up...") +physicalLayer.physicalLayerUp() +print(" SL : link up...waiting...") +while canLink.pollState() != CanLink.State.Permitted: + physicalLayer.receiveAll(sock, verbose=settings['trace']) + physicalLayer.sendAll(sock, verbose=True) + precise_sleep(.02) +print(" SL : link up") +# request that nodes identify themselves so that we can print their node IDs +message = Message(MTI.Verify_NodeID_Number_Global, + NodeID(settings['localNodeID']), None) +canLink.sendMessage(message) + +# process resulting activity +while True: + count = 0 + count += physicalLayer.sendAll(sock, verbose=True) + count += physicalLayer.receiveAll(sock, verbose=settings['trace']) + if count < 1: + precise_sleep(.01) + # else skip sleep to avoid latency (port already delayed) + +physicalLayer.physicalLayerDown() diff --git a/openlcb/__init__.py b/openlcb/__init__.py index dc8ae025..3e7a8ac0 100644 --- a/openlcb/__init__.py +++ b/openlcb/__init__.py @@ -1,4 +1,6 @@ from enum import Enum +import os +import platform import re import time @@ -23,6 +25,18 @@ ORD_z = 0x7A +def get_config_dir(unique_software_name: str): + """Get a configuration directory for any program + (In the parent directory recommended by the specific platform). + """ + CONFIGS = os.path.expanduser("~/.config") + if platform.system() == "Darwin": + CONFIGS = os.path.expanduser("~/Library/Application Support") + elif platform.system() == "Windows": + CONFIGS = os.environ['APPDATA'] + return os.path.join(CONFIGS, unique_software_name) + + def only_hex_pairs(value: str): # type: (str) -> Union[re.Match[bytes], re.Match[str], None] """Check if string contains only machine-readable hex pairs. diff --git a/openlcb/cdimemo.py b/openlcb/cdimemo.py index 9803d1f1..5ef49b1f 100644 --- a/openlcb/cdimemo.py +++ b/openlcb/cdimemo.py @@ -233,10 +233,12 @@ def toCDIVar(self): result.signed = True # if self.size is not None: if result.size not in [1, 2, 4, 8]: + children_msg = json.dumps(self.children, sort_keys=True, + indent=2, + default=CDIMemo.to_dict) raise AttributeError( f"expected 1,2,4,8 for int size, got {result.size}" - f" in children={json.dumps(self.children, sort_keys=True, indent=2, - default=CDIMemo.to_dict)}") + f" in children={children_msg}") if result.max is None: if result.signed: result.max = math.pow(2, result.size * 8 - 1) - 1 diff --git a/openlcb/cdivar.py b/openlcb/cdivar.py index 3a51db90..26c374a6 100644 --- a/openlcb/cdivar.py +++ b/openlcb/cdivar.py @@ -195,7 +195,8 @@ def setInt(self, value: int): self.data = self.intToData(value) def floatToData(self, value: float) -> bytes: - assert self.className == "float" + assert self.className == "float", \ + f"floatToData attempted on non-float: {self.className}" assert isinstance(value, float) return struct.pack(self.packFormat(), value) diff --git a/openlcb/localnode.py b/openlcb/localnode.py new file mode 100644 index 00000000..8f44a29e --- /dev/null +++ b/openlcb/localnode.py @@ -0,0 +1,227 @@ +import os + +from logging import getLogger +from typing import Union +from openlcb import emit_cast +from openlcb.node import PIP, SNIP, Node + +from openlcb.localnodeprocessor import LocalNodeProcessor +from openlcb.nodeid import ( + NodeID, +) +from openlcb.xmldataprocessor import ( + CanLink, + CDIMemo, + CDIVar, + # CLASSNAME_TYPES, + d_quote, + MemoryReadMemo, + MemorySpace, + XMLDataProcessor, +) + +logger = getLogger(__name__) + + +class LocalNode(Node): + """A Node with its own virtual memory + (emulate memory spaces such as for creating a virtual + signal node with settings)""" + def __init__(self, id: NodeID, snip: SNIP, pipSet: set, + linkLayer: CanLink): + Node.__init__(self, id, snip, pipSet) + self.cdi = None # type: XMLDataProcessor|None + self._replicated_cdi_tree = None # type: CDIMemo|None + if PIP.CONFIGURATION_DESCRIPTION_INFORMATION in pipSet: + self.cdi = XMLDataProcessor(linkLayer, MemorySpace.CDI) + else: + logger.warning( + "PIP.CONFIGURATION_DESCRIPTION_INFORMATION is not in pipSet" + f" for new LocalNode {self.cdi}, so XMLDataProcessor" + " will not be initialized (functioning as Node, unless" + " remote user knows addresses apart from CDI)") + self.spaces = {} # type: dict[int, bytearray] + self.localNodeProcessor = LocalNodeProcessor(linkLayer, self) + linkLayer.registerMessageReceivedListener( + self.localNodeProcessor.process) + + def loadCDIFile(self, path, memo=None): + """Load a CDI file to generate virtual memory spaces + (to create a virtual node, not representing a remote one) + + Args: + path (str): Location of original file, also used + to generate cache dir (parent of path). + memo (MemoryReadMemo): Typically left blank, + This would provide a success or fail message, + but this method can be called asynchronously + since LocalNode assumes local data is loaded, + not network data. + """ + assert isinstance(path, str) + if not os.path.isfile(path): + raise FileNotFoundError(path) + + # self.cdi.load(self.id, path, MemorySpace.CDI, memo) + xml_data = None + with open(path, "wb") as stream: + xml_data = stream.read() + return self.loadCDIString(xml_data, path, memo=memo) + + def loadCDIString(self, xml_data, path, memo=None): + """Load raw XML data from a string. + Args: + xml_data (Union[bytes, bytearray, str]): Raw XML + path (str): Location of original file, for + reference and use as cache dir (parent of path). + memo (Optional[MemoryReadMemo]): Typically left blank, + This would provide a success or fail message, + but this method can be called asynchronously + since LocalNode assumes local data is loaded, + not network data. + """ + self.cdiBackupDir = os.path.dirname(path) + assert self.cdi is not None, \ + ("PIP.CONFIGURATION_DESCRIPTION_INFORMATION is not in pipSet" + f" for LocalNode {self.id}") + if memo is None: + memo = MemoryReadMemo( + self.id, 0, MemorySpace.CDI.value, 0, + self.onCDILoadFailed, self.onCDILoaded) + self.cdi.load(self.id, path, MemorySpace.CDI, memo, data=xml_data) + # with open(path, "r") as stream: + # data = stream.read() + # self.tree = etree.fromstring(data) + self.reserveSpaces() + + def setMemory(self, memo: CDIMemo, var: CDIVar): + """Set a memory address at memo to the value in var""" + assert memo.space is not None + size = memo.getSize() + assert size is not None + assert size > 0, f"size={repr(size)}" + assert memo.address is not None + # if var is None: + # var = memo.toCDIVar() + assert var is not None + assert var.data is not None + assert len(var.data) == memo.getSize() + self.setMemoryAt(memo.space, memo.address, var.data) + + def setMemoryAt(self, space, address, data): + """Set address in virtual memory space to data""" + assert isinstance(data, (bytearray, bytes)) + if isinstance(space, MemorySpace): + space = space.value + assert isinstance(space, int) + assert isinstance(address, int) + assert address >= 0 + size = len(data) + end = address + size + + if space not in self.spaces: + self.spaces[space] = bytearray() + else: + assert isinstance(self.spaces[space], bytearray) + + newRegionLen = end - len(self.spaces[space]) + if newRegionLen > 0: + logger.warning( + f"Extending LocalNode data from {len(self.spaces[space])}" + f" byte(s) to {end} byte(s).") + self.spaces[space] += b'\0' * newRegionLen + assert end - address == len(data) + self.spaces[space][address:end] = data + print(f"Set LocalNode {self.id} space {space}" + f" address {space} (length {len(data)}).") + + def reserveSpaces(self, parent: Union[CDIMemo, None] = None): + + assert self.cdi is not None, \ + ("PIP.CONFIGURATION_DESCRIPTION_INFORMATION is not in pipSet" + f" for LocalNode {self.id}") + + if parent is None: + parent = self.cdi.getRootMemo() + assert parent is not None + assert parent.tag == "cdi", f"Expected cdi, got {parent.tag}" + assert parent.element is not None + if (('replicated' in parent.element.attrib) + and (parent.element.attrib['replicated'] == "true")): + # Caller already used expandedTree + return self._reserveSpaces(parent=parent) + expanded_root_memo, expanded_root = self.cdi.expandedTree() + self.cdi.expanded_root_memo = expanded_root_memo + self.cdi.expanded_root = expanded_root + # ^ self.cdi.expanded_root_memo can also be set via + # self.cdi.extractCDIVarMemos. + return self._reserveSpaces( + parent=self.cdi.expanded_root_memo, + ) + + def _reserveSpaces(self, parent: Union[CDIMemo, None] = None, level=0): + assert parent is not None + assert parent.tag is not None + tag = parent.tag.lower() + if tag == "cdi": + assert parent.element is not None + assert parent.element.attrib['replicated'] == "true", \ + "expanded_root_memo accounting for replication must be used." + if tag in ("int", "float"): # CLASSNAME_TYPES: + # cast_fn = int if tag == "int" else float + var = parent.toCDIVar() + # _min = parent.getChildContentN("min", tag) + # _max = parent.getChildContentN("max", tag) + value = parent.getChildContentN("default", tag) + # size = parent.getSize() + # assert size is not None + # var = CDIVar(parent.tag, _min=_min, _max=_max, _size=size) + if value is not None: + if tag == "float": + var.setFloat(value) + elif tag == "int": + assert isinstance(value, int), \ + f"tried to use {emit_cast(value)} for int tag" + var.setInt(value) + assert var.data is not None + # var.default = bytearray(copy.deepcopy(var.data)) + assert parent.space is not None, \ + f"No space defined in CDI for a(n) {tag}" + self.setMemory(parent, var) + return + for child in parent.children: + self._reserveSpaces(child, level=level+1) + if level == 0: + if not self.cdiBackupDir: + logger.warning( + f"Not backing up virtual node {self.id} memory since" + " no cdiBackupDir is set for the LocalNode instance.") + return + if not os.path.isdir(self.cdiBackupDir): + logger.warning( + f"Creating cdiBackupDir {self.cdiBackupDir}") + os.makedirs(self.cdiBackupDir) + for space, data in self.spaces.items(): + name = f"{self.id}.lcc-link-virtual-node.space={space}.xml" + path = os.path.join(self.cdiBackupDir, name) + with open(path, "wb") as stream: + stream.write(data) + print(f"Wrote {d_quote(path)}") + + def onCDILoaded(self, memo: MemoryReadMemo): + """Default handler, typically enough since CDI is local + in the case of LocalNode""" + assert self.cdi is not None, \ + ("PIP.CONFIGURATION_DESCRIPTION_INFORMATION is not in pipSet" + f" for LocalNode {self.id}") + print(f"LocalNode onFileLoaded {self.cdi.getPath()}: {memo}") + + def onCDILoadFailed(self, memo: MemoryReadMemo): + """Default handler for file load failed. + Shouldn't happen unless application has provided malformed XML, + since CDI is local in the case of LocalNode + """ + assert self.cdi is not None, \ + ("PIP.CONFIGURATION_DESCRIPTION_INFORMATION is not in pipSet" + f" for LocalNode {self.id}") + print(f"LocalNode onCDILoadFailed {self.cdi.getPath()}: {memo}") diff --git a/openlcb/memoryservice.py b/openlcb/memoryservice.py index a65ad148..997706bd 100644 --- a/openlcb/memoryservice.py +++ b/openlcb/memoryservice.py @@ -194,7 +194,7 @@ def assertMemoOK(memo: Union[MemoryReadMemo, MemoryWriteMemo]): assert isinstance(memo.address, int), \ f"Expected int, got address={emit_cast(memo.address)}" assert len(memo.data) <= 64 - assert isinstance(memo.data, Union[bytes, bytearray]), \ + assert isinstance(memo.data, (bytes, bytearray)), \ f"Expected bytearray, got data={emit_cast(memo.data)}" diff --git a/openlcb/xmldataprocessor.py b/openlcb/xmldataprocessor.py index 8c0ed7ad..4e638ee4 100644 --- a/openlcb/xmldataprocessor.py +++ b/openlcb/xmldataprocessor.py @@ -313,8 +313,18 @@ def _feedNext(self, memo: MemoryReadMemo): def load(self, node_id: NodeID, path, space: Union[MemorySpace, int], memo: Union[MemoryReadMemo, None] = None, - format: Union[DataFormat, None] = None): - """Load instead of downloading.""" + format: Union[DataFormat, None] = None, + data: Union[bytes, bytearray, str, None] = None): + """Load instead of downloading. + Args: + path (str): Location of original xml data (unused if + data is specified, but may be used for tracing; + Always sets self._path). + data (Optional[Union[bytes, bytearray, str]]): Actual XML data, + optional if path exists. If None, + path will be loaded, otherwise it will not, + and data will be used instead. + """ assert not self._data self._is_from_cache = True self.onStartDownload() @@ -334,9 +344,13 @@ def load(self, node_id: NodeID, path, space: Union[MemorySpace, int], self._format = format self._space = space # type:ignore # int if device-specific logger.warning(f"Using device-specific space: {space}") - data = None - with open(path, "rb") as stream: - data = stream.read() # type:ignore + if data is None: + with open(path, "rb") as stream: + data = stream.read() # type:ignore + else: + if isinstance(data, str): + # Mimic network data by converting to bytes: + data = data.encode('utf-8') self._path = path if self._format is DataFormat.XML: if memo is not None: @@ -543,6 +557,9 @@ def startElement(self, name: str, if self._tag_stack: parent_cm = self._tag_stack[-1] cm = CDIMemo(tag=name, element=el, parent=parent_cm, document=self) + # cm.space = self._tmp_space Commented since not replicated! + # - address and space should be set by expandedTree or the + # node processing the CDI, accounting for replication. if name == "segment": self._tmp_space = attrib.get('space') self._tmp_address = int(attrib.get('origin', 0)) @@ -705,12 +722,22 @@ def expandedTree(self) -> Tuple[CDIMemo, ET.Element]: new_root = ET.Element("cdi") # always new: children added from memos new_root.attrib.update(root_memo.element.attrib) + new_root.attrib['replicated'] = "true" + if new_root.tag != "cdi": + logger.warning( + f"expected cdi got {new_root.tag} from {root_memo.tag}") - new_root_memo = copy.deepcopy(root_memo) # deepcopy to edit children! + tmp_el = root_memo.element + root_memo.element = None # avoid deepcopy. Temporarily erase it. + new_root_memo = copy.deepcopy(root_memo) # copy to avoid affecting old + root_memo.element = tmp_el # restore the old copy. new_root_memo.document = self - size = self._expanded_tree_recursive(new_root_memo, new_root, address=0) + new_root_memo.element = new_root # use this not root_memo.element copy + size = self._expanded_tree_recursive(new_root_memo, new_root, + address=0) if size < 1: - logger.warning(f"No space used by CDI after replication (size={size})") + logger.warning( + f"No space used by CDI after replication (size={size})") return new_root_memo, new_root def _expanded_tree_recursive( From 82326de86743f72e96f31d382d431f735bd05e0a Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Tue, 19 May 2026 18:21:03 -0400 Subject: [PATCH 33/64] (NOOP) Rename expanded* to replicated* for consistency. --- examples/example_cdi_access.py | 2 +- openlcb/cdimemo.py | 6 ++-- openlcb/localnode.py | 14 ++++----- openlcb/xmldataprocessor.py | 56 +++++++++++++++++----------------- 4 files changed, 39 insertions(+), 39 deletions(-) diff --git a/examples/example_cdi_access.py b/examples/example_cdi_access.py index 4bbf5f27..c27b75bf 100644 --- a/examples/example_cdi_access.py +++ b/examples/example_cdi_access.py @@ -191,7 +191,7 @@ class MyHandler(xml.sax.handler.ContentHandler): This is implementation-specific, and not required if streaming (parser.feed). _tmp_address (int|None): For sanity check, not actual address. - See expandedTree docstring. + See replicatedTree docstring. """ def __init__(self): diff --git a/openlcb/cdimemo.py b/openlcb/cdimemo.py index 5ef49b1f..ba6ad9b0 100644 --- a/openlcb/cdimemo.py +++ b/openlcb/cdimemo.py @@ -47,7 +47,7 @@ class CDIMemo(DataProcessorMemo): name (str): Name (determined by `name` child element content). space (int|None): The memory space address (May be one in MemorySpace values, or not if vendor-specific such as - defined in CDI etc. See expandedTree in XMLDataProcessor). + defined in CDI etc. See replicatedTree in XMLDataProcessor). stray (bool): The end tag is misplaced (doesn't match a start tag) due to bad xml or incorrect parsing. tail (str|None): Content following the end tag (not used in @@ -188,7 +188,7 @@ def toCDIVar(self): See LCC "Configuration Description Information" Standard. NOTE: The `address` is only correct if this CDIMemo has been - replicated (such as in expandedTree or self.expanded_root). + replicated (such as in replicatedTree or self.replicated_root). """ # result = CDIVar(self.tag) assert (self.tag is not None) and (self.tag.strip()) @@ -219,7 +219,7 @@ def toCDIVar(self): # enforces size: result = CDIVar(self.tag, _min=result_min, _max=result_max, _size=result_size, _default=result_default) - result.address = self.address # only set in expandedTree() + result.address = self.address # only set in replicatedTree() result.space = self.space result.floatFormat = result_floatFormat result.name = self.getChildContent("name") diff --git a/openlcb/localnode.py b/openlcb/localnode.py index 8f44a29e..8053085d 100644 --- a/openlcb/localnode.py +++ b/openlcb/localnode.py @@ -148,15 +148,15 @@ def reserveSpaces(self, parent: Union[CDIMemo, None] = None): assert parent.element is not None if (('replicated' in parent.element.attrib) and (parent.element.attrib['replicated'] == "true")): - # Caller already used expandedTree + # Caller already used replicatedTree return self._reserveSpaces(parent=parent) - expanded_root_memo, expanded_root = self.cdi.expandedTree() - self.cdi.expanded_root_memo = expanded_root_memo - self.cdi.expanded_root = expanded_root - # ^ self.cdi.expanded_root_memo can also be set via + replicated_root_memo, replicated_root = self.cdi.replicatedTree() + self.cdi.replicated_root_memo = replicated_root_memo + self.cdi.replicated_root = replicated_root + # ^ self.cdi.replicated_root_memo can also be set via # self.cdi.extractCDIVarMemos. return self._reserveSpaces( - parent=self.cdi.expanded_root_memo, + parent=self.cdi.replicated_root_memo, ) def _reserveSpaces(self, parent: Union[CDIMemo, None] = None, level=0): @@ -166,7 +166,7 @@ def _reserveSpaces(self, parent: Union[CDIMemo, None] = None, level=0): if tag == "cdi": assert parent.element is not None assert parent.element.attrib['replicated'] == "true", \ - "expanded_root_memo accounting for replication must be used." + "replicated_root_memo accounting for replication must be used." if tag in ("int", "float"): # CLASSNAME_TYPES: # cast_fn = int if tag == "int" else float var = parent.toCDIVar() diff --git a/openlcb/xmldataprocessor.py b/openlcb/xmldataprocessor.py index 4e638ee4..541a0484 100644 --- a/openlcb/xmldataprocessor.py +++ b/openlcb/xmldataprocessor.py @@ -110,7 +110,7 @@ class XMLDataProcessor(xml.sax.handler.ContentHandler, DataProcessor): _tmp_space (int|None): What space we are currently on (of data described by Element(s), not of XML data itself). _tmp_address (int|None): For sanity check, not actual address - (no replication)! See expandedTree docstring. + (no replication)! See replicatedTree docstring. """ XML_TOP_TAGS = ("cdi", "fdi") DEFAULT_EXT = ".cdi.xml" # override in subclass @@ -120,8 +120,8 @@ class XMLDataProcessor(xml.sax.handler.ContentHandler, DataProcessor): def __init__(self, linkLayer: CanLink, space: MemorySpace): self.canLink: CanLink = linkLayer # caches_dir = SysDirs.Cache - self.expanded_root = None # type: ET.Element|None - self.expanded_root_memo = None # type: CDIMemo|None + self.replicated_root = None # type: ET.Element|None + self.replicated_root_memo = None # type: CDIMemo|None self._root_memos = None # type: list[CDIMemo]|None self._root_memo = None # type: CDIMemo|None self._space: Union[MemorySpace, None] = None @@ -558,7 +558,7 @@ def startElement(self, name: str, parent_cm = self._tag_stack[-1] cm = CDIMemo(tag=name, element=el, parent=parent_cm, document=self) # cm.space = self._tmp_space Commented since not replicated! - # - address and space should be set by expandedTree or the + # - address and space should be set by replicatedTree or the # node processing the CDI, accounting for replication. if name == "segment": self._tmp_space = attrib.get('space') @@ -575,7 +575,7 @@ def startElement(self, name: str, raise AttributeError( f"Node specifies {name} offset before segment origin") self._tmp_address += offset - # NOTE: ^ Sanity check only! For real address see expandedTree. + # NOTE: ^ Sanity check only! For real address see replicatedTree. self.onPushScope(cm) if len(self._tag_stack) < 1: @@ -703,7 +703,7 @@ def characters(self, content: str): "Expected str, got {}".format(type(content).__name__)) self._chunks.append(content) - def expandedTree(self) -> Tuple[CDIMemo, ET.Element]: + def replicatedTree(self) -> Tuple[CDIMemo, ET.Element]: """Build an expanded XML tree with replication and addresses. Starting from the root CDIMemo (via :meth:`getRootMemo`), this @@ -715,7 +715,7 @@ def expandedTree(self) -> Tuple[CDIMemo, ET.Element]: elements in the new tree. The original tree is left unchanged. Returns: - ET.Element: Root of the new expanded tree. + ET.Element: Root of the new replicated tree. """ root_memo = self.getRootMemo() assert root_memo is not None and root_memo.element is not None @@ -733,20 +733,20 @@ def expandedTree(self) -> Tuple[CDIMemo, ET.Element]: root_memo.element = tmp_el # restore the old copy. new_root_memo.document = self new_root_memo.element = new_root # use this not root_memo.element copy - size = self._expanded_tree_recursive(new_root_memo, new_root, + size = self._replicated_tree_recursive(new_root_memo, new_root, address=0) if size < 1: logger.warning( f"No space used by CDI after replication (size={size})") return new_root_memo, new_root - def _expanded_tree_recursive( + def _replicated_tree_recursive( self, parent: CDIMemo, parent_el: ET.Element, allow_non_standard=False, address: int = 0, space: Union[int, None] = None, ) -> int: - """Recursive helper for :meth:`expandedTree`. + """Recursive helper for :meth:`replicatedTree`. Copies the element, handles replication, sets addresses, and recurses into children. Removes ``replication`` attribute from @@ -762,7 +762,7 @@ def _expanded_tree_recursive( parent_el.text = parent.content elif parent_tag_lower in ("name", "description"): logger.warning( - f"expanded {parent_tag_lower} has no content.") + f"replicated {parent_tag_lower} has no content.") if parent_el.tail: parent.tail = parent_el.tail elif parent.tail: # new_root @@ -807,8 +807,8 @@ def _expanded_tree_recursive( # copy_child_el = child_el # copy_child_memo = child_memo # NOTE: ^ Why commented: We don't want to modify - # self.etree children (if we modify expandedTree - # result such as self.expanded_root)! + # self.etree children (if we modify replicatedTree + # result such as self.replicated_root)! # - Don't even chance it by keeping the memo # (otherwise child_memo.element would be from tree). # - Also, we always add child to parent_el below. @@ -817,7 +817,7 @@ def _expanded_tree_recursive( # for child_el in new_parent: # copy_parent_el.append(child_el) - # Remove replication from the expanded copy + # Remove replication from the replicated copy if "replication" in copy_child_el.attrib: del copy_child_el.attrib["replication"] @@ -865,7 +865,7 @@ def _expanded_tree_recursive( else: assert not size, el_error - address = self._expanded_tree_recursive( + address = self._replicated_tree_recursive( copy_child_memo, copy_child_el, address=address, space=space) if c_tag_lower == "segment": @@ -874,14 +874,14 @@ def _expanded_tree_recursive( parent.children = new_children # Same references if no replication return address - def extractCDIVarMemos(self, expanded_root=None, root_memo=None) -> List[CDIMemo]: # noqa: E501 + def extractCDIVarMemos(self, replicated_root=None, root_memo=None) -> List[CDIMemo]: # noqa: E501 # type: (ET.Element|None, CDIMemo|None) -> List[CDIMemo] """Build a flat list of CDIMemo objects for all variables. - Uses the expanded tree (replication expanded, replication + Uses the replicated tree (replication expanded, replication attribute removed, addresses set). Returns original-style memos (with .content) but with .element pointing into the - expanded tree so that modifications (setData etc.) affect + replicated tree so that modifications (setData etc.) affect the saved XML. """ # TODO: Implement ACDI vars if present (See OpenLCB @@ -889,18 +889,18 @@ def extractCDIVarMemos(self, expanded_root=None, root_memo=None) -> List[CDIMemo if not hasattr(self, "etree") or self.etree is None: logger.error("processor has no etree") return [] - if expanded_root is not None: + if replicated_root is not None: if root_memo is not None: # reserved assert isinstance(root_memo, CDIMemo) - assert isinstance(expanded_root, ET.Element) + assert isinstance(replicated_root, ET.Element) root_memo = root_memo # reserved - root_el = expanded_root + root_el = replicated_root else: - root_memo, root_el = self.expandedTree() - self.expanded_root = root_el - self.expanded_root_memo = root_memo + root_memo, root_el = self.replicatedTree() + self.replicated_root = root_el + self.replicated_root_memo = root_memo - assert isinstance(self.expanded_root_memo, CDIMemo) + assert isinstance(self.replicated_root_memo, CDIMemo) cdivar_memos: List[CDIMemo] = [] @@ -908,11 +908,11 @@ def traverse(memo: CDIMemo) -> None: tag = memo.getTag() tag_lower = tag.lower() if tag else "" if tag_lower in CLASSNAME_TYPES: - # Use the existing expanded memo (has correct .content) + # Use the existing replicated memo (has correct .content) cdivar_memos.append(memo) for child in memo.children: traverse(child) - traverse(self.expanded_root_memo) - assert root_el is self.expanded_root # concurrent modification check + traverse(self.replicated_root_memo) + assert root_el is self.replicated_root # concurrent modification check return cdivar_memos From 1a1e10cdbc31b0de5582f530ee1794f009c9929a Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Wed, 20 May 2026 14:43:33 -0400 Subject: [PATCH 34/64] Remove lint. --- openlcb/cdimemo.py | 2 +- openlcb/cdivar.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/openlcb/cdimemo.py b/openlcb/cdimemo.py index ba6ad9b0..84a37c4c 100644 --- a/openlcb/cdimemo.py +++ b/openlcb/cdimemo.py @@ -290,7 +290,7 @@ def addChildren(self) -> None: if self.element.tail: self.tail = self.element.tail - for child_elem in list(self.element): # list() to avoid modification issues + for child_elem in list(self.element): # list() fixes concurrency issue child_memo = CDIMemo( tag=child_elem.tag, element=child_elem, diff --git a/openlcb/cdivar.py b/openlcb/cdivar.py index 26c374a6..38dd44e7 100644 --- a/openlcb/cdivar.py +++ b/openlcb/cdivar.py @@ -181,6 +181,7 @@ def getSerializable(self): elif self.className == "string": return self.getString() assert self.className in ("blob", "eventid", "action") + assert self.data is not None, "CDIVar data not initialized" return base64.b64encode(self.data) def getDict(self, add_name=True): From 52ffae28f8a129ad26c130c95406bd91f0747d52 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Thu, 21 May 2026 15:39:56 -0400 Subject: [PATCH 35/64] Fix: Don't miss parsing if realtime and also loaded from CDI file/string. --- openlcb/xmldataprocessor.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/openlcb/xmldataprocessor.py b/openlcb/xmldataprocessor.py index 541a0484..48627f64 100644 --- a/openlcb/xmldataprocessor.py +++ b/openlcb/xmldataprocessor.py @@ -166,6 +166,7 @@ def getRootMemo(self) -> Union[CDIMemo, None]: (_feed) mode. """ if not self._root_memos: + logger.warning("[getRootMemo] No root memos.") return None if len(self._root_memos) > 1: summaries = [] @@ -347,10 +348,17 @@ def load(self, node_id: NodeID, path, space: Union[MemorySpace, int], if data is None: with open(path, "rb") as stream: data = stream.read() # type:ignore + print(f"[XMLDataProcessor] Loading {d_quote(path)}") else: if isinstance(data, str): # Mimic network data by converting to bytes: data = data.encode('utf-8') + print("[XMLDataProcessor] Loading string length" + " {} as bytes from memory." + .format(len(data))) + else: + print("[XMLDataProcessor] Loading {} length {} from memory." + .format(type(data).__name__, len(data))) self._path = path if self._format is DataFormat.XML: if memo is not None: @@ -380,6 +388,9 @@ def memoryReadFail(memo: MemoryReadMemo): # based on "else" (done) case in _memoryReadSuccess # in OpenLCBNetwork: self._stringTerminated = True + if self._realtime: + # Don't miss calling parser if realtime + self._feedNext(memo) self._feedLast(memo, enable_cache=False) self.onStop() # sets self._format to DataFormat.EOF else: @@ -533,7 +544,8 @@ def startElement(self, name: str, self._tag_stack[-1].content += content else: logger.warning( - f"Stray characters before {repr(name)}: {repr(content)}") + f"Stray characters before {repr(name)}:" + f" {repr(content)}") if self._ended_memo is not None: self._ended_memo = None attrib = attrs_to_dict(attrs) @@ -575,7 +587,7 @@ def startElement(self, name: str, raise AttributeError( f"Node specifies {name} offset before segment origin") self._tmp_address += offset - # NOTE: ^ Sanity check only! For real address see replicatedTree. + # NOTE: ^ Sanity check only! real address: see replicatedTree self.onPushScope(cm) if len(self._tag_stack) < 1: @@ -718,7 +730,8 @@ def replicatedTree(self) -> Tuple[CDIMemo, ET.Element]: ET.Element: Root of the new replicated tree. """ root_memo = self.getRootMemo() - assert root_memo is not None and root_memo.element is not None + assert root_memo is not None + assert root_memo.element is not None new_root = ET.Element("cdi") # always new: children added from memos new_root.attrib.update(root_memo.element.attrib) @@ -730,11 +743,11 @@ def replicatedTree(self) -> Tuple[CDIMemo, ET.Element]: tmp_el = root_memo.element root_memo.element = None # avoid deepcopy. Temporarily erase it. new_root_memo = copy.deepcopy(root_memo) # copy to avoid affecting old - root_memo.element = tmp_el # restore the old copy. + root_memo.element = tmp_el # Put the old root back into the old memo. new_root_memo.document = self new_root_memo.element = new_root # use this not root_memo.element copy size = self._replicated_tree_recursive(new_root_memo, new_root, - address=0) + address=0) if size < 1: logger.warning( f"No space used by CDI after replication (size={size})") From 8d6ae23fb910d2e07557134c3e6452e61e5cb4cc Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Thu, 21 May 2026 15:45:01 -0400 Subject: [PATCH 36/64] Implement more dunder methods (comparison operators, str conversion) and set method to CDIVar. Make min, max, and default into CDIVar for explicitness (less duck typing; better example for users esp. for direct translation to other languages), and check min & max on set. --- openlcb/cdivar.py | 373 ++++++++++++++++++++++++++++++-- tests/test_cdivar.py | 143 ++++++++++-- tests/test_cdivar_edge_cases.py | 18 +- 3 files changed, 495 insertions(+), 39 deletions(-) diff --git a/openlcb/cdivar.py b/openlcb/cdivar.py index 38dd44e7..66518463 100644 --- a/openlcb/cdivar.py +++ b/openlcb/cdivar.py @@ -2,12 +2,13 @@ import base64 from collections import OrderedDict import copy +from enum import Enum import struct from logging import getLogger from typing import Any, List, Type, Union -from openlcb import emit_cast +from openlcb import emit_cast, formatted_ex from openlcb.eventid import EventID from openlcb.openlcbaction import OpenLCBAction @@ -15,7 +16,14 @@ NUM_TYPES = {'int': int, 'float': float} # type: dict[str, Type] # Assumes "IEEE" in OpenLCB CDI Standard means IEEE 754-2008: -FLOAT_MAXIMUMS = {16: 65504.0, 32: 3.40e38, 64: 1.80e308} # type: dict[int, float] # noqa: E501 +FLOAT_MAXIMUMS = {2: 65504.0, 4: 3.40e38, 8: 1.80e308} # type: dict[int, float] # noqa: E501 +UNSIGNED_INT_MAXIMUMS = { # type: dict[int, int] + 1: 0xFF, 2: 0xFFFF, 4: 0xFFFF_FFFF, 8: 0xFFFF_FFFF_FFFF_FFFF} +SIGNED_INT_MINIMUMS = {} +SIGNED_INT_MAXIMUMS = {} +for k, v in UNSIGNED_INT_MAXIMUMS.items(): + SIGNED_INT_MINIMUMS[k] = - ((v + 1) // 2) # - is 1 further from 0 than + + SIGNED_INT_MAXIMUMS[k] = v // 2 # floor div removes the extra from odd # CLASSNAME_TYPES = {'int': int, 'float': float, 'string': str, 'blob': bytearray, 'eventid': EventID, 'action': OpenLCBAction} @@ -38,6 +46,12 @@ } +class CompareOp(Enum): + LessThan = -1 + EqualTo = 0 + GreaterThan = 1 + + class CDIVar: """A byte array representing a single configuration variable. Arguments: @@ -60,30 +74,115 @@ class CDIVar: must be .size. element (xml.etree.Element): An associated element in an XML tree. + assert_range (bool): raise AssertionError if default + is out of range, such as for tests, instead of + showing an error. Defaults to False (fault-tolerant + to avoid crash for malformed CDI). + _no_min (bool): For internal use only (Prevent infinite + recursion of the constructor when constructing CDIVar for a + standard default). + _no_max (bool): For internal use only (Prevent infinite + recursion of the constructor when constructing CDIVar for a + standard default). """ TYPED_KEYS = ['min', 'max', 'default'] def __init__(self, className, _min=None, _max=None, - _size=None, _default=None): + _size=None, _default=None, assert_range=False, + _no_min=False, _no_max=False, _default_data=None): + self.data = None # type: bytes|None + self.min = _min # type: CDIVar|None + self.max = _max # type: CDIVar|None + assert isinstance(className, str), \ f"Expected {CLASSNAME_TYPES.keys()} got {emit_cast(className)}" assert className, f"Expected {CLASSNAME_TYPES.keys()} got {className}" assert className in CLASSNAME_TYPES, \ f"Expected {list(CLASSNAME_TYPES.keys())} got {className}" + if _default is not None: + assert isinstance(_default, CDIVar) + if className in NUM_TYPES: + _default.assertNumberFormat() + assert _default_data is None, \ + "Can only set _default or _default_data" + elif _default_data is not None: + if isinstance(_default_data, bytes): + _default_data = bytearray(_default_data) + assert isinstance(_default_data, bytearray) + _default = CDIVar(className, _size=_size, + _no_max=True, _no_min=True) # prevent recursion + _default.data = _default_data + self.name = None # type: str|None self.className = className # type: str - self.data = None # type: bytes|None - self.min = _min # type: int|float|None self.signed = False # type: bool - if self.min and self.min < 0: - self.signed = True - self.max = _max # type: int|float|None - self.default = _default # type: bytearray|None + assert isinstance(_no_min, bool) + assert isinstance(_no_max, bool) + self._no_min = _no_min + self._no_max = _no_max + thisType = CLASSNAME_TYPES.get(className) + num_types = tuple(NUM_TYPES.values()) + if _min is not None: + assert isinstance(_min, CDIVar) + _min.assertNumberFormat() + assert thisType is not None + min_value = _min.value() + assert isinstance(min_value, thisType) + assert isinstance(min_value, num_types) # types valid for min + assert isinstance(min_value, (int, float)) # types valid for min + # ^ assert (int, float) to help (Pylance) linting of comparisons + if min_value and min_value < 0: + self.signed = True + if _default is not None: + if _default < _min: + error = f"default for {_default} < min {_min}" + if assert_range: + raise AssertionError(error) + else: + logger.error(error) + elif not _no_min: + self.min = CDIVar(className, _size=_size, + _no_min=True, _no_max=True) # prevent inf recurs + if _max is not None: + assert isinstance(_max, CDIVar) + _max.assertNumberFormat() + assert thisType is not None + max_value = _max.value() + assert isinstance(max_value, thisType) + assert isinstance(max_value, num_types) # types valid for max + if _default is not None: + if _default > self.max: + error = f"default for {_default} > max {self.max}" + if assert_range: + raise AssertionError(error) + else: + logger.error(error) + elif (className in NUM_TYPES) and not _no_max: + assert isinstance(_size, int) + self.max = CDIVar(className, _size=_size, + _no_min=True, _no_max=True) # prevent inf recur + if className == "int": + if self.signed: + self.max.setInt(SIGNED_INT_MAXIMUMS[_size]) + else: + self.max.setInt(UNSIGNED_INT_MAXIMUMS[_size]) + elif className == "float": + self.max.setFloat(FLOAT_MAXIMUMS[_size]) + else: + NotImplementedError() + if _default is not None: + if _default > self.max: + logger.error(f"default for {_default} > max {self.max}") + self.default = _default # type: CDIVar|None self.size = _size # type: int|None self.branch_size = None # type: int|None # size including children if self.size is None: - if self.default is not None: - self.size = len(self.default) + if className == "eventid": + self.size = 8 + else: + raise ValueError(f"size must be specified for {className}") + # elif self.default is not None: + # self.size = len(self.default.data) if self.className in ("int", "float"): self.assertNumberFormat() elif self.className == "eventid": @@ -101,6 +200,178 @@ def __init__(self, className, _min=None, _max=None, self.element = None # type: Any|None self.space = None # type: int|None + @staticmethod + def cmp_float(left: 'CDIVar', right: Union['CDIVar', float]) -> CompareOp: + assert left.className == "float" + l_value = left.getFloat() + assert l_value is not None + if isinstance(right, (float, int)): + # ^ int is only allowed since Python reverts 0 or 1 etc to int + r_value = right + else: + assert isinstance(right, CDIVar) + assert left.className == right.className + r_value = right.getFloat() + assert r_value is not None + if l_value < r_value: + return CompareOp.LessThan + elif l_value > r_value: + return CompareOp.GreaterThan + return CompareOp.EqualTo + + @staticmethod + def cmp_int(left: 'CDIVar', right: Union['CDIVar', int]) -> CompareOp: + assert left.className == "int" + l_value = left.getInt() + assert l_value is not None + if isinstance(right, int): + r_value = right + else: + assert isinstance(right, CDIVar) + assert left.className == right.className + r_value = right.getInt() + assert r_value is not None + if l_value < r_value: + return CompareOp.LessThan + elif l_value > r_value: + return CompareOp.GreaterThan + return CompareOp.EqualTo + + def __le__(self, other): + return (self < other) or (self == other) + + def __ge__(self, other): + return (self > other) or (self == other) + + def __add__(self, other): + if isinstance(other, float): + assert self.className == "float" + return self._getFloat() + other + elif isinstance(other, int): + assert self.className == "int" + return self._getInt() + other + else: + assert isinstance(other, CDIVar) + assert self.className == other.className + if self.className == "float": + return self._getFloat() + other._getFloat() + elif self.className == "int": + return self._getInt() + other._getInt() + raise TypeError( + f"Cannot add {self.className} CDIVar and {type(other).__name__}") + + def __sub__(self, other): # required by assertAlmostEqual + if isinstance(other, (int, float)): + value = self + (-other) + assert isinstance(value, (int, float)) + return value + assert isinstance(other, CDIVar) + assert self.className == other.className + if self.className == "int": + return self._getInt() + other._getInt() + if self.className == "float": + return self._getFloat() + other._getFloat() + raise TypeError( + f"Cannot '-' {self.className} CDIVar and {type(other).__name__}") + + def __eq__(self, other): + if self.className == "string": + # Don't carelessly compare data in case a + # null-terminated string has junk after it. + if isinstance(other, str): + return self.getString() == other + assert isinstance(other, CDIVar) + return self.getString() == other.getString() + assert not isinstance(other, str) + r_value = other + if isinstance(other, CDIVar): + assert self.className == other.className + if self.size != other.size: + if self.className == "int": + r_value = other.getInt() + elif self.className == "float": + r_value = other.getFloat() + else: + assert self.className not in NUM_TYPES, \ + f"comparing value of {self.className} not implemented" + return False + assert r_value is not None + if self.className in NUM_TYPES: + return type(self).cmp_any(self, r_value, CompareOp.EqualTo) + return self.data == other.data + # return type(self).cmp_any(self, other, CompareOp.EqualTo) + + def __gt__(self, other): + return type(self).cmp_any(self, other, CompareOp.GreaterThan) + + def __lt__(self, other): + return type(self).cmp_any(self, other, CompareOp.LessThan) + + def __str__(self): + return str(self.value()) + + def __repr__(self): + return repr(self.value()) + + @classmethod + def cmp_any(cls, self: 'CDIVar', other: Union['CDIVar', float, int], + compare_op: CompareOp): + if isinstance(other, CDIVar): + assert self.className == other.className, \ + f"Cannot compare {self.className} to {other.className} CDIVars" + if self.className == "float": + return self.cmp_float(self, other) == compare_op + elif self.className == "int": + return self.cmp_int(self, other) == compare_op + else: + raise TypeError( + f"Can only compare value types, got {self.className}") + elif isinstance(other, float): + assert self.className == "float", \ + f"Can't compare float to {self.className} CDIVar" + return self.cmp_float(self, other) == compare_op + elif isinstance(other, int): + assert self.className == "int", \ + f"Can't compare int to {self.className} CDIVar" + return self.cmp_int(self, other) == compare_op + else: + raise TypeError( + f"Cannot compare {type(other).__name__}" + f" to CDIVar with {compare_op}") + + @classmethod + def fromNumber(cls, value: Union[int, float], + className: str, _size: int) -> 'CDIVar': + var = CDIVar(className, _size=_size) + if value < 0: + var.signed = True + if className == "int": + assert isinstance(value, int) + var.setInt(value) + elif className == "float": + assert isinstance(value, (int, float)) + # ^ int is allowed only since in Python an int literal such + # as 0 or 1 is common in place of float. + var.setFloat(value) + else: + raise TypeError( + f"fromNumber requires {list(NUM_TYPES.keys())}" + f" but got {className} for className") + return var + + @classmethod + def fromInt(cls, value: int, _size: int) -> 'CDIVar': + return cls.fromNumber(value, "int", _size) + + @classmethod + def fromFloat(cls, value: float, _size: int) -> 'CDIVar': + return cls.fromNumber(value, "float", _size) + + @classmethod + def fromString(cls, value: str, _size: int) -> 'CDIVar': + var = CDIVar(className="string", _size=_size) + return var + def setData(self, data: Union[bytes, bytearray]): assert isinstance(data, (bytes, bytearray)) if isinstance(data, bytes): @@ -120,6 +391,46 @@ def setData(self, data: Union[bytes, bytearray]): raise NotImplementedError(f"Type {self.className} not implemented") self.data = data + def set(self, other: Union['CDIVar', int, float]): + assert isinstance(other, (CDIVar, int, float, str)) + if isinstance(other, CDIVar): + assert self.className == other.className, \ + f"tried to set {self.className} using {other.className}" + if self.className == "int": + r_value = other.getInt() + assert r_value is not None + if self.min is not None: + assert r_value >= self.min + if self.max is not None: + assert r_value <= self.max + self.setInt(r_value) + elif self.className == "float": + r_value = other.getFloat() + assert r_value is not None + if self.min is not None: + assert r_value >= self.min + if self.max is not None: + assert r_value <= self.max + self.setFloat(r_value) + else: + assert self.className not in NUM_TYPES, \ + f"bounds check not implemented for {self.className}" + # No bounds checking necessary + self.data = copy.deepcopy(other.data) + else: + assert isinstance(other, CLASSNAME_TYPES[self.className]), \ + f"Tried to set {self.className} to a(n) {type(other).__name__}" + if self.className == "int": + assert isinstance(other, int) # for linter (Pylance) + self.setInt(other) # asserts type as well + elif self.className == "float": + assert isinstance(other, float) # for linter (Pylance) + self.setFloat(other) # asserts type as well + else: + raise NotImplementedError( + f"Tried to set {self.className} to a(n)" + f" {type(other).__name__}") + def getData(self): return self.data @@ -170,10 +481,18 @@ def packFormat(self) -> str: def intToData(self, value: int) -> bytes: assert self.className == "int" assert isinstance(value, int) - return struct.pack(self.packFormat(), value) + try: + return struct.pack(self.packFormat(), value) + except Exception as ex: + logger.error(formatted_ex(ex)) + logger.error( + f"Tried to set a(n) {self.subtype()} CDIVar" + f" (packed via {self.packFormat()}) to {value}") + raise def getSerializable(self): - """Get a value in the corresponding Python type""" + """Get a value in the corresponding Python type, + or a string representing binary data.""" if self.className == "int": return self.getInt() elif self.className == "float": @@ -184,6 +503,22 @@ def getSerializable(self): assert self.data is not None, "CDIVar data not initialized" return base64.b64encode(self.data) + def signedMsg(self) -> str: + return "signed" if self.signed else "unsigned" + + def value(self): + """Get the value as a directly comparable Python type""" + if self.className == "int": + return self.getInt() + elif self.className == "float": + return self.getFloat() + elif self.className == "string": + return self.getString() + assert self.className in ("blob", "eventid", "action") + assert self.data is not None, "CDIVar data not initialized" + raise NotImplementedError( + f"Converting binary {self.className} to a Python type") + def getDict(self, add_name=True): result = OrderedDict() if add_name and self.name: @@ -218,6 +553,18 @@ def setString(self, value: str): assert len(encoded) + 1 <= self.size # size is max *only* if "string" self.data = encoded + b"\x00" # null-terminated for OpenLCB network + def _getFloat(self) -> float: + assert self.className == "float" + value = self.getFloat() + assert value is not None + return value + + def _getInt(self) -> int: + assert self.className == "int" + value = self.getInt() + assert value is not None + return value + def dataToInt(self, data) -> Union[int, None]: assert self.className == "int" if (data is None) or (len(data) < 1): diff --git a/tests/test_cdivar.py b/tests/test_cdivar.py index 51682dc6..8de67561 100644 --- a/tests/test_cdivar.py +++ b/tests/test_cdivar.py @@ -1,18 +1,27 @@ +import math import unittest -from openlcb.cdivar import CDIVar, SUBTYPE_FORMATS +from openlcb.cdivar import FLOAT_MAXIMUMS, SIGNED_INT_MAXIMUMS, SIGNED_INT_MINIMUMS, UNSIGNED_INT_MAXIMUMS, CDIVar, SUBTYPE_FORMATS class TestCDIVar(unittest.TestCase): def test_initialization_valid(self): - cdivar_int = CDIVar(className='int', _min=0, _max=100, _size=4, - _default=bytearray(b'\x00\x00\x00\x00')) + size = 4 + cdivar_int = CDIVar(className='int', _min=CDIVar.fromInt(0, size), + _max=CDIVar.fromInt(100, size), + _size=size, + _default_data=bytearray(b'\x00\x00\x00\x00')) self.assertEqual(cdivar_int.className, 'int') - self.assertEqual(cdivar_int.min, 0) - self.assertEqual(cdivar_int.max, 100) - self.assertEqual(cdivar_int.size, 4) - - cdivar_float = CDIVar(className='float', _min=0.0, _max=100.0, _size=4) + self.assertEqual(cdivar_int.min, CDIVar.fromInt(0, size), + f"got {cdivar_int.min}") + self.assertEqual(cdivar_int.max, CDIVar.fromInt(100, size)) + self.assertEqual(cdivar_int.size, CDIVar.fromInt(4, size)) + + size = 4 + cdivar_float = CDIVar(className='float', + _min=CDIVar.fromFloat(0.0, size), + _max=CDIVar.fromFloat(100.0, size), + _size=size) self.assertEqual(cdivar_float.className, 'float') self.assertEqual(cdivar_float.min, 0.0) self.assertEqual(cdivar_float.max, 100.0) @@ -20,10 +29,11 @@ def test_initialization_valid(self): maxSize = 100 cdivar_string = CDIVar(className='string', - _default=bytearray(b'Hello'), + _default_data=bytearray(b'Hello\0'), _size=maxSize) self.assertEqual(cdivar_string.className, 'string') - self.assertEqual(cdivar_string.default, bytearray(b'Hello')) + self.assertEqual(cdivar_string.default.data, bytearray(b'Hello\0'), + f"got {cdivar_string.default}") assert cdivar_string.default is not None # self.assertEqual(cdivar_string.size, len(cdivar_string.default)) self.assertEqual(cdivar_string.size, maxSize) @@ -33,55 +43,144 @@ def test_initialization_invalid_classname(self): CDIVar(className='invalid_class') def test_subtype(self): - cdivar_signed_int = CDIVar(className='int', _size=4, _min=-100, - _max=100) + size = 4 + cdivar_signed_int = CDIVar(className='int', _size=size, + _min=CDIVar.fromInt(-100, size), + _max=CDIVar.fromInt(100, size)) self.assertEqual(cdivar_signed_int.subtype(), 'int32') - cdivar_unsigned_int = CDIVar(className='int', _size=4) + cdivar_unsigned_int = CDIVar(className='int', _size=size) self.assertEqual(cdivar_unsigned_int.subtype(), 'uint32') - cdivar_signed_float = CDIVar(className='float', _size=4) + cdivar_signed_float = CDIVar(className='float', _size=size) self.assertEqual(cdivar_signed_float.subtype(), 'float32') def test_pack_format(self): - cdivar_int = CDIVar(className='int', _size=4) + size = 4 + cdivar_int = CDIVar(className='int', _size=size) self.assertEqual(cdivar_int.packFormat(), SUBTYPE_FORMATS[cdivar_int.subtype()]) - cdivar_float = CDIVar(className='float', _size=4) + cdivar_float = CDIVar(className='float', _size=size) self.assertEqual(cdivar_float.packFormat(), SUBTYPE_FORMATS[cdivar_float.subtype()]) def test_set_get_int(self): - cdivar_int = CDIVar(className='int', _size=4) + size = 4 + cdivar_int = CDIVar(className='int', _size=size) cdivar_int.setInt(42) self.assertEqual(cdivar_int.getInt(), 42) def test_set_get_float(self): - cdivar_float = CDIVar(className='float', _size=4) + size = 4 + cdivar_float = CDIVar(className='float', _size=size) cdivar_float.setFloat(3.14) got = cdivar_float.getFloat() assert got is not None self.assertAlmostEqual(got, 3.14, places=6) def test_set_get_string(self): - cdivar_string = CDIVar(className='string', _size=100) + size = 100 + cdivar_string = CDIVar(className='string', _size=size) cdivar_string.setString("Hello") self.assertEqual(cdivar_string.getString(), "Hello") def test_invalid_set_int(self): - cdivar_int = CDIVar(className='int', _size=4) + size = 4 + cdivar_int = CDIVar(className='int', _size=size) with self.assertRaises(AssertionError): cdivar_int.setInt("not an int") # type:ignore (assertRaises) def test_invalid_set_float(self): - cdivar_float = CDIVar(className='float', _size=4) + size = 4 + cdivar_float = CDIVar(className='float', _size=size) with self.assertRaises(AssertionError): cdivar_float.setFloat("not a float") # type:ignore (assertRaises) def test_invalid_set_string(self): - cdivar_string = CDIVar(className='string', _size=100) + size = 100 + cdivar_string = CDIVar(className='string', _size=size) with self.assertRaises(AttributeError): # number has no attribute 'encode' cdivar_string.setString(12345) # type:ignore (assertRaises) + def test_ranges(self): + # See https://learn.microsoft.com/en-us/cpp/cpp/data-type-ranges?view=msvc-170 + # size 1 byte is 8-bit + self.assertEqual(SIGNED_INT_MINIMUMS[1], -128) + self.assertEqual(SIGNED_INT_MAXIMUMS[1], 127) + # size 2 bytes is 16-bit + self.assertEqual(SIGNED_INT_MINIMUMS[2], -32768) + self.assertEqual(SIGNED_INT_MAXIMUMS[2], 32767) + # size 1 byte is 8-bit + self.assertEqual(UNSIGNED_INT_MAXIMUMS[1], 255) + # size 2 byte is 16-bit + self.assertEqual(UNSIGNED_INT_MAXIMUMS[2], 65535) + # size 4 bytes is 32-bit + self.assertEqual(UNSIGNED_INT_MAXIMUMS[4], 4_294_967_295) + # size 8 bytes is 64-bit + self.assertEqual(UNSIGNED_INT_MAXIMUMS[8], 18_446_744_073_709_551_615) + # size 4 bytes is 32-bit + self.assertEqual(SIGNED_INT_MINIMUMS[4], -2_147_483_648) + self.assertEqual(SIGNED_INT_MAXIMUMS[4], 2_147_483_647) + # size 8 bytes is 64-bit + self.assertEqual(SIGNED_INT_MINIMUMS[8], -9_223_372_036_854_775_808) + self.assertEqual(SIGNED_INT_MAXIMUMS[8], 9_223_372_036_854_775_807) + # size 2 bytes is 16-bit + self.assertEqual(SIGNED_INT_MAXIMUMS[2], (1 << 15) - 1) + + def test_compare_float(self): + size = 4 + left = CDIVar("float", _size=size) + left.setFloat(0.5) # 0.5 can be stored precisely in IEEE float format + rightG = CDIVar("float", _size=size) + rightG.setFloat(0.6) + rightGE = CDIVar("float", _size=size) + # NOTE: almost equal due to float imprecision + # (3.3999999521443642e+38 != 3.4e+38) + assert rightGE.max is not None + assert rightGE.max.className == "float" + assert rightGE.size == size + assert left.getFloat() == 0.5 + assert isinstance(FLOAT_MAXIMUMS[4], float) + rightGE_max_value = rightGE.max.getFloat() + assert rightGE_max_value is not None + # self.assertAlmostEqual(rightGE.max, FLOAT_MAXIMUMS[4], + # places=5, + # msg=f"got {rightGE.max}") + # ^ "AssertionError: 3.3999999521443642e+38 != 3.4e+38" so: + self.assertTrue(math.isclose(rightGE_max_value, FLOAT_MAXIMUMS[4], + rel_tol=1e-6), + msg=f"got {rightGE.max}") + rightGE = CDIVar("float", _size=2) + self.assertEqual(rightGE.max, FLOAT_MAXIMUMS[2], + msg=f"got {rightGE.max}") + rightGE.setFloat(0.5) + self.assertTrue(left < rightG) + self.assertTrue(left == rightGE) + self.assertTrue(rightGE >= left) + self.assertTrue(left <= rightGE) + self.assertTrue(rightG > left) + self.assertFalse(rightG < left) + self.assertFalse(left == rightG) + self.assertFalse(left > rightG) + + def test_compare_int(self): + size = 4 + left = CDIVar("int", _size=size) + left.setInt(5) # 0.5 can be stored precisely in IEEE float format + rightG = CDIVar("int", _size=size) + rightG.setInt(6) + rightGE = CDIVar("int", _size=size) + self.assertEqual(rightGE.max, UNSIGNED_INT_MAXIMUMS[4], + msg=f"got {rightGE.max}") + rightGE.setInt(5) + self.assertTrue(left < rightG) + self.assertTrue(left == rightGE) + self.assertTrue(rightGE >= left) + self.assertTrue(left <= rightGE) + self.assertTrue(rightG > left) + self.assertFalse(rightG < left) + self.assertFalse(left == rightG) + self.assertFalse(left > rightG) + if __name__ == '__main__': unittest.main() diff --git a/tests/test_cdivar_edge_cases.py b/tests/test_cdivar_edge_cases.py index 7820c43b..960b5a4b 100644 --- a/tests/test_cdivar_edge_cases.py +++ b/tests/test_cdivar_edge_cases.py @@ -29,9 +29,11 @@ def test_setInt_getInt_4byte_edge_cases(self): (0x12345678, [0x12, 0x34, 0x56, 0x78]), ] + size = 4 for value, expected_bytes in cases: + cvNeg1 = CDIVar.fromInt(-1, _size=size) with self.subTest(f"int {value} → bytes"): - var = CDIVar("int", _size=4, _min=-1) # signed + var = CDIVar("int", _size=size, _min=cvNeg1) # signed var.setInt(value) assert var.data is not None self.assertBytesEqual(expected_bytes, var.data) @@ -52,8 +54,11 @@ def test_small_int_sizes_sign_extension(self): ] for val, size, signed, exp_bytes, exp_restored in cases: + cvNeg1 = CDIVar.fromInt(-1, _size=size) + cv0 = CDIVar.fromInt(0, _size=size) with self.subTest(f"{val} @ {size} bytes signed={signed}"): - var = CDIVar("int", _size=size, _min=-1 if signed else 0) + var = CDIVar("int", _size=size, + _min=cvNeg1 if signed else cv0) var.setInt(val) assert var.data is not None self.assertBytesEqual(exp_bytes, var.data) @@ -128,17 +133,22 @@ def test_string_null_terminated(self): ("café π", "café π".encode("utf-8") + b"\x00"), ] + size = 100 for s, expected_bytes in cases: with self.subTest(f"setString({s!r})"): - var = CDIVar("string", _size=100) + var = CDIVar("string", _size=size) var.setString(s) + expected_padded = expected_bytes + if len(expected_padded) < size: + slack = size - len(expected_bytes) + expected_padded = bytearray(expected_bytes) + bytearray(b'\0'*slack) self.assertEqual(expected_bytes, var.data) restored = var.getString() self.assertEqual(s, restored) # Extra data after null is ignored - var = CDIVar("string") + var = CDIVar("string", _size=100) var.data = b"test\x00junk" self.assertEqual("test", var.getString()) From a9fd767f78520cc2be20757e25f23357f1d6d3d9 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Thu, 21 May 2026 16:00:06 -0400 Subject: [PATCH 37/64] Remove lint. --- python-openlcb.code-workspace | 1 + tests/test_cdivar.py | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/python-openlcb.code-workspace b/python-openlcb.code-workspace index bf30c553..db0ae80d 100644 --- a/python-openlcb.code-workspace +++ b/python-openlcb.code-workspace @@ -71,6 +71,7 @@ "memoryservice", "metas", "MSGLEN", + "msvc", "nodeid", "nodestore", "offvalue", diff --git a/tests/test_cdivar.py b/tests/test_cdivar.py index 8de67561..201c83e8 100644 --- a/tests/test_cdivar.py +++ b/tests/test_cdivar.py @@ -1,6 +1,8 @@ import math import unittest -from openlcb.cdivar import FLOAT_MAXIMUMS, SIGNED_INT_MAXIMUMS, SIGNED_INT_MINIMUMS, UNSIGNED_INT_MAXIMUMS, CDIVar, SUBTYPE_FORMATS +from openlcb.cdivar import ( + FLOAT_MAXIMUMS, SIGNED_INT_MAXIMUMS, SIGNED_INT_MINIMUMS, + UNSIGNED_INT_MAXIMUMS, CDIVar, SUBTYPE_FORMATS) class TestCDIVar(unittest.TestCase): @@ -32,6 +34,7 @@ def test_initialization_valid(self): _default_data=bytearray(b'Hello\0'), _size=maxSize) self.assertEqual(cdivar_string.className, 'string') + assert cdivar_string.default is not None self.assertEqual(cdivar_string.default.data, bytearray(b'Hello\0'), f"got {cdivar_string.default}") assert cdivar_string.default is not None @@ -98,11 +101,11 @@ def test_invalid_set_float(self): def test_invalid_set_string(self): size = 100 cdivar_string = CDIVar(className='string', _size=size) - with self.assertRaises(AttributeError): # number has no attribute 'encode' + with self.assertRaises(AttributeError): # int, no attribute 'encode' cdivar_string.setString(12345) # type:ignore (assertRaises) def test_ranges(self): - # See https://learn.microsoft.com/en-us/cpp/cpp/data-type-ranges?view=msvc-170 + # See learn.microsoft.com/en-us/cpp/cpp/data-type-ranges?view=msvc-170 # size 1 byte is 8-bit self.assertEqual(SIGNED_INT_MINIMUMS[1], -128) self.assertEqual(SIGNED_INT_MAXIMUMS[1], 127) From cd9bac5890a43f020f4da75d9421860e87b34fa3 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Thu, 21 May 2026 17:07:31 -0400 Subject: [PATCH 38/64] Move spaces to new MemoryManager class for reuse. --- openlcb/localnode.py | 36 ++++++----------------------------- openlcb/memorymanager.py | 35 ++++++++++++++++++++++++++++++++++ python-openlcb.code-workspace | 1 + 3 files changed, 42 insertions(+), 30 deletions(-) create mode 100644 openlcb/memorymanager.py diff --git a/openlcb/localnode.py b/openlcb/localnode.py index 8053085d..8a494371 100644 --- a/openlcb/localnode.py +++ b/openlcb/localnode.py @@ -3,6 +3,7 @@ from logging import getLogger from typing import Union from openlcb import emit_cast +from openlcb.memorymanager import MemoryManager from openlcb.node import PIP, SNIP, Node from openlcb.localnodeprocessor import LocalNodeProcessor @@ -40,10 +41,10 @@ def __init__(self, id: NodeID, snip: SNIP, pipSet: set, f" for new LocalNode {self.cdi}, so XMLDataProcessor" " will not be initialized (functioning as Node, unless" " remote user knows addresses apart from CDI)") - self.spaces = {} # type: dict[int, bytearray] self.localNodeProcessor = LocalNodeProcessor(linkLayer, self) linkLayer.registerMessageReceivedListener( self.localNodeProcessor.process) + self.memory = MemoryManager() def loadCDIFile(self, path, memo=None): """Load a CDI file to generate virtual memory spaces @@ -106,34 +107,9 @@ def setMemory(self, memo: CDIMemo, var: CDIVar): assert var is not None assert var.data is not None assert len(var.data) == memo.getSize() - self.setMemoryAt(memo.space, memo.address, var.data) - - def setMemoryAt(self, space, address, data): - """Set address in virtual memory space to data""" - assert isinstance(data, (bytearray, bytes)) - if isinstance(space, MemorySpace): - space = space.value - assert isinstance(space, int) - assert isinstance(address, int) - assert address >= 0 - size = len(data) - end = address + size - - if space not in self.spaces: - self.spaces[space] = bytearray() - else: - assert isinstance(self.spaces[space], bytearray) - - newRegionLen = end - len(self.spaces[space]) - if newRegionLen > 0: - logger.warning( - f"Extending LocalNode data from {len(self.spaces[space])}" - f" byte(s) to {end} byte(s).") - self.spaces[space] += b'\0' * newRegionLen - assert end - address == len(data) - self.spaces[space][address:end] = data - print(f"Set LocalNode {self.id} space {space}" - f" address {space} (length {len(data)}).") + self.memory.set(memo.space, memo.address, var.data) + print(f"Set LocalNode {self.id} space {memo.space}" + f" address {memo.space} (length {len(var.data)}).") def reserveSpaces(self, parent: Union[CDIMemo, None] = None): @@ -201,7 +177,7 @@ def _reserveSpaces(self, parent: Union[CDIMemo, None] = None, level=0): logger.warning( f"Creating cdiBackupDir {self.cdiBackupDir}") os.makedirs(self.cdiBackupDir) - for space, data in self.spaces.items(): + for space, data in self.memory.spaces.items(): name = f"{self.id}.lcc-link-virtual-node.space={space}.xml" path = os.path.join(self.cdiBackupDir, name) with open(path, "wb") as stream: diff --git a/openlcb/memorymanager.py b/openlcb/memorymanager.py new file mode 100644 index 00000000..043d4886 --- /dev/null +++ b/openlcb/memorymanager.py @@ -0,0 +1,35 @@ +from logging import getLogger + +from openlcb.memoryservice import MemorySpace + +logger = getLogger(__name__) + + +class MemoryManager: + def __init__(self): + self.spaces = {} # type: dict[int, bytearray] + + def set(self, space, address, data): + """Set address in virtual memory space to data""" + assert isinstance(data, (bytearray, bytes)) + if isinstance(space, MemorySpace): + space = space.value + assert isinstance(space, int) + assert isinstance(address, int) + assert address >= 0 + size = len(data) + end = address + size + + if space not in self.spaces: + self.spaces[space] = bytearray() + else: + assert isinstance(self.spaces[space], bytearray) + + newRegionLen = end - len(self.spaces[space]) + if newRegionLen > 0: + logger.warning( + f"Extending LocalNode data from {len(self.spaces[space])}" + f" byte(s) to {end} byte(s).") + self.spaces[space] += b'\0' * newRegionLen + assert end - address == len(data) + self.spaces[space][address:end] = data diff --git a/python-openlcb.code-workspace b/python-openlcb.code-workspace index db0ae80d..7d33412c 100644 --- a/python-openlcb.code-workspace +++ b/python-openlcb.code-workspace @@ -68,6 +68,7 @@ "localoverrides", "MDNS", "mdnsconventions", + "memorymanager", "memoryservice", "metas", "MSGLEN", From 4a3283cfe631a624017b9c922caede0756044e83 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Fri, 22 May 2026 18:54:16 -0400 Subject: [PATCH 39/64] Make LocalNode a MemoryManager subclass so it can be inserted into MemoryService to make CDI available to MemoryService. --- openlcb/localnode.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/openlcb/localnode.py b/openlcb/localnode.py index 8a494371..60c1226d 100644 --- a/openlcb/localnode.py +++ b/openlcb/localnode.py @@ -24,13 +24,14 @@ logger = getLogger(__name__) -class LocalNode(Node): +class LocalNode(Node, MemoryManager): """A Node with its own virtual memory (emulate memory spaces such as for creating a virtual signal node with settings)""" def __init__(self, id: NodeID, snip: SNIP, pipSet: set, linkLayer: CanLink): Node.__init__(self, id, snip, pipSet) + MemoryManager.__init__(self) self.cdi = None # type: XMLDataProcessor|None self._replicated_cdi_tree = None # type: CDIMemo|None if PIP.CONFIGURATION_DESCRIPTION_INFORMATION in pipSet: @@ -44,7 +45,6 @@ def __init__(self, id: NodeID, snip: SNIP, pipSet: set, self.localNodeProcessor = LocalNodeProcessor(linkLayer, self) linkLayer.registerMessageReceivedListener( self.localNodeProcessor.process) - self.memory = MemoryManager() def loadCDIFile(self, path, memo=None): """Load a CDI file to generate virtual memory spaces @@ -107,7 +107,7 @@ def setMemory(self, memo: CDIMemo, var: CDIVar): assert var is not None assert var.data is not None assert len(var.data) == memo.getSize() - self.memory.set(memo.space, memo.address, var.data) + self.set(memo.space, memo.address, var.data) print(f"Set LocalNode {self.id} space {memo.space}" f" address {memo.space} (length {len(var.data)}).") @@ -177,7 +177,7 @@ def _reserveSpaces(self, parent: Union[CDIMemo, None] = None, level=0): logger.warning( f"Creating cdiBackupDir {self.cdiBackupDir}") os.makedirs(self.cdiBackupDir) - for space, data in self.memory.spaces.items(): + for space, data in self.spaces.items(): name = f"{self.id}.lcc-link-virtual-node.space={space}.xml" path = os.path.join(self.cdiBackupDir, name) with open(path, "wb") as stream: From 9569133571968568f3a251879331526f19373b29 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Fri, 22 May 2026 18:55:41 -0400 Subject: [PATCH 40/64] Move bitfields to MemoryConfigurationHeader class and MemorySpaceIndex enum to make usage and enforcement more clear. --- openlcb/convert.py | 39 ----------- openlcb/memoryconfigurationheader.py | 81 +++++++++++++++++++++++ openlcb/memoryservice.py | 97 ++++++++++++++++++---------- python-openlcb.code-workspace | 1 + tests/test_convert.py | 31 +++++---- 5 files changed, 162 insertions(+), 87 deletions(-) create mode 100644 openlcb/memoryconfigurationheader.py diff --git a/openlcb/convert.py b/openlcb/convert.py index 58754777..63f85fff 100644 --- a/openlcb/convert.py +++ b/openlcb/convert.py @@ -19,45 +19,6 @@ class Convert: - @staticmethod - def deserializeMC2ndByte(datagramByte1): - """Decode byte[1] (2nd) of Memory Configuration Datagram""" - has_byte6 = False - if datagramByte1 & 0x03 == 0: - has_byte6 = True - return has_byte6, datagramByte1 & 0xFC - # ^ 0xFC = 11111100 - - # formerly spaceDecode, but it serializes a space for datagram byte2 - @staticmethod - def serializeSpace(space): - """Convert from a space number to either - False and control number or True and standard memory space - for use in a Datagram. - - Args: - space (int): Sequential memory space identifier, where values: - - 0xFF to 0xFD are special spaces, and only the least significant - 2 bits will be used in a datagram. - - 0x00 to 0xFC represent standard memory spaces directly. - - Returns: - tuple(bool, byte): (is custom space, control | space) - - (False, control number 1 to 3 inclusive) : - spaces 0xFF - 0xFD (Except bits beyond 0x00000011 - differ for each datagram type. See 4.2 Address - Space Selection in OpenLCB Memory Configuration - Standard) - - or (True, space number) : spaces 0 - 0xFC - (NOTE: type of space may affect type of output) - """ - # TODO: Maybe check type of space & raise TypeError if not - # something valid, whether byte, int, or what is ok [add - # more _description_ to space in docstring]. - if space >= 0xFD: - return (False, space & 0x03) - return (True, space) - @staticmethod def arrayToInt(data: Union[bytes, bytearray, List[int]]) -> int: """Convert an array in MSB-first order to an integer diff --git a/openlcb/memoryconfigurationheader.py b/openlcb/memoryconfigurationheader.py new file mode 100644 index 00000000..b8cb3861 --- /dev/null +++ b/openlcb/memoryconfigurationheader.py @@ -0,0 +1,81 @@ + +from enum import Enum +from typing import Union +# FDI = 0xFA +# Configuration = 0xFD +# All = 0xFE +# CDI = 0xFF # ~~decodes~~ encoded (in header) as 0x03 + + +class MemorySpaceIndex(Enum): + Uninitialized = -1 + Custom = 0 + Configuration = 1 # 0xFD & 0x03 == 1 + All = 2 # 0xFE & 0x03 == 2 + CDI = 3 # 0xFF & 0x03 == 3 + + @classmethod + def fromNumber(cls, num: int): + """Return the MemorySpace member with the given numeric value, + or None if no match is found. + """ + assert isinstance(num, int) + for member in cls: + if member.value == num: + return member + return cls.Custom + + +class MemoryConfigurationHeader: + """Manage data corresponding to bitfields in Memory Configuration. + See OpenLCB "Memory Configuration" Standard + + Arguments: + space (int): Space number (MemorySpaceIndex.*.Value, + MemorySpace.*.value, or raw number including a custom + space). + - 0xFF to 0xFD are special spaces, and only the least + significant 2 bits will be used in a datagram. + - 0x00 to 0xFC represent standard memory spaces directly. + """ + def __init__(self, space: int): + # formerly Convert.serializeSpace + # formerly spaceDecode, but it serializes a space for datagram byte2 + assert isinstance(space, int) + spaceIndexValue = space & 0x03 + self.spaceIndex = \ + MemorySpaceIndex.fromNumber( + spaceIndexValue) # type: MemorySpaceIndex + self.customSpace = None # type: int|None + if self.spaceIndex is MemorySpaceIndex.Custom: + self.customSpace = space + self.highBits = 0 # type: int + + @classmethod + def fromMC2ndByte(cls, datagramByte1: int, space: Union[int, None] = None) -> 'MemoryConfigurationHeader': # noqa: E501 + """Deserialize Memory Configuration byte 1. + + For serializing a space (such as packing a datagram header), + use constructor instead. + + Args: + datagramByte1 (int): byte[1] (2nd) of Memory Configuration Datagram + space (int): Only applies for custom space (datagramByte) + """ + if space is not None: + assert isinstance(space, int) + assert datagramByte1 & 0x03 == 0, \ + 'custom space requires datagramByte1 with last 2 bits 00' + else: + # space is None + assert datagramByte1 & 0x03 != 0, \ + 'a standard space must be in last 2 bits datagramByte1' + space = -1 + # formerly deserializeMC2ndByte + result = cls(datagramByte1 & 0x03) + if datagramByte1 & 0x03 == 0: + # Default (-1) means not enough information + # (space not known, but is MemorySpaceIndex.Custom) + result.customSpace = space + result.highBits = datagramByte1 & 0xFC # 0xFC = 0b11111100 + return result diff --git a/openlcb/memoryservice.py b/openlcb/memoryservice.py index 997706bd..fd5e06c7 100644 --- a/openlcb/memoryservice.py +++ b/openlcb/memoryservice.py @@ -40,23 +40,30 @@ DatagramService, ) from openlcb.convert import Convert +from openlcb.memoryconfigurationheader import MemoryConfigurationHeader, MemorySpaceIndex +from openlcb.memorymanager import MemoryManager from openlcb.nodeid import NodeID logger = getLogger(__name__) - +MODE_BYTES[''] MODE_BYTES = { - 'Read_Command': {0x40, 0x41, 0x42, 0x43}, + # order determines meaning for lists (See ) + 'Read_Command': {0x40, 0x41, 0x42, 0x43}, # TODO: Use memoryManagers 'Read_Reply': {0x50, 0x51, 0x52, 0x53}, - 'Read_Stream_Command': {0x60, 0x61, 0x62, 0x63}, - 'Read_Stream_Reply': {0x70, 0x71, 0x72, 0x73}, - 'Write_Command': {0x00, 0x01, 0x02, 0x03}, + 'Read_Stream_Command': {0x60, 0x61, 0x62, 0x63}, # TODO: Use memoryManagers + 'Read_Stream_Reply': {0x70, 0x71, 0x72, 0x73}, # TODO + 'Write_Command': [0x00, 0x01, 0x02, 0x03], # TODO: Use memoryManagers 'Write_Reply': {0x10, 0x11, 0x12, 0x13}, - 'Write_Under_Mask_Command': {0x08, 0x09, 0x0A, 0x0B}, + 'Write_Under_Mask_Command': {0x08, 0x09, 0x0A, 0x0B}, # TODO: Use memoryManagers 'Write_Stream_Command': {0x20, 0x21, 0x22, 0x23}, - 'Write_Stream_Reply': {0x30, 0x31, 0x32, 0x33}, + 'Write_Stream_Reply': {0x30, 0x31, 0x32, 0x33}, # TODO + 'Get_Address_Space_Info_Command': {0x84, }, + 'Get_Address_Space_Info_Reply': {0x86, 0x87, }, + 'Lock_Reserve_Command': {0x88, }, } + MODE_ERROR_BYTES = { 'Read_Reply': {0x58, 0x59, 0x5A, 0x5B}, 'Read_Stream_Reply': {0x78, 0x79, 0x7A, 0x7B}, @@ -85,10 +92,10 @@ class MemorySpace(Enum): (See OpenLCB Memory Configuration Standard 4.2). """ Uninitialized = -1 - CDI = 0xFF # decodes to 0x03 FDI = 0xFA - All = 0xFE Configuration = 0xFD + All = 0xFE + CDI = 0xFF # decodes to 0x03 @classmethod def fromNumber(cls, num: int): @@ -101,6 +108,22 @@ def fromNumber(cls, num: int): return member return None + @classmethod + def fromIndex(cls, msi: MemorySpaceIndex): + """Return the MemorySpace member with the given numeric value, + or None if no match is found. + """ + assert isinstance(msi, MemorySpaceIndex) + if msi is MemorySpaceIndex.Custom: + return None + elif msi is MemorySpaceIndex.Configuration: + return cls.Configuration + elif msi is MemorySpaceIndex.All: + return cls.All + elif msi is MemorySpaceIndex.CDI: + return cls.CDI + return None + class MemoryReadMemo: """Memo carries request and reply. @@ -211,10 +234,16 @@ def parseReplyDatagram(memo: Union[MemoryReadMemo, MemoryWriteMemo], "Datagram is truncated to 1 byte:" f" it is {hex(dmemo.data[0])}") return - (hasByte6, _) = Convert.deserializeMC2ndByte(dmemo.data[1]) + mcHeader = MemoryConfigurationHeader.fromMC2ndByte( + dmemo.data[1], + # space=memo.space, + ) offset = 6 error = None - if hasByte6: + assert mcHeader.spaceIndex is not MemorySpaceIndex.Uninitialized + if mcHeader.spaceIndex is MemorySpaceIndex.Custom: + # mcHeader.customSpace = memo.space + mcHeader.customSpace = dmemo.data[6] offset = 7 memo.error = None memo.errorCode = None @@ -256,9 +285,15 @@ def parseReplyDatagram(memo: Union[MemoryReadMemo, MemoryWriteMemo], error += f" ({list(dmemo.data[message_idx:])})" else: error = f"(2nd byte = {hex(dmemo.data[1])})" - error += f" (hasByte6={hasByte6})" - if hasByte6: - error += f" (space={hex(dmemo.data[6])})" + error += f" (spaceIndex={mcHeader.spaceIndex})" + if mcHeader.spaceIndex is mcHeader.customSpace: + if mcHeader.customSpace is not None: + if mcHeader.customSpace != dmemo.data[6]: + error += f" (mcHeader.customSpace={hex(mcHeader.customSpace)} != space={hex(dmemo.data[6])} !)" # noqa: E501 + else: + error += f" (mcHeader.customSpace={hex(mcHeader.customSpace)})" + else: + error += f" (space={hex(dmemo.data[6])} mcHeader.customSpace=None!)" # noqa: E501 memo.error = error @@ -268,6 +303,12 @@ class MemoryService: Args: service (DatagramService): See DatagramService. + + Attributes: + memoryManagers (dict[str, MemoryManager]): The storage where + other nodes can read and write memory. Each element can be + changed to a specific nodeid's memory manager. They key is + the NodeID in string form (dotted notation). """ def __init__(self, service: DatagramService): @@ -280,6 +321,7 @@ def __init__(self, service: DatagramService): self.service.registerDatagramReceivedListener( self.datagramReceivedListener ) + self.memoryManagers = {} # type: dict[str, MemoryManager] def requestMemoryRead(self, memo, stream: bool = False): # type: (MemoryReadMemo, Optional[bool]) -> None @@ -308,16 +350,10 @@ def requestMemoryReadNext(self, memo, stream: bool = False): memo (MemoryReadMemo): Request to send. """ assert isinstance(stream, bool) - hasByte6 = False # if custom space is defined in byte 6 - flag = 0 - (hasByte6, flag) = Convert.serializeSpace(memo.space) - if stream: - # Encoding: 0x60=custom, 0x61=0xFD, 0x62=0xFE, 0x63=0xFF - spaceFlag = 0x60 if hasByte6 else (flag | 0x60) - else: - # Encoding: 0x40=custom, 0x41=0xFD, 0x42=0xFE, 0x43=0xFF - spaceFlag = 0x40 if hasByte6 else (flag | 0x40) # | 0b11111100 - # ^ In else case, flag is 1-3, so re-add 0xFC (0b11111100) + mcHeader = MemoryConfigurationHeader(memo.space) + assert mcHeader.spaceIndex is not None + spaceFlag = (0x60 if stream else 0x40) | mcHeader.spaceIndex.value + # NOTE: Why was there commented: | 0xFC (0b11111100) if not stream? addr2 = ((memo.address >> 24) & 0xFF) addr3 = ((memo.address >> 16) & 0xFF) addr4 = ((memo.address >> 8) & 0xFF) @@ -326,7 +362,7 @@ def requestMemoryReadNext(self, memo, stream: bool = False): DatagramService.ProtocolID.MemoryOperation.value, spaceFlag, addr2, addr3, addr4, addr5]) # NOTE: list[int] is ok for bytearray extend (`+` requires cast) - if hasByte6: + if mcHeader.customSpace is not None: assert memo.space <= 0xFF, f"Space {memo.space} out of byte range" data.extend([(memo.space & 0xFF)]) data.extend([memo.size]) @@ -352,7 +388,6 @@ def datagramReceivedListener(self, dmemo: DatagramReadMemo) -> bool: if self.service.datagramType(dmemo.data) \ != DatagramService.ProtocolID.MemoryOperation : return False - # datagram must has a command value if len(dmemo.data) < 2: logger.error("Memory service datagram too short: {}" @@ -449,14 +484,8 @@ def requestMemoryWrite(self, memo: MemoryWriteMemo, stream: bool = False): self.writeMemos.append(memo) # create & send a write datagram hasByte6 = False # if custom space is defined in byte 6 - flag = 0 - (hasByte6, flag) = Convert.serializeSpace(memo.space) - if stream: - # Encoding: 0x20=custom, 0x21=0xFD, 0x22=0xFE, 0x23=0xFF - spaceFlag = 0x20 if hasByte6 else (flag | 0x20) - else: - # Encoding: 0x00=custom, 0x01=0xFD, 0x02=0xFE, 0x03=0xFF - spaceFlag = 0x00 if hasByte6 else (flag | 0x00) + header = MemoryConfigurationHeader(memo.space) + spaceFlag = (0x20 if stream else 0) | header.spaceIndex.value addr2 = ((memo.address >> 24) & 0xFF) addr3 = ((memo.address >> 16) & 0xFF) addr4 = ((memo.address >> 8) & 0xFF) diff --git a/python-openlcb.code-workspace b/python-openlcb.code-workspace index 7d33412c..0c651920 100644 --- a/python-openlcb.code-workspace +++ b/python-openlcb.code-workspace @@ -68,6 +68,7 @@ "localoverrides", "MDNS", "mdnsconventions", + "memoryconfigurationheader", "memorymanager", "memoryservice", "metas", diff --git a/tests/test_convert.py b/tests/test_convert.py index 4271edd5..1d7bcce0 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -3,6 +3,7 @@ import unittest from openlcb.convert import Convert +from openlcb.memoryconfigurationheader import MemoryConfigurationHeader, MemorySpaceIndex class TestConvertClass(unittest.TestCase): @@ -88,17 +89,19 @@ def testIntToArrayFail(self): Convert.intToArray(value, length) def testSerializeSpace(self): - byte6 = False - space = 0x00 - - (byte6, space) = Convert.serializeSpace(0xF8) - self.assertEqual(space, 0xF8) - self.assertTrue(byte6) - - (byte6, space) = Convert.serializeSpace(0xFF) - self.assertEqual(space, 0x03) - self.assertFalse(byte6) - - (byte6, space) = Convert.serializeSpace(0xFD) - self.assertEqual(space, 0x01) - self.assertFalse(byte6) + # byte6 = False + # space = 0x00 + + mcHeader = MemoryConfigurationHeader(0xF8) + self.assertEqual(mcHeader.customSpace, 0xF8) + self.assertEqual(mcHeader.spaceIndex, MemorySpaceIndex.Custom) + + mcHeader = MemoryConfigurationHeader(0xFF) + self.assertIs(mcHeader.spaceIndex, MemorySpaceIndex.CDI) + self.assertEqual(mcHeader.spaceIndex.value, 0x03) + self.assertIsNone(mcHeader.customSpace) + + mcHeader = MemoryConfigurationHeader(0xFD) + self.assertIs(mcHeader.spaceIndex, MemorySpaceIndex.Configuration) + self.assertEqual(mcHeader.spaceIndex.value, 0x01) + self.assertIsNone(mcHeader.customSpace) From 9693beaf5f6e7c8e7b5b7d1a40214fe2628d328d Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Wed, 27 May 2026 17:56:31 -0400 Subject: [PATCH 41/64] (NOOP) Rename MemoryManager to StoragePool for clarity and terseness; Systematize MemoryConfiguration bitfields. --- openlcb/localnode.py | 6 +- openlcb/memorymanager.py | 2 +- openlcb/memoryservice.py | 118 ++++++++++++++++++++++++++++------ python-openlcb.code-workspace | 2 +- tests/test_convert.py | 2 +- 5 files changed, 103 insertions(+), 27 deletions(-) diff --git a/openlcb/localnode.py b/openlcb/localnode.py index 60c1226d..70a42b2a 100644 --- a/openlcb/localnode.py +++ b/openlcb/localnode.py @@ -3,7 +3,7 @@ from logging import getLogger from typing import Union from openlcb import emit_cast -from openlcb.memorymanager import MemoryManager +from openlcb.memorymanager import StoragePool from openlcb.node import PIP, SNIP, Node from openlcb.localnodeprocessor import LocalNodeProcessor @@ -24,14 +24,14 @@ logger = getLogger(__name__) -class LocalNode(Node, MemoryManager): +class LocalNode(Node, StoragePool): """A Node with its own virtual memory (emulate memory spaces such as for creating a virtual signal node with settings)""" def __init__(self, id: NodeID, snip: SNIP, pipSet: set, linkLayer: CanLink): Node.__init__(self, id, snip, pipSet) - MemoryManager.__init__(self) + StoragePool.__init__(self) self.cdi = None # type: XMLDataProcessor|None self._replicated_cdi_tree = None # type: CDIMemo|None if PIP.CONFIGURATION_DESCRIPTION_INFORMATION in pipSet: diff --git a/openlcb/memorymanager.py b/openlcb/memorymanager.py index 043d4886..99f24073 100644 --- a/openlcb/memorymanager.py +++ b/openlcb/memorymanager.py @@ -5,7 +5,7 @@ logger = getLogger(__name__) -class MemoryManager: +class StoragePool: def __init__(self): self.spaces = {} # type: dict[int, bytearray] diff --git a/openlcb/memoryservice.py b/openlcb/memoryservice.py index fd5e06c7..ec84de3e 100644 --- a/openlcb/memoryservice.py +++ b/openlcb/memoryservice.py @@ -41,34 +41,110 @@ ) from openlcb.convert import Convert from openlcb.memoryconfigurationheader import MemoryConfigurationHeader, MemorySpaceIndex -from openlcb.memorymanager import MemoryManager +from openlcb.memorymanager import StoragePool from openlcb.nodeid import NodeID logger = getLogger(__name__) -MODE_BYTES[''] + +class MCOp(Enum): + """Byte 1 & 0b11111100 values (assumes byte 0 is 0x20)""" + Read_Command = 0x40 # 01000000 + Read_Reply = 0x50 # 01010000 + Read_Reply_Failure = 0x58 # 01011000 + Read_Stream_Command = 0x60 # 01100000 + Read_Stream_Reply = 0x70 # 01110000 + Read_Stream_Reply_Failure = 0x78 # 01111000 + Write_Command = 0x00 + Write_Reply = 0x10 # 00010000 + Write_Reply_Failure = 0x18 + Write_Under_Mask_Command = 0x08 # 00001000 + Write_Stream_Command = 0x20 # 01000000 + Write_Stream_Reply = 0x30 # 00110000 + Write_Stream_Reply_Failure = 0x38 # 0b111000 + Get_Configuration_Options_Command = 0x80 # 10000000 + # 1 sub-operation (same as above using MCOpMasks.Default): + Get_Configuration_Options_Reply = 0x82 # 10000010 + # ^ special datagram format follows (in later bytes) + Get_Address_Space_Info_Command = 0x84 # 10000100 + # 2 sub-operations (same as above using MCOpMasks.Default): + Get_Address_Space_Info_Reply = 0x86 # 10000110 + Get_Address_Space_Info_Reply_Command = 0x87 # 10000111 + Lock_or_Reserve_Command = 0x88 # 10001000 + # 1 sub-operation (same as above using MCOpMasks.Default): + Lock_or_Reserve_Reply = 0x8A # 10001010 + Get_Unique_ID_Command = 0x8C # 10001100 + # 1 sub-operation (same as above using MCOpMasks.Default): + Get_Unique_ID_Reply = 0x8D # 10001101 + Unfreeze_Command = 0xA0 # 10100000 + # 1 sub-operation (same as above using MCOpMasks.Default): + Freeze_Command = 0xA1 # 10100001 + Update_Complete_Command = 0xA8 # 10101000 + # 2 sub-operations (same as above using MCOpMasks.Default): + Reset_or_Reboot_Command = 0xA9 # 10101001 + Reinitialize_or_Factory_Reset_Command = 0xAA # 10101010 + + @classmethod + def fromNumber(cls, num: int): + """Return the MemorySpace member with the given numeric value, + or None if no match is found. + """ + assert isinstance(num, int) + for member in cls: + if member.value == num: + return member + return None + + +class MCOpBits: + Failure_Bit = 0x08 + + +class MCOpMasks: + Default = 0x11111100 + # The following aren't necessary since + # they can be broken down with + # sub-checks even if Default is used + # (See any set with more than one entry + # in MODE_BYTES): + # Get_Configuration_Options = 0x11111110 + # Get_Address_Space_Info = 0x11111110 + + MODE_BYTES = { # order determines meaning for lists (See ) - 'Read_Command': {0x40, 0x41, 0x42, 0x43}, # TODO: Use memoryManagers - 'Read_Reply': {0x50, 0x51, 0x52, 0x53}, - 'Read_Stream_Command': {0x60, 0x61, 0x62, 0x63}, # TODO: Use memoryManagers - 'Read_Stream_Reply': {0x70, 0x71, 0x72, 0x73}, # TODO - 'Write_Command': [0x00, 0x01, 0x02, 0x03], # TODO: Use memoryManagers - 'Write_Reply': {0x10, 0x11, 0x12, 0x13}, - 'Write_Under_Mask_Command': {0x08, 0x09, 0x0A, 0x0B}, # TODO: Use memoryManagers - 'Write_Stream_Command': {0x20, 0x21, 0x22, 0x23}, - 'Write_Stream_Reply': {0x30, 0x31, 0x32, 0x33}, # TODO - 'Get_Address_Space_Info_Command': {0x84, }, - 'Get_Address_Space_Info_Reply': {0x86, 0x87, }, - 'Lock_Reserve_Command': {0x88, }, + MCOp.Read_Command.value: {0x40, 0x41, 0x42, 0x43}, # pools + MCOp.Read_Reply.value: {0x50, 0x51, 0x52, 0x53}, + MCOp.Read_Stream_Command.value: {0x60, 0x61, 0x62, 0x63}, # pools + MCOp.Read_Stream_Reply.value: {0x70, 0x71, 0x72, 0x73}, # TODO + MCOp.Write_Command.value: [0x00, 0x01, 0x02, 0x03], # pools + MCOp.Write_Reply.value: {0x10, 0x11, 0x12, 0x13}, + MCOp.Write_Under_Mask_Command.value: {0x08, 0x09, 0x0A, 0x0B}, # pools + MCOp.Write_Stream_Command.value: {0x20, 0x21, 0x22, 0x23}, + MCOp.Write_Stream_Reply.value: {0x30, 0x31, 0x32, 0x33}, # TODO + MCOp.Get_Configuration_Options_Command.value: {0x80, }, + MCOp.Get_Configuration_Options_Reply.value: {0x82, }, + MCOp.Get_Address_Space_Info_Command.value: { + MCOp.Get_Address_Space_Info_Command.value, + MCOp.Get_Address_Space_Info_Reply.value, + MCOp.Get_Address_Space_Info_Reply_Command.value, + }, + MCOp.Lock_or_Reserve_Command.value: {0x88, }, + MCOp.Get_Unique_ID_Command.value: {0x8C, }, + MCOp.Get_Unique_ID_Reply.value: {0x8D, }, + MCOp.Unfreeze_Command.value: {0xA1, 0xA0}, # unfreeze, freeze respectively + MCOp.Update_Complete_Command.value: { # all match using MCOpMasks.Default + MCOp.Update_Complete_Command.value, + MCOp.Reset_or_Reboot_Command.value, + MCOp.Reinitialize_or_Factory_Reset_Command.value, + } } - MODE_ERROR_BYTES = { - 'Read_Reply': {0x58, 0x59, 0x5A, 0x5B}, - 'Read_Stream_Reply': {0x78, 0x79, 0x7A, 0x7B}, - 'Write_Reply': {0x18, 0x19, 0x1A, 0x1B}, - 'Write_Stream_Reply': {0x38, 0x39, 0x3A, 0x3B}, + MCOp.Read_Reply.value: {0x58, 0x59, 0x5A, 0x5B}, + MCOp.Read_Stream_Reply.value: {0x78, 0x79, 0x7A, 0x7B}, + MCOp.Write_Reply.value: {0x18, 0x19, 0x1A, 0x1B}, + MCOp.Write_Stream_Reply.value: {0x38, 0x39, 0x3A, 0x3B}, } @@ -305,7 +381,7 @@ class MemoryService: service (DatagramService): See DatagramService. Attributes: - memoryManagers (dict[str, MemoryManager]): The storage where + pools (dict[str, StoragePool]): The storage where other nodes can read and write memory. Each element can be changed to a specific nodeid's memory manager. They key is the NodeID in string form (dotted notation). @@ -321,7 +397,7 @@ def __init__(self, service: DatagramService): self.service.registerDatagramReceivedListener( self.datagramReceivedListener ) - self.memoryManagers = {} # type: dict[str, MemoryManager] + self.pools = {} # type: dict[str, StoragePool] def requestMemoryRead(self, memo, stream: bool = False): # type: (MemoryReadMemo, Optional[bool]) -> None diff --git a/python-openlcb.code-workspace b/python-openlcb.code-workspace index 0c651920..a2e89083 100644 --- a/python-openlcb.code-workspace +++ b/python-openlcb.code-workspace @@ -69,7 +69,7 @@ "MDNS", "mdnsconventions", "memoryconfigurationheader", - "memorymanager", + "StoragePool", "memoryservice", "metas", "MSGLEN", diff --git a/tests/test_convert.py b/tests/test_convert.py index 1d7bcce0..1d8f8c98 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -11,7 +11,7 @@ class TestConvertClass(unittest.TestCase): def testReturnCyrillicStrings(self): # See also testReturnCyrillicStrings in test_snip # If you have characters specific to UTF-8 (either in code or comment) - # add the following as the 1st or 2nd line of the py file: + # add the following as 1st line of py file (or line after shebang): # -*- coding: utf-8 -*- data = bytearray([0xd0, 0x94, 0xd0, 0xbc, 0xd0, 0xb8, 0xd1, 0x82, 0xd1, 0x80, 0xd0, 0xb8, 0xd0, 0xb9]) # Cyrillic spelling of the name Dmitry (7 characters becomes 14 bytes) # noqa: E501 self.assertEqual(Convert.arrayToString(data, len(data)), "Дмитрий") # Cyrillic spelling of the name Dmitry. This string should appear as 7 Cyrillic characters like Cyrillic-demo-Dmitry.png in doc (14 bytes in a hex editor), otherwise your editor does not support utf-8 and editing this file with it could break it. # noqa:E501 From d40d408c7b8d81ba100452f8e6c7551b9da133d6 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Wed, 27 May 2026 17:58:50 -0400 Subject: [PATCH 42/64] Rename module to match new StoragePool class name. --- openlcb/localnode.py | 2 +- openlcb/memoryservice.py | 2 +- openlcb/{memorymanager.py => storagepool.py} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename openlcb/{memorymanager.py => storagepool.py} (100%) diff --git a/openlcb/localnode.py b/openlcb/localnode.py index 70a42b2a..3a7a834b 100644 --- a/openlcb/localnode.py +++ b/openlcb/localnode.py @@ -3,7 +3,7 @@ from logging import getLogger from typing import Union from openlcb import emit_cast -from openlcb.memorymanager import StoragePool +from openlcb.storagepool import StoragePool from openlcb.node import PIP, SNIP, Node from openlcb.localnodeprocessor import LocalNodeProcessor diff --git a/openlcb/memoryservice.py b/openlcb/memoryservice.py index ec84de3e..df4a3db3 100644 --- a/openlcb/memoryservice.py +++ b/openlcb/memoryservice.py @@ -41,7 +41,7 @@ ) from openlcb.convert import Convert from openlcb.memoryconfigurationheader import MemoryConfigurationHeader, MemorySpaceIndex -from openlcb.memorymanager import StoragePool +from openlcb.storagepool import StoragePool from openlcb.nodeid import NodeID logger = getLogger(__name__) diff --git a/openlcb/memorymanager.py b/openlcb/storagepool.py similarity index 100% rename from openlcb/memorymanager.py rename to openlcb/storagepool.py From 572137ecc4473440e53b70e755569d25d8ab55b0 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Wed, 27 May 2026 18:28:07 -0400 Subject: [PATCH 43/64] Add "set" method to StoragePool and related tests. Move classes to submodules to avoid circular imports. --- examples/examples_gui.py | 3 +- examples/tkexamples/cdiform.py | 2 +- openlcb/localnode.py | 2 +- openlcb/memoryconfigurationheader.py | 21 +------- openlcb/memoryservice.py | 56 +--------------------- openlcb/memoryspace.py | 56 ++++++++++++++++++++++ openlcb/memoryspaceindex.py | 22 +++++++++ openlcb/openlcbnetwork.py | 3 +- openlcb/storagepool.py | 23 ++++++++- openlcb/xmldataprocessor.py | 2 +- python-openlcb.code-workspace | 5 +- tests/test_convert.py | 3 +- tests/test_openlcbnetwork.py | 2 +- tests/test_storagepool.py | 71 ++++++++++++++++++++++++++++ 14 files changed, 189 insertions(+), 82 deletions(-) create mode 100644 openlcb/memoryspace.py create mode 100644 openlcb/memoryspaceindex.py create mode 100644 tests/test_storagepool.py diff --git a/examples/examples_gui.py b/examples/examples_gui.py index a38338e3..036c0c7b 100644 --- a/examples/examples_gui.py +++ b/examples/examples_gui.py @@ -20,6 +20,8 @@ from logging import getLogger +from openlcb.memoryspace import MemorySpace + try: import tkinter as tk except ImportError: @@ -35,7 +37,6 @@ from openlcb.cdimemo import CDIMemo -from openlcb.memoryservice import MemorySpace from openlcb.message import Message from openlcb.xmldataprocessor import XMLDataProcessor from openlcb.mti import MTI diff --git a/examples/tkexamples/cdiform.py b/examples/tkexamples/cdiform.py index 09fce4f3..166cd0cb 100644 --- a/examples/tkexamples/cdiform.py +++ b/examples/tkexamples/cdiform.py @@ -44,7 +44,7 @@ from openlcb.xmldataprocessor import XMLDataProcessor from openlcb.cdimemo import CDIMemo from openlcb.linklayer import LinkLayer - from openlcb.memoryservice import MemorySpace + from openlcb.memoryspace import MemorySpace from openlcb.xmldataprocessor import element_to_dict except ImportError as ex: print("{}: {}".format(type(ex).__name__, ex), file=sys.stderr) diff --git a/openlcb/localnode.py b/openlcb/localnode.py index 3a7a834b..8678b7f1 100644 --- a/openlcb/localnode.py +++ b/openlcb/localnode.py @@ -3,6 +3,7 @@ from logging import getLogger from typing import Union from openlcb import emit_cast +from openlcb.memoryspace import MemorySpace from openlcb.storagepool import StoragePool from openlcb.node import PIP, SNIP, Node @@ -17,7 +18,6 @@ # CLASSNAME_TYPES, d_quote, MemoryReadMemo, - MemorySpace, XMLDataProcessor, ) diff --git a/openlcb/memoryconfigurationheader.py b/openlcb/memoryconfigurationheader.py index b8cb3861..5c1c1168 100644 --- a/openlcb/memoryconfigurationheader.py +++ b/openlcb/memoryconfigurationheader.py @@ -1,31 +1,14 @@ from enum import Enum from typing import Union + +from openlcb.memoryspaceindex import MemorySpaceIndex # FDI = 0xFA # Configuration = 0xFD # All = 0xFE # CDI = 0xFF # ~~decodes~~ encoded (in header) as 0x03 -class MemorySpaceIndex(Enum): - Uninitialized = -1 - Custom = 0 - Configuration = 1 # 0xFD & 0x03 == 1 - All = 2 # 0xFE & 0x03 == 2 - CDI = 3 # 0xFF & 0x03 == 3 - - @classmethod - def fromNumber(cls, num: int): - """Return the MemorySpace member with the given numeric value, - or None if no match is found. - """ - assert isinstance(num, int) - for member in cls: - if member.value == num: - return member - return cls.Custom - - class MemoryConfigurationHeader: """Manage data corresponding to bitfields in Memory Configuration. See OpenLCB "Memory Configuration" Standard diff --git a/openlcb/memoryservice.py b/openlcb/memoryservice.py index df4a3db3..fd7b016e 100644 --- a/openlcb/memoryservice.py +++ b/openlcb/memoryservice.py @@ -40,7 +40,8 @@ DatagramService, ) from openlcb.convert import Convert -from openlcb.memoryconfigurationheader import MemoryConfigurationHeader, MemorySpaceIndex +from openlcb.memoryconfigurationheader import MemoryConfigurationHeader +from openlcb.memoryspaceindex import MemorySpaceIndex from openlcb.storagepool import StoragePool from openlcb.nodeid import NodeID @@ -148,59 +149,6 @@ class MCOpMasks: } -class MemorySpace(Enum): - """The memory space to read. - In practice, XMLDataProcessor (or a non-XML parser if necessary) - uses this to track what data type and format is to be assumed in a - received Message. It is assumed to have the same space as the - request (MemoryReadMemo). - - A datagram's `space` attribute's type should be `int` not - MemorySpace, because CDI specifies variables' space arbitrarily. - - Attributes: - Uninitialized: No data (memory read request response) is expected. - CDI: The data expected from the memory read is CDI XML. - FDI: The data expected from the memory read is FDI XML. - All: All memory of the device, where all is defined by its designer - (See OpenLCB Memory Configuration Standard 4.2). - Configuration: A writeable basic configuration space, with - the structure of the 32-bit space defined by the designer - (See OpenLCB Memory Configuration Standard 4.2). - """ - Uninitialized = -1 - FDI = 0xFA - Configuration = 0xFD - All = 0xFE - CDI = 0xFF # decodes to 0x03 - - @classmethod - def fromNumber(cls, num: int): - """Return the MemorySpace member with the given numeric value, - or None if no match is found. - """ - assert isinstance(num, int) - for member in cls: - if member.value == num: - return member - return None - - @classmethod - def fromIndex(cls, msi: MemorySpaceIndex): - """Return the MemorySpace member with the given numeric value, - or None if no match is found. - """ - assert isinstance(msi, MemorySpaceIndex) - if msi is MemorySpaceIndex.Custom: - return None - elif msi is MemorySpaceIndex.Configuration: - return cls.Configuration - elif msi is MemorySpaceIndex.All: - return cls.All - elif msi is MemorySpaceIndex.CDI: - return cls.CDI - return None - - class MemoryReadMemo: """Memo carries request and reply. diff --git a/openlcb/memoryspace.py b/openlcb/memoryspace.py new file mode 100644 index 00000000..81fc3177 --- /dev/null +++ b/openlcb/memoryspace.py @@ -0,0 +1,56 @@ +from enum import Enum + +from openlcb.memoryspaceindex import MemorySpaceIndex + + +class MemorySpace(Enum): + """The memory space to read. + In practice, XMLDataProcessor (or a non-XML parser if necessary) + uses this to track what data type and format is to be assumed in a + received Message. It is assumed to have the same space as the + request (MemoryReadMemo). + - A datagram's `space` attribute's type should be `int` not + MemorySpace, because CDI specifies variables' space arbitrarily. + + Attributes: + Uninitialized: No data (memory read request response) is expected. + CDI: The data expected from the memory read is CDI XML. + FDI: The data expected from the memory read is FDI XML. + All: All memory of the device, where all is defined by its designer + (See OpenLCB Memory Configuration Standard 4.2). + Configuration: A writeable basic configuration space, with + the structure of the 32-bit space defined by the designer + (See OpenLCB Memory Configuration Standard 4.2). + """ + Uninitialized = -1 + FDI = 0xFA + Configuration = 0xFD + All = 0xFE + CDI = 0xFF # decodes to 0x03 + + @classmethod + def fromNumber(cls, num: int): + """Return the MemorySpace member with the given numeric value, + or None if no match is found. + """ + assert isinstance(num, int) + for member in cls: + if member.value == num: + return member + return None + + @classmethod + def fromIndex(cls, msi: MemorySpaceIndex): + """Return the MemorySpace member with the given numeric value, + or None if no match is found. + """ + assert isinstance(msi, MemorySpaceIndex) + if msi is MemorySpaceIndex.Custom: + return None + elif msi is MemorySpaceIndex.Configuration: + return cls.Configuration + elif msi is MemorySpaceIndex.All: + return cls.All + elif msi is MemorySpaceIndex.CDI: + return cls.CDI + return None diff --git a/openlcb/memoryspaceindex.py b/openlcb/memoryspaceindex.py new file mode 100644 index 00000000..a7613421 --- /dev/null +++ b/openlcb/memoryspaceindex.py @@ -0,0 +1,22 @@ + + +from enum import Enum + + +class MemorySpaceIndex(Enum): + Uninitialized = -1 + Custom = 0 + Configuration = 1 # 0xFD & 0x03 == 1 + All = 2 # 0xFE & 0x03 == 2 + CDI = 3 # 0xFF & 0x03 == 3 + + @classmethod + def fromNumber(cls, num: int): + """Return the MemorySpace member with the given numeric value, + or None if no match is found. + """ + assert isinstance(num, int) + for member in cls: + if member.value == num: + return member + return cls.Custom diff --git a/openlcb/openlcbnetwork.py b/openlcb/openlcbnetwork.py index 33f28067..8efeff16 100644 --- a/openlcb/openlcbnetwork.py +++ b/openlcb/openlcbnetwork.py @@ -29,7 +29,8 @@ from openlcb.datagramservice import DatagramReadMemo, DatagramService from openlcb.dataprocessor import DataFormat from openlcb.dataprocessormemo import DataProcessorMemo -from openlcb.memoryservice import MemoryReadMemo, MemoryService, MemorySpace +from openlcb.memoryservice import MemoryReadMemo, MemoryService +from openlcb.memoryspace import MemorySpace from openlcb.message import Message from openlcb.xmldataprocessor import XMLDataProcessor from openlcb.mti import MTI diff --git a/openlcb/storagepool.py b/openlcb/storagepool.py index 99f24073..d6e692c4 100644 --- a/openlcb/storagepool.py +++ b/openlcb/storagepool.py @@ -1,6 +1,6 @@ from logging import getLogger -from openlcb.memoryservice import MemorySpace +from openlcb.memoryspace import MemorySpace logger = getLogger(__name__) @@ -33,3 +33,24 @@ def set(self, space, address, data): self.spaces[space] += b'\0' * newRegionLen assert end - address == len(data) self.spaces[space][address:end] = data + + def get(self, space, address, size, force=False) -> bytearray: + if isinstance(space, MemorySpace): + space = space.value + assert isinstance(space, int) + assert isinstance(address, int) + assert isinstance(size, int) + if space not in self.spaces: + if force: + self.spaces[space] = bytearray() + else: + raise KeyError(f"Space {hex(space)} does not exist.") + + end = address + size + if address >= len(self.spaces[space]): + self.set(space, address, size*b"\0") + elif end > len(self.spaces[space]): + slack = end - len(self.spaces[space]) + offset = size - slack + self.set(space, address + offset, slack*b"\0") + return self.spaces[space][address:end] diff --git a/openlcb/xmldataprocessor.py b/openlcb/xmldataprocessor.py index 48627f64..e51d7dbc 100644 --- a/openlcb/xmldataprocessor.py +++ b/openlcb/xmldataprocessor.py @@ -14,6 +14,7 @@ from openlcb.canbus.canlink import CanLink from openlcb.cdimemo import CDIMemo, DataProcessorMemo from openlcb.dataprocessor import DataFormat, DataProcessor +from openlcb.memoryspace import MemorySpace from openlcb.nodeid import NodeID from openlcb.platformextras import ( SysDirs, @@ -21,7 +22,6 @@ ) from openlcb.memoryservice import ( MemoryReadMemo, - MemorySpace, ) # from openlcb.remotenodeprocessor import RemoteNodeProcessor from openlcb.cdivar import ( diff --git a/python-openlcb.code-workspace b/python-openlcb.code-workspace index a2e89083..cc672e0c 100644 --- a/python-openlcb.code-workspace +++ b/python-openlcb.code-workspace @@ -48,6 +48,7 @@ "datagrams", "datagramservice", "dataprocessor", + "dataprocessormemo", "deque", "distros", "dmemo", @@ -69,8 +70,9 @@ "MDNS", "mdnsconventions", "memoryconfigurationheader", - "StoragePool", "memoryservice", + "memoryspace", + "memoryspaceindex", "metas", "MSGLEN", "msvc", @@ -103,6 +105,7 @@ "settingtypes", "setuptools", "SOCK_DGRAM", + "StoragePool", "sysdirs", "tcplink", "tcpsocket", diff --git a/tests/test_convert.py b/tests/test_convert.py index 1d8f8c98..1ea859f2 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -3,7 +3,8 @@ import unittest from openlcb.convert import Convert -from openlcb.memoryconfigurationheader import MemoryConfigurationHeader, MemorySpaceIndex +from openlcb.memoryconfigurationheader import MemoryConfigurationHeader +from openlcb.memoryspaceindex import MemorySpaceIndex class TestConvertClass(unittest.TestCase): diff --git a/tests/test_openlcbnetwork.py b/tests/test_openlcbnetwork.py index 46a6a683..0a7a1c2e 100644 --- a/tests/test_openlcbnetwork.py +++ b/tests/test_openlcbnetwork.py @@ -1,6 +1,6 @@ import unittest -from openlcb.memoryservice import MemorySpace +from openlcb.memoryspace import MemorySpace class OpenLCBNetworkTest(unittest.TestCase): diff --git a/tests/test_storagepool.py b/tests/test_storagepool.py new file mode 100644 index 00000000..239e78bf --- /dev/null +++ b/tests/test_storagepool.py @@ -0,0 +1,71 @@ +import os +import struct +import sys +import unittest + +from openlcb.storagepool import StoragePool + + +from logging import getLogger +if __name__ == "__main__": + logger = getLogger(__file__) +else: + logger = getLogger(__name__) + +if __name__ == "__main__": + # Allow importing repo copy of openlcb if running tests from repo manually. + TESTS_DIR = os.path.dirname(os.path.realpath(__file__)) + REPO_DIR = os.path.dirname(TESTS_DIR) + if os.path.isfile(os.path.join(REPO_DIR, "openlcb", "__init__.py")): + sys.path.insert(0, REPO_DIR) + else: + logger.warning( + "Reverting to installed copy if present (or imports will fail)," + " since test running from repo but could not find openlcb in {}." + .format(repr(REPO_DIR))) + + +class TestStoragePool(unittest.TestCase): + + def testGetNothing(self): + pool = StoragePool() + value_bytes = pool.get(4, 40, 4, force=True) + self.assertEqual(len(value_bytes), 4) + value = struct.unpack(">I", value_bytes)[0] + assert isinstance(value, int) + # i or I: int32 + # capital letter: unsigned + self.assertEqual(value, 0) + + def test_get_raises_keyerror(self): + pool = StoragePool() + with self.assertRaises(KeyError): + pool.get(4, 40, 4) # adjust arguments as needed + + def testUnsignedInt(self): + in_value = 9999999 + value_bytes = struct.pack(">I", in_value) + self.assertEqual(len(value_bytes), 4) + assert isinstance(value_bytes, (bytes, bytearray)) + pool = StoragePool() + pool.set(1, 10, value_bytes) + out_bytes = pool.get(1, 10, 4) + self.assertEqual(len(out_bytes), 4) + out_value = struct.unpack(">I", out_bytes)[0] + self.assertEqual(in_value, out_value) + + def testSignedInt(self): + in_value = -9999999 + value_bytes = struct.pack(">i", in_value) + self.assertEqual(len(value_bytes), 4) + assert isinstance(value_bytes, (bytes, bytearray)) + pool = StoragePool() + pool.set(1, 10, value_bytes) + out_bytes = pool.get(1, 10, 4) + self.assertEqual(len(out_bytes), 4) + out_value = struct.unpack(">i", out_bytes)[0] + self.assertEqual(in_value, out_value) + + +if __name__ == "__main__": + unittest.main() From 1a90cdc94b6c74a2cd155749426aa6b4cfdaf1d4 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Thu, 28 May 2026 11:16:58 -0400 Subject: [PATCH 44/64] Add set and get number methods to StoragePool. --- openlcb/localnode.py | 2 +- openlcb/storagepool.py | 81 +++++++++++++++++++++++++++++++++++++-- tests/test_storagepool.py | 80 ++++++++++++++++++++++++++++++++++---- 3 files changed, 150 insertions(+), 13 deletions(-) diff --git a/openlcb/localnode.py b/openlcb/localnode.py index 8678b7f1..1a1fa6de 100644 --- a/openlcb/localnode.py +++ b/openlcb/localnode.py @@ -107,7 +107,7 @@ def setMemory(self, memo: CDIMemo, var: CDIVar): assert var is not None assert var.data is not None assert len(var.data) == memo.getSize() - self.set(memo.space, memo.address, var.data) + self.setData(memo.space, memo.address, var.data) print(f"Set LocalNode {self.id} space {memo.space}" f" address {memo.space} (length {len(var.data)}).") diff --git a/openlcb/storagepool.py b/openlcb/storagepool.py index d6e692c4..dc16685a 100644 --- a/openlcb/storagepool.py +++ b/openlcb/storagepool.py @@ -1,5 +1,8 @@ from logging import getLogger +import struct +from typing import Union +from openlcb.cdivar import SUBTYPE_FORMATS from openlcb.memoryspace import MemorySpace logger = getLogger(__name__) @@ -9,7 +12,8 @@ class StoragePool: def __init__(self): self.spaces = {} # type: dict[int, bytearray] - def set(self, space, address, data): + def setData(self, space: Union[MemorySpace, int], address: int, + data: Union[bytes, bytearray]): """Set address in virtual memory space to data""" assert isinstance(data, (bytearray, bytes)) if isinstance(space, MemorySpace): @@ -17,6 +21,7 @@ def set(self, space, address, data): assert isinstance(space, int) assert isinstance(address, int) assert address >= 0 + # if size is None: size = len(data) end = address + size @@ -34,7 +39,8 @@ def set(self, space, address, data): assert end - address == len(data) self.spaces[space][address:end] = data - def get(self, space, address, size, force=False) -> bytearray: + def getData(self, space: Union[MemorySpace, int], address: int, + size: int, force=False) -> bytearray: if isinstance(space, MemorySpace): space = space.value assert isinstance(space, int) @@ -48,9 +54,76 @@ def get(self, space, address, size, force=False) -> bytearray: end = address + size if address >= len(self.spaces[space]): - self.set(space, address, size*b"\0") + self.setData(space, address, size*b"\0") elif end > len(self.spaces[space]): slack = end - len(self.spaces[space]) offset = size - slack - self.set(space, address + offset, slack*b"\0") + self.setData(space, address + offset, slack*b"\0") return self.spaces[space][address:end] + + def setInt(self, space: Union[MemorySpace, int], address: int, + value: int, size: int, signed: bool): + if isinstance(space, MemorySpace): + space = space.value + assert isinstance(space, int) + assert isinstance(address, int) + assert isinstance(size, int) + assert isinstance(signed, bool) + assert size in (1, 2, 4, 8) + typeStr = f"int{size*8}" + dataFormat = SUBTYPE_FORMATS[typeStr] + data = struct.pack(dataFormat, value) + print(f"packing {dataFormat}") + assert len(data) == size, \ + f"Expected {size} byte(s) for {typeStr}, got {len(data)}" + return self.setData(space, address, data) + + def getInt(self, space: Union[MemorySpace, int], address: int, + size: int, signed: bool) -> int: + if isinstance(space, MemorySpace): + space = space.value + assert isinstance(space, int) + assert isinstance(address, int) + assert isinstance(size, int) + assert isinstance(signed, bool) + data = self.getData(space, address, size) + assert size in (1, 2, 4, 8) + typeStr = f"int{size*8}" + if not signed: + typeStr = "u" + typeStr + dataFormat = SUBTYPE_FORMATS[typeStr] + values = struct.unpack(dataFormat, data) + assert len(values) == 1, f"Expected 1 {typeStr}, got {len(values)}" + assert isinstance(values[0], int) + return values[0] + + def setFloat(self, space: Union[MemorySpace, int], address: int, + value: float, size: int): + if isinstance(space, MemorySpace): + space = space.value + assert isinstance(space, int) + assert isinstance(address, int) + assert isinstance(size, int) + assert size in (2, 4, 8) + typeStr = f"float{size*8}" + dataFormat = SUBTYPE_FORMATS[typeStr] + data = struct.pack(dataFormat, value) + assert len(data) == size, \ + f"Expected {size} byte(s) for {typeStr}, got {len(data)}" + return self.setData(space, address, data) + + def getFloat(self, space: Union[MemorySpace, int], address: int, + size: int) -> float: + if isinstance(space, MemorySpace): + space = space.value + assert isinstance(space, int) + assert isinstance(address, int) + assert isinstance(size, int) + data = self.getData(space, address, size) + assert size in (2, 4, 8) + typeStr = f"float{size*8}" + dataFormat = SUBTYPE_FORMATS[typeStr] + values = struct.unpack(dataFormat, data) + assert len(values) == 1, f"Expected 1 {typeStr}, got {len(values)}" + assert isinstance(values[0], float) + return values[0] diff --git a/tests/test_storagepool.py b/tests/test_storagepool.py index 239e78bf..89acf10f 100644 --- a/tests/test_storagepool.py +++ b/tests/test_storagepool.py @@ -29,7 +29,7 @@ class TestStoragePool(unittest.TestCase): def testGetNothing(self): pool = StoragePool() - value_bytes = pool.get(4, 40, 4, force=True) + value_bytes = pool.getData(4, 40, 4, force=True) self.assertEqual(len(value_bytes), 4) value = struct.unpack(">I", value_bytes)[0] assert isinstance(value, int) @@ -40,32 +40,96 @@ def testGetNothing(self): def test_get_raises_keyerror(self): pool = StoragePool() with self.assertRaises(KeyError): - pool.get(4, 40, 4) # adjust arguments as needed + # KeyError is necessary because space 4 was not defined + # (pool.set* is not called above, so no spaces exist). + pool.getData(4, 40, 4) # adjust arguments as needed - def testUnsignedInt(self): + def testUnsignedIntData(self): in_value = 9999999 value_bytes = struct.pack(">I", in_value) self.assertEqual(len(value_bytes), 4) assert isinstance(value_bytes, (bytes, bytearray)) pool = StoragePool() - pool.set(1, 10, value_bytes) - out_bytes = pool.get(1, 10, 4) + pool.setData(1, 10, value_bytes) + out_bytes = pool.getData(1, 10, 4) self.assertEqual(len(out_bytes), 4) out_value = struct.unpack(">I", out_bytes)[0] self.assertEqual(in_value, out_value) - def testSignedInt(self): + def testSignedIntData(self): in_value = -9999999 value_bytes = struct.pack(">i", in_value) self.assertEqual(len(value_bytes), 4) assert isinstance(value_bytes, (bytes, bytearray)) pool = StoragePool() - pool.set(1, 10, value_bytes) - out_bytes = pool.get(1, 10, 4) + pool.setData(1, 10, value_bytes) + out_bytes = pool.getData(1, 10, 4) self.assertEqual(len(out_bytes), 4) out_value = struct.unpack(">i", out_bytes)[0] self.assertEqual(in_value, out_value) + def testUnsignedInt(self): + in_value = 9999999 + pool = StoragePool() + size = 4 + signed = False + pool.setInt(1, 10, in_value, size, signed) + out_bytes = pool.getData(1, 10, size) + self.assertEqual(len(out_bytes), size) + out_value = struct.unpack(">I", out_bytes)[0] + self.assertEqual(in_value, out_value) + out_value = pool.getInt(1, 10, size, signed) + self.assertEqual(in_value, out_value) + + def testSignedInt(self): + in_value = -9999999 + pool = StoragePool() + size = 4 + signed = True + pool.setInt(1, 10, in_value, size, signed) + out_bytes = pool.getData(1, 10, size) + self.assertEqual(len(out_bytes), size) + out_value = struct.unpack(">i", out_bytes)[0] + self.assertEqual(in_value, out_value) + out_value = pool.getInt(1, 10, size, signed) + self.assertEqual(in_value, out_value) + + def testFloat(self): + pool = StoragePool() + sizeFormats = { + 2: ">e", + 4: ">f", + 8: ">d", + } + in_value = -999 + size = 2 + pool.setFloat(1, 10, in_value, size) + out_bytes = pool.getData(1, 10, size) + self.assertEqual(len(out_bytes), size) + out_value = struct.unpack(sizeFormats[size], out_bytes)[0] + self.assertEqual(in_value, out_value) + out_value = pool.getFloat(1, 10, size) + self.assertEqual(in_value, out_value) + + size = 4 + in_value = -9999999 # NOTE: f32 fits -9999999 f16 does not + pool.setFloat(1, 10, in_value, size) + out_bytes = pool.getData(1, 10, size) + self.assertEqual(len(out_bytes), size) + out_value = struct.unpack(sizeFormats[size], out_bytes)[0] + self.assertEqual(in_value, out_value) + out_value = pool.getFloat(1, 10, size) + self.assertEqual(in_value, out_value) + + size = 8 + pool.setFloat(1, 10, in_value, size) + out_bytes = pool.getData(1, 10, size) + self.assertEqual(len(out_bytes), size) + out_value = struct.unpack(sizeFormats[size], out_bytes)[0] + self.assertEqual(in_value, out_value) + out_value = pool.getFloat(1, 10, size) + self.assertEqual(in_value, out_value) + if __name__ == "__main__": unittest.main() From e6cbf68e0509bfe91d45b58bb32f5bc9d1911d31 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Thu, 28 May 2026 12:50:21 -0400 Subject: [PATCH 45/64] Implement minimums by size as per CDI Standard. --- openlcb/cdivar.py | 88 +++++++++++++++++++++++++++++++++++---- openlcb/storagepool.py | 39 ++++++++++++++--- tests/test_storagepool.py | 61 +++++++++++++++++++++++++++ 3 files changed, 174 insertions(+), 14 deletions(-) diff --git a/openlcb/cdivar.py b/openlcb/cdivar.py index 66518463..5905b2e9 100644 --- a/openlcb/cdivar.py +++ b/openlcb/cdivar.py @@ -17,6 +17,50 @@ NUM_TYPES = {'int': int, 'float': float} # type: dict[str, Type] # Assumes "IEEE" in OpenLCB CDI Standard means IEEE 754-2008: FLOAT_MAXIMUMS = {2: 65504.0, 4: 3.40e38, 8: 1.80e308} # type: dict[int, float] # noqa: E501 +# Float minimums (https://en.wikipedia.org/wiki/IEEE_754): +# 16-bit smallest normal 6.10×10−5 subnormal 5.96×10−8 +# 32-bit smallest normal 1.18×10−38 subnormal 1.40×10−45 +# 64-bit smallest normal 2.23×10−308 subnormal 4.94×10−324 +F_MIN_BITS = { + 2: [0] * 16, + 4: [0] * 32, + 8: [0] * 64, +} + +# Set bits for the most negative finite number +for size, bits in F_MIN_BITS.items(): + bits[0] = 1 # Sign bit = 1 (negative) + if size == 2: # binary16: 1 sign + 5 exp + 10 mant + for i in range(1, 6): bits[i] = 1 # exp = 11110 + bits[5] = 0 # ← important: clear LSB of exponent + for i in range(6, 16): bits[i] = 1 # mantissa all 1s + elif size == 4: # binary32: 1 sign + 8 exp + 23 mant + for i in range(1, 9): bits[i] = 1 # exp = 11111110 + bits[8] = 0 # ← clear LSB of exponent + for i in range(9, 32): bits[i] = 1 + else: # binary64: 1 sign + 11 exp + 52 mant + for i in range(1, 12): bits[i] = 1 # exp = 111...1110 + bits[11] = 0 # ← clear LSB of exponent + for i in range(12, 64): bits[i] = 1 + +FLOAT_MINIMUMS = {} # F_MIN_DATA = {} + +for k, bits in F_MIN_BITS.items(): + # Create traceable binary string (e.g. "0b000...001") + bit_str = "0b" + "".join(map(str, bits)) + # Convert to integer then to bytes + value = int(bit_str, 2) + data_bytes = value.to_bytes(k, 'big') + fmt = {2: ">e", 4: ">f", 8: ">d"}[k] + # F_MIN_DATA[k] = data_bytes + FLOAT_MINIMUMS[k] = struct.unpack(fmt, data_bytes)[0] + # print(f"binary{k*8:2d} bits: {bit_str}") + # print(f"binary{k*8:2d} value: {F_MIN_DATA[k]:.20e}\n") + # results: + # -65504.0 + # -3.4028234663852886e+38 + # -1.7976931348623157e+308 + UNSIGNED_INT_MAXIMUMS = { # type: dict[int, int] 1: 0xFF, 2: 0xFFFF, 4: 0xFFFF_FFFF, 8: 0xFFFF_FFFF_FFFF_FFFF} SIGNED_INT_MINIMUMS = {} @@ -89,7 +133,8 @@ class CDIVar: def __init__(self, className, _min=None, _max=None, _size=None, _default=None, assert_range=False, - _no_min=False, _no_max=False, _default_data=None): + _no_min=False, _no_max=False, _default_data=None, + signed=None): self.data = None # type: bytes|None self.min = _min # type: CDIVar|None self.max = _max # type: CDIVar|None @@ -101,8 +146,6 @@ def __init__(self, className, _min=None, _max=None, f"Expected {list(CLASSNAME_TYPES.keys())} got {className}" if _default is not None: assert isinstance(_default, CDIVar) - if className in NUM_TYPES: - _default.assertNumberFormat() assert _default_data is None, \ "Can only set _default or _default_data" elif _default_data is not None: @@ -113,9 +156,18 @@ def __init__(self, className, _min=None, _max=None, _no_max=True, _no_min=True) # prevent recursion _default.data = _default_data + if _default is not None: + if className in NUM_TYPES: + _default.assertNumberFormat() + if _default < 0: + if signed is None: + signed = True + self.name = None # type: str|None self.className = className # type: str - self.signed = False # type: bool + if signed is None: + signed = False + self.signed = signed # type: bool assert isinstance(_no_min, bool) assert isinstance(_no_max, bool) self._no_min = _no_min @@ -140,9 +192,26 @@ def __init__(self, className, _min=None, _max=None, raise AssertionError(error) else: logger.error(error) - elif not _no_min: - self.min = CDIVar(className, _size=_size, - _no_min=True, _no_max=True) # prevent inf recurs + elif (className in NUM_TYPES) and not _no_min: + # self.min = CDIVar(className, _size=_size, + # _no_min=True, _no_max=True) # prevent inf recurs + # Set minimum based on size, + # as per Configuration Description Information Standard. + assert _size is not None + if className == "int": + if signed: + # self.min.setInt(SIGNED_INT_MINIMUMS[_size]) + self.min = CDIVar.fromInt(SIGNED_INT_MINIMUMS[_size], + _size) + else: + # self.min.setInt(0) + self.min = CDIVar.fromInt(0, _size) + elif className == "float": + # self.min.setFloat(FLOAT_MINIMUMS[_size]) + self.min = CDIVar.fromFloat(FLOAT_MINIMUMS[_size], _size) + else: + raise NotImplementedError(f"no default minimum {className}") + if _max is not None: assert isinstance(_max, CDIVar) _max.assertNumberFormat() @@ -342,9 +411,11 @@ def cmp_any(cls, self: 'CDIVar', other: Union['CDIVar', float, int], @classmethod def fromNumber(cls, value: Union[int, float], className: str, _size: int) -> 'CDIVar': - var = CDIVar(className, _size=_size) + var = CDIVar(className, _size=_size, _no_min=True) + # ^ _no_min prevents infinite recursion generating min if value < 0: var.signed = True + var.min = None # remove default (0) if className == "int": assert isinstance(value, int) var.setInt(value) @@ -484,6 +555,7 @@ def intToData(self, value: int) -> bytes: try: return struct.pack(self.packFormat(), value) except Exception as ex: + logger.error("") logger.error(formatted_ex(ex)) logger.error( f"Tried to set a(n) {self.subtype()} CDIVar" diff --git a/openlcb/storagepool.py b/openlcb/storagepool.py index dc16685a..0e6e82e5 100644 --- a/openlcb/storagepool.py +++ b/openlcb/storagepool.py @@ -2,7 +2,7 @@ import struct from typing import Union -from openlcb.cdivar import SUBTYPE_FORMATS +from openlcb.cdivar import SUBTYPE_FORMATS, CDIVar from openlcb.memoryspace import MemorySpace logger = getLogger(__name__) @@ -12,8 +12,31 @@ class StoragePool: def __init__(self): self.spaces = {} # type: dict[int, bytearray] + def set(self, var: CDIVar): + assert isinstance(var, CDIVar) + assert var.space is not None + assert var.address + data = var.getData() + assert data is not None + self.setData(var.space, var.address, data, size=var.size) + + def get(self, var: CDIVar) -> CDIVar: + """Modify var in place. + + Returns: + CDIVar: Same var instance (returned by reference) modified. + """ + assert isinstance(var, CDIVar) + assert var.space is not None + assert var.address + assert var.size is not None + data = self.getData(var.space, var.address, var.size) + assert data is not None + var.setData(data) + return var + def setData(self, space: Union[MemorySpace, int], address: int, - data: Union[bytes, bytearray]): + data: Union[bytes, bytearray], size=None): """Set address in virtual memory space to data""" assert isinstance(data, (bytearray, bytes)) if isinstance(space, MemorySpace): @@ -21,8 +44,10 @@ def setData(self, space: Union[MemorySpace, int], address: int, assert isinstance(space, int) assert isinstance(address, int) assert address >= 0 - # if size is None: - size = len(data) + if size is None: + size = len(data) + else: + assert size <= len(data) end = address + size if space not in self.spaces: @@ -37,7 +62,10 @@ def setData(self, space: Union[MemorySpace, int], address: int, f" byte(s) to {end} byte(s).") self.spaces[space] += b'\0' * newRegionLen assert end - address == len(data) - self.spaces[space][address:end] = data + if size < len(data): + self.spaces[space][address:end] = data[:size] + else: + self.spaces[space][address:end] = data def getData(self, space: Union[MemorySpace, int], address: int, size: int, force=False) -> bytearray: @@ -73,7 +101,6 @@ def setInt(self, space: Union[MemorySpace, int], address: int, typeStr = f"int{size*8}" dataFormat = SUBTYPE_FORMATS[typeStr] data = struct.pack(dataFormat, value) - print(f"packing {dataFormat}") assert len(data) == size, \ f"Expected {size} byte(s) for {typeStr}, got {len(data)}" return self.setData(space, address, data) diff --git a/tests/test_storagepool.py b/tests/test_storagepool.py index 89acf10f..50244a41 100644 --- a/tests/test_storagepool.py +++ b/tests/test_storagepool.py @@ -3,6 +3,8 @@ import sys import unittest +from openlcb import emit_cast +from openlcb.cdivar import SIGNED_INT_MINIMUMS, CDIVar from openlcb.storagepool import StoragePool @@ -130,6 +132,65 @@ def testFloat(self): out_value = pool.getFloat(1, 10, size) self.assertEqual(in_value, out_value) + def testCDIVarUInt(self): + size = 4 + var = CDIVar("int", _size=size) + var.space = 1 + var.address = 10 + in_value = 999 + # signed = False + var.setInt(in_value) + self.assertEqual(var.getInt(), in_value) + pool = StoragePool() + pool.set(var) + var = pool.get(var) + self.assertEqual(var.getInt(), in_value) + assert var.space is not None + assert var.address is not None + assert var.size is not None + assert var.signed is not None + out_value = pool.getInt(var.space, var.address, var.size, var.signed) + self.assertEqual(out_value, in_value) + + def testCDIVarSInt(self): + size = 4 + in_value = -999 + signed = True if in_value < 0 else False + # defaultVar = CDIVar("int", _size=size, _no_min=True, _no_max=True, + # signed=signed) + # defaultVar.setInt(in_value) + # simplified construction: + defaultVar = CDIVar.fromInt(in_value, size) + self.assertTrue(defaultVar.signed) + self.assertIsNone(defaultVar.min) + var = CDIVar( + "int", + _size=size, + _default=defaultVar, # forces signed since negative + ) + self.assertTrue(var.signed) + self.assertIsInstance(var.min, CDIVar) + self.assertIsNotNone( + var.min, + msg=f"{emit_cast(var.min)} should be min for {size*8}-bit") + self.assertEqual(var.min, SIGNED_INT_MINIMUMS[size]) + # ^ == allowed since __eq__ is defined for CDIVar (var.min) + var.space = 1 + var.address = 10 + # signed = False + var.setInt(in_value) + self.assertEqual(var.getInt(), in_value) + pool = StoragePool() + pool.set(var) + var = pool.get(var) + self.assertEqual(var.getInt(), in_value) + assert var.space is not None + assert var.address is not None + assert var.size is not None + assert var.signed is True + out_value = pool.getInt(var.space, var.address, var.size, var.signed) + self.assertEqual(out_value, in_value) + if __name__ == "__main__": unittest.main() From 87cc7f76b52928ef3d138dda04c1cb1623e499c3 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Thu, 28 May 2026 14:13:06 -0400 Subject: [PATCH 46/64] Fix: Don't load CDI twice. --- openlcb/xmldataprocessor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openlcb/xmldataprocessor.py b/openlcb/xmldataprocessor.py index e51d7dbc..20587990 100644 --- a/openlcb/xmldataprocessor.py +++ b/openlcb/xmldataprocessor.py @@ -388,9 +388,9 @@ def memoryReadFail(memo: MemoryReadMemo): # based on "else" (done) case in _memoryReadSuccess # in OpenLCBNetwork: self._stringTerminated = True - if self._realtime: - # Don't miss calling parser if realtime - self._feedNext(memo) + # if self._realtime: + # # Don't miss calling parser if realtime + # self._feedNext(memo) self._feedLast(memo, enable_cache=False) self.onStop() # sets self._format to DataFormat.EOF else: From 009f53d9b1824c1b9ae5d8d922137020043bc5fa Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Thu, 28 May 2026 14:14:33 -0400 Subject: [PATCH 47/64] Fix: Compare int to CDI float (more Pythonic). --- openlcb/cdivar.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/openlcb/cdivar.py b/openlcb/cdivar.py index 5905b2e9..9bf245ce 100644 --- a/openlcb/cdivar.py +++ b/openlcb/cdivar.py @@ -145,7 +145,8 @@ def __init__(self, className, _min=None, _max=None, assert className in CLASSNAME_TYPES, \ f"Expected {list(CLASSNAME_TYPES.keys())} got {className}" if _default is not None: - assert isinstance(_default, CDIVar) + assert isinstance(_default, CDIVar), \ + f"expected CDIVar got {type(_default).__name__}" assert _default_data is None, \ "Can only set _default or _default_data" elif _default_data is not None: @@ -400,9 +401,13 @@ def cmp_any(cls, self: 'CDIVar', other: Union['CDIVar', float, int], f"Can't compare float to {self.className} CDIVar" return self.cmp_float(self, other) == compare_op elif isinstance(other, int): - assert self.className == "int", \ - f"Can't compare int to {self.className} CDIVar" - return self.cmp_int(self, other) == compare_op + # assert self.className == "int", \ + # f"Can't compare int to {self.className} CDIVar" + # Allow int such as 0 to be compared to float CDIVar: + if self.className == "float": + return self.cmp_float(self, float(other)) == compare_op + else: + return self.cmp_int(self, other) == compare_op else: raise TypeError( f"Cannot compare {type(other).__name__}" From c8bbb4fc4161188b161f15c20402d457e34d6fcb Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Thu, 28 May 2026 14:15:46 -0400 Subject: [PATCH 48/64] Use simplified CDIVar construction for `default`. --- openlcb/cdimemo.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/openlcb/cdimemo.py b/openlcb/cdimemo.py index 84a37c4c..343b035d 100644 --- a/openlcb/cdimemo.py +++ b/openlcb/cdimemo.py @@ -202,19 +202,23 @@ def toCDIVar(self): result_default = None result_size = self.getSize() if this_t is not None: + assert result_size is not None, \ + f"size is required for {this_t.__name__}" result_min = self.getChildContentN("min", className) result_max = self.getChildContentN("max", className) default_n = self.getChildContentN("default", className) if default_n is not None: - default_var = CDIVar(className, _size=result_size) - if isinstance(default_n, int): - assert self.tag == "int" - default_var.setInt(default_n) - else: - assert self.tag == "float" - default_var.setFloat(default_n) - assert default_var.data is not None - result_default = bytearray(default_var.data) + result_default = CDIVar.fromNumber(default_n, className, + result_size) + # default_var = CDIVar(className, _size=result_size) + # if isinstance(default_n, int): + # assert self.tag == "int" + # default_var.setInt(default_n) + # else: + # assert self.tag == "float" + # default_var.setFloat(default_n) + # assert default_var.data is not None + # result_default = bytearray(default_var.data) # Size must be gotten ahead of time since CDIVar constructor # enforces size: result = CDIVar(self.tag, _min=result_min, _max=result_max, From c67dc4f599212d87031670750ebe6dc5353aa2b2 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Thu, 28 May 2026 14:16:25 -0400 Subject: [PATCH 49/64] Set CDI memory space on loadCDI*. --- examples/example_node_memory_implementation.py | 15 +++++++++++---- openlcb/localnode.py | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/examples/example_node_memory_implementation.py b/examples/example_node_memory_implementation.py index d5011c11..8de82c22 100644 --- a/examples/example_node_memory_implementation.py +++ b/examples/example_node_memory_implementation.py @@ -24,7 +24,8 @@ # region same code as other examples from examples_settings import Settings -from openlcb.localnode import LocalNode # do 1st to fix path if no pip install +from openlcb.localnode import LocalNode +from openlcb.memoryspace import MemorySpace # do 1st to fix path if no pip install settings = Settings() if __name__ == "__main__": @@ -179,11 +180,13 @@ def memoryReadFail(memo): # create a node and connect it update # This is a very minimal node, which just takes part in the low-level common # protocols +localNodeID = NodeID(settings['localNodeID']) localNode = LocalNode( - NodeID(settings['localNodeID']), + localNodeID, SNIP("python-openlcb example authors", "example_node_memory_implementation", - "1.0", "1.0", "Custom Name Here", "Custom Description Here"), + "1.0", "1.0", "example_node_memory_implementation", + "python-openlcb example node with memory"), set([ PIP.SIMPLE_NODE_IDENTIFICATION_PROTOCOL, PIP.DATAGRAM_PROTOCOL, @@ -193,12 +196,16 @@ def memoryReadFail(memo): ]), canLink ) +memoryService.pools[str(localNodeID)] = localNode my_conf_dir = os.path.join(get_config_dir("python-openlcb")) backup_name = "example_node_memory_implementation.cdi.xml" backup_path = os.path.join(my_conf_dir, backup_name) localNode.loadCDIString(cdi, backup_path) - +# NOTE: loadCDI or loadCDIString sets Element tree and +# localNode.spaces[MemorySpace.CDI.value] +assert MemorySpace.CDI.value in localNode.spaces +assert isinstance(localNode.spaces[MemorySpace.CDI.value], (bytearray, bytes)) # localNodeProcessor = LocalNodeProcessor(canLink, localNode) # canLink.registerMessageReceivedListener(localNodeProcessor.process) localNodeProcessor = localNode.localNodeProcessor diff --git a/openlcb/localnode.py b/openlcb/localnode.py index 1a1fa6de..bdaacffb 100644 --- a/openlcb/localnode.py +++ b/openlcb/localnode.py @@ -94,6 +94,18 @@ def loadCDIString(self, xml_data, path, memo=None): # data = stream.read() # self.tree = etree.fromstring(data) self.reserveSpaces() + if MemorySpace.CDI.value in self.spaces: + logger.warning(f"CDI defined {MemorySpace.CDI.value}") + + # NOTE: self.cdi._data is None after load is done! + if isinstance(xml_data, bytes): + xml_data = bytearray(xml_data) + elif isinstance(xml_data, str): + xml_data = bytearray(xml_data.encode("utf-8")) + assert isinstance(xml_data, bytearray), \ + f"expected bytearray got {type(xml_data).__name__}" + # assert isinstance(xml_data, (bytes, bytearray)) + self.spaces[MemorySpace.CDI.value] = xml_data def setMemory(self, memo: CDIMemo, var: CDIVar): """Set a memory address at memo to the value in var""" @@ -191,6 +203,8 @@ def onCDILoaded(self, memo: MemoryReadMemo): ("PIP.CONFIGURATION_DESCRIPTION_INFORMATION is not in pipSet" f" for LocalNode {self.id}") print(f"LocalNode onFileLoaded {self.cdi.getPath()}: {memo}") + # NOTE: self.cdi._data is None after load is done! + # self.setData is done during loadCDIString since not multi-threaded. def onCDILoadFailed(self, memo: MemoryReadMemo): """Default handler for file load failed. From e7aca7f5140136cd716a7f4d40baec089a118086 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Thu, 28 May 2026 17:02:54 -0400 Subject: [PATCH 50/64] Implement Get Address Space Information Reply. Fix: MCOpMasks.Default value. --- .../example_node_memory_implementation.py | 12 +- openlcb/localnode.py | 15 +- openlcb/memoryservice.py | 90 ++++++- openlcb/storagepool.py | 229 +++++++++++++++--- tests/test_storagepool.py | 22 +- 5 files changed, 294 insertions(+), 74 deletions(-) diff --git a/examples/example_node_memory_implementation.py b/examples/example_node_memory_implementation.py index 8de82c22..5d2e0d5a 100644 --- a/examples/example_node_memory_implementation.py +++ b/examples/example_node_memory_implementation.py @@ -25,7 +25,8 @@ # region same code as other examples from examples_settings import Settings from openlcb.localnode import LocalNode -from openlcb.memoryspace import MemorySpace # do 1st to fix path if no pip install +from openlcb.memoryspace import MemorySpace +from openlcb.storagepool import StorageSpace # do 1st to fix path if no pip install settings = Settings() if __name__ == "__main__": @@ -196,16 +197,17 @@ def memoryReadFail(memo): ]), canLink ) -memoryService.pools[str(localNodeID)] = localNode +memoryService.pool = localNode my_conf_dir = os.path.join(get_config_dir("python-openlcb")) backup_name = "example_node_memory_implementation.cdi.xml" backup_path = os.path.join(my_conf_dir, backup_name) localNode.loadCDIString(cdi, backup_path) # NOTE: loadCDI or loadCDIString sets Element tree and -# localNode.spaces[MemorySpace.CDI.value] -assert MemorySpace.CDI.value in localNode.spaces -assert isinstance(localNode.spaces[MemorySpace.CDI.value], (bytearray, bytes)) +# localNode._spaces[MemorySpace.CDI.value] +storage = localNode.getStorage(MemorySpace.CDI.value) +assert isinstance(storage, StorageSpace) +assert isinstance(storage._data, (bytearray, bytes)) # localNodeProcessor = LocalNodeProcessor(canLink, localNode) # canLink.registerMessageReceivedListener(localNodeProcessor.process) localNodeProcessor = localNode.localNodeProcessor diff --git a/openlcb/localnode.py b/openlcb/localnode.py index bdaacffb..8533c478 100644 --- a/openlcb/localnode.py +++ b/openlcb/localnode.py @@ -4,7 +4,7 @@ from typing import Union from openlcb import emit_cast from openlcb.memoryspace import MemorySpace -from openlcb.storagepool import StoragePool +from openlcb.storagepool import StoragePool, StorageSpace from openlcb.node import PIP, SNIP, Node from openlcb.localnodeprocessor import LocalNodeProcessor @@ -94,7 +94,7 @@ def loadCDIString(self, xml_data, path, memo=None): # data = stream.read() # self.tree = etree.fromstring(data) self.reserveSpaces() - if MemorySpace.CDI.value in self.spaces: + if MemorySpace.CDI.value in self._spaces: logger.warning(f"CDI defined {MemorySpace.CDI.value}") # NOTE: self.cdi._data is None after load is done! @@ -105,7 +105,10 @@ def loadCDIString(self, xml_data, path, memo=None): assert isinstance(xml_data, bytearray), \ f"expected bytearray got {type(xml_data).__name__}" # assert isinstance(xml_data, (bytes, bytearray)) - self.spaces[MemorySpace.CDI.value] = xml_data + storage = StorageSpace() + self._spaces[MemorySpace.CDI.value] = storage + storage.setData(0, xml_data, force=True) + storage.markReadOnly(True) def setMemory(self, memo: CDIMemo, var: CDIVar): """Set a memory address at memo to the value in var""" @@ -119,7 +122,7 @@ def setMemory(self, memo: CDIMemo, var: CDIVar): assert var is not None assert var.data is not None assert len(var.data) == memo.getSize() - self.setData(memo.space, memo.address, var.data) + self.setSlice(memo.space, memo.address, var.data) print(f"Set LocalNode {self.id} space {memo.space}" f" address {memo.space} (length {len(var.data)}).") @@ -189,11 +192,11 @@ def _reserveSpaces(self, parent: Union[CDIMemo, None] = None, level=0): logger.warning( f"Creating cdiBackupDir {self.cdiBackupDir}") os.makedirs(self.cdiBackupDir) - for space, data in self.spaces.items(): + for space, storage in self._spaces.items(): name = f"{self.id}.lcc-link-virtual-node.space={space}.xml" path = os.path.join(self.cdiBackupDir, name) with open(path, "wb") as stream: - stream.write(data) + stream.write(storage._data) print(f"Wrote {d_quote(path)}") def onCDILoaded(self, memo: MemoryReadMemo): diff --git a/openlcb/memoryservice.py b/openlcb/memoryservice.py index fd7b016e..5dd716e4 100644 --- a/openlcb/memoryservice.py +++ b/openlcb/memoryservice.py @@ -23,6 +23,7 @@ from enum import Enum from logging import getLogger +import struct from typing import ( Callable, List, @@ -58,13 +59,14 @@ class MCOp(Enum): Read_Stream_Reply_Failure = 0x78 # 01111000 Write_Command = 0x00 Write_Reply = 0x10 # 00010000 + # or 00010001 (0x11) and so on Write_Reply_Failure = 0x18 Write_Under_Mask_Command = 0x08 # 00001000 Write_Stream_Command = 0x20 # 01000000 Write_Stream_Reply = 0x30 # 00110000 Write_Stream_Reply_Failure = 0x38 # 0b111000 Get_Configuration_Options_Command = 0x80 # 10000000 - # 1 sub-operation (same as above using MCOpMasks.Default): + # 1 sub-operation (same as above using MCOpMasks.Default) Get_Configuration_Options_Reply = 0x82 # 10000010 # ^ special datagram format follows (in later bytes) Get_Address_Space_Info_Command = 0x84 # 10000100 @@ -102,7 +104,7 @@ class MCOpBits: class MCOpMasks: - Default = 0x11111100 + Default = 0b11111100 # The following aren't necessary since # they can be broken down with # sub-checks even if Default is used @@ -114,13 +116,13 @@ class MCOpMasks: MODE_BYTES = { # order determines meaning for lists (See ) - MCOp.Read_Command.value: {0x40, 0x41, 0x42, 0x43}, # pools + MCOp.Read_Command.value: {0x40, 0x41, 0x42, 0x43}, # pool MCOp.Read_Reply.value: {0x50, 0x51, 0x52, 0x53}, - MCOp.Read_Stream_Command.value: {0x60, 0x61, 0x62, 0x63}, # pools + MCOp.Read_Stream_Command.value: {0x60, 0x61, 0x62, 0x63}, # pool MCOp.Read_Stream_Reply.value: {0x70, 0x71, 0x72, 0x73}, # TODO - MCOp.Write_Command.value: [0x00, 0x01, 0x02, 0x03], # pools + MCOp.Write_Command.value: [0x00, 0x01, 0x02, 0x03], # pool MCOp.Write_Reply.value: {0x10, 0x11, 0x12, 0x13}, - MCOp.Write_Under_Mask_Command.value: {0x08, 0x09, 0x0A, 0x0B}, # pools + MCOp.Write_Under_Mask_Command.value: {0x08, 0x09, 0x0A, 0x0B}, # pool MCOp.Write_Stream_Command.value: {0x20, 0x21, 0x22, 0x23}, MCOp.Write_Stream_Reply.value: {0x30, 0x31, 0x32, 0x33}, # TODO MCOp.Get_Configuration_Options_Command.value: {0x80, }, @@ -329,10 +331,11 @@ class MemoryService: service (DatagramService): See DatagramService. Attributes: - pools (dict[str, StoragePool]): The storage where - other nodes can read and write memory. Each element can be - changed to a specific nodeid's memory manager. They key is - the NodeID in string form (dotted notation). + pool (Union[StoragePool, LocalNode]): The storage where + other nodes can read and write memory. Since a datagram + doesn't have a destination, there must be a MemoryService + for each local node (the local Configuration tool or node + and any virtual nodes). """ def __init__(self, service: DatagramService): @@ -345,7 +348,7 @@ def __init__(self, service: DatagramService): self.service.registerDatagramReceivedListener( self.datagramReceivedListener ) - self.pools = {} # type: dict[str, StoragePool] + self.pool = StoragePool() def requestMemoryRead(self, memo, stream: bool = False): # type: (MemoryReadMemo, Optional[bool]) -> None @@ -422,9 +425,12 @@ def datagramReceivedListener(self, dmemo: DatagramReadMemo) -> bool: return True # error, but for our service; sent negative reply # Acknowledge the datagram self.service.positiveReplyToDatagram(dmemo, 0x0000) - + mcOp = MCOp.fromNumber(dmemo.data[1] & MCOpMasks.Default) # decode if read, write or some other reply if dmemo.data[1] in (0x50, 0x51, 0x52, 0x53, 0x58, 0x59, 0x5A, 0x5B): + assert mcOp is MCOp.Read_Reply, \ + "self-test failed (bad constant(s))" + # MCOp.Read_Reply # read or read-error reply # return data to requestor: first find matching memory read @@ -459,6 +465,9 @@ def datagramReceivedListener(self, dmemo: DatagramReadMemo) -> bool: tMemoryMemo.rejectedReply(tMemoryMemo) break elif dmemo.data[1] in (0x10, 0x11, 0x12, 0x13, 0x18, 0x19, 0x1A, 0x1B): + assert mcOp is MCOp.Write_Reply, \ + (f"self-test failed (bad constant(s));" + f" got op {mcOp} for sub-op {hex(dmemo.data[1])}") # write reply good, bad # return data to requestor: first find matching memory write @@ -473,7 +482,64 @@ def datagramReceivedListener(self, dmemo: DatagramReadMemo) -> bool: else: writeMemo.rejectedReply(writeMemo) break + elif dmemo.data[1] == MCOp.Get_Address_Space_Info_Command.value: + # 0x84 (A node sent us a command requesting space info) + assert mcOp is MCOp.Get_Address_Space_Info_Command, \ + "self-test failed (bad constant(s))" + space = dmemo.data[2] + last = self.pool.getLast(space) + if last is not None: + first = self.pool.getFirst(space) + assert isinstance(first, int) + assert last - first >= 0, \ + (f"{type(self.pool).__name__} incorrectly implemented:" + " first>last") + ReadOnly = 0b10000000 + HasLowestAddress = 0b01000000 + highestAddrBytes = struct.pack(">I", last) + assert len(highestAddrBytes) == 4 + # TODO: ^ Memory Configuration Standard doesn't say + # unsigned/signed, so assuming unsigned for + # highestAddrBytes (and lowestAddrBytes below) + replyData = bytearray([ + DatagramService.ProtocolID.MemoryOperation.value, + 0x87, # If 0x86, bytes after first three can be omitted. + space, + ]) + if replyData[1] == 0x87: + replyData += highestAddrBytes # bytes 3-6 (0 indexed) + replyData.append(0x00) # flags (set below) + if self.pool.isReadOnly(space): + replyData[-1] |= ReadOnly + if first != 0: + replyData[-1] |= HasLowestAddress + lowestAddrBytes = struct.pack(">I", first) + assert len(lowestAddrBytes) == 4 + replyData += lowestAddrBytes + description = self.pool.getDescription(space) + if description: + descBytes = bytearray(description.encode()) + else: + descBytes = bytearray() + descBytes.append(0) + # FIXME: is null-terminator is required if no + # description (See + # https://github.com/openlcb/documents/issues/190)? + replyData += descBytes + spaceInfoReplyMemo = DatagramWriteMemo( + dmemo.srcID, + replyData + ) + self.service.sendDatagram(spaceInfoReplyMemo) + else: + # TODO: rejected + pass elif dmemo.data[1] in (0x86, 0x87): # Address Space Information Reply + assert mcOp is MCOp.Get_Address_Space_Info_Command, \ + "self-test failed (bad constant(s))" + # ^ same first 6 bits as Get_Address_Space_Info_Command, + # but in this case actually a reply to a command. + if self.spaceLengthCallback is None: logger.error("Address Space Information Reply" " received with no callback") diff --git a/openlcb/storagepool.py b/openlcb/storagepool.py index 0e6e82e5..37b3b8ee 100644 --- a/openlcb/storagepool.py +++ b/openlcb/storagepool.py @@ -8,9 +8,106 @@ logger = getLogger(__name__) +class StorageSpace: + def __init__(self, size=0, readOnly=False): + assert isinstance(size, int) + assert size >= 0 + assert isinstance(readOnly, bool) + self._description = None + self._first = None + self._last = None + if size == 0: + self._data = bytearray() + else: + self._data = bytearray(b"\0"*size) + self._readOnly = readOnly + + def getFirst(self): + if self._first is None: + return 0 + assert isinstance(self._first, int) + return self._first + + def getLast(self): + if self._last is None: + return self.getFirst() + len(self._data) - 1 + # ^ -1 since inclusive + assert isinstance(self._last, int) + return self._last + + def getLength(self): + return self.getLast() - self.getFirst() + 1 + + def getSlice(self, address: int, size: int, force=False): + end = address + size + if address >= len(self._data): + if force: + data = bytearray(size*b"\0") + self.setData(address, data, force=force) + return data + else: + raise IndexError( + f"Tried to get address {address}" + f"in {self.getLength()}-long space") + elif end > len(self._data): + slack = end - len(self._data) + offset = size - slack + if force: + self.setData(address + offset, slack*b"\0") + else: + raise IndexError( + f"Tried to get address {address}" + f"in {self.getLength()}-long space") + return self._data[address:end] + + def getDescription(self) -> Union[str, None]: + return self._description + + def setDescription(self, description: str): + assert isinstance(description, str) + self._description = description + + def isReadOnly(self) -> bool: + return self._readOnly + + def markReadOnly(self, readOnly: bool): + assert isinstance(readOnly, bool) + self._readOnly = readOnly + + def extend(self, data): + self._data += data + + def setData(self, address: int, data: Union[bytearray, bytes], + size: Union[int, None] = None, force=True): + assert isinstance(data, (bytearray, bytes)) + assert isinstance(address, int) + assert address >= 0 + if size is None: + size = len(data) + else: + assert size <= len(data) + end = address + size + newRegionLen = end - len(self._data) + if newRegionLen > 0: + if force: + logger.warning( + f"Extending LocalNode data from {len(self._data)}" + f" byte(s) to {end} byte(s).") + self.extend(b'\0' * newRegionLen) + else: + raise IndexError( + f"Tried to set address {address}" + f"in {self.getLength()}-long space") + assert end - address == len(data) + if size < len(data): + self._data[address:end] = data[:size] + else: + self._data[address:end] = data + + class StoragePool: def __init__(self): - self.spaces = {} # type: dict[int, bytearray] + self._spaces = {} # type: dict[int, StorageSpace] def set(self, var: CDIVar): assert isinstance(var, CDIVar) @@ -18,7 +115,77 @@ def set(self, var: CDIVar): assert var.address data = var.getData() assert data is not None - self.setData(var.space, var.address, data, size=var.size) + self.setSlice(var.space, var.address, data, size=var.size) + + def getFirst(self, space: Union[MemorySpace, int]): + """Get first address""" + if isinstance(space, MemorySpace): + space = space.value + storage = self._spaces.get(space) + if storage is None: + return None + return storage.getFirst() + + def markReadOnly(self, space: Union[MemorySpace, int], readOnly): + if isinstance(space, MemorySpace): + space = space.value + storage = self._spaces.get(space) + if storage is None: + return + return storage.markReadOnly(readOnly) + + def isReadOnly(self, space: Union[MemorySpace, int]): + if isinstance(space, MemorySpace): + space = space.value + + storage = self._spaces.get(space) + if storage is None: + return True # True since can't write if not present + return storage.isReadOnly() + + def getDescription(self, space): + if isinstance(space, MemorySpace): + space = space.value + storage = self._spaces.get(space) + if storage is None: + return None + return storage.getDescription() + + def setDescription(self, space, description: str): + if isinstance(space, MemorySpace): + space = space.value + assert isinstance(description, str) + storage = self._spaces.get(space) + if storage is None: + return + storage.setDescription(description) + + def getLength(self, space: Union[MemorySpace, int]): + """Get size of space""" + if isinstance(space, MemorySpace): + space = space.value + if space not in self._spaces: + return None + return self._spaces[space].getLength() + + def getLast(self, space: Union[MemorySpace, int]): + """Get last address + (may differ from length-1 on actual hardware) + """ + if isinstance(space, MemorySpace): + space = space.value + storage = self._spaces.get(space) + if storage is None: + return None + return storage.getLast() + + def getStorage(self, space: Union[MemorySpace, int]) -> Union[StorageSpace, None]: # noqa: E501 + """Get last address + (may differ from length-1 on actual hardware) + """ + if isinstance(space, MemorySpace): + space = space.value + return self._spaces.get(space) def get(self, var: CDIVar) -> CDIVar: """Modify var in place. @@ -30,13 +197,13 @@ def get(self, var: CDIVar) -> CDIVar: assert var.space is not None assert var.address assert var.size is not None - data = self.getData(var.space, var.address, var.size) + data = self.getSlice(var.space, var.address, var.size) assert data is not None var.setData(data) return var - def setData(self, space: Union[MemorySpace, int], address: int, - data: Union[bytes, bytearray], size=None): + def setSlice(self, space: Union[MemorySpace, int], address: int, + data: Union[bytes, bytearray], size=None): """Set address in virtual memory space to data""" assert isinstance(data, (bytearray, bytes)) if isinstance(space, MemorySpace): @@ -48,46 +215,28 @@ def setData(self, space: Union[MemorySpace, int], address: int, size = len(data) else: assert size <= len(data) - end = address + size - - if space not in self.spaces: - self.spaces[space] = bytearray() - else: - assert isinstance(self.spaces[space], bytearray) + storage = self._spaces.get(space) + if storage is None: + storage = StorageSpace() + self._spaces[space] = storage + storage.setData(address, data, size=size) - newRegionLen = end - len(self.spaces[space]) - if newRegionLen > 0: - logger.warning( - f"Extending LocalNode data from {len(self.spaces[space])}" - f" byte(s) to {end} byte(s).") - self.spaces[space] += b'\0' * newRegionLen - assert end - address == len(data) - if size < len(data): - self.spaces[space][address:end] = data[:size] - else: - self.spaces[space][address:end] = data - - def getData(self, space: Union[MemorySpace, int], address: int, - size: int, force=False) -> bytearray: + def getSlice(self, space: Union[MemorySpace, int], address: int, + size: int, force=False) -> bytearray: if isinstance(space, MemorySpace): space = space.value assert isinstance(space, int) assert isinstance(address, int) assert isinstance(size, int) - if space not in self.spaces: + storage = self._spaces.get(space) + if storage is None: if force: - self.spaces[space] = bytearray() + storage = StorageSpace(size=address+size) + storage.setData(address, b"\0"*size, force=True) + self._spaces[space] = storage else: raise KeyError(f"Space {hex(space)} does not exist.") - - end = address + size - if address >= len(self.spaces[space]): - self.setData(space, address, size*b"\0") - elif end > len(self.spaces[space]): - slack = end - len(self.spaces[space]) - offset = size - slack - self.setData(space, address + offset, slack*b"\0") - return self.spaces[space][address:end] + return storage.getSlice(address, size, force=force) def setInt(self, space: Union[MemorySpace, int], address: int, value: int, size: int, signed: bool): @@ -103,7 +252,7 @@ def setInt(self, space: Union[MemorySpace, int], address: int, data = struct.pack(dataFormat, value) assert len(data) == size, \ f"Expected {size} byte(s) for {typeStr}, got {len(data)}" - return self.setData(space, address, data) + return self.setSlice(space, address, data) def getInt(self, space: Union[MemorySpace, int], address: int, size: int, signed: bool) -> int: @@ -113,7 +262,7 @@ def getInt(self, space: Union[MemorySpace, int], address: int, assert isinstance(address, int) assert isinstance(size, int) assert isinstance(signed, bool) - data = self.getData(space, address, size) + data = self.getSlice(space, address, size) assert size in (1, 2, 4, 8) typeStr = f"int{size*8}" if not signed: @@ -137,7 +286,7 @@ def setFloat(self, space: Union[MemorySpace, int], address: int, data = struct.pack(dataFormat, value) assert len(data) == size, \ f"Expected {size} byte(s) for {typeStr}, got {len(data)}" - return self.setData(space, address, data) + return self.setSlice(space, address, data) def getFloat(self, space: Union[MemorySpace, int], address: int, size: int) -> float: @@ -146,7 +295,7 @@ def getFloat(self, space: Union[MemorySpace, int], address: int, assert isinstance(space, int) assert isinstance(address, int) assert isinstance(size, int) - data = self.getData(space, address, size) + data = self.getSlice(space, address, size) assert size in (2, 4, 8) typeStr = f"float{size*8}" dataFormat = SUBTYPE_FORMATS[typeStr] diff --git a/tests/test_storagepool.py b/tests/test_storagepool.py index 50244a41..18f1bd48 100644 --- a/tests/test_storagepool.py +++ b/tests/test_storagepool.py @@ -31,7 +31,7 @@ class TestStoragePool(unittest.TestCase): def testGetNothing(self): pool = StoragePool() - value_bytes = pool.getData(4, 40, 4, force=True) + value_bytes = pool.getSlice(4, 40, 4, force=True) self.assertEqual(len(value_bytes), 4) value = struct.unpack(">I", value_bytes)[0] assert isinstance(value, int) @@ -44,7 +44,7 @@ def test_get_raises_keyerror(self): with self.assertRaises(KeyError): # KeyError is necessary because space 4 was not defined # (pool.set* is not called above, so no spaces exist). - pool.getData(4, 40, 4) # adjust arguments as needed + pool.getSlice(4, 40, 4) # adjust arguments as needed def testUnsignedIntData(self): in_value = 9999999 @@ -52,8 +52,8 @@ def testUnsignedIntData(self): self.assertEqual(len(value_bytes), 4) assert isinstance(value_bytes, (bytes, bytearray)) pool = StoragePool() - pool.setData(1, 10, value_bytes) - out_bytes = pool.getData(1, 10, 4) + pool.setSlice(1, 10, value_bytes) + out_bytes = pool.getSlice(1, 10, 4) self.assertEqual(len(out_bytes), 4) out_value = struct.unpack(">I", out_bytes)[0] self.assertEqual(in_value, out_value) @@ -64,8 +64,8 @@ def testSignedIntData(self): self.assertEqual(len(value_bytes), 4) assert isinstance(value_bytes, (bytes, bytearray)) pool = StoragePool() - pool.setData(1, 10, value_bytes) - out_bytes = pool.getData(1, 10, 4) + pool.setSlice(1, 10, value_bytes) + out_bytes = pool.getSlice(1, 10, 4) self.assertEqual(len(out_bytes), 4) out_value = struct.unpack(">i", out_bytes)[0] self.assertEqual(in_value, out_value) @@ -76,7 +76,7 @@ def testUnsignedInt(self): size = 4 signed = False pool.setInt(1, 10, in_value, size, signed) - out_bytes = pool.getData(1, 10, size) + out_bytes = pool.getSlice(1, 10, size) self.assertEqual(len(out_bytes), size) out_value = struct.unpack(">I", out_bytes)[0] self.assertEqual(in_value, out_value) @@ -89,7 +89,7 @@ def testSignedInt(self): size = 4 signed = True pool.setInt(1, 10, in_value, size, signed) - out_bytes = pool.getData(1, 10, size) + out_bytes = pool.getSlice(1, 10, size) self.assertEqual(len(out_bytes), size) out_value = struct.unpack(">i", out_bytes)[0] self.assertEqual(in_value, out_value) @@ -106,7 +106,7 @@ def testFloat(self): in_value = -999 size = 2 pool.setFloat(1, 10, in_value, size) - out_bytes = pool.getData(1, 10, size) + out_bytes = pool.getSlice(1, 10, size) self.assertEqual(len(out_bytes), size) out_value = struct.unpack(sizeFormats[size], out_bytes)[0] self.assertEqual(in_value, out_value) @@ -116,7 +116,7 @@ def testFloat(self): size = 4 in_value = -9999999 # NOTE: f32 fits -9999999 f16 does not pool.setFloat(1, 10, in_value, size) - out_bytes = pool.getData(1, 10, size) + out_bytes = pool.getSlice(1, 10, size) self.assertEqual(len(out_bytes), size) out_value = struct.unpack(sizeFormats[size], out_bytes)[0] self.assertEqual(in_value, out_value) @@ -125,7 +125,7 @@ def testFloat(self): size = 8 pool.setFloat(1, 10, in_value, size) - out_bytes = pool.getData(1, 10, size) + out_bytes = pool.getSlice(1, 10, size) self.assertEqual(len(out_bytes), size) out_value = struct.unpack(sizeFormats[size], out_bytes)[0] self.assertEqual(in_value, out_value) From 853e73fc0a8e3f3a3e18945e6fc4a33933282be0 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Fri, 29 May 2026 15:12:14 -0400 Subject: [PATCH 51/64] (NOOP) Systematize Memory Configuration bitfield constants (Make them fully cross-checkable and add related tests). --- openlcb/memoryservice.py | 67 ++++++++++++++++++++++++------------ tests/test_memoryservice.py | 68 +++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 21 deletions(-) diff --git a/openlcb/memoryservice.py b/openlcb/memoryservice.py index 5dd716e4..ec843044 100644 --- a/openlcb/memoryservice.py +++ b/openlcb/memoryservice.py @@ -53,14 +53,14 @@ class MCOp(Enum): """Byte 1 & 0b11111100 values (assumes byte 0 is 0x20)""" Read_Command = 0x40 # 01000000 Read_Reply = 0x50 # 01010000 - Read_Reply_Failure = 0x58 # 01011000 + Read_Reply_Failure = 0x58 # 01011000; See OP_FAILURE_BYTES Read_Stream_Command = 0x60 # 01100000 Read_Stream_Reply = 0x70 # 01110000 - Read_Stream_Reply_Failure = 0x78 # 01111000 + Read_Stream_Reply_Failure = 0x78 # 01111000; See OP_FAILURE_BYTES Write_Command = 0x00 Write_Reply = 0x10 # 00010000 # or 00010001 (0x11) and so on - Write_Reply_Failure = 0x18 + Write_Reply_Failure = 0x18 # 00011000; See OP_FAILURE_BYTES Write_Under_Mask_Command = 0x08 # 00001000 Write_Stream_Command = 0x20 # 01000000 Write_Stream_Reply = 0x30 # 00110000 @@ -109,22 +109,31 @@ class MCOpMasks: # they can be broken down with # sub-checks even if Default is used # (See any set with more than one entry - # in MODE_BYTES): + # in TWO_BIT_PARAMS): # Get_Configuration_Options = 0x11111110 # Get_Address_Space_Info = 0x11111110 -MODE_BYTES = { - # order determines meaning for lists (See ) - MCOp.Read_Command.value: {0x40, 0x41, 0x42, 0x43}, # pool - MCOp.Read_Reply.value: {0x50, 0x51, 0x52, 0x53}, - MCOp.Read_Stream_Command.value: {0x60, 0x61, 0x62, 0x63}, # pool - MCOp.Read_Stream_Reply.value: {0x70, 0x71, 0x72, 0x73}, # TODO +TWO_BIT_PARAMS = { + # The combined values in these lists include all 6-bit + # operation groups *except* those in OP_FAILURE_BYTES + # order determines meaning for lists: + # - first indicates custom space follows (in later byte) + # - Others are standard spaces + # See MemorySpaceIndex enum for meaning of last 2 bits + # (and Configuration Description Information Standard). + # - does *not* apply to *sets* below! + # *lists* (last 2 bits are MemorySpaceIndex.fromNumber param) + MCOp.Read_Command.value: [0x40, 0x41, 0x42, 0x43], # pool + MCOp.Read_Reply.value: [0x50, 0x51, 0x52, 0x53], + MCOp.Read_Stream_Command.value: [0x60, 0x61, 0x62, 0x63], # pool + MCOp.Read_Stream_Reply.value: [0x70, 0x71, 0x72, 0x73], # TODO MCOp.Write_Command.value: [0x00, 0x01, 0x02, 0x03], # pool - MCOp.Write_Reply.value: {0x10, 0x11, 0x12, 0x13}, - MCOp.Write_Under_Mask_Command.value: {0x08, 0x09, 0x0A, 0x0B}, # pool - MCOp.Write_Stream_Command.value: {0x20, 0x21, 0x22, 0x23}, - MCOp.Write_Stream_Reply.value: {0x30, 0x31, 0x32, 0x33}, # TODO + MCOp.Write_Reply.value: [0x10, 0x11, 0x12, 0x13], + MCOp.Write_Under_Mask_Command.value: [0x08, 0x09, 0x0A, 0x0B], # pool + MCOp.Write_Stream_Command.value: [0x20, 0x21, 0x22, 0x23], + MCOp.Write_Stream_Reply.value: [0x30, 0x31, 0x32, 0x33], # TODO + # *sets* (no standard meaning to last 2 bits) MCOp.Get_Configuration_Options_Command.value: {0x80, }, MCOp.Get_Configuration_Options_Reply.value: {0x82, }, MCOp.Get_Address_Space_Info_Command.value: { @@ -132,7 +141,10 @@ class MCOpMasks: MCOp.Get_Address_Space_Info_Reply.value, MCOp.Get_Address_Space_Info_Reply_Command.value, }, - MCOp.Lock_or_Reserve_Command.value: {0x88, }, + MCOp.Lock_or_Reserve_Command.value: { + MCOp.Lock_or_Reserve_Command.value, + MCOp.Lock_or_Reserve_Reply.value, + }, MCOp.Get_Unique_ID_Command.value: {0x8C, }, MCOp.Get_Unique_ID_Reply.value: {0x8D, }, MCOp.Unfreeze_Command.value: {0xA1, 0xA0}, # unfreeze, freeze respectively @@ -143,11 +155,21 @@ class MCOpMasks: } } -MODE_ERROR_BYTES = { - MCOp.Read_Reply.value: {0x58, 0x59, 0x5A, 0x5B}, - MCOp.Read_Stream_Reply.value: {0x78, 0x79, 0x7A, 0x7B}, - MCOp.Write_Reply.value: {0x18, 0x19, 0x1A, 0x1B}, - MCOp.Write_Stream_Reply.value: {0x38, 0x39, 0x3A, 0x3B}, +OP_FAILURE_BYTES = { + # *lists* (last 2 bits are standard memory space): + MCOp.Read_Reply.value: [0x58, 0x59, 0x5A, 0x5B], + MCOp.Read_Stream_Reply.value: [0x78, 0x79, 0x7A, 0x7B], + MCOp.Write_Reply.value: [0x18, 0x19, 0x1A, 0x1B], + MCOp.Write_Stream_Reply.value: [0x38, 0x39, 0x3A, 0x3B], + # *sets* (last 2 bits have no special meaning): + MCOp.Read_Reply_Failure.value: { + MCOp.Read_Reply_Failure.value}, + MCOp.Write_Reply_Failure.value: { + MCOp.Write_Reply_Failure.value}, + MCOp.Read_Stream_Reply_Failure.value: { + MCOp.Read_Stream_Reply_Failure.value}, + MCOp.Write_Stream_Reply_Failure.value: { + MCOp.Write_Stream_Reply_Failure.value}, } @@ -280,7 +302,7 @@ def parseReplyDatagram(memo: Union[MemoryReadMemo, MemoryWriteMemo], pass # 0x08 (0b00001000) is error bit # mode = None - # for k, values in MODE_ERROR_BYTES.items(): + # for k, values in OP_FAILURE_BYTES.items(): # if dmemo.data[1] in values: # mode = k # break @@ -430,6 +452,9 @@ def datagramReceivedListener(self, dmemo: DatagramReadMemo) -> bool: if dmemo.data[1] in (0x50, 0x51, 0x52, 0x53, 0x58, 0x59, 0x5A, 0x5B): assert mcOp is MCOp.Read_Reply, \ "self-test failed (bad constant(s))" + assert (dmemo.data[1] in TWO_BIT_PARAMS[mcOp.value] + or dmemo.data[1] in OP_FAILURE_BYTES[mcOp.value]), \ + "self-test failed (bad constant(s))" # MCOp.Read_Reply # read or read-error reply diff --git a/tests/test_memoryservice.py b/tests/test_memoryservice.py index 8033d20a..219b7bdb 100644 --- a/tests/test_memoryservice.py +++ b/tests/test_memoryservice.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from collections import OrderedDict import os import struct import sys @@ -30,6 +31,9 @@ from openlcb.mti import MTI # noqa: E402 from openlcb.message import Message # noqa: E402 from openlcb.memoryservice import ( # noqa: E402 + OP_FAILURE_BYTES, + TWO_BIT_PARAMS, + MCOp, MemoryReadMemo, MemoryWriteMemo, MemoryService, @@ -183,6 +187,70 @@ def testMultipleRead(self): self.assertEqual(len(LinkMockLayer.sentMessages), 5) # read reply datagram reply sent and next datagram sent # noqa: E501 self.assertEqual(len(self.returnedMemoryReadMemo), 2) # memory read returned # noqa: E501 + def testProtocolGroupUniqueness(self): + """Ensure each 6-bit field is unique""" + opCounts = OrderedDict() + + def incrementKey(key, counts): + if key not in counts: + counts[key] = 1 + return + raise AssertionError( + f"Bitfield {hex(key)} applies to more than one parent op") + # counts[key] += 1 + + for op in MCOp: + incrementKey(op.value, opCounts) + # Ensure that 6-bit fields are systematized (correct constants) + if ((op.value in OP_FAILURE_BYTES) + and (len(OP_FAILURE_BYTES[op.value]) == 1)): + assert op.value not in TWO_BIT_PARAMS, \ + (f"{op} last 2 bits are not significant" + " so it should not be in TWO_BIT_PARAMS dict") + # elif op.value & 0b11111100 in OP_FAILURE_BYTES: + # for _, failureBytes in \ + # OP_FAILURE_BYTES[op.value & 0b11111100].items(): + # for failureByte in failureBytes: + # assert failureByte not in TWO_BIT_PARAMS[op.value], \ + # (f"Failure bytes should not be in two-bit" + # f" params for {op}") + # assert isinstance(OP_FAILURE_BYTES[op.value], list), \ + # (f"If {op} use last 2 bits as index," + # " it should be recorded as a list in OP_FAILURE_BYTES") + # elif op.value in OP_FAILURE_BYTES: + # assert isinstance(OP_FAILURE_BYTES[op.value], set), \ + # (f"If {op} doesn't use last 2 bits as index," + # " it should be recorded as a set") + elif op.value in TWO_BIT_PARAMS: + if op.value in OP_FAILURE_BYTES: + assert isinstance(OP_FAILURE_BYTES[op.value], list), \ + (f"If {op} use last 2 bits as index, it" + " should be recorded as a list in OP_FAILURE_BYTES") + # elif "Reply" in str(op): + # # commented since not a real problem + # # MCOp.Get_Configuration_Options_Reply doesn't have a + # # corresponding error for the same 6-bit op field. + # raise AssertionError( + # (f"{op} is a reply but does not have an error reply." + # " Does this follow the standard? If so," + # "remove this assertion.")) + else: + # assert op.value in TWO_BIT_PARAMS, \ + # f"There is no list of two-bit params for {op}" + # ^ Incorrect assertion since some share 1st 6 bits and + # don't have params. See below instead. + parentValue = op.value & 0b11111100 + assert (op.value in TWO_BIT_PARAMS + or ((parentValue in TWO_BIT_PARAMS) + and (op.value in TWO_BIT_PARAMS[parentValue])) + ), \ + f"There is no list of two-bit params for {op}" + + for opValue, _ in TWO_BIT_PARAMS.items(): + parentValue = opValue & 0b11111100 + assert opCounts.get(parentValue) == 1, \ + f"op {hex(opValue)} is not in MCOp parents enum" + if __name__ == '__main__': unittest.main() From e658489c7adc24d1125db0858acef5f0fda1655f Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Fri, 29 May 2026 16:09:52 -0400 Subject: [PATCH 52/64] (NOOP) Group Memory Configuration operations correctly and add a related test (and remove such invariant assertions from the code). Clarify related comments and docstrings. --- openlcb/memoryconfigurationheader.py | 15 ++++- openlcb/memoryservice.py | 93 ++++++++++++++++++++-------- openlcb/memoryspaceindex.py | 8 +++ tests/test_memoryservice.py | 20 +++++- 4 files changed, 105 insertions(+), 31 deletions(-) diff --git a/openlcb/memoryconfigurationheader.py b/openlcb/memoryconfigurationheader.py index 5c1c1168..4574ac5c 100644 --- a/openlcb/memoryconfigurationheader.py +++ b/openlcb/memoryconfigurationheader.py @@ -52,8 +52,10 @@ def fromMC2ndByte(cls, datagramByte1: int, space: Union[int, None] = None) -> 'M else: # space is None assert datagramByte1 & 0x03 != 0, \ - 'a standard space must be in last 2 bits datagramByte1' + 'a standard space index must be in last 2 bits datagramByte1' space = -1 + # NOTE: Third option is that space isn't known yet + # (must be set later, if spaceIsCustom()) # formerly deserializeMC2ndByte result = cls(datagramByte1 & 0x03) if datagramByte1 & 0x03 == 0: @@ -62,3 +64,14 @@ def fromMC2ndByte(cls, datagramByte1: int, space: Union[int, None] = None) -> 'M result.customSpace = space result.highBits = datagramByte1 & 0xFC # 0xFC = 0b11111100 return result + + def spaceIsCustom(self): + """Is MemorySpaceIndex.Custom? + Detected as True if 0 was in last 2 bits of datagramByte1 + (2nd byte of datagram bitwise-and 6-high-bit mask), + so only use fromMC2ndByte (or MemorySpaceIndex.fromNumber) if + `isinstance(TWO_BIT_PARAMS[datagramByte1 & McOpMasks.Default], list)` + (list is used as a convention in TWO_BIT_PARAMS values to + indicate a meaningful index in last 2 bits). + """ + return self.spaceIndex is MemorySpaceIndex.Custom diff --git a/openlcb/memoryservice.py b/openlcb/memoryservice.py index ec843044..4dfbb915 100644 --- a/openlcb/memoryservice.py +++ b/openlcb/memoryservice.py @@ -50,7 +50,25 @@ class MCOp(Enum): - """Byte 1 & 0b11111100 values (assumes byte 0 is 0x20)""" + """Byte 1 values where first 6 bits are unique *or* + last 2 bits do *not* have a separate meaning + (first 6 bits can't be used in isolation to determine + meaning, but first 6 bits typically imply a set of + more loosely related commands). + - Use datagram memo's `data[1] & MCOpMasks.Default` (use + "bitwise and" with the 6-high-bit mask) to get first 6 bits, which + can be used as a param for fromNumber (For operations where last 2 + bits do not have a separate meaning, do not use the 6-high-bit + mask). + - Byte 0 is 0x20 for Memory Configuration. + - Byte 1 values not in here (not masked) are in TWO_BIT_PARAMS + and/or OP_FAILURE_BYTES. + - Any value in here indicating *failure*, whether or not a key in + TWO_BIT_PARAMS, is a key in OP_FAILURE_BYTES. + - For a full understanding of how the Memory Configuration protocol + constants are systematized, See testProtocolGroupUniqueness and/or + the constant dicts using these values as keys. + """ Read_Command = 0x40 # 01000000 Read_Reply = 0x50 # 01010000 Read_Reply_Failure = 0x58 # 01011000; See OP_FAILURE_BYTES @@ -114,16 +132,30 @@ class MCOpMasks: # Get_Address_Space_Info = 0x11111110 +""" +The combined *values* TWO_BIT_PARAMS include all 6-high-bit operation +groups *except* those in OP_FAILURE_BYTES order determines meaning for +lists: +- First ([0]) indicates custom space follows (in later byte) +- Others are standard spaces. For meaning of last 2 bits see + MemorySpaceIndex Enum (derived from Configuration Description + Information Standard). + - Meaningful last 2 bits are *not* applicable if value of dict is a + *set* (set is used by convention here to indicate no ordered + meaning)! + - Ones with *pool* comment (typically with "Command" in the name) + require a StoragePool/subclass. Only setup memoryservice instance's + pool if your device itself should be remotely configurable (not + required and not typical for a Configuration Tool, since its purpose + it to configure other nodes, but for an actual/virtual node, see + example_node_memory_implementation). Such a Node should set the + MEMORY_CONFIGURATION_PROTOCOL and other applicable protocols in its + PIP set (See Node constructor). + - Each node must have its own MemoryService instance since + DatagramReadMemo does not carry a destination address. +""" TWO_BIT_PARAMS = { - # The combined values in these lists include all 6-bit - # operation groups *except* those in OP_FAILURE_BYTES - # order determines meaning for lists: - # - first indicates custom space follows (in later byte) - # - Others are standard spaces - # See MemorySpaceIndex enum for meaning of last 2 bits - # (and Configuration Description Information Standard). - # - does *not* apply to *sets* below! - # *lists* (last 2 bits are MemorySpaceIndex.fromNumber param) + # region *lists* (last 2 bits are MemorySpaceIndex.fromNumber param) MCOp.Read_Command.value: [0x40, 0x41, 0x42, 0x43], # pool MCOp.Read_Reply.value: [0x50, 0x51, 0x52, 0x53], MCOp.Read_Stream_Command.value: [0x60, 0x61, 0x62, 0x63], # pool @@ -133,9 +165,13 @@ class MCOpMasks: MCOp.Write_Under_Mask_Command.value: [0x08, 0x09, 0x0A, 0x0B], # pool MCOp.Write_Stream_Command.value: [0x20, 0x21, 0x22, 0x23], MCOp.Write_Stream_Reply.value: [0x30, 0x31, 0x32, 0x33], # TODO - # *sets* (no standard meaning to last 2 bits) - MCOp.Get_Configuration_Options_Command.value: {0x80, }, - MCOp.Get_Configuration_Options_Reply.value: {0x82, }, + # endregion + + # region *sets* (no standard meaning to last 2 bits) + MCOp.Get_Configuration_Options_Command.value: { + MCOp.Get_Configuration_Options_Command.value, + MCOp.Get_Configuration_Options_Reply.value, + }, MCOp.Get_Address_Space_Info_Command.value: { MCOp.Get_Address_Space_Info_Command.value, MCOp.Get_Address_Space_Info_Reply.value, @@ -145,14 +181,17 @@ class MCOpMasks: MCOp.Lock_or_Reserve_Command.value, MCOp.Lock_or_Reserve_Reply.value, }, - MCOp.Get_Unique_ID_Command.value: {0x8C, }, - MCOp.Get_Unique_ID_Reply.value: {0x8D, }, + MCOp.Get_Unique_ID_Command.value: { + MCOp.Get_Unique_ID_Command.value, + MCOp.Get_Unique_ID_Reply.value + }, MCOp.Unfreeze_Command.value: {0xA1, 0xA0}, # unfreeze, freeze respectively MCOp.Update_Complete_Command.value: { # all match using MCOpMasks.Default MCOp.Update_Complete_Command.value, MCOp.Reset_or_Reboot_Command.value, MCOp.Reinitialize_or_Factory_Reset_Command.value, } + # endregion } OP_FAILURE_BYTES = { @@ -450,11 +489,11 @@ def datagramReceivedListener(self, dmemo: DatagramReadMemo) -> bool: mcOp = MCOp.fromNumber(dmemo.data[1] & MCOpMasks.Default) # decode if read, write or some other reply if dmemo.data[1] in (0x50, 0x51, 0x52, 0x53, 0x58, 0x59, 0x5A, 0x5B): - assert mcOp is MCOp.Read_Reply, \ - "self-test failed (bad constant(s))" - assert (dmemo.data[1] in TWO_BIT_PARAMS[mcOp.value] - or dmemo.data[1] in OP_FAILURE_BYTES[mcOp.value]), \ - "self-test failed (bad constant(s))" + # assert mcOp is MCOp.Read_Reply, \ + # "self-test failed (bad constant(s))" + # assert (dmemo.data[1] in TWO_BIT_PARAMS[mcOp.value] + # or dmemo.data[1] in OP_FAILURE_BYTES[mcOp.value]), \ + # "self-test failed (bad constant(s))" # MCOp.Read_Reply # read or read-error reply @@ -490,9 +529,9 @@ def datagramReceivedListener(self, dmemo: DatagramReadMemo) -> bool: tMemoryMemo.rejectedReply(tMemoryMemo) break elif dmemo.data[1] in (0x10, 0x11, 0x12, 0x13, 0x18, 0x19, 0x1A, 0x1B): - assert mcOp is MCOp.Write_Reply, \ - (f"self-test failed (bad constant(s));" - f" got op {mcOp} for sub-op {hex(dmemo.data[1])}") + # assert mcOp is MCOp.Write_Reply, \ + # (f"self-test failed (bad constant(s));" + # f" got op {mcOp} for sub-op {hex(dmemo.data[1])}") # write reply good, bad # return data to requestor: first find matching memory write @@ -509,8 +548,8 @@ def datagramReceivedListener(self, dmemo: DatagramReadMemo) -> bool: break elif dmemo.data[1] == MCOp.Get_Address_Space_Info_Command.value: # 0x84 (A node sent us a command requesting space info) - assert mcOp is MCOp.Get_Address_Space_Info_Command, \ - "self-test failed (bad constant(s))" + # assert mcOp is MCOp.Get_Address_Space_Info_Command, \ + # "self-test failed (bad constant(s))" space = dmemo.data[2] last = self.pool.getLast(space) if last is not None: @@ -560,8 +599,8 @@ def datagramReceivedListener(self, dmemo: DatagramReadMemo) -> bool: # TODO: rejected pass elif dmemo.data[1] in (0x86, 0x87): # Address Space Information Reply - assert mcOp is MCOp.Get_Address_Space_Info_Command, \ - "self-test failed (bad constant(s))" + # assert mcOp is MCOp.Get_Address_Space_Info_Command, \ + # "self-test failed (bad constant(s))" # ^ same first 6 bits as Get_Address_Space_Info_Command, # but in this case actually a reply to a command. diff --git a/openlcb/memoryspaceindex.py b/openlcb/memoryspaceindex.py index a7613421..de9b9bb8 100644 --- a/openlcb/memoryspaceindex.py +++ b/openlcb/memoryspaceindex.py @@ -14,6 +14,14 @@ class MemorySpaceIndex(Enum): def fromNumber(cls, num: int): """Return the MemorySpace member with the given numeric value, or None if no match is found. + Args: + num (int): Typically datagramByte1 & McOpMasks.Default + (2nd byte of datagram bitwise-and 6-high-bit mask), so + only use this method if if + `isinstance(TWO_BIT_PARAMS[datagramByte1 & + McOpMasks.Default], list)` + (list is used as a convention in TWO_BIT_PARAMS values + to indicate a meaningful index in last 2 bits). """ assert isinstance(num, int) for member in cls: diff --git a/tests/test_memoryservice.py b/tests/test_memoryservice.py index 219b7bdb..ebac3a79 100644 --- a/tests/test_memoryservice.py +++ b/tests/test_memoryservice.py @@ -34,6 +34,7 @@ OP_FAILURE_BYTES, TWO_BIT_PARAMS, MCOp, + MCOpMasks, MemoryReadMemo, MemoryWriteMemo, MemoryService, @@ -188,7 +189,7 @@ def testMultipleRead(self): self.assertEqual(len(self.returnedMemoryReadMemo), 2) # memory read returned # noqa: E501 def testProtocolGroupUniqueness(self): - """Ensure each 6-bit field is unique""" + """Ensure each 6-high-bit field is unique""" opCounts = OrderedDict() def incrementKey(key, counts): @@ -201,7 +202,8 @@ def incrementKey(key, counts): for op in MCOp: incrementKey(op.value, opCounts) - # Ensure that 6-bit fields are systematized (correct constants) + # Ensure that 6-high-bit fields are systematized (correct + # constants) if ((op.value in OP_FAILURE_BYTES) and (len(OP_FAILURE_BYTES[op.value]) == 1)): assert op.value not in TWO_BIT_PARAMS, \ @@ -229,7 +231,8 @@ def incrementKey(key, counts): # elif "Reply" in str(op): # # commented since not a real problem # # MCOp.Get_Configuration_Options_Reply doesn't have a - # # corresponding error for the same 6-bit op field. + # # corresponding error for the same 6-high-bit + # # op field. # raise AssertionError( # (f"{op} is a reply but does not have an error reply." # " Does this follow the standard? If so," @@ -251,6 +254,17 @@ def incrementKey(key, counts): assert opCounts.get(parentValue) == 1, \ f"op {hex(opValue)} is not in MCOp parents enum" + def testProtocolGroupCompleteness(self): + for statedParentValue, values in TWO_BIT_PARAMS.items(): + for byte1 in values: + # second byte. + computableParentValue = byte1 & MCOpMasks.Default + assert computableParentValue == statedParentValue, \ + (f"parent value of {hex(byte1)} is computed as" + f" {hex(computableParentValue)} but dict records it under" + f" {hex(statedParentValue)}," + " so constants aren't systematized") + if __name__ == '__main__': unittest.main() From 2fcc13a7f348a2b8d9f425f788a8ca7b26f073c0 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Fri, 29 May 2026 17:13:00 -0400 Subject: [PATCH 53/64] (NOOP) Rename StorageSpace to Segment and StoragePool to MemoryManager to use terms similarly to CDI Standard. --- .../example_node_memory_implementation.py | 12 +-- openlcb/localnode.py | 20 ++--- openlcb/memoryconfigurationheader.py | 4 +- openlcb/memoryservice.py | 41 +++++----- openlcb/storagepool.py | 72 ++++++++--------- python-openlcb.code-workspace | 2 +- tests/test_storagepool.py | 78 +++++++++---------- 7 files changed, 117 insertions(+), 112 deletions(-) diff --git a/examples/example_node_memory_implementation.py b/examples/example_node_memory_implementation.py index 5d2e0d5a..c577dc41 100644 --- a/examples/example_node_memory_implementation.py +++ b/examples/example_node_memory_implementation.py @@ -26,7 +26,7 @@ from examples_settings import Settings from openlcb.localnode import LocalNode from openlcb.memoryspace import MemorySpace -from openlcb.storagepool import StorageSpace # do 1st to fix path if no pip install +from openlcb.storagepool import Segment # do 1st to fix path if no pip install settings = Settings() if __name__ == "__main__": @@ -197,17 +197,17 @@ def memoryReadFail(memo): ]), canLink ) -memoryService.pool = localNode +memoryService.memory = localNode my_conf_dir = os.path.join(get_config_dir("python-openlcb")) backup_name = "example_node_memory_implementation.cdi.xml" backup_path = os.path.join(my_conf_dir, backup_name) localNode.loadCDIString(cdi, backup_path) # NOTE: loadCDI or loadCDIString sets Element tree and -# localNode._spaces[MemorySpace.CDI.value] -storage = localNode.getStorage(MemorySpace.CDI.value) -assert isinstance(storage, StorageSpace) -assert isinstance(storage._data, (bytearray, bytes)) +# localNode._segments[MemorySpace.CDI.value] +segment = localNode.getStorage(MemorySpace.CDI.value) +assert isinstance(segment, Segment) +assert isinstance(segment._data, (bytearray, bytes)) # localNodeProcessor = LocalNodeProcessor(canLink, localNode) # canLink.registerMessageReceivedListener(localNodeProcessor.process) localNodeProcessor = localNode.localNodeProcessor diff --git a/openlcb/localnode.py b/openlcb/localnode.py index 8533c478..dfcac802 100644 --- a/openlcb/localnode.py +++ b/openlcb/localnode.py @@ -4,7 +4,7 @@ from typing import Union from openlcb import emit_cast from openlcb.memoryspace import MemorySpace -from openlcb.storagepool import StoragePool, StorageSpace +from openlcb.storagepool import MemoryManager, Segment from openlcb.node import PIP, SNIP, Node from openlcb.localnodeprocessor import LocalNodeProcessor @@ -24,14 +24,14 @@ logger = getLogger(__name__) -class LocalNode(Node, StoragePool): +class LocalNode(Node, MemoryManager): """A Node with its own virtual memory (emulate memory spaces such as for creating a virtual signal node with settings)""" def __init__(self, id: NodeID, snip: SNIP, pipSet: set, linkLayer: CanLink): Node.__init__(self, id, snip, pipSet) - StoragePool.__init__(self) + MemoryManager.__init__(self) self.cdi = None # type: XMLDataProcessor|None self._replicated_cdi_tree = None # type: CDIMemo|None if PIP.CONFIGURATION_DESCRIPTION_INFORMATION in pipSet: @@ -94,7 +94,7 @@ def loadCDIString(self, xml_data, path, memo=None): # data = stream.read() # self.tree = etree.fromstring(data) self.reserveSpaces() - if MemorySpace.CDI.value in self._spaces: + if MemorySpace.CDI.value in self._segments: logger.warning(f"CDI defined {MemorySpace.CDI.value}") # NOTE: self.cdi._data is None after load is done! @@ -105,10 +105,10 @@ def loadCDIString(self, xml_data, path, memo=None): assert isinstance(xml_data, bytearray), \ f"expected bytearray got {type(xml_data).__name__}" # assert isinstance(xml_data, (bytes, bytearray)) - storage = StorageSpace() - self._spaces[MemorySpace.CDI.value] = storage - storage.setData(0, xml_data, force=True) - storage.markReadOnly(True) + segment = Segment() + self._segments[MemorySpace.CDI.value] = segment + segment.setData(0, xml_data, force=True) + segment.markReadOnly(True) def setMemory(self, memo: CDIMemo, var: CDIVar): """Set a memory address at memo to the value in var""" @@ -192,11 +192,11 @@ def _reserveSpaces(self, parent: Union[CDIMemo, None] = None, level=0): logger.warning( f"Creating cdiBackupDir {self.cdiBackupDir}") os.makedirs(self.cdiBackupDir) - for space, storage in self._spaces.items(): + for space, segment in self._segments.items(): name = f"{self.id}.lcc-link-virtual-node.space={space}.xml" path = os.path.join(self.cdiBackupDir, name) with open(path, "wb") as stream: - stream.write(storage._data) + stream.write(segment._data) print(f"Wrote {d_quote(path)}") def onCDILoaded(self, memo: MemoryReadMemo): diff --git a/openlcb/memoryconfigurationheader.py b/openlcb/memoryconfigurationheader.py index 4574ac5c..569f8112 100644 --- a/openlcb/memoryconfigurationheader.py +++ b/openlcb/memoryconfigurationheader.py @@ -65,7 +65,7 @@ def fromMC2ndByte(cls, datagramByte1: int, space: Union[int, None] = None) -> 'M result.highBits = datagramByte1 & 0xFC # 0xFC = 0b11111100 return result - def spaceIsCustom(self): + def spaceIsCustom(self) -> bool: """Is MemorySpaceIndex.Custom? Detected as True if 0 was in last 2 bits of datagramByte1 (2nd byte of datagram bitwise-and 6-high-bit mask), @@ -74,4 +74,6 @@ def spaceIsCustom(self): (list is used as a convention in TWO_BIT_PARAMS values to indicate a meaningful index in last 2 bits). """ + assert self.spaceIndex is not None, \ + "Constructor failed (space index not computed)" return self.spaceIndex is MemorySpaceIndex.Custom diff --git a/openlcb/memoryservice.py b/openlcb/memoryservice.py index 4dfbb915..5351675f 100644 --- a/openlcb/memoryservice.py +++ b/openlcb/memoryservice.py @@ -43,7 +43,7 @@ from openlcb.convert import Convert from openlcb.memoryconfigurationheader import MemoryConfigurationHeader from openlcb.memoryspaceindex import MemorySpaceIndex -from openlcb.storagepool import StoragePool +from openlcb.storagepool import MemoryManager from openlcb.nodeid import NodeID logger = getLogger(__name__) @@ -143,9 +143,9 @@ class MCOpMasks: - Meaningful last 2 bits are *not* applicable if value of dict is a *set* (set is used by convention here to indicate no ordered meaning)! - - Ones with *pool* comment (typically with "Command" in the name) - require a StoragePool/subclass. Only setup memoryservice instance's - pool if your device itself should be remotely configurable (not + - Ones with *remote* comment (typically with "Command" in the name) + require a MemoryManager/subclass. Only setup memoryservice instance's + .memory if your device itself should be remotely configurable (not required and not typical for a Configuration Tool, since its purpose it to configure other nodes, but for an actual/virtual node, see example_node_memory_implementation). Such a Node should set the @@ -156,13 +156,13 @@ class MCOpMasks: """ TWO_BIT_PARAMS = { # region *lists* (last 2 bits are MemorySpaceIndex.fromNumber param) - MCOp.Read_Command.value: [0x40, 0x41, 0x42, 0x43], # pool + MCOp.Read_Command.value: [0x40, 0x41, 0x42, 0x43], # remote MCOp.Read_Reply.value: [0x50, 0x51, 0x52, 0x53], - MCOp.Read_Stream_Command.value: [0x60, 0x61, 0x62, 0x63], # pool + MCOp.Read_Stream_Command.value: [0x60, 0x61, 0x62, 0x63], # remote MCOp.Read_Stream_Reply.value: [0x70, 0x71, 0x72, 0x73], # TODO - MCOp.Write_Command.value: [0x00, 0x01, 0x02, 0x03], # pool + MCOp.Write_Command.value: [0x00, 0x01, 0x02, 0x03], # remote MCOp.Write_Reply.value: [0x10, 0x11, 0x12, 0x13], - MCOp.Write_Under_Mask_Command.value: [0x08, 0x09, 0x0A, 0x0B], # pool + MCOp.Write_Under_Mask_Command.value: [0x08, 0x09, 0x0A, 0x0B], # remote MCOp.Write_Stream_Command.value: [0x20, 0x21, 0x22, 0x23], MCOp.Write_Stream_Reply.value: [0x30, 0x31, 0x32, 0x33], # TODO # endregion @@ -392,11 +392,14 @@ class MemoryService: service (DatagramService): See DatagramService. Attributes: - pool (Union[StoragePool, LocalNode]): The storage where - other nodes can read and write memory. Since a datagram - doesn't have a destination, there must be a MemoryService - for each local node (the local Configuration tool or node - and any virtual nodes). + memory (Union[MemoryManager, LocalNode]): The storage + (all segments specific to this node) where other nodes can + read and write memory. Since a datagram doesn't have a + destination, there must be a MemoryService for each local + node (the local Configuration Tool or node and any virtual + nodes. Though typically a Configuration Tool isn't itself + configured remotely, that is technically possible, and + more typical in cases where it generates virtual nodes). """ def __init__(self, service: DatagramService): @@ -409,7 +412,7 @@ def __init__(self, service: DatagramService): self.service.registerDatagramReceivedListener( self.datagramReceivedListener ) - self.pool = StoragePool() + self.memory = MemoryManager() def requestMemoryRead(self, memo, stream: bool = False): # type: (MemoryReadMemo, Optional[bool]) -> None @@ -551,12 +554,12 @@ def datagramReceivedListener(self, dmemo: DatagramReadMemo) -> bool: # assert mcOp is MCOp.Get_Address_Space_Info_Command, \ # "self-test failed (bad constant(s))" space = dmemo.data[2] - last = self.pool.getLast(space) + last = self.memory.getLast(space) if last is not None: - first = self.pool.getFirst(space) + first = self.memory.getFirst(space) assert isinstance(first, int) assert last - first >= 0, \ - (f"{type(self.pool).__name__} incorrectly implemented:" + (f"{type(self.memory).__name__} incorrectly implemented:" " first>last") ReadOnly = 0b10000000 HasLowestAddress = 0b01000000 @@ -573,14 +576,14 @@ def datagramReceivedListener(self, dmemo: DatagramReadMemo) -> bool: if replyData[1] == 0x87: replyData += highestAddrBytes # bytes 3-6 (0 indexed) replyData.append(0x00) # flags (set below) - if self.pool.isReadOnly(space): + if self.memory.isReadOnly(space): replyData[-1] |= ReadOnly if first != 0: replyData[-1] |= HasLowestAddress lowestAddrBytes = struct.pack(">I", first) assert len(lowestAddrBytes) == 4 replyData += lowestAddrBytes - description = self.pool.getDescription(space) + description = self.memory.getDescription(space) if description: descBytes = bytearray(description.encode()) else: diff --git a/openlcb/storagepool.py b/openlcb/storagepool.py index 37b3b8ee..bcfd7306 100644 --- a/openlcb/storagepool.py +++ b/openlcb/storagepool.py @@ -8,7 +8,7 @@ logger = getLogger(__name__) -class StorageSpace: +class Segment: def __init__(self, size=0, readOnly=False): assert isinstance(size, int) assert size >= 0 @@ -105,9 +105,9 @@ def setData(self, address: int, data: Union[bytearray, bytes], self._data[address:end] = data -class StoragePool: +class MemoryManager: def __init__(self): - self._spaces = {} # type: dict[int, StorageSpace] + self._segments = {} # type: dict[int, Segment] def set(self, var: CDIVar): assert isinstance(var, CDIVar) @@ -121,52 +121,52 @@ def getFirst(self, space: Union[MemorySpace, int]): """Get first address""" if isinstance(space, MemorySpace): space = space.value - storage = self._spaces.get(space) - if storage is None: + segment = self._segments.get(space) + if segment is None: return None - return storage.getFirst() + return segment.getFirst() def markReadOnly(self, space: Union[MemorySpace, int], readOnly): if isinstance(space, MemorySpace): space = space.value - storage = self._spaces.get(space) - if storage is None: + segment = self._segments.get(space) + if segment is None: return - return storage.markReadOnly(readOnly) + return segment.markReadOnly(readOnly) def isReadOnly(self, space: Union[MemorySpace, int]): if isinstance(space, MemorySpace): space = space.value - storage = self._spaces.get(space) - if storage is None: + segment = self._segments.get(space) + if segment is None: return True # True since can't write if not present - return storage.isReadOnly() + return segment.isReadOnly() def getDescription(self, space): if isinstance(space, MemorySpace): space = space.value - storage = self._spaces.get(space) - if storage is None: + segment = self._segments.get(space) + if segment is None: return None - return storage.getDescription() + return segment.getDescription() def setDescription(self, space, description: str): if isinstance(space, MemorySpace): space = space.value assert isinstance(description, str) - storage = self._spaces.get(space) - if storage is None: + segment = self._segments.get(space) + if segment is None: return - storage.setDescription(description) + segment.setDescription(description) def getLength(self, space: Union[MemorySpace, int]): """Get size of space""" if isinstance(space, MemorySpace): space = space.value - if space not in self._spaces: + if space not in self._segments: return None - return self._spaces[space].getLength() + return self._segments[space].getLength() def getLast(self, space: Union[MemorySpace, int]): """Get last address @@ -174,18 +174,18 @@ def getLast(self, space: Union[MemorySpace, int]): """ if isinstance(space, MemorySpace): space = space.value - storage = self._spaces.get(space) - if storage is None: + segment = self._segments.get(space) + if segment is None: return None - return storage.getLast() + return segment.getLast() - def getStorage(self, space: Union[MemorySpace, int]) -> Union[StorageSpace, None]: # noqa: E501 + def getStorage(self, space: Union[MemorySpace, int]) -> Union[Segment, None]: # noqa: E501 """Get last address (may differ from length-1 on actual hardware) """ if isinstance(space, MemorySpace): space = space.value - return self._spaces.get(space) + return self._segments.get(space) def get(self, var: CDIVar) -> CDIVar: """Modify var in place. @@ -215,11 +215,11 @@ def setSlice(self, space: Union[MemorySpace, int], address: int, size = len(data) else: assert size <= len(data) - storage = self._spaces.get(space) - if storage is None: - storage = StorageSpace() - self._spaces[space] = storage - storage.setData(address, data, size=size) + segment = self._segments.get(space) + if segment is None: + segment = Segment() + self._segments[space] = segment + segment.setData(address, data, size=size) def getSlice(self, space: Union[MemorySpace, int], address: int, size: int, force=False) -> bytearray: @@ -228,15 +228,15 @@ def getSlice(self, space: Union[MemorySpace, int], address: int, assert isinstance(space, int) assert isinstance(address, int) assert isinstance(size, int) - storage = self._spaces.get(space) - if storage is None: + segment = self._segments.get(space) + if segment is None: if force: - storage = StorageSpace(size=address+size) - storage.setData(address, b"\0"*size, force=True) - self._spaces[space] = storage + segment = Segment(size=address+size) + segment.setData(address, b"\0"*size, force=True) + self._segments[space] = segment else: raise KeyError(f"Space {hex(space)} does not exist.") - return storage.getSlice(address, size, force=force) + return segment.getSlice(address, size, force=force) def setInt(self, space: Union[MemorySpace, int], address: int, value: int, size: int, signed: bool): diff --git a/python-openlcb.code-workspace b/python-openlcb.code-workspace index cc672e0c..40d1aaa5 100644 --- a/python-openlcb.code-workspace +++ b/python-openlcb.code-workspace @@ -105,7 +105,7 @@ "settingtypes", "setuptools", "SOCK_DGRAM", - "StoragePool", + "MemoryManager", "sysdirs", "tcplink", "tcpsocket", diff --git a/tests/test_storagepool.py b/tests/test_storagepool.py index 18f1bd48..aa42337b 100644 --- a/tests/test_storagepool.py +++ b/tests/test_storagepool.py @@ -5,7 +5,7 @@ from openlcb import emit_cast from openlcb.cdivar import SIGNED_INT_MINIMUMS, CDIVar -from openlcb.storagepool import StoragePool +from openlcb.storagepool import MemoryManager from logging import getLogger @@ -27,11 +27,11 @@ .format(repr(REPO_DIR))) -class TestStoragePool(unittest.TestCase): +class TestMemoryManager(unittest.TestCase): def testGetNothing(self): - pool = StoragePool() - value_bytes = pool.getSlice(4, 40, 4, force=True) + memory = MemoryManager() + value_bytes = memory.getSlice(4, 40, 4, force=True) self.assertEqual(len(value_bytes), 4) value = struct.unpack(">I", value_bytes)[0] assert isinstance(value, int) @@ -40,20 +40,20 @@ def testGetNothing(self): self.assertEqual(value, 0) def test_get_raises_keyerror(self): - pool = StoragePool() + memory = MemoryManager() with self.assertRaises(KeyError): # KeyError is necessary because space 4 was not defined - # (pool.set* is not called above, so no spaces exist). - pool.getSlice(4, 40, 4) # adjust arguments as needed + # (memory.set* is not called above, so no spaces exist). + memory.getSlice(4, 40, 4) # adjust arguments as needed def testUnsignedIntData(self): in_value = 9999999 value_bytes = struct.pack(">I", in_value) self.assertEqual(len(value_bytes), 4) assert isinstance(value_bytes, (bytes, bytearray)) - pool = StoragePool() - pool.setSlice(1, 10, value_bytes) - out_bytes = pool.getSlice(1, 10, 4) + memory = MemoryManager() + memory.setSlice(1, 10, value_bytes) + out_bytes = memory.getSlice(1, 10, 4) self.assertEqual(len(out_bytes), 4) out_value = struct.unpack(">I", out_bytes)[0] self.assertEqual(in_value, out_value) @@ -63,41 +63,41 @@ def testSignedIntData(self): value_bytes = struct.pack(">i", in_value) self.assertEqual(len(value_bytes), 4) assert isinstance(value_bytes, (bytes, bytearray)) - pool = StoragePool() - pool.setSlice(1, 10, value_bytes) - out_bytes = pool.getSlice(1, 10, 4) + memory = MemoryManager() + memory.setSlice(1, 10, value_bytes) + out_bytes = memory.getSlice(1, 10, 4) self.assertEqual(len(out_bytes), 4) out_value = struct.unpack(">i", out_bytes)[0] self.assertEqual(in_value, out_value) def testUnsignedInt(self): in_value = 9999999 - pool = StoragePool() + memory = MemoryManager() size = 4 signed = False - pool.setInt(1, 10, in_value, size, signed) - out_bytes = pool.getSlice(1, 10, size) + memory.setInt(1, 10, in_value, size, signed) + out_bytes = memory.getSlice(1, 10, size) self.assertEqual(len(out_bytes), size) out_value = struct.unpack(">I", out_bytes)[0] self.assertEqual(in_value, out_value) - out_value = pool.getInt(1, 10, size, signed) + out_value = memory.getInt(1, 10, size, signed) self.assertEqual(in_value, out_value) def testSignedInt(self): in_value = -9999999 - pool = StoragePool() + memory = MemoryManager() size = 4 signed = True - pool.setInt(1, 10, in_value, size, signed) - out_bytes = pool.getSlice(1, 10, size) + memory.setInt(1, 10, in_value, size, signed) + out_bytes = memory.getSlice(1, 10, size) self.assertEqual(len(out_bytes), size) out_value = struct.unpack(">i", out_bytes)[0] self.assertEqual(in_value, out_value) - out_value = pool.getInt(1, 10, size, signed) + out_value = memory.getInt(1, 10, size, signed) self.assertEqual(in_value, out_value) def testFloat(self): - pool = StoragePool() + memory = MemoryManager() sizeFormats = { 2: ">e", 4: ">f", @@ -105,31 +105,31 @@ def testFloat(self): } in_value = -999 size = 2 - pool.setFloat(1, 10, in_value, size) - out_bytes = pool.getSlice(1, 10, size) + memory.setFloat(1, 10, in_value, size) + out_bytes = memory.getSlice(1, 10, size) self.assertEqual(len(out_bytes), size) out_value = struct.unpack(sizeFormats[size], out_bytes)[0] self.assertEqual(in_value, out_value) - out_value = pool.getFloat(1, 10, size) + out_value = memory.getFloat(1, 10, size) self.assertEqual(in_value, out_value) size = 4 in_value = -9999999 # NOTE: f32 fits -9999999 f16 does not - pool.setFloat(1, 10, in_value, size) - out_bytes = pool.getSlice(1, 10, size) + memory.setFloat(1, 10, in_value, size) + out_bytes = memory.getSlice(1, 10, size) self.assertEqual(len(out_bytes), size) out_value = struct.unpack(sizeFormats[size], out_bytes)[0] self.assertEqual(in_value, out_value) - out_value = pool.getFloat(1, 10, size) + out_value = memory.getFloat(1, 10, size) self.assertEqual(in_value, out_value) size = 8 - pool.setFloat(1, 10, in_value, size) - out_bytes = pool.getSlice(1, 10, size) + memory.setFloat(1, 10, in_value, size) + out_bytes = memory.getSlice(1, 10, size) self.assertEqual(len(out_bytes), size) out_value = struct.unpack(sizeFormats[size], out_bytes)[0] self.assertEqual(in_value, out_value) - out_value = pool.getFloat(1, 10, size) + out_value = memory.getFloat(1, 10, size) self.assertEqual(in_value, out_value) def testCDIVarUInt(self): @@ -141,15 +141,15 @@ def testCDIVarUInt(self): # signed = False var.setInt(in_value) self.assertEqual(var.getInt(), in_value) - pool = StoragePool() - pool.set(var) - var = pool.get(var) + memory = MemoryManager() + memory.set(var) + var = memory.get(var) self.assertEqual(var.getInt(), in_value) assert var.space is not None assert var.address is not None assert var.size is not None assert var.signed is not None - out_value = pool.getInt(var.space, var.address, var.size, var.signed) + out_value = memory.getInt(var.space, var.address, var.size, var.signed) self.assertEqual(out_value, in_value) def testCDIVarSInt(self): @@ -180,15 +180,15 @@ def testCDIVarSInt(self): # signed = False var.setInt(in_value) self.assertEqual(var.getInt(), in_value) - pool = StoragePool() - pool.set(var) - var = pool.get(var) + memory = MemoryManager() + memory.set(var) + var = memory.get(var) self.assertEqual(var.getInt(), in_value) assert var.space is not None assert var.address is not None assert var.size is not None assert var.signed is True - out_value = pool.getInt(var.space, var.address, var.size, var.signed) + out_value = memory.getInt(var.space, var.address, var.size, var.signed) self.assertEqual(out_value, in_value) From cd3466d5ffe1928d9fabce5cbf03f7337304596e Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Fri, 29 May 2026 17:24:32 -0400 Subject: [PATCH 54/64] (NOOP) Rename test file using new class name. --- tests/{test_storagepool.py => test_memorymanager.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{test_storagepool.py => test_memorymanager.py} (100%) diff --git a/tests/test_storagepool.py b/tests/test_memorymanager.py similarity index 100% rename from tests/test_storagepool.py rename to tests/test_memorymanager.py From 33934c2799ef9b755d65d65a7b29d654c0a64166 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Fri, 29 May 2026 17:25:50 -0400 Subject: [PATCH 55/64] (NOOP) Rename submodule using new class name. --- examples/example_node_memory_implementation.py | 2 +- openlcb/localnode.py | 2 +- openlcb/{storagepool.py => memorymanager.py} | 0 openlcb/memoryservice.py | 2 +- tests/test_memorymanager.py | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) rename openlcb/{storagepool.py => memorymanager.py} (100%) diff --git a/examples/example_node_memory_implementation.py b/examples/example_node_memory_implementation.py index c577dc41..6944c7df 100644 --- a/examples/example_node_memory_implementation.py +++ b/examples/example_node_memory_implementation.py @@ -26,7 +26,7 @@ from examples_settings import Settings from openlcb.localnode import LocalNode from openlcb.memoryspace import MemorySpace -from openlcb.storagepool import Segment # do 1st to fix path if no pip install +from openlcb.memorymanager import Segment # do 1st to fix path if no pip install settings = Settings() if __name__ == "__main__": diff --git a/openlcb/localnode.py b/openlcb/localnode.py index dfcac802..1706116e 100644 --- a/openlcb/localnode.py +++ b/openlcb/localnode.py @@ -4,7 +4,7 @@ from typing import Union from openlcb import emit_cast from openlcb.memoryspace import MemorySpace -from openlcb.storagepool import MemoryManager, Segment +from openlcb.memorymanager import MemoryManager, Segment from openlcb.node import PIP, SNIP, Node from openlcb.localnodeprocessor import LocalNodeProcessor diff --git a/openlcb/storagepool.py b/openlcb/memorymanager.py similarity index 100% rename from openlcb/storagepool.py rename to openlcb/memorymanager.py diff --git a/openlcb/memoryservice.py b/openlcb/memoryservice.py index 5351675f..f1a1145f 100644 --- a/openlcb/memoryservice.py +++ b/openlcb/memoryservice.py @@ -43,7 +43,7 @@ from openlcb.convert import Convert from openlcb.memoryconfigurationheader import MemoryConfigurationHeader from openlcb.memoryspaceindex import MemorySpaceIndex -from openlcb.storagepool import MemoryManager +from openlcb.memorymanager import MemoryManager from openlcb.nodeid import NodeID logger = getLogger(__name__) diff --git a/tests/test_memorymanager.py b/tests/test_memorymanager.py index aa42337b..2baa2658 100644 --- a/tests/test_memorymanager.py +++ b/tests/test_memorymanager.py @@ -5,7 +5,7 @@ from openlcb import emit_cast from openlcb.cdivar import SIGNED_INT_MINIMUMS, CDIVar -from openlcb.storagepool import MemoryManager +from openlcb.memorymanager import MemoryManager from logging import getLogger From 34abc9e025b35bf13837e5d8efa57011937be4dd Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Fri, 29 May 2026 17:56:53 -0400 Subject: [PATCH 56/64] Accept and process Memory Read command (FIXME: test this). --- openlcb/memoryservice.py | 75 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/openlcb/memoryservice.py b/openlcb/memoryservice.py index f1a1145f..8793f7ea 100644 --- a/openlcb/memoryservice.py +++ b/openlcb/memoryservice.py @@ -623,6 +623,81 @@ def datagramReceivedListener(self, dmemo: DatagramReadMemo) -> bool: + int(dmemo.data[6])) self.spaceLengthCallback(address) self.spaceLengthCallback = None + elif mcOp is MCOp.Read_Command: + # assert dmemo.data[1] in TWO_BIT_PARAMS[MCOp.Read_Command.value], \ + # "self-test failed (bad constant(s))" + mcHeader = MemoryConfigurationHeader.fromMC2ndByte(dmemo.data[1]) + addressBytes = dmemo.data[2:6] + address = struct.unpack(">I", addressBytes)[0] + # ^ [0] since always returns list even when reading 1 value. + # ^ capital assumes unsigned, "I" assumes 32-bit (4 bytes) + space = dmemo.data[1] & 0b00000011 + offset = 0 + if mcHeader.spaceIsCustom(): + assert space == 0 + space = dmemo.data[6] + offset = 1 + size = dmemo.data[6+offset] # requested read count + datagramBytes = bytearray([0x20, MCOp.Read_Reply.value]) + # ^ byte1 (2nd) changed to error below if applicable + assert isinstance(space, int), \ + (f"Logic missing, space should be number here," + f" got {emit_cast(space)}") + assert isinstance(address, int) + datagramBytes += addressBytes + assert space is not None, \ + f"space not computed from datagram: {dmemo.data}" + if mcHeader.spaceIsCustom(): + datagramBytes.append(space) + assert len(datagramBytes) == 7, "space goes in index [6]" + else: + spaceIndex = (space & 0b00000011) + assert spaceIndex == space + assert space is not None + datagramBytes[1] = \ + datagramBytes[1] | spaceIndex + assert len(datagramBytes) == 6, "should not have space in [6]" + payload = None + try: + payload = self.memory.getSlice(space, address, size) + except (IndexError, KeyError) as ex: + # address out of range (See Segment's getSlice) + datagramBytes[1] = MCOp.Read_Reply_Failure.value + message = None + if isinstance(ex, KeyError): + message = f"space {space} not valid" + elif isinstance(ex, KeyError): + if space is not None: + message = (f"address {hex(address)} not valid" + f" in space {hex(space)}") + else: + raise NotImplementedError( + f"space not computed from datagram: {dmemo.data}") + errorCode = 1 + errorBytes = struct.pack(">H", errorCode) + # FIXME: ^ Standard doesn't specify signed/unsigned + datagramBytes += errorBytes # 2 required bytes + messageBytes = None + if message is not None: + messageBytes = bytearray(message.encode("utf-8")) + messageBytes.append(0x00) # null terminator + datagramBytes += messageBytes + failedRequestedMemoryMemo = DatagramWriteMemo( + dmemo.srcID, + datagramBytes + ) + self.service.sendDatagram(failedRequestedMemoryMemo) + return True # handled (early in this case) + assert payload is not None, \ + ("getSlice failed to raise IndexError or KeyError" + " on invalid request (expected bytes, got None)") + assert len(payload) <= 64 + datagramBytes += payload + requestedMemoryMemo = DatagramWriteMemo( + dmemo.srcID, + datagramBytes + ) + self.service.sendDatagram(requestedMemoryMemo) else: logger.error("Did not expect reply of type 0x{:02X}" .format(dmemo.data[1])) From 8441d0f9ac84f7de15b7da22efef26694c04383e Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Sun, 31 May 2026 16:42:24 -0400 Subject: [PATCH 57/64] Fix circular import. --- openlcb/memoryservice.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openlcb/memoryservice.py b/openlcb/memoryservice.py index 8793f7ea..63eebef1 100644 --- a/openlcb/memoryservice.py +++ b/openlcb/memoryservice.py @@ -41,6 +41,7 @@ DatagramService, ) from openlcb.convert import Convert +# from openlcb.localnode import LocalNode # circular import from openlcb.memoryconfigurationheader import MemoryConfigurationHeader from openlcb.memoryspaceindex import MemorySpaceIndex from openlcb.memorymanager import MemoryManager @@ -412,7 +413,7 @@ def __init__(self, service: DatagramService): self.service.registerDatagramReceivedListener( self.datagramReceivedListener ) - self.memory = MemoryManager() + self.memory = MemoryManager() # type: MemoryManager|LocalNode def requestMemoryRead(self, memo, stream: bool = False): # type: (MemoryReadMemo, Optional[bool]) -> None From d52bc4fd5e6ffe3ca160979bca709173f1207038 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Sun, 31 May 2026 17:00:30 -0400 Subject: [PATCH 58/64] Fix: min and max type. --- openlcb/cdimemo.py | 10 ++++++++++ openlcb/cdivar.py | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/openlcb/cdimemo.py b/openlcb/cdimemo.py index 343b035d..9801a8d0 100644 --- a/openlcb/cdimemo.py +++ b/openlcb/cdimemo.py @@ -221,6 +221,16 @@ def toCDIVar(self): # result_default = bytearray(default_var.data) # Size must be gotten ahead of time since CDIVar constructor # enforces size: + if result_min is not None: + assert result_size is not None, \ + f"size is required with min (className={className})" + result_min = CDIVar.fromNumber(result_min, className, + _size=result_size) + if result_max is not None: + assert result_size is not None, \ + f"size is required with min (className={className})" + result_max = CDIVar.fromNumber(result_max, className, + _size=result_size) result = CDIVar(self.tag, _min=result_min, _max=result_max, _size=result_size, _default=result_default) result.address = self.address # only set in replicatedTree() diff --git a/openlcb/cdivar.py b/openlcb/cdivar.py index 9bf245ce..2cee56e2 100644 --- a/openlcb/cdivar.py +++ b/openlcb/cdivar.py @@ -176,7 +176,8 @@ def __init__(self, className, _min=None, _max=None, thisType = CLASSNAME_TYPES.get(className) num_types = tuple(NUM_TYPES.values()) if _min is not None: - assert isinstance(_min, CDIVar) + assert isinstance(_min, CDIVar), \ + f"Expected CDIVar got {emit_cast(_min)}" _min.assertNumberFormat() assert thisType is not None min_value = _min.value() From 24d09297bbae2f706ad9db0b4a7579774f85af69 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Sun, 31 May 2026 17:01:22 -0400 Subject: [PATCH 59/64] Fix: wait for far node alias before using farNodeID. (NOOP) Move example multi-chunk read into reusable (OO) MemoryReadJob. --- examples/example_cdi_access.py | 105 ++++++--------------------------- openlcb/dataprocessor.py | 3 +- openlcb/memoryreadjob.py | 99 +++++++++++++++++++++++++++++++ python-openlcb.code-workspace | 4 +- 4 files changed, 121 insertions(+), 90 deletions(-) create mode 100644 openlcb/memoryreadjob.py diff --git a/examples/example_cdi_access.py b/examples/example_cdi_access.py index c27b75bf..ba768e06 100644 --- a/examples/example_cdi_access.py +++ b/examples/example_cdi_access.py @@ -21,6 +21,8 @@ from examples_settings import Settings # do 1st to fix path if no pip install from openlcb import precise_sleep +from openlcb.dataprocessor import DataFormat +from openlcb.memoryreadjob import MemoryReadJob from openlcb.convert import Convert from openlcb.xmldataprocessor import attrs_to_dict from openlcb.tcplink.tcpsocket import TcpSocket @@ -108,70 +110,6 @@ def printDatagram(memo): memoryService = MemoryService(datagramService) - -# accumulate the CDI information -resultingCDI = bytearray() - -# callbacks to get results of memory read - -complete_data = False -read_failed = False - - -def memoryReadSuccess(memo): - """Handle a successful read - Invoked when the memory read successfully returns, - this queues a new read until the entire CDI has been - returned. At that point, it invokes the XML processing below. - - Args: - memo (MemoryReadMemo): Successful MemoryReadMemo - """ - # print("successful memory read: {}".format(memo.data)) - - global resultingCDI - global complete_data - - # is this done? - if len(memo.data) == 64 and 0 not in memo.data: - # save content - resultingCDI += memo.data - logger.debug( - f"[{memo.address}] successful read" - f" {Convert.arrayToString(memo.data, len(memo.data))}" - "; next = address + 64") - # update the address - memo.address = memo.address+64 - # and read again - memoryService.requestMemoryRead(memo) - # The last packet is not yet reached, so don't parse (However, - # parser.feed could be called for realtime processing). - else : - # and we're done! - # save content - resultingCDI += memo.data - # concert resultingCDI to a string up to 1st zero - cdiString = "" - null_i = resultingCDI.find(b'\0') - terminate_i = len(resultingCDI) - if null_i > -1: - terminate_i = min(null_i, terminate_i) - cdiString = resultingCDI[:terminate_i].decode("utf-8") - # print (cdiString) - - # and process that - processXML(cdiString) - complete_data = True - - # done - - -def memoryReadFail(memo): - global read_failed - print("memory read failed: {}".format(memo.data)) - read_failed = True - - ####################### # The XML parsing section. # @@ -269,26 +207,6 @@ def characters(self, content: str): handler = MyHandler() - -def processXML(content: str) : - """process the XML and invoke callbacks - - Args: - content (str): Raw XML data - """ - # NOTE: The data is complete in this example since processXML is - # only called when there is a null terminator, which indicates the - # last packet was reached for the requested read. - # - See memoryReadSuccess comments for details. - with open("cached-cdi.xml", 'w') as stream: - # NOTE: Actual caching should key by all SNIP info that could - # affect CDI/FDI: manufacturer, model, and version. Without - # all 3 being present in SNIP, the cache may be incorrect. - stream.write(content) - xml.sax.parseString(content, handler) - print("\nParser done") - - ####################### # have the socket layer report up to bring the link layer up and get an alias @@ -313,6 +231,9 @@ def processXML(content: str) : print(" SENT frames : link up") +job = MemoryReadJob(memoryService, DataFormat.XML, handler=handler) + + def memoryRead(): """Create and send a read datagram. This is a read of 20 bytes from the start of CDI space. @@ -329,10 +250,18 @@ def memoryRead(): # This delay could be .2 (per alias collision), but longer to # reduce console messages: time.sleep(.5) + farNodeID = NodeID(settings['farNodeID']) + waited = 0 + delaySec = 1 + while farNodeID not in canLink.nodeIdToAlias: + time.sleep(delaySec) + waited += delaySec + print(f"Connected nodes: {canLink.nodeIdToAlias}") + print(f"Waiting for {farNodeID} ({waited}s)...") print("Requesting memory read. Please wait...") # read 64 bytes from the CDI space starting at address zero - memMemo = MemoryReadMemo(NodeID(settings['farNodeID']), 64, 0xFF, 0, - memoryReadFail, memoryReadSuccess) + memMemo = MemoryReadMemo(farNodeID, 64, 0xFF, 0, + job.memoryReadFail, job.memoryReadSuccess) memoryService.requestMemoryRead(memMemo) @@ -343,7 +272,7 @@ def memoryRead(): # process resulting activity print() print("This example will exit on failure or complete data.") -while not complete_data and not read_failed: +while not job.completeData and not job.failed: # In this example, requests are initiate by the # memoryRead thread, and receiveAll actually # receives the data from the requested memory space (CDI in this @@ -362,7 +291,7 @@ def memoryRead(): physicalLayer.physicalLayerDown() -if read_failed: +if job.failed: print("Read complete (FAILED)") else: print("Read complete (OK)") diff --git a/openlcb/dataprocessor.py b/openlcb/dataprocessor.py index f3f71090..c16507c3 100644 --- a/openlcb/dataprocessor.py +++ b/openlcb/dataprocessor.py @@ -3,7 +3,8 @@ class DataFormat(Enum): - EOF = 0 + EOF = -1 + Blob = 0 XML = 1 diff --git a/openlcb/memoryreadjob.py b/openlcb/memoryreadjob.py new file mode 100644 index 00000000..c2338367 --- /dev/null +++ b/openlcb/memoryreadjob.py @@ -0,0 +1,99 @@ +from logging import getLogger +import xml.sax + +from openlcb.convert import Convert +from openlcb.dataprocessor import DataFormat + +logger = getLogger(__name__) + + +class MemoryReadJob: + """Reusable multi-chunk memory read job. + Callbacks get results of memory read + (override in subclass for specific behavior such as progress). + """ + def __init__(self, memoryService, dataFormat: DataFormat, handler=None): + assert isinstance(dataFormat, DataFormat) + if dataFormat is DataFormat.XML: + assert handler is not None, \ + "XML needs handler (xml.sax.handler.ContentHandler/subclass)" + self.memoryService = memoryService + # accumulate the CDI information + self.resultingCDI = bytearray() + + self.handler = handler + self.dataFormat = dataFormat + + self.completeData = False + self.failed = False + + def memoryReadSuccess(self, memo): + """Handle a successful read + Invoked when the memory read successfully returns, + this queues a new read until the entire CDI has been + returned. At that point, it invokes the XML processing below. + + Args: + memo (MemoryReadMemo): Successful MemoryReadMemo + """ + # print("successful memory read: {}".format(memo.data)) + + # is this done? + if len(memo.data) == 64 and 0 not in memo.data: + # save content + self.resultingCDI += memo.data + logger.debug( + f"[{memo.address}] successful read" + f" {Convert.arrayToString(memo.data, len(memo.data))}" + "; next = address + 64") + # update the address + memo.address = memo.address+64 + # and read again + self.memoryService.requestMemoryRead(memo) + # The last packet is not yet reached, so don't parse (However, + # parser.feed could be called for realtime processing). + else : + # and we're done! + # save content + self.resultingCDI += memo.data + # concert resultingCDI to a string up to 1st zero + cdiString = "" + null_i = self.resultingCDI.find(b'\0') + terminate_i = len(self.resultingCDI) + if null_i > -1: + terminate_i = min(null_i, terminate_i) + cdiString = self.resultingCDI[:terminate_i].decode("utf-8") + # print (cdiString) + + # and process that + if self.dataFormat is DataFormat.XML: + self.processXML(cdiString) + else: + print( + f"Skipping processing for misc. format: {self.dataFormat}") + self.completeData = True + # done + + def memoryReadFail(self, memo): + print("memory read failed: {}".format(memo.data)) + self.failed = True + + def processXML(self, content: str) : + """process the XML and invoke callbacks + + Args: + content (str): Raw XML data + """ + # NOTE: The data is complete in this example since processXML is + # only called when there is a null terminator, which indicates the + # last packet was reached for the requested read. + # - See memoryReadSuccess comments for details. + with open("cached-cdi.xml", 'w') as stream: + # NOTE: Actual caching should key by all SNIP info that could + # affect CDI/FDI: manufacturer, model, and version. Without + # all 3 being present in SNIP, the cache may be incorrect. + stream.write(content) + assert self.handler is not None, \ + "XML needs handler (xml.sax.handler.ContentHandler/subclass)" + xml.sax.parseString(content, self.handler) + print("\nParser done") diff --git a/python-openlcb.code-workspace b/python-openlcb.code-workspace index 40d1aaa5..9e22c65b 100644 --- a/python-openlcb.code-workspace +++ b/python-openlcb.code-workspace @@ -67,9 +67,12 @@ "localeventstore", "localnodeprocessor", "localoverrides", + "mant", "MDNS", "mdnsconventions", "memoryconfigurationheader", + "MemoryManager", + "memoryreadjob", "memoryservice", "memoryspace", "memoryspaceindex", @@ -105,7 +108,6 @@ "settingtypes", "setuptools", "SOCK_DGRAM", - "MemoryManager", "sysdirs", "tcplink", "tcpsocket", From 525e9f8ab24749460b5a4ee050d42b43dcadc187 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Sun, 31 May 2026 17:30:29 -0400 Subject: [PATCH 60/64] Move memoryRead to reusable MemoryReadJob class. --- examples/example_cdi_access.py | 74 ++++++++++++++++++---------------- openlcb/memoryreadjob.py | 60 +++++++++++++++++++++++---- 2 files changed, 92 insertions(+), 42 deletions(-) diff --git a/examples/example_cdi_access.py b/examples/example_cdi_access.py index ba768e06..334820f4 100644 --- a/examples/example_cdi_access.py +++ b/examples/example_cdi_access.py @@ -13,6 +13,8 @@ # region same code as other examples import copy import sys +import time +from typing import Union import xml.sax import xml.sax.handler import xml.sax.xmlreader # for static type hints, autocomplete in this case @@ -22,8 +24,10 @@ from examples_settings import Settings # do 1st to fix path if no pip install from openlcb import precise_sleep from openlcb.dataprocessor import DataFormat +from openlcb.dataprocessormemo import DataProcessorMemo from openlcb.memoryreadjob import MemoryReadJob from openlcb.convert import Convert +from openlcb.memoryspace import MemorySpace from openlcb.xmldataprocessor import attrs_to_dict from openlcb.tcplink.tcpsocket import TcpSocket settings = Settings() @@ -231,43 +235,43 @@ def characters(self, content: str): print(" SENT frames : link up") -job = MemoryReadJob(memoryService, DataFormat.XML, handler=handler) +job = MemoryReadJob(memoryService) -def memoryRead(): - """Create and send a read datagram. - This is a read of 20 bytes from the start of CDI space. - We will fire it on a separate thread to give time for other nodes to reply - to AME - """ - import time - time.sleep(.21) - # ^ 200ms: See section 6.2.1 of CAN Frame Transfer Standard - # (CanLink.State.Permitted will only occur after that, but waiting - # now will reduce output & delays below in this example). - while canLink.getState() != CanLink.State.Permitted: - print("Waiting for connection sequence to complete...") - # This delay could be .2 (per alias collision), but longer to - # reduce console messages: - time.sleep(.5) - farNodeID = NodeID(settings['farNodeID']) - waited = 0 - delaySec = 1 - while farNodeID not in canLink.nodeIdToAlias: - time.sleep(delaySec) - waited += delaySec - print(f"Connected nodes: {canLink.nodeIdToAlias}") - print(f"Waiting for {farNodeID} ({waited}s)...") - print("Requesting memory read. Please wait...") - # read 64 bytes from the CDI space starting at address zero - memMemo = MemoryReadMemo(farNodeID, 64, 0xFF, 0, - job.memoryReadFail, job.memoryReadSuccess) - memoryService.requestMemoryRead(memMemo) - - -import threading # noqa E402 -thread = threading.Thread(target=memoryRead) -thread.start() +def statusCallback(memo: DataProcessorMemo): + if memo.status: + print(memo.status) + + +farNodeID = NodeID(settings['farNodeID']) + +time.sleep(.21) +# ^ 200ms: See section 6.2.1 of CAN Frame Transfer Standard +# (CanLink.State.Permitted will only occur after that, but waiting +# now will reduce output & delays below in this example). +while canLink.getState() != CanLink.State.Permitted: + print("Waiting for connection sequence to complete...") + # This delay could be .2 (per alias collision), but longer to + # reduce console messages: + time.sleep(.5) + +print(f"CanLink state={canLink.getState()}") + +waited = 0 +delaySec = 1 +while farNodeID not in canLink.nodeIdToAlias: + # canLink.pollState() + physicalLayer.receiveAll(sock) + physicalLayer.sendAll(sock) + time.sleep(delaySec) + waited += delaySec + print(f"Connected nodes: {canLink.nodeIdToAlias}") + print(f"Waiting for {farNodeID} ({waited}s)...") + +job.readMemory(canLink, farNodeID, MemorySpace.CDI, + handler=handler, + callback=statusCallback) + previous_nodes = copy.deepcopy(canLink.nodeIdToAlias) # process resulting activity print() diff --git a/openlcb/memoryreadjob.py b/openlcb/memoryreadjob.py index c2338367..107a941c 100644 --- a/openlcb/memoryreadjob.py +++ b/openlcb/memoryreadjob.py @@ -1,8 +1,14 @@ from logging import getLogger +from typing import Callable, Union import xml.sax +from openlcb.canbus.canlink import CanLink from openlcb.convert import Convert from openlcb.dataprocessor import DataFormat +from openlcb.dataprocessormemo import DataProcessorMemo +from openlcb.memoryservice import MemoryReadMemo +from openlcb.memoryspace import MemorySpace +from openlcb.nodeid import NodeID logger = getLogger(__name__) @@ -12,20 +18,57 @@ class MemoryReadJob: Callbacks get results of memory read (override in subclass for specific behavior such as progress). """ - def __init__(self, memoryService, dataFormat: DataFormat, handler=None): - assert isinstance(dataFormat, DataFormat) - if dataFormat is DataFormat.XML: - assert handler is not None, \ - "XML needs handler (xml.sax.handler.ContentHandler/subclass)" + def __init__(self, memoryService, ): self.memoryService = memoryService # accumulate the CDI information self.resultingCDI = bytearray() + self.handler = None + self.dataFormat = None + self.completeData = False + self.failed = False + self.memMemo = None + self.statusCallback = None + def readMemory(self, canLink, farNodeID: NodeID, + space: Union[int, MemorySpace], + dataFormat: Union[DataFormat, None] = None, handler=None, + callback: Union[Callable[[DataProcessorMemo], None], None] = None): # noqa: E501 + """Create and send a read datagram. + This is a read of 20 bytes from the start of CDI space. We will + fire it on a separate thread to give time for other nodes to + reply to AME. + """ + memo = DataProcessorMemo() self.handler = handler + if dataFormat is None: + if isinstance(space, int): + spaceID = MemorySpace.fromNumber(space) + else: + assert isinstance(space, MemorySpace) + spaceID = space + if spaceID in (MemorySpace.CDI, MemorySpace.FDI): + dataFormat = DataFormat.XML + assert isinstance(dataFormat, DataFormat) + if dataFormat is DataFormat.XML: + assert handler is not None, \ + "XML needs handler (xml.sax.handler.ContentHandler/subclass)" self.dataFormat = dataFormat + self.statusCallback = callback - self.completeData = False - self.failed = False + def echoS(message: str): + """push a message""" + memo.status = message + if callback is not None: + callback(memo) + + if isinstance(space, MemorySpace): + space = space.value + echoS("Requesting memory read. Please wait...") + # read 64 bytes from the CDI space starting at address zero + self.memMemo = MemoryReadMemo(farNodeID, 64, space, 0, + self.memoryReadFail, + self.memoryReadSuccess) + self.memoryService.requestMemoryRead(self.memMemo) def memoryReadSuccess(self, memo): """Handle a successful read @@ -72,6 +115,9 @@ def memoryReadSuccess(self, memo): print( f"Skipping processing for misc. format: {self.dataFormat}") self.completeData = True + memo = DataProcessorMemo() + memo.status = "" + memo.done = True # done def memoryReadFail(self, memo): From f42443e35da8d9486612e57b01e8a98c918e7eb4 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Sun, 31 May 2026 21:36:33 -0400 Subject: [PATCH 61/64] Fix local memory read (Encode and decode space index correctly; Pad with zeroes if last chunk less than 64 bytes). Add a related test case. Add increment option to generate_node_id. --- openlcb/conventions.py | 10 +- openlcb/linklayer.py | 4 +- openlcb/memoryconfigurationheader.py | 7 +- openlcb/memorymanager.py | 3 + openlcb/memoryreadjob.py | 14 +- openlcb/memoryservice.py | 46 +++-- openlcb/nodeid.py | 10 +- openlcb/openlcbnetwork.py | 3 +- openlcb/physicallayer.py | 9 +- openlcb/portinterface.py | 15 +- openlcb/xmldataprocessor.py | 2 +- python-openlcb.code-workspace | 1 + tests/test_memorymanager.py | 289 ++++++++++++++++++++++++++- 13 files changed, 379 insertions(+), 34 deletions(-) diff --git a/openlcb/conventions.py b/openlcb/conventions.py index 842cf97f..df1be392 100644 --- a/openlcb/conventions.py +++ b/openlcb/conventions.py @@ -146,6 +146,7 @@ def get_local_ip() -> Optional[str]: previous_three_octets = None initial_three_octets = None +used_ids = set() def increment_octets(octets: bytearray): @@ -241,6 +242,8 @@ def generate_node_id_str(id_range_prefix: str, increment: bool = False) -> str: python-openlcb (or as otherwise assigned by OpenLCB Group which reserves 05.* range). See . + increment (bool): Increment such as more multiple + local IDs (virtual node(s) aside from local Node). Returns: str: Full 48-bit node ID in dotted hex string notation (Example: '05.01.01.4A.B7.19') that is unique (very likely...). @@ -260,5 +263,10 @@ def generate_node_id_str(id_range_prefix: str, increment: bool = False) -> str: " (preferably less to increase likelihood of uniqueness). Got {}" .format(id_range_prefix)) uniqueCount = 6 - len(prefixParts) - return ".".join(prefixParts+lastParts[-uniqueCount:]) + id_str = ".".join(prefixParts+lastParts[-uniqueCount:]) # ^ negative to keep last uniqueCount pairs + if used_ids in used_ids: + logger.warning(f"{id_str} was already used. Set increment=True" + " if there is more than one local node!") + used_ids.add(id_str) + return id_str diff --git a/openlcb/linklayer.py b/openlcb/linklayer.py index 7ab2a005..7aeb12eb 100644 --- a/openlcb/linklayer.py +++ b/openlcb/linklayer.py @@ -48,7 +48,9 @@ class State(Enum): # (enforced using type(self).__name__ != "LinkLayer" checks in methods) def __init__(self, physicalLayer: PhysicalLayer, localNodeID): - assert isinstance(physicalLayer, PhysicalLayer) # allows any subclass + assert isinstance(physicalLayer, PhysicalLayer), \ + f"Expected PhysicalLayer/subclass, got a(n) {type(physicalLayer)}" + # ^ allows any subclass # subclass should check type of localNodeID technically self.localNodeID = localNodeID self._messageReceivedListeners = [] diff --git a/openlcb/memoryconfigurationheader.py b/openlcb/memoryconfigurationheader.py index 569f8112..fc8c7eff 100644 --- a/openlcb/memoryconfigurationheader.py +++ b/openlcb/memoryconfigurationheader.py @@ -51,8 +51,11 @@ def fromMC2ndByte(cls, datagramByte1: int, space: Union[int, None] = None) -> 'M 'custom space requires datagramByte1 with last 2 bits 00' else: # space is None - assert datagramByte1 & 0x03 != 0, \ - 'a standard space index must be in last 2 bits datagramByte1' + # assert datagramByte1 & 0x03 != 0, \ + # ("standard space index req. in low 2 bits of datagramByte1" + # f" but got {hex(datagramByte1)}") + # ^ commented to allow indeterminate state, + # so caller can set space later from the later space byte space = -1 # NOTE: Third option is that space isn't known yet # (must be set later, if spaceIsCustom()) diff --git a/openlcb/memorymanager.py b/openlcb/memorymanager.py index bcfd7306..b7a7c913 100644 --- a/openlcb/memorymanager.py +++ b/openlcb/memorymanager.py @@ -104,6 +104,9 @@ def setData(self, address: int, data: Union[bytearray, bytes], else: self._data[address:end] = data + def size(self) -> int: + return len(self._data) + class MemoryManager: def __init__(self): diff --git a/openlcb/memoryreadjob.py b/openlcb/memoryreadjob.py index 107a941c..8c564381 100644 --- a/openlcb/memoryreadjob.py +++ b/openlcb/memoryreadjob.py @@ -63,7 +63,8 @@ def echoS(message: str): if isinstance(space, MemorySpace): space = space.value - echoS("Requesting memory read. Please wait...") + echoS("") + echoS(f"Requesting memory read (space={space}). Please wait...") # read 64 bytes from the CDI space starting at address zero self.memMemo = MemoryReadMemo(farNodeID, 64, space, 0, self.memoryReadFail, @@ -85,9 +86,9 @@ def memoryReadSuccess(self, memo): if len(memo.data) == 64 and 0 not in memo.data: # save content self.resultingCDI += memo.data - logger.debug( + logger.info( f"[{memo.address}] successful read" - f" {Convert.arrayToString(memo.data, len(memo.data))}" + f" `{Convert.arrayToString(memo.data, len(memo.data))}`" "; next = address + 64") # update the address memo.address = memo.address+64 @@ -120,8 +121,11 @@ def memoryReadSuccess(self, memo): memo.done = True # done - def memoryReadFail(self, memo): - print("memory read failed: {}".format(memo.data)) + def memoryReadFail(self, memo: MemoryReadMemo): + assert isinstance(memo, MemoryReadMemo) + print(f"memory read failed: id={memo.nodeID}" + f" data={memo.data} space={memo.space} address={memo.address}" + f" error={memo.error} code={memo.errorCode}") self.failed = True def processXML(self, content: str) : diff --git a/openlcb/memoryservice.py b/openlcb/memoryservice.py index 63eebef1..795c4e86 100644 --- a/openlcb/memoryservice.py +++ b/openlcb/memoryservice.py @@ -43,6 +43,7 @@ from openlcb.convert import Convert # from openlcb.localnode import LocalNode # circular import from openlcb.memoryconfigurationheader import MemoryConfigurationHeader +from openlcb.memoryspace import MemorySpace from openlcb.memoryspaceindex import MemorySpaceIndex from openlcb.memorymanager import MemoryManager from openlcb.nodeid import NodeID @@ -328,13 +329,17 @@ def parseReplyDatagram(memo: Union[MemoryReadMemo, MemoryWriteMemo], ) offset = 6 error = None - assert mcHeader.spaceIndex is not MemorySpaceIndex.Uninitialized - if mcHeader.spaceIndex is MemorySpaceIndex.Custom: + if mcHeader.spaceIndex in (MemorySpaceIndex.Custom, + MemorySpaceIndex.Uninitialized): # mcHeader.customSpace = memo.space mcHeader.customSpace = dmemo.data[6] offset = 7 + else: + assert mcHeader.customSpace is None, \ + "fromMC2ndByte should not set customSpace in this case" memo.error = None memo.errorCode = None + print(f"reply datagram: {mcHeader.spaceIndex} customSpace={mcHeader.customSpace}") if (dmemo.data[1] & 0x08 == 0): # ok reply return @@ -625,42 +630,55 @@ def datagramReceivedListener(self, dmemo: DatagramReadMemo) -> bool: self.spaceLengthCallback(address) self.spaceLengthCallback = None elif mcOp is MCOp.Read_Command: - # assert dmemo.data[1] in TWO_BIT_PARAMS[MCOp.Read_Command.value], \ + # assert dmemo.data[1] in TWO_BIT_PARAMS[MCOp.Read_Command.value],\ # "self-test failed (bad constant(s))" mcHeader = MemoryConfigurationHeader.fromMC2ndByte(dmemo.data[1]) addressBytes = dmemo.data[2:6] address = struct.unpack(">I", addressBytes)[0] # ^ [0] since always returns list even when reading 1 value. # ^ capital assumes unsigned, "I" assumes 32-bit (4 bytes) - space = dmemo.data[1] & 0b00000011 + spaceIndex = dmemo.data[1] & 0b00000011 offset = 0 + space = None if mcHeader.spaceIsCustom(): - assert space == 0 + assert spaceIndex == 0 space = dmemo.data[6] offset = 1 size = dmemo.data[6+offset] # requested read count datagramBytes = bytearray([0x20, MCOp.Read_Reply.value]) # ^ byte1 (2nd) changed to error below if applicable - assert isinstance(space, int), \ - (f"Logic missing, space should be number here," - f" got {emit_cast(space)}") + assert isinstance(spaceIndex, int), \ + (f"Logic missing, spaceIndex should be number here," + f" got {emit_cast(spaceIndex)}") assert isinstance(address, int) datagramBytes += addressBytes - assert space is not None, \ - f"space not computed from datagram: {dmemo.data}" if mcHeader.spaceIsCustom(): + assert space is not None datagramBytes.append(space) assert len(datagramBytes) == 7, "space goes in index [6]" else: + space = MemorySpace.fromIndex(MemorySpaceIndex(spaceIndex)) + assert space is not None + space = space.value spaceIndex = (space & 0b00000011) - assert spaceIndex == space assert space is not None datagramBytes[1] = \ datagramBytes[1] | spaceIndex assert len(datagramBytes) == 6, "should not have space in [6]" + assert space is not None, \ + f"space not computed from datagram: {dmemo.data}" payload = None try: - payload = self.memory.getSlice(space, address, size) + segment = self.memory.getStorage(space) + if segment is None: + raise KeyError(f"space {space} is not valid") + if address >= segment.size(): + raise IndexError( + f"address {address} past end of {hex(space)}") + payload = self.memory.getSlice(space, address, size, + force=True) + # ^ force=True because reading past end is normal + # (pad with zeroes to indicate end) except (IndexError, KeyError) as ex: # address out of range (See Segment's getSlice) datagramBytes[1] = MCOp.Read_Reply_Failure.value @@ -698,6 +716,10 @@ def datagramReceivedListener(self, dmemo: DatagramReadMemo) -> bool: dmemo.srcID, datagramBytes ) + # hexStrings = [] + # for b in datagramBytes: + # hexStrings.append(hex(b)) + # print(f"Sending read reply: {hexStrings}") self.service.sendDatagram(requestedMemoryMemo) else: logger.error("Did not expect reply of type 0x{:02X}" diff --git a/openlcb/nodeid.py b/openlcb/nodeid.py index f4c2517f..abc2c583 100644 --- a/openlcb/nodeid.py +++ b/openlcb/nodeid.py @@ -1,6 +1,10 @@ +from logging import getLogger + from openlcb import emit_cast from openlcb.conventions import generate_node_id_str +logger = getLogger(__name__) + class NodeID: """A 6-byte (48-bit) Node ID. @@ -88,7 +92,7 @@ def __hash__(self): return hash(self.value) -def generate_node_id(id_range_prefix): +def generate_node_id(id_range_prefix, increment=False): """Generate a unique NodeID for the session to ensure each instance (even of python-openlcb on same device) or locally-generated virtual node is unique. @@ -99,7 +103,9 @@ def generate_node_id(id_range_prefix): python-openlcb (or as otherwise assigned by OpenLCB Group which reserves 05.* range). See . + increment (bool): Increment such as more multiple + local IDs (virtual node(s) aside from local Node). Returns: NodeID: A NodeID that is unique (very likely...). """ - return NodeID(generate_node_id_str(id_range_prefix)) + return NodeID(generate_node_id_str(id_range_prefix, increment=increment)) diff --git a/openlcb/openlcbnetwork.py b/openlcb/openlcbnetwork.py index 8efeff16..51f3276e 100644 --- a/openlcb/openlcbnetwork.py +++ b/openlcb/openlcbnetwork.py @@ -368,7 +368,8 @@ def _fireStatus(self, status, print("OpenLCBNetwork callback_msg({})".format(repr(status))) callback(CDIMemo(status=status)) else: - logger.warning("No callback, but set status: {}".format(status)) + logger.warning( + f"[OpenLCBNetwork] No callback, but set status: {status}") def _memoryReadSuccess(self, memo: MemoryReadMemo, force_end=False): """Handle a successful read diff --git a/openlcb/physicallayer.py b/openlcb/physicallayer.py index c683307d..5e996005 100644 --- a/openlcb/physicallayer.py +++ b/openlcb/physicallayer.py @@ -226,15 +226,18 @@ def encodeFrameAsData(self, frame) -> Union[bytearray, bytes]: def physicalLayerUp(self): """abstract method""" - raise NotImplementedError("Each subclass must implement this.") + raise NotImplementedError( + f"{type(self).__name__} subclass must implement physicalLayerUp.") def physicalLayerRestart(self): """abstract method""" - raise NotImplementedError("Each subclass must implement this.") + raise NotImplementedError( + f"{type(self).__name__} subclass must implement physicalLayerRestart.") def physicalLayerDown(self): """abstract method""" - raise NotImplementedError("Each subclass must implement this.") + raise NotImplementedError( + f"{type(self).__name__} subclass must implement physicalLayerDown.") def handleData(self, data: Union[bytes, bytearray]) -> int: """abstract method (accept data, return # of frames created)""" diff --git a/openlcb/portinterface.py b/openlcb/portinterface.py index e708bba0..41a5c30e 100644 --- a/openlcb/portinterface.py +++ b/openlcb/portinterface.py @@ -69,7 +69,8 @@ def setListeners(self, onReadyToSend, onReadyToReceive): def _settimeout(self, seconds): """Abstract method. Return: implementation-specific or None.""" - raise NotImplementedError("Subclass must implement this.") + raise NotImplementedError( + f"{type(self).__name__} subclass must implement _settimeout.") def settimeout(self, seconds): return self._settimeout(seconds) @@ -79,7 +80,8 @@ def _connect(self, host: Any, port: Any, device: Any = None): See connect for details. raise exception on failure to prevent self._open = True. """ - raise NotImplementedError("Subclass must implement this.") + raise NotImplementedError( + f"{type(self).__name__} subclass must implement _connect.") def connect(self, host, port, device=None): """Connect to a port. @@ -109,7 +111,8 @@ def connectLocal(self, port): def _send(self, data: Union[bytes, bytearray]) -> None: """Abstract method. Return: implementation-specific or None""" - raise NotImplementedError("Subclass must implement this.") + raise NotImplementedError( + f"{type(self).__name__} subclass must implement _send.") def send(self, data: Union[bytes, bytearray]) -> None: """ @@ -132,7 +135,8 @@ def send(self, data: Union[bytes, bytearray]) -> None: def _receive(self) -> Union[bytearray, bytes, None]: """Abstract method. Return (bytes): data""" - raise NotImplementedError("Subclass must implement this.") + raise NotImplementedError( + f"{type(self).__name__} subclass must implement _receive.") def receive(self) -> Union[bytearray, bytes, None]: self._setBusy("receive") @@ -147,7 +151,8 @@ def receive(self) -> Union[bytearray, bytes, None]: def _close(self) -> None: """Abstract method. Return: implementation-specific or None""" - raise NotImplementedError("Subclass must implement this.") + raise NotImplementedError( + f"{type(self).__name__} subclass must implement _close.") def setOpen(self, is_open): if self._open != is_open: diff --git a/openlcb/xmldataprocessor.py b/openlcb/xmldataprocessor.py index 20587990..6af80269 100644 --- a/openlcb/xmldataprocessor.py +++ b/openlcb/xmldataprocessor.py @@ -290,7 +290,7 @@ def _fireStatusMemo(self, statusMemo, logger.info(f"OpenLCBNetwork callback_msg({statusMemo})") callback(statusMemo) else: - logger.warning(f"No callback, but set status: {statusMemo}") + logger.warning(f"No callback, but set status memo: {statusMemo}") def _feedNext(self, memo: MemoryReadMemo): """Handle partial CDI XML (any packet except last) diff --git a/python-openlcb.code-workspace b/python-openlcb.code-workspace index 9e22c65b..52d75088 100644 --- a/python-openlcb.code-workspace +++ b/python-openlcb.code-workspace @@ -65,6 +65,7 @@ "linklayer", "LOCALAPPDATA", "localeventstore", + "localnode", "localnodeprocessor", "localoverrides", "mant", diff --git a/tests/test_memorymanager.py b/tests/test_memorymanager.py index 2baa2658..be970ff4 100644 --- a/tests/test_memorymanager.py +++ b/tests/test_memorymanager.py @@ -1,14 +1,35 @@ import os import struct import sys +import time import unittest +import xml.sax.handler + +from typing import Any, Union from openlcb import emit_cast +from openlcb.canbus.canlink import CanLink +from openlcb.canbus.canphysicallayergridconnect import CanPhysicalLayerGridConnect from openlcb.cdivar import SIGNED_INT_MINIMUMS, CDIVar +from openlcb.datagramservice import DatagramService, DatagramWriteMemo +from openlcb.dataprocessormemo import DataProcessorMemo +from openlcb.localnode import LocalNode +from openlcb.localnodeprocessor import LocalNodeProcessor from openlcb.memorymanager import MemoryManager - +from openlcb.memoryspace import MemorySpace from logging import getLogger + +from openlcb.memoryreadjob import MemoryReadJob +from openlcb.memoryservice import MemoryReadMemo, MemoryService +from openlcb.message import Message +from openlcb.mti import MTI +from openlcb.node import Node +from openlcb.nodeid import generate_node_id +from openlcb.openlcbnetwork import OpenLCBNetwork +from openlcb.pip import PIP +from openlcb.portinterface import PortInterface +from openlcb.snip import SNIP if __name__ == "__main__": logger = getLogger(__file__) else: @@ -26,6 +47,71 @@ " since test running from repo but could not find openlcb in {}." .format(repr(REPO_DIR))) +# demo_virtual_node_cdi: same as example_node_memory_implementation.py +demo_virtual_node_cdi = """ + + + python-openlcb example authors + example_node_memory_implementation + 1.0 + 1.0 + + + + + Port + Network port of remote hub (2-byte unsigned short) + 12021 + + + Timeout + Network timeout (2-byte binary16 value). + 0.5 + + + +""" # noqa: E501 + + +class MockPort(PortInterface): + def __init__(self, name="MockPort"): + PortInterface.__init__(self) + self.name = name + self.data = bytearray() # type: bytearray + + def _settimeout(self, seconds): + """Abstract method. Return: implementation-specific or None.""" + raise NotImplementedError( + f"{type(self).__name__} subclass must implement _settimeout.") + + def _connect(self, host: Any, port: Any, device: Any = None): + """Abstract interface. Return: implementation-specific or None + See connect for details. + raise exception on failure to prevent self._open = True. + """ + pass + + def _send(self, data: Union[bytes, bytearray]) -> None: + """Abstract method. Return: implementation-specific or None""" + print("[MockPort] Ran dummy version of _send method.") + pass + + def _receive(self) -> Union[bytearray, bytes, None]: + """Abstract method. Return (bytes): data""" + end = len(self.data) # concurrency issue mitigation + data = self.data[:end] + del self.data[:end] + if data: + # GridConnect (hex notation) bytes (already human-readable): + logger.debug(f"{self.name} Received {len(data)} byte(s): {data}") + # print(f"{self.name} Received {len(data)} byte(s)") + return data + + def _close(self) -> None: + """Abstract method. Return: implementation-specific or None""" + pass + class TestMemoryManager(unittest.TestCase): @@ -191,6 +277,207 @@ def testCDIVarSInt(self): out_value = memory.getInt(var.space, var.address, var.size, var.signed) self.assertEqual(out_value, in_value) + def testGetCDI(self): + def readReply(memo: MemoryReadMemo): + print(f"GOT: {memo.data}") + + def readRejected(memo: MemoryReadMemo): + print(f"REJECTED: {memo}") + + localNodeID = generate_node_id("05.01.05") # "05.01.01" is only for OpenLCB Group. See # noqa: E501 + network = OpenLCBNetwork(localNodeID) + localNode = Node( + localNodeID, + SNIP("python-openlcb authors", + "test_memorymanager CT", + "1.0", "1.0", "test_memorymanager CT", + "python-openlcb configuration tool for test_memorymanager"), + set([ + PIP.SIMPLE_NODE_IDENTIFICATION_PROTOCOL, + PIP.DATAGRAM_PROTOCOL, + PIP.CONFIGURATION_DESCRIPTION_INFORMATION, + PIP.ADCDI_PROTOCOL, + PIP.MEMORY_CONFIGURATION_PROTOCOL, + ]) + ) + # ^ Same as (Except SNIP not necessary for CT if MemoryManager isn't used): + + virtualPhysicalLayer = CanPhysicalLayerGridConnect() + + mockLocalPort = MockPort(name="localMockPort") + virtualMockPort = MockPort(name="virtualMockPort") + + network._port = mockLocalPort + + def local_send(data): + """Make physicalLayer into loopback device""" + virtualMockPort.data += data + + def virtual_send(data): + mockLocalPort.data += data + + # Setup loopback: + virtualMockPort.send = virtual_send + mockLocalPort.send = local_send + + virtualNodeID = generate_node_id("05.01.05", increment=True) # "05.01.01" is only for OpenLCB Group. See # noqa: E501 + assert virtualNodeID != localNodeID + + print(f"localNodeID: {localNodeID}") + print(f"virtualNodeID: {virtualNodeID}") + + virtualCanLink = CanLink(virtualPhysicalLayer, virtualNodeID) + + virtualNode = LocalNode( + virtualNodeID, + SNIP("python-openlcb authors", + "test_memorymanager VN", + "1.0", "1.0", "test_memorymanager VN", + "python-openlcb virtual node with memory for test_memorymanager"), # noqa: E501 + set([ + PIP.SIMPLE_NODE_IDENTIFICATION_PROTOCOL, + PIP.DATAGRAM_PROTOCOL, + PIP.CONFIGURATION_DESCRIPTION_INFORMATION, + PIP.ADCDI_PROTOCOL, + PIP.MEMORY_CONFIGURATION_PROTOCOL, + ]), + virtualCanLink + ) + # dgService = DatagramService(canLink) + # localMemoryService = MemoryService(dgService) + virtualNodeProcessor = LocalNodeProcessor(virtualCanLink, virtualNode) + virtualCanLink.registerMessageReceivedListener( + virtualNodeProcessor.process) + virtualDGService = DatagramService(virtualCanLink) + virtualCanLink.registerMessageReceivedListener( + virtualDGService.process) + + def debug_virtual_incoming(memo): + logger.debug( + f"🔍 VIRTUAL RECEIVED DATAGRAM: {type(memo).__name__} - {memo}") + + # def debug_virtual_outgoing(memo): + # print(f"📤 VIRTUAL SENDING REPLY: {memo}") + + virtualDGService.registerDatagramReceivedListener( + debug_virtual_incoming) + + def debug_local_reply(memo): + logger.debug(f"🔙 LOCAL RECEIVED REPLY: {memo}") + + network._datagramService.registerDatagramReceivedListener( + debug_local_reply) + + virtualMemoryService = MemoryService(virtualDGService) + virtualMemoryService.memory = virtualNode + virtualNode.loadCDIString(demo_virtual_node_cdi, __file__) + got = virtualNode.getSlice(MemorySpace.CDI, + 0, len(demo_virtual_node_cdi)) + assert got.decode() == demo_virtual_node_cdi + segment = virtualNode.getStorage(MemorySpace.CDI) + assert segment._data == demo_virtual_node_cdi.encode() + # ^ not itself threaded, so no memo for callback is necessary + # network.startListening(mockLocalPort) + # commenting this requires your own physicalLayerUp + # and listen loop (sendAll and receiveAll calls) + network.physicalLayer.physicalLayerUp() + virtualPhysicalLayer.physicalLayerUp() + + localState = network.canLink.pollState() + network._port = None # since using manual sendAll and receiveAll + while localState != CanLink.State.Permitted: + network.physicalLayer.sendAll(mockLocalPort) + network.physicalLayer.receiveAll(mockLocalPort) + virtualPhysicalLayer.sendAll(virtualMockPort) + virtualPhysicalLayer.receiveAll(virtualMockPort) + # NOTE: startListening will be doing sendAll and receiveAll + # for network.physicalLayer in another thread. + print(f"Waiting for Permitted (state={localState})") + localState = network.canLink.pollState() + time.sleep(.02) + print(f"Network state is {localState}") + + virtualState = virtualCanLink.pollState() + while True: + network.physicalLayer.sendAll(mockLocalPort) + network.physicalLayer.receiveAll(mockLocalPort) + virtualPhysicalLayer.sendAll(virtualMockPort) + virtualPhysicalLayer.receiveAll(virtualMockPort) + # NOTE: startListening will be doing sendAll and receiveAll + # for network.physicalLayer in another thread. + network.canLink.pollState() + virtualState = virtualCanLink.pollState() + if virtualState == CanLink.State.Permitted: + if virtualNodeID in network.canLink.nodeIdToAlias: + break + else: + print( + f"Waiting for alias {virtualNodeID}" + f" in {network.canLink.nodeIdToAlias}...") + else: + print(f"Waiting for virtual Permitted (state={virtualState})") + time.sleep(.02) + + print(f"Virtual network state is {localState}") + + localNodeProcessor = LocalNodeProcessor( + network.canLink, localNode) + network.canLink.registerMessageReceivedListener( + localNodeProcessor.process) + + def displayOtherNodeIds(message: Message) : + """Listener to identify connected nodes + + Args: + message (Message): A response from the network + """ + print(f"[displayOtherNodeIds] {type(message).__name__}: {message.mti}") + if message.mti == MTI.Verified_NodeID : + print("Detected farNodeID is {}".format(message.source)) + + network.canLink.registerMessageReceivedListener(displayOtherNodeIds) + + job = MemoryReadJob(network._memoryService) + + class DummyHandler(xml.sax.handler.ContentHandler): + pass + + handler = DummyHandler() + + done = False + + def statusCallback(memo: DataProcessorMemo): + nonlocal done + print(f"statusCallback(memo): {memo.status}") + done = memo.done + + job.readMemory(network.canLink, virtualNodeID, MemorySpace.CDI, + handler=handler, + callback=statusCallback) + while not done: + if job.failed: + print(f"MemoryReadJob failed" + f" (expected {len(demo_virtual_node_cdi)} byte(s))") + break + if job.completeData: + print("MemoryReadJob completed") + break + network.physicalLayer.sendAll(mockLocalPort) + network.physicalLayer.receiveAll(mockLocalPort) + virtualPhysicalLayer.sendAll(virtualMockPort) + virtualPhysicalLayer.receiveAll(virtualMockPort) + assert network._port is mockLocalPort or network._port is None + # NOTE: startListening will be doing sendAll and receiveAll + # for network.physicalLayer in another thread. + localState = network.canLink.pollState() + virtualState = virtualCanLink.pollState() + if localState != CanLink.State.Permitted: + print(f"Warning: local network state is {localState}") + if virtualState != CanLink.State.Permitted: + print(f"Warning: virtual network state is {virtualState}") + print("Waiting for done...") + time.sleep(.02) + if __name__ == "__main__": unittest.main() From 3368a1eadf46c1eea7dccaf3404494a6a6bb4800 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Sun, 31 May 2026 21:40:26 -0400 Subject: [PATCH 62/64] (NOOP) Rename getStorage to getSegment. --- examples/example_node_memory_implementation.py | 2 +- openlcb/memorymanager.py | 2 +- openlcb/memoryservice.py | 2 +- tests/test_memorymanager.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/example_node_memory_implementation.py b/examples/example_node_memory_implementation.py index 6944c7df..aa91c776 100644 --- a/examples/example_node_memory_implementation.py +++ b/examples/example_node_memory_implementation.py @@ -205,7 +205,7 @@ def memoryReadFail(memo): localNode.loadCDIString(cdi, backup_path) # NOTE: loadCDI or loadCDIString sets Element tree and # localNode._segments[MemorySpace.CDI.value] -segment = localNode.getStorage(MemorySpace.CDI.value) +segment = localNode.getSegment(MemorySpace.CDI.value) assert isinstance(segment, Segment) assert isinstance(segment._data, (bytearray, bytes)) # localNodeProcessor = LocalNodeProcessor(canLink, localNode) diff --git a/openlcb/memorymanager.py b/openlcb/memorymanager.py index b7a7c913..296b5bed 100644 --- a/openlcb/memorymanager.py +++ b/openlcb/memorymanager.py @@ -182,7 +182,7 @@ def getLast(self, space: Union[MemorySpace, int]): return None return segment.getLast() - def getStorage(self, space: Union[MemorySpace, int]) -> Union[Segment, None]: # noqa: E501 + def getSegment(self, space: Union[MemorySpace, int]) -> Union[Segment, None]: # noqa: E501 """Get last address (may differ from length-1 on actual hardware) """ diff --git a/openlcb/memoryservice.py b/openlcb/memoryservice.py index 795c4e86..329589fa 100644 --- a/openlcb/memoryservice.py +++ b/openlcb/memoryservice.py @@ -669,7 +669,7 @@ def datagramReceivedListener(self, dmemo: DatagramReadMemo) -> bool: f"space not computed from datagram: {dmemo.data}" payload = None try: - segment = self.memory.getStorage(space) + segment = self.memory.getSegment(space) if segment is None: raise KeyError(f"space {space} is not valid") if address >= segment.size(): diff --git a/tests/test_memorymanager.py b/tests/test_memorymanager.py index be970ff4..41ef2723 100644 --- a/tests/test_memorymanager.py +++ b/tests/test_memorymanager.py @@ -374,7 +374,7 @@ def debug_local_reply(memo): got = virtualNode.getSlice(MemorySpace.CDI, 0, len(demo_virtual_node_cdi)) assert got.decode() == demo_virtual_node_cdi - segment = virtualNode.getStorage(MemorySpace.CDI) + segment = virtualNode.getSegment(MemorySpace.CDI) assert segment._data == demo_virtual_node_cdi.encode() # ^ not itself threaded, so no memo for callback is necessary # network.startListening(mockLocalPort) From 2e14df380c0b40ddaaaec0b9dcc7766df0a1c014 Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Mon, 1 Jun 2026 14:03:40 -0400 Subject: [PATCH 63/64] Rename setData to setSlice in Segment. --- openlcb/localnode.py | 2 +- openlcb/memorymanager.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/openlcb/localnode.py b/openlcb/localnode.py index 1706116e..8691d5be 100644 --- a/openlcb/localnode.py +++ b/openlcb/localnode.py @@ -107,7 +107,7 @@ def loadCDIString(self, xml_data, path, memo=None): # assert isinstance(xml_data, (bytes, bytearray)) segment = Segment() self._segments[MemorySpace.CDI.value] = segment - segment.setData(0, xml_data, force=True) + segment.setSlice(0, xml_data, force=True) segment.markReadOnly(True) def setMemory(self, memo: CDIMemo, var: CDIVar): diff --git a/openlcb/memorymanager.py b/openlcb/memorymanager.py index 296b5bed..671831b5 100644 --- a/openlcb/memorymanager.py +++ b/openlcb/memorymanager.py @@ -43,7 +43,7 @@ def getSlice(self, address: int, size: int, force=False): if address >= len(self._data): if force: data = bytearray(size*b"\0") - self.setData(address, data, force=force) + self.setSlice(address, data, force=force) return data else: raise IndexError( @@ -53,7 +53,7 @@ def getSlice(self, address: int, size: int, force=False): slack = end - len(self._data) offset = size - slack if force: - self.setData(address + offset, slack*b"\0") + self.setSlice(address + offset, slack*b"\0") else: raise IndexError( f"Tried to get address {address}" @@ -77,7 +77,7 @@ def markReadOnly(self, readOnly: bool): def extend(self, data): self._data += data - def setData(self, address: int, data: Union[bytearray, bytes], + def setSlice(self, address: int, data: Union[bytearray, bytes], size: Union[int, None] = None, force=True): assert isinstance(data, (bytearray, bytes)) assert isinstance(address, int) @@ -222,7 +222,7 @@ def setSlice(self, space: Union[MemorySpace, int], address: int, if segment is None: segment = Segment() self._segments[space] = segment - segment.setData(address, data, size=size) + segment.setSlice(address, data, size=size) def getSlice(self, space: Union[MemorySpace, int], address: int, size: int, force=False) -> bytearray: @@ -235,7 +235,7 @@ def getSlice(self, space: Union[MemorySpace, int], address: int, if segment is None: if force: segment = Segment(size=address+size) - segment.setData(address, b"\0"*size, force=True) + segment.setSlice(address, b"\0"*size, force=True) self._segments[space] = segment else: raise KeyError(f"Space {hex(space)} does not exist.") From b9f73dbd11f13a50f5915742528596f73a9fe50e Mon Sep 17 00:00:00 2001 From: Jacob Gustafson <7557867+Poikilos@users.noreply.github.com> Date: Mon, 1 Jun 2026 14:54:58 -0400 Subject: [PATCH 64/64] Fix: Clear upper bit of size byte (reserved bit in Standard). Implement Memory Write command. --- openlcb/memoryservice.py | 190 ++++++++++++++++++++++++++------------- 1 file changed, 126 insertions(+), 64 deletions(-) diff --git a/openlcb/memoryservice.py b/openlcb/memoryservice.py index 329589fa..594e9e79 100644 --- a/openlcb/memoryservice.py +++ b/openlcb/memoryservice.py @@ -599,6 +599,7 @@ def datagramReceivedListener(self, dmemo: DatagramReadMemo) -> bool: # description (See # https://github.com/openlcb/documents/issues/190)? replyData += descBytes + # FIXME: Need to send Datagram Received OK datagram 1st? spaceInfoReplyMemo = DatagramWriteMemo( dmemo.srcID, replyData @@ -629,9 +630,8 @@ def datagramReceivedListener(self, dmemo: DatagramReadMemo) -> bool: + int(dmemo.data[6])) self.spaceLengthCallback(address) self.spaceLengthCallback = None - elif mcOp is MCOp.Read_Command: - # assert dmemo.data[1] in TWO_BIT_PARAMS[MCOp.Read_Command.value],\ - # "self-test failed (bad constant(s))" + elif mcOp in (MCOp.Read_Command, MCOp.Write_Command): + # See OpenLCB Memory Configuration Standard, section 4.4 & 4.8 mcHeader = MemoryConfigurationHeader.fromMC2ndByte(dmemo.data[1]) addressBytes = dmemo.data[2:6] address = struct.unpack(">I", addressBytes)[0] @@ -644,89 +644,151 @@ def datagramReceivedListener(self, dmemo: DatagramReadMemo) -> bool: assert spaceIndex == 0 space = dmemo.data[6] offset = 1 - size = dmemo.data[6+offset] # requested read count - datagramBytes = bytearray([0x20, MCOp.Read_Reply.value]) + replyHighBits = MCOp.Read_Reply.value + if mcOp is MCOp.Write_Command: + replyHighBits = MCOp.Write_Reply.value + replyBytes = bytearray([0x20, replyHighBits]) # ^ byte1 (2nd) changed to error below if applicable assert isinstance(spaceIndex, int), \ (f"Logic missing, spaceIndex should be number here," - f" got {emit_cast(spaceIndex)}") + f" got {emit_cast(spaceIndex)}") assert isinstance(address, int) - datagramBytes += addressBytes + replyBytes += addressBytes if mcHeader.spaceIsCustom(): assert space is not None - datagramBytes.append(space) - assert len(datagramBytes) == 7, "space goes in index [6]" + replyBytes.append(space) + assert len(replyBytes) == 7, "space goes in index [6]" else: space = MemorySpace.fromIndex(MemorySpaceIndex(spaceIndex)) assert space is not None space = space.value spaceIndex = (space & 0b00000011) assert space is not None - datagramBytes[1] = \ - datagramBytes[1] | spaceIndex - assert len(datagramBytes) == 6, "should not have space in [6]" + replyBytes[1] = \ + replyBytes[1] | spaceIndex + assert len(replyBytes) == 6, "should not have space in [6]" assert space is not None, \ f"space not computed from datagram: {dmemo.data}" - payload = None - try: - segment = self.memory.getSegment(space) - if segment is None: - raise KeyError(f"space {space} is not valid") - if address >= segment.size(): - raise IndexError( - f"address {address} past end of {hex(space)}") - payload = self.memory.getSlice(space, address, size, - force=True) - # ^ force=True because reading past end is normal - # (pad with zeroes to indicate end) - except (IndexError, KeyError) as ex: - # address out of range (See Segment's getSlice) - datagramBytes[1] = MCOp.Read_Reply_Failure.value - message = None - if isinstance(ex, KeyError): - message = f"space {space} not valid" - elif isinstance(ex, KeyError): - if space is not None: - message = (f"address {hex(address)} not valid" - f" in space {hex(space)}") - else: - raise NotImplementedError( - f"space not computed from datagram: {dmemo.data}") - errorCode = 1 - errorBytes = struct.pack(">H", errorCode) - # FIXME: ^ Standard doesn't specify signed/unsigned - datagramBytes += errorBytes # 2 required bytes - messageBytes = None - if message is not None: - messageBytes = bytearray(message.encode("utf-8")) - messageBytes.append(0x00) # null terminator - datagramBytes += messageBytes - failedRequestedMemoryMemo = DatagramWriteMemo( + if mcOp is MCOp.Read_Command: + # assert dmemo.data[1] \ + # in TWO_BIT_PARAMS[MCOp.Read_Command.value],\ + # "self-test failed (bad constant(s))" + size = dmemo.data[6+offset] # requested read count + size = size & 0b01111111 # upper bit is reserved + + payload = None + try: + segment = self.memory.getSegment(space) + if segment is None: + raise KeyError(f"space {space} is not valid") + if address >= segment.size(): + raise IndexError( + f"address {address} past end of {hex(space)}") + payload = self.memory.getSlice(space, address, size, + force=True) + # ^ force=True because reading past end is normal + # (pad with zeroes to indicate end) + except (IndexError, KeyError) as ex: + # address out of range (See Segment's getSlice) + replyBytes[1] = MCOp.Read_Reply_Failure.value + message = None + if isinstance(ex, KeyError): + message = f"space {space} not valid" + elif isinstance(ex, KeyError): + if space is not None: + message = (f"address {hex(address)} not valid" + f" in space {hex(space)}") + else: + raise NotImplementedError( + f"space not computed from datagram:" + f" {dmemo.data}") + errorCode = 1 + failedMemo = MemoryService.failedMemo(mcOp, dmemo.srcID, + address, + space, errorCode, + message) + self.service.sendDatagram(failedMemo) + return True # handled (early, in error case) + assert payload is not None, \ + ("getSlice failed to raise IndexError or KeyError" + " on invalid request (expected bytes, got None)") + assert len(payload) <= 64 + replyBytes += payload + requestedMemoryMemo = DatagramWriteMemo( dmemo.srcID, - datagramBytes + replyBytes ) - self.service.sendDatagram(failedRequestedMemoryMemo) + # FIXME: Must we manually send Datagram Received OK first? + self.service.sendDatagram(requestedMemoryMemo) return True # handled (early in this case) - assert payload is not None, \ - ("getSlice failed to raise IndexError or KeyError" - " on invalid request (expected bytes, got None)") - assert len(payload) <= 64 - datagramBytes += payload - requestedMemoryMemo = DatagramWriteMemo( - dmemo.srcID, - datagramBytes - ) - # hexStrings = [] - # for b in datagramBytes: - # hexStrings.append(hex(b)) - # print(f"Sending read reply: {hexStrings}") - self.service.sendDatagram(requestedMemoryMemo) + elif mcOp is MCOp.Write_Command: + try: + segment = self.memory.getSegment(space) + if segment is None: + raise KeyError(f"space {space} is not valid") + if address >= segment.size(): + raise IndexError( + f"address {address} past end of {hex(space)}") + receivedPayload = dmemo.data[6+offset:] + if len(receivedPayload) < 1: + logger.warning(f"Got {mcOp} with payload size of 0") + self.memory.setSlice(space, address, receivedPayload) + # TODO: force=true is ok for read, not sure if ok for write + except (IndexError, KeyError) as ex: + # address out of range (See Segment's getSlice) + replyBytes[1] = MCOp.Read_Reply_Failure.value + message = None + if isinstance(ex, KeyError): + message = f"space {space} not valid" + elif isinstance(ex, KeyError): + if space is not None: + message = (f"address {hex(address)} not valid" + f" in space {hex(space)}") + else: + raise NotImplementedError( + f"space not computed from datagram:" + f" {dmemo.data}") + errorCode = 1 + failedMemo = MemoryService.failedMemo(mcOp, dmemo.srcID, + address, + space, errorCode, + message) + self.service.sendDatagram(failedMemo) + return True # handled (early, in error case) else: logger.error("Did not expect reply of type 0x{:02X}" .format(dmemo.data[1])) - return True + @staticmethod + def failedMemo(mcOp: MCOp, srcID: NodeID, address: int, space: int, + errorCode: int, message) -> DatagramWriteMemo: + assert isinstance(mcOp, MCOp) + assert isinstance(srcID, NodeID) + assert isinstance(address, int) + if isinstance(space, MemorySpaceIndex): + space = space.value + elif isinstance(space, MemorySpace): + space = space.value + assert isinstance(space, int) + assert isinstance(errorCode, int) + space = space & 0b00000011 + replyBytes = bytearray([0x20, mcOp.value | space]) + assert errorCode <= 65535 + errorBytes = struct.pack(">H", errorCode) + # FIXME: ^ Standard doesn't specify signed/unsigned + replyBytes += errorBytes # 2 required bytes + + messageBytes = None + if message is not None: + messageBytes = bytearray(message.encode("utf-8")) + messageBytes.append(0x00) # null terminator + replyBytes += messageBytes + return DatagramWriteMemo( + srcID, + replyBytes + ) + def requestMemoryWrite(self, memo: MemoryWriteMemo, stream: bool = False): # type: (MemoryWriteMemo, Optional[bool]) -> None """Request memory write.