diff --git a/CHANGELOG.md b/CHANGELOG.md index e138410b..776e9446 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/). ## [Unreleased] +### Fixed +- `from_solis`: accepts metadata listed before or after data + ## [3.6.4] ### Added diff --git a/WrightTools/data/_solis.py b/WrightTools/data/_solis.py index ae0018cc..33fe8cff 100644 --- a/WrightTools/data/_solis.py +++ b/WrightTools/data/_solis.py @@ -7,7 +7,7 @@ import pathlib import time import warnings - +import string import numpy as np from ._data import Data @@ -59,7 +59,6 @@ def from_Solis(filepath, name=None, parent=None, verbose=True) -> Data: """ # parse filepath - filestr = os.fspath(filepath) filepath = pathlib.Path(filepath) if not ".asc" in filepath.suffixes: @@ -69,52 +68,19 @@ def from_Solis(filepath, name=None, parent=None, verbose=True) -> Data: name = filepath.name.split(".")[0] # create data ds = DataSource(None) - f = ds.open(filestr, "rt") axis0 = [] arr = [] attrs = {} - line0 = f.readline().strip()[:-1] - line0 = [float(x) for x in line0.split(",")] # TODO: robust to space, tab, comma - axis0.append(line0.pop(0)) - arr.append(line0) - - def get_frames(f, arr, axis0): - axis0_written = False - while True: - line = f.readline().strip()[:-1] - if len(line) == 0: - break - else: - line = [float(x) for x in line.split(",")] - # signature of new frames is restart of axis0 - if not axis0_written and (line[0] == axis0[0]): - axis0_written = True - if axis0_written: - line.pop(0) - else: - axis0.append(line.pop(0)) - arr.append(line) - return arr, axis0 - + # extract the contents of the file + f = ds.open(str(filepath), "rt") + attrs = parse_metadata(f) arr, axis0 = get_frames(f, arr, axis0) nframes = len(arr) // len(axis0) - - i = 0 - while i < 3: - line = f.readline().strip() - if len(line) == 0: - i += 1 - else: - try: - key, val = line.split(":", 1) - except ValueError: - pass - else: - attrs[key.strip()] = val.strip() - + attrs.update(parse_metadata(f)) f.close() + # construct the data object try: created = attrs["Date and Time"] # is this UTC? created = time.strptime(created, "%a %b %d %H:%M:%S %Y") @@ -126,7 +92,7 @@ def get_frames(f, arr, axis0): f"{filepath.name} has no 'Date and Time' field: using file modified time instead: {created}" ) - kwargs = {"name": name, "kind": "Solis", "source": filestr, "created": created} + kwargs = {"name": name, "kind": "Solis", "source": str(filepath), "created": created} if parent is None: data = Data(**kwargs) else: @@ -184,3 +150,55 @@ def get_frames(f, arr, axis0): print(" axes: {0}".format(data.axis_names)) print(" shape: {0}".format(data.shape)) return data + + +def get_frames(f, arr, axis0): + axis0_written = False + line0 = f.readline().strip()[:-1] + line0 = [float(x) for x in line0.split(",")] # TODO: robust to space, tab, comma + axis0.append(line0.pop(0)) + arr.append(line0) + + while True: + line = f.readline().strip()[:-1] + # data ends at a blank line (potentially EOF) + if not line: + break + else: + line = [float(x) for x in line.split(",")] + # signature of new frames is restart of axis0 + if not axis0_written and (line[0] == axis0[0]): + axis0_written = True + if axis0_written: + line.pop(0) + else: + axis0.append(line.pop(0)) + arr.append(line) + + return arr, axis0 + + +def parse_metadata(f) -> dict: + """ + readlines for key value pairs until data or EOF is encountered + when readlines is finished, reverts to last valid line so no data is missed by subsequent readlines + """ + attrs = {} + + while True: + pos = f.tell() + line = f.readline() + if (not line) or line[0].isdigit(): # EOF or numeric data + break + if line[0] not in string.ascii_letters: + continue + line = line.strip() + try: + key, val = line.split(":", 1) + except ValueError: + print(f"could not parse line {line}") + else: + attrs[key.strip()] = val.strip() + + f.seek(pos) + return attrs diff --git a/tests/data/from_Solis.py b/tests/data/from_Solis.py index d8d2bfc2..7f22eab5 100644 --- a/tests/data/from_Solis.py +++ b/tests/data/from_Solis.py @@ -15,6 +15,8 @@ def test_wm_ypos_fluorescence_with_filter(): assert data.shape == (2560, 2160) assert data.axis_expressions == ("wm", "yindex") assert data.units == ("nm", None) + assert data.created.human == "2018-06-30 01:52:35" + data.close()