From e2a5137107430ff02947c8aa761853f14f7f05a4 Mon Sep 17 00:00:00 2001 From: Olaf Targowski Date: Sun, 19 Apr 2026 09:21:50 +0200 Subject: [PATCH 1/4] Support listing remote files properly Make the server adhere to the spec regarding last-modified query parameter format (RFC 2822 instead of timestamp) and add support in the client. --- filetracker/client/client.py | 10 ++++++++++ filetracker/client/remote_data_store.py | 16 ++++++++++++++++ filetracker/servers/files.py | 10 +++++++--- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/filetracker/client/client.py b/filetracker/client/client.py index 2ecbf2c..ae34b5c 100644 --- a/filetracker/client/client.py +++ b/filetracker/client/client.py @@ -338,3 +338,13 @@ def list_local_files(self): if self.local_store: result.extend(self.local_store.list_files()) return result + + def list_remote_files(self, version_cutoff_timestamp=None, subpath=""): + """Returns list of all stored remote files under `subpath` + not newer than `version_cutoff_timestamp`. + + Each element of this list is just a string with the full path. + """ + if not self.remote_store: + return [] + return self.remote_store.list_files(version_cutoff_timestamp, subpath) diff --git a/filetracker/client/remote_data_store.py b/filetracker/client/remote_data_store.py index 04e54fe..4df111e 100644 --- a/filetracker/client/remote_data_store.py +++ b/filetracker/client/remote_data_store.py @@ -40,6 +40,9 @@ # The server supports deleting files SERVER_ACCEPTS_DELETE = 4 +# The server supports listing files +SERVER_ACCEPTS_LIST = 5 + _PROTOCOL_CAPABILITIES = { 1: [ SERVER_REQUIRES_VERSION_HEADER, @@ -48,6 +51,7 @@ SERVER_ACCEPTS_GZIP, SERVER_ACCEPTS_SHA256_DIGEST, SERVER_ACCEPTS_DELETE, + SERVER_ACCEPTS_LIST, ], } @@ -217,6 +221,18 @@ def delete_file(self, filename): response = requests.delete(url, headers=headers) response.raise_for_status() + @_verbose_http_errors + def list_files(self, version, subpath): + if not self._has_capability(SERVER_ACCEPTS_LIST): + return + url = self.base_url + '/list' + pathname2url(subpath) + url, headers = self._add_version_to_request(url, {}, version) + response = requests.get(url, headers=headers) + response.raise_for_status() + result = response.content.decode('utf-8').split('\n') + assert len(result.pop()) == 0 + return result + def _add_version_to_request(self, url, headers, version): """Adds version to either url or headers, depending on protocol.""" if self._has_capability(SERVER_REQUIRES_VERSION_HEADER): diff --git a/filetracker/servers/files.py b/filetracker/servers/files.py index 4938229..a07cd3f 100644 --- a/filetracker/servers/files.py +++ b/filetracker/servers/files.py @@ -156,9 +156,13 @@ def handle_list(self, environ, start_response): query_params = self.parse_query_params(environ) last_modified = query_params.get('last_modified', (None,))[0] - if not last_modified: - last_modified = int(time.time()) - + if last_modified: + last_modified = email.utils.parsedate_tz(last_modified) + last_modified = email.utils.mktime_tz(last_modified) + else: + last_modified = time.time() + last_modified = int(last_modified) + logger.debug('Handling GET /list/%s (@%d)', path, last_modified) root_dir = os.path.join(self.dir, path) From 85cded0de049c42a4e87a45dcade0cf1b5893728 Mon Sep 17 00:00:00 2001 From: Olaf Targowski Date: Sun, 19 Apr 2026 09:56:07 +0200 Subject: [PATCH 2/4] Loosen the spec for /list version cutoff No implementation was this strict anyway. --- PROTOCOL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PROTOCOL.md b/PROTOCOL.md index 9f64c4c..89c2f84 100644 --- a/PROTOCOL.md +++ b/PROTOCOL.md @@ -97,7 +97,7 @@ slashes. The response is plain-text, with one line for every file path. -Version cutoff must be specified in RFC 2822 format through +Version cutoff may be specified in RFC 2822 format through `?last_modified=` query parameter. Only files with modification time older than this version will be listed. From 1f5ecf371a355b1e9fd528c622edc9d941b30290 Mon Sep 17 00:00:00 2001 From: Olaf Targowski Date: Sun, 19 Apr 2026 10:48:45 +0200 Subject: [PATCH 3/4] Allow for getting absolute paths from subdir list --- filetracker/client/client.py | 4 ++-- filetracker/client/remote_data_store.py | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/filetracker/client/client.py b/filetracker/client/client.py index ae34b5c..9235677 100644 --- a/filetracker/client/client.py +++ b/filetracker/client/client.py @@ -339,7 +339,7 @@ def list_local_files(self): result.extend(self.local_store.list_files()) return result - def list_remote_files(self, version_cutoff_timestamp=None, subpath=""): + def list_remote_files(self, version_cutoff_timestamp=None, subpath="", absolute_paths=False): """Returns list of all stored remote files under `subpath` not newer than `version_cutoff_timestamp`. @@ -347,4 +347,4 @@ def list_remote_files(self, version_cutoff_timestamp=None, subpath=""): """ if not self.remote_store: return [] - return self.remote_store.list_files(version_cutoff_timestamp, subpath) + return self.remote_store.list_files(version_cutoff_timestamp, subpath, absolute_paths) diff --git a/filetracker/client/remote_data_store.py b/filetracker/client/remote_data_store.py index 4df111e..5b5c09e 100644 --- a/filetracker/client/remote_data_store.py +++ b/filetracker/client/remote_data_store.py @@ -222,7 +222,7 @@ def delete_file(self, filename): response.raise_for_status() @_verbose_http_errors - def list_files(self, version, subpath): + def list_files(self, version, subpath, absolute_paths): if not self._has_capability(SERVER_ACCEPTS_LIST): return url = self.base_url + '/list' + pathname2url(subpath) @@ -231,7 +231,11 @@ def list_files(self, version, subpath): response.raise_for_status() result = response.content.decode('utf-8').split('\n') assert len(result.pop()) == 0 - return result + if absolute_paths and subpath and subpath != "/": + prefix = subpath.rstrip("/").lstrip("/") + "/" + return [prefix + path for path in result] + else: + return result def _add_version_to_request(self, url, headers, version): """Adds version to either url or headers, depending on protocol.""" From 477d37f65cb716cbdc2a3bc2e89bb7efb52e2a77 Mon Sep 17 00:00:00 2001 From: Olaf Targowski Date: Sun, 19 Apr 2026 10:49:18 +0200 Subject: [PATCH 4/4] Add tests --- filetracker/tests/interaction_test.py | 29 +++++++++++++++++++++++++++ filetracker/tests/protocol_test.py | 26 ++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/filetracker/tests/interaction_test.py b/filetracker/tests/interaction_test.py index 93e9c8b..9581da2 100644 --- a/filetracker/tests/interaction_test.py +++ b/filetracker/tests/interaction_test.py @@ -148,6 +148,35 @@ def test_every_link_should_have_independent_version(self): self.assertNotEqual(version_a, version_b) + def test_list_remote_files(self): + src_file = os.path.join(self.temp_dir, 'older.txt') + with open(src_file, 'wb') as sf: + sf.write(b'these tests are so bad') + + self.client.put_file('/A@1', src_file) + self.client.put_file('/B@5', src_file) + self.client.put_file('/C/D@10', src_file) + + def check(expected, *args): + result_raw = self.client.list_remote_files(*args) + # Filter out files from other tests. + result = sorted(filter(lambda x: not x.endswith(".txt"), result_raw)) + self.assertEqual(result, sorted(expected)) + + check(["A", "B", "C/D"]) + check(["A"], 4) + check(["A", "B"], 5) + check(["A", "B"], 9) + check(["A", "B", "C/D"], 10) + with self.assertRaises(FiletrackerError): + check(["D"], 10, "C") + check(["D"], 10, "/C") + check(["D"], 10, "/C/") + # absolute_paths=True + check(["C/D"], 10, "/C", True) + check(["C/D"], 10, "/C/", True) + check([], 9, "/C") + def test_put_older_should_fail(self): """This test assumes file version is stored in mtime.""" src_file = os.path.join(self.temp_dir, 'older.txt') diff --git a/filetracker/tests/protocol_test.py b/filetracker/tests/protocol_test.py index df78b42..cf9fb88 100644 --- a/filetracker/tests/protocol_test.py +++ b/filetracker/tests/protocol_test.py @@ -5,6 +5,7 @@ from __future__ import print_function from multiprocessing import Process +import email.utils import os import shutil import tempfile @@ -67,6 +68,31 @@ def test_list_files_in_root_should_work(self): self.assertEqual(lines.count('list_a.txt'), 1) self.assertEqual(lines.count('list_b.txt'), 1) + def test_list_files_version_cutoff_should_work(self): + src_file = os.path.join(self.temp_dir, 'list.txt') + with open(src_file, 'wb') as sf: + sf.write(b'hello list') + + self.client.put_file('/A@1', src_file) + self.client.put_file('/B@100', src_file) + self.client.put_file('/C@200', src_file) + + def check(timestamp, expected): + date = email.utils.formatdate(timestamp) + params = {'last_modified': date} + res = requests.get('http://127.0.0.1:{}/list/'.format(_TEST_PORT_NUMBER), params=params) + self.assertEqual(res.status_code, 200) + # Filter out files from other tests. + lines = [l for l in res.text.split('\n') if l] + self.assertCountEqual(lines, expected) + + check(0, []) + check(1, ['A']) + check(99, ['A']) + check(100, ['A', 'B']) + check(199, ['A', 'B']) + check(200, ['A', 'B', 'C']) + def test_list_files_in_subdirectory_should_work(self): src_file = os.path.join(self.temp_dir, 'list_sub.txt') with open(src_file, 'wb') as sf: