Add support for listing files in client and fix it in server

PROTOCOL.md

@@ -97,7 +97,7 @@ slashes.
 
 The response is plain-text, with one line for every file path.
 
-Version cutoff must be specified in RFC 2822 format through
+Version cutoff may be specified in RFC 2822 format through
 `?last_modified=` query parameter. Only files with modification time
 older than this version will be listed.
 

filetracker/client/client.py

@@ -338,3 +338,13 @@ def list_local_files(self):
         if self.local_store:
             result.extend(self.local_store.list_files())
         return result
+
+    def list_remote_files(self, version_cutoff_timestamp=None, subpath="", absolute_paths=False):
+        """Returns list of all stored remote files under `subpath`
+        not newer than `version_cutoff_timestamp`.
+
+        Each element of this list is just a string with the full path.
+        """
+        if not self.remote_store:
+            return []
+        return self.remote_store.list_files(version_cutoff_timestamp, subpath, absolute_paths)

filetracker/client/remote_data_store.py

@@ -40,6 +40,9 @@
 # The server supports deleting files
 SERVER_ACCEPTS_DELETE = 4
 
+# The server supports listing files
+SERVER_ACCEPTS_LIST = 5
+
 _PROTOCOL_CAPABILITIES = {
     1: [
         SERVER_REQUIRES_VERSION_HEADER,
@@ -48,6 +51,7 @@
         SERVER_ACCEPTS_GZIP,
         SERVER_ACCEPTS_SHA256_DIGEST,
         SERVER_ACCEPTS_DELETE,
+        SERVER_ACCEPTS_LIST,
     ],
 }
 
@@ -217,6 +221,22 @@ def delete_file(self, filename):
         response = requests.delete(url, headers=headers)
         response.raise_for_status()
 
+    @_verbose_http_errors
+    def list_files(self, version, subpath, absolute_paths):
+        if not self._has_capability(SERVER_ACCEPTS_LIST):
+            return
+        url = self.base_url + '/list' + pathname2url(subpath)
+        url, headers = self._add_version_to_request(url, {}, version)
+        response = requests.get(url, headers=headers)
+        response.raise_for_status()
+        result = response.content.decode('utf-8').split('\n')
+        assert len(result.pop()) == 0
+        if absolute_paths and subpath and subpath != "/":
+            prefix = subpath.rstrip("/").lstrip("/") + "/"
+            return [prefix + path for path in result]
+        else:
+            return result
+
     def _add_version_to_request(self, url, headers, version):
         """Adds version to either url or headers, depending on protocol."""
         if self._has_capability(SERVER_REQUIRES_VERSION_HEADER):

filetracker/servers/files.py

@@ -156,9 +156,13 @@ def handle_list(self, environ, start_response):
         query_params = self.parse_query_params(environ)
 
         last_modified = query_params.get('last_modified', (None,))[0]
-        if not last_modified:
-            last_modified = int(time.time())
-
+        if last_modified:
+            last_modified = email.utils.parsedate_tz(last_modified)
+            last_modified = email.utils.mktime_tz(last_modified)
+        else:
+            last_modified = time.time()
+        last_modified = int(last_modified)
+
         logger.debug('Handling GET /list/%s (@%d)', path, last_modified)
 
         root_dir = os.path.join(self.dir, path)

filetracker/tests/interaction_test.py

@@ -148,6 +148,35 @@ def test_every_link_should_have_independent_version(self):
 
         self.assertNotEqual(version_a, version_b)
 
+    def test_list_remote_files(self):
+        src_file = os.path.join(self.temp_dir, 'older.txt')
+        with open(src_file, 'wb') as sf:
+            sf.write(b'these tests are so bad')
+
+        self.client.put_file('/A@1', src_file)
+        self.client.put_file('/B@5', src_file)
+        self.client.put_file('/C/D@10', src_file)
+
+        def check(expected, *args):
+            result_raw = self.client.list_remote_files(*args)
+            # Filter out files from other tests.
+            result = sorted(filter(lambda x: not x.endswith(".txt"), result_raw))
+            self.assertEqual(result, sorted(expected))
+
+        check(["A", "B", "C/D"])
+        check(["A"], 4)
+        check(["A", "B"], 5)
+        check(["A", "B"], 9)
+        check(["A", "B", "C/D"], 10)
+        with self.assertRaises(FiletrackerError):
+            check(["D"], 10, "C")
+        check(["D"], 10, "/C")
+        check(["D"], 10, "/C/")
+        # absolute_paths=True
+        check(["C/D"], 10, "/C", True)
+        check(["C/D"], 10, "/C/", True)
+        check([], 9, "/C")
+
     def test_put_older_should_fail(self):
         """This test assumes file version is stored in mtime."""
         src_file = os.path.join(self.temp_dir, 'older.txt')

filetracker/tests/protocol_test.py

@@ -5,6 +5,7 @@
 from __future__ import print_function
 
 from multiprocessing import Process
+import email.utils
 import os
 import shutil
 import tempfile
@@ -67,6 +68,31 @@ def test_list_files_in_root_should_work(self):
         self.assertEqual(lines.count('list_a.txt'), 1)
         self.assertEqual(lines.count('list_b.txt'), 1)
 
+    def test_list_files_version_cutoff_should_work(self):
+        src_file = os.path.join(self.temp_dir, 'list.txt')
+        with open(src_file, 'wb') as sf:
+            sf.write(b'hello list')
+
+        self.client.put_file('/A@1', src_file)
+        self.client.put_file('/B@100', src_file)
+        self.client.put_file('/C@200', src_file)
+
+        def check(timestamp, expected):
+            date = email.utils.formatdate(timestamp)
+            params = {'last_modified': date}
+            res = requests.get('http://127.0.0.1:{}/list/'.format(_TEST_PORT_NUMBER), params=params)
+            self.assertEqual(res.status_code, 200)
+            # Filter out files from other tests.
+            lines = [l for l in res.text.split('\n') if l]
+            self.assertCountEqual(lines, expected)
+
+        check(0, [])
+        check(1, ['A'])
+        check(99, ['A'])
+        check(100, ['A', 'B'])
+        check(199, ['A', 'B'])
+        check(200, ['A', 'B', 'C'])
+
     def test_list_files_in_subdirectory_should_work(self):
         src_file = os.path.join(self.temp_dir, 'list_sub.txt')
         with open(src_file, 'wb') as sf: